4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2016 by Delphix. All rights reserved.
24 * Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
28 #include <sys/spa_impl.h>
30 #include <sys/vdev_impl.h>
31 #include <sys/vdev_trim.h>
32 #include <sys/metaslab_impl.h>
33 #include <sys/dsl_synctask.h>
35 #include <sys/dmu_tx.h>
36 #include <sys/arc_impl.h>
39 * TRIM is a feature which is used to notify a SSD that some previously
40 * written space is no longer allocated by the pool. This is useful because
41 * writes to a SSD must be performed to blocks which have first been erased.
42 * Ensuring the SSD always has a supply of erased blocks for new writes
43 * helps prevent the performance from deteriorating.
45 * There are two supported TRIM methods; manual and automatic.
49 * A manual TRIM is initiated by running the 'zpool trim' command. A single
50 * 'vdev_trim' thread is created for each leaf vdev, and it is responsible for
51 * managing that vdev TRIM process. This involves iterating over all the
52 * metaslabs, calculating the unallocated space ranges, and then issuing the
55 * While a metaslab is being actively trimmed it is not eligible to perform
56 * new allocations. After traversing all of the metaslabs the thread is
57 * terminated. Finally, both the requested options and current progress of
58 * the TRIM are regularly written to the pool. This allows the TRIM to be
59 * suspended and resumed as needed.
63 * An automatic TRIM is enabled by setting the 'autotrim' pool property
64 * to 'on'. When enabled, a `vdev_autotrim' thread is created for each
65 * top-level (not leaf) vdev in the pool. These threads perform the same
66 * core TRIM process as a manual TRIM, but with a few key differences.
68 * 1) Automatic TRIM happens continuously in the background and operates
69 * solely on recently freed blocks (ms_trim not ms_allocatable).
71 * 2) Each thread is associated with a top-level (not leaf) vdev. This has
72 * the benefit of simplifying the threading model, it makes it easier
73 * to coordinate administrative commands, and it ensures only a single
74 * metaslab is disabled at a time. Unlike manual TRIM, this means each
75 * 'vdev_autotrim' thread is responsible for issuing TRIM I/Os for its
78 * 3) There is no automatic TRIM progress information stored on disk, nor
79 * is it reported by 'zpool status'.
81 * While the automatic TRIM process is highly effective it is more likely
82 * than a manual TRIM to encounter tiny ranges. Ranges less than or equal to
83 * 'zfs_trim_extent_bytes_min' (32k) are considered too small to efficiently
84 * TRIM and are skipped. This means small amounts of freed space may not
85 * be automatically trimmed.
87 * Furthermore, devices with attached hot spares and devices being actively
88 * replaced are skipped. This is done to avoid adding additional stress to
89 * a potentially unhealthy device and to minimize the required rebuild time.
91 * For this reason it may be beneficial to occasionally manually TRIM a pool
92 * even when automatic TRIM is enabled.
96 * Maximum size of TRIM I/O, ranges will be chunked in to 128MiB lengths.
98 unsigned int zfs_trim_extent_bytes_max = 128 * 1024 * 1024;
101 * Minimum size of TRIM I/O, extents smaller than 32Kib will be skipped.
103 unsigned int zfs_trim_extent_bytes_min = 32 * 1024;
106 * Skip uninitialized metaslabs during the TRIM process. This option is
107 * useful for pools constructed from large thinly-provisioned devices where
108 * TRIM operations are slow. As a pool ages an increasing fraction of
109 * the pools metaslabs will be initialized progressively degrading the
110 * usefulness of this option. This setting is stored when starting a
111 * manual TRIM and will persist for the duration of the requested TRIM.
113 unsigned int zfs_trim_metaslab_skip = 0;
116 * Maximum number of queued TRIM I/Os per leaf vdev. The number of
117 * concurrent TRIM I/Os issued to the device is controlled by the
118 * zfs_vdev_trim_min_active and zfs_vdev_trim_max_active module options.
120 unsigned int zfs_trim_queue_limit = 10;
123 * The minimum number of transaction groups between automatic trims of a
124 * metaslab. This setting represents a trade-off between issuing more
125 * efficient TRIM operations, by allowing them to be aggregated longer,
126 * and issuing them promptly so the trimmed space is available. Note
127 * that this value is a minimum; metaslabs can be trimmed less frequently
128 * when there are a large number of ranges which need to be trimmed.
130 * Increasing this value will allow frees to be aggregated for a longer
131 * time. This can result is larger TRIM operations, and increased memory
132 * usage in order to track the ranges to be trimmed. Decreasing this value
133 * has the opposite effect. The default value of 32 was determined though
134 * testing to be a reasonable compromise.
136 unsigned int zfs_trim_txg_batch = 32;
139 * The trim_args are a control structure which describe how a leaf vdev
140 * should be trimmed. The core elements are the vdev, the metaslab being
141 * trimmed and a range tree containing the extents to TRIM. All provided
142 * ranges must be within the metaslab.
144 typedef struct trim_args {
146 * These fields are set by the caller of vdev_trim_ranges().
148 vdev_t *trim_vdev; /* Leaf vdev to TRIM */
149 metaslab_t *trim_msp; /* Disabled metaslab */
150 range_tree_t *trim_tree; /* TRIM ranges (in metaslab) */
151 trim_type_t trim_type; /* Manual or auto TRIM */
152 uint64_t trim_extent_bytes_max; /* Maximum TRIM I/O size */
153 uint64_t trim_extent_bytes_min; /* Minimum TRIM I/O size */
154 enum trim_flag trim_flags; /* TRIM flags (secure) */
157 * These fields are updated by vdev_trim_ranges().
159 hrtime_t trim_start_time; /* Start time */
160 uint64_t trim_bytes_done; /* Bytes trimmed */
164 * Determines whether a vdev_trim_thread() should be stopped.
167 vdev_trim_should_stop(vdev_t *vd)
169 return (vd->vdev_trim_exit_wanted || !vdev_writeable(vd) ||
170 vd->vdev_detached || vd->vdev_top->vdev_removing);
174 * Determines whether a vdev_autotrim_thread() should be stopped.
177 vdev_autotrim_should_stop(vdev_t *tvd)
179 return (tvd->vdev_autotrim_exit_wanted ||
180 !vdev_writeable(tvd) || tvd->vdev_removing ||
181 spa_get_autotrim(tvd->vdev_spa) == SPA_AUTOTRIM_OFF);
185 * The sync task for updating the on-disk state of a manual TRIM. This
186 * is scheduled by vdev_trim_change_state().
189 vdev_trim_zap_update_sync(void *arg, dmu_tx_t *tx)
192 * We pass in the guid instead of the vdev_t since the vdev may
193 * have been freed prior to the sync task being processed. This
194 * happens when a vdev is detached as we call spa_config_vdev_exit(),
195 * stop the trimming thread, schedule the sync task, and free
196 * the vdev. Later when the scheduled sync task is invoked, it would
197 * find that the vdev has been freed.
199 uint64_t guid = *(uint64_t *)arg;
200 uint64_t txg = dmu_tx_get_txg(tx);
201 kmem_free(arg, sizeof (uint64_t));
203 vdev_t *vd = spa_lookup_by_guid(tx->tx_pool->dp_spa, guid, B_FALSE);
204 if (vd == NULL || vd->vdev_top->vdev_removing || !vdev_is_concrete(vd))
207 uint64_t last_offset = vd->vdev_trim_offset[txg & TXG_MASK];
208 vd->vdev_trim_offset[txg & TXG_MASK] = 0;
210 VERIFY3U(vd->vdev_leaf_zap, !=, 0);
212 objset_t *mos = vd->vdev_spa->spa_meta_objset;
214 if (last_offset > 0 || vd->vdev_trim_last_offset == UINT64_MAX) {
216 if (vd->vdev_trim_last_offset == UINT64_MAX)
219 vd->vdev_trim_last_offset = last_offset;
220 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
221 VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
222 sizeof (last_offset), 1, &last_offset, tx));
225 if (vd->vdev_trim_action_time > 0) {
226 uint64_t val = (uint64_t)vd->vdev_trim_action_time;
227 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
228 VDEV_LEAF_ZAP_TRIM_ACTION_TIME, sizeof (val),
232 if (vd->vdev_trim_rate > 0) {
233 uint64_t rate = (uint64_t)vd->vdev_trim_rate;
235 if (rate == UINT64_MAX)
238 VERIFY0(zap_update(mos, vd->vdev_leaf_zap,
239 VDEV_LEAF_ZAP_TRIM_RATE, sizeof (rate), 1, &rate, tx));
242 uint64_t partial = vd->vdev_trim_partial;
243 if (partial == UINT64_MAX)
246 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
247 sizeof (partial), 1, &partial, tx));
249 uint64_t secure = vd->vdev_trim_secure;
250 if (secure == UINT64_MAX)
253 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
254 sizeof (secure), 1, &secure, tx));
257 uint64_t trim_state = vd->vdev_trim_state;
258 VERIFY0(zap_update(mos, vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
259 sizeof (trim_state), 1, &trim_state, tx));
263 * Update the on-disk state of a manual TRIM. This is called to request
264 * that a TRIM be started/suspended/canceled, or to change one of the
265 * TRIM options (partial, secure, rate).
268 vdev_trim_change_state(vdev_t *vd, vdev_trim_state_t new_state,
269 uint64_t rate, boolean_t partial, boolean_t secure)
271 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
272 spa_t *spa = vd->vdev_spa;
274 if (new_state == vd->vdev_trim_state)
278 * Copy the vd's guid, this will be freed by the sync task.
280 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
281 *guid = vd->vdev_guid;
284 * If we're suspending, then preserve the original start time.
286 if (vd->vdev_trim_state != VDEV_TRIM_SUSPENDED) {
287 vd->vdev_trim_action_time = gethrestime_sec();
291 * If we're activating, then preserve the requested rate and trim
292 * method. Setting the last offset and rate to UINT64_MAX is used
293 * as a sentinel to indicate they should be reset to default values.
295 if (new_state == VDEV_TRIM_ACTIVE) {
296 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE ||
297 vd->vdev_trim_state == VDEV_TRIM_CANCELED) {
298 vd->vdev_trim_last_offset = UINT64_MAX;
299 vd->vdev_trim_rate = UINT64_MAX;
300 vd->vdev_trim_partial = UINT64_MAX;
301 vd->vdev_trim_secure = UINT64_MAX;
305 vd->vdev_trim_rate = rate;
308 vd->vdev_trim_partial = partial;
311 vd->vdev_trim_secure = secure;
314 boolean_t resumed = !!(vd->vdev_trim_state == VDEV_TRIM_SUSPENDED);
315 vd->vdev_trim_state = new_state;
317 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
318 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
319 dsl_sync_task_nowait(spa_get_dsl(spa), vdev_trim_zap_update_sync,
323 case VDEV_TRIM_ACTIVE:
324 spa_event_notify(spa, vd, NULL,
325 resumed ? ESC_ZFS_TRIM_RESUME : ESC_ZFS_TRIM_START);
326 spa_history_log_internal(spa, "trim", tx,
327 "vdev=%s activated", vd->vdev_path);
329 case VDEV_TRIM_SUSPENDED:
330 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_SUSPEND);
331 spa_history_log_internal(spa, "trim", tx,
332 "vdev=%s suspended", vd->vdev_path);
334 case VDEV_TRIM_CANCELED:
335 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_CANCEL);
336 spa_history_log_internal(spa, "trim", tx,
337 "vdev=%s canceled", vd->vdev_path);
339 case VDEV_TRIM_COMPLETE:
340 spa_event_notify(spa, vd, NULL, ESC_ZFS_TRIM_FINISH);
341 spa_history_log_internal(spa, "trim", tx,
342 "vdev=%s complete", vd->vdev_path);
345 panic("invalid state %llu", (unsigned long long)new_state);
350 if (new_state != VDEV_TRIM_ACTIVE)
351 spa_notify_waiters(spa);
355 * The zio_done_func_t done callback for each manual TRIM issued. It is
356 * responsible for updating the TRIM stats, reissuing failed TRIM I/Os,
357 * and limiting the number of in flight TRIM I/Os.
360 vdev_trim_cb(zio_t *zio)
362 vdev_t *vd = zio->io_vd;
364 mutex_enter(&vd->vdev_trim_io_lock);
365 if (zio->io_error == ENXIO && !vdev_writeable(vd)) {
367 * The I/O failed because the vdev was unavailable; roll the
368 * last offset back. (This works because spa_sync waits on
369 * spa_txg_zio before it runs sync tasks.)
372 &vd->vdev_trim_offset[zio->io_txg & TXG_MASK];
373 *offset = MIN(*offset, zio->io_offset);
375 if (zio->io_error != 0) {
376 vd->vdev_stat.vs_trim_errors++;
377 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
378 0, 0, 0, 0, 1, zio->io_orig_size);
380 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_MANUAL,
381 1, zio->io_orig_size, 0, 0, 0, 0);
384 vd->vdev_trim_bytes_done += zio->io_orig_size;
387 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_MANUAL], >, 0);
388 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL]--;
389 cv_broadcast(&vd->vdev_trim_io_cv);
390 mutex_exit(&vd->vdev_trim_io_lock);
392 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
396 * The zio_done_func_t done callback for each automatic TRIM issued. It
397 * is responsible for updating the TRIM stats and limiting the number of
398 * in flight TRIM I/Os. Automatic TRIM I/Os are best effort and are
399 * never reissued on failure.
402 vdev_autotrim_cb(zio_t *zio)
404 vdev_t *vd = zio->io_vd;
406 mutex_enter(&vd->vdev_trim_io_lock);
408 if (zio->io_error != 0) {
409 vd->vdev_stat.vs_trim_errors++;
410 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
411 0, 0, 0, 0, 1, zio->io_orig_size);
413 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_AUTO,
414 1, zio->io_orig_size, 0, 0, 0, 0);
417 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_AUTO], >, 0);
418 vd->vdev_trim_inflight[TRIM_TYPE_AUTO]--;
419 cv_broadcast(&vd->vdev_trim_io_cv);
420 mutex_exit(&vd->vdev_trim_io_lock);
422 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
426 * The zio_done_func_t done callback for each TRIM issued via
427 * vdev_trim_simple(). It is responsible for updating the TRIM stats and
428 * limiting the number of in flight TRIM I/Os. Simple TRIM I/Os are best
429 * effort and are never reissued on failure.
432 vdev_trim_simple_cb(zio_t *zio)
434 vdev_t *vd = zio->io_vd;
436 mutex_enter(&vd->vdev_trim_io_lock);
438 if (zio->io_error != 0) {
439 vd->vdev_stat.vs_trim_errors++;
440 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,
441 0, 0, 0, 0, 1, zio->io_orig_size);
443 spa_iostats_trim_add(vd->vdev_spa, TRIM_TYPE_SIMPLE,
444 1, zio->io_orig_size, 0, 0, 0, 0);
447 ASSERT3U(vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE], >, 0);
448 vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE]--;
449 cv_broadcast(&vd->vdev_trim_io_cv);
450 mutex_exit(&vd->vdev_trim_io_lock);
452 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
455 * Returns the average trim rate in bytes/sec for the ta->trim_vdev.
458 vdev_trim_calculate_rate(trim_args_t *ta)
460 return (ta->trim_bytes_done * 1000 /
461 (NSEC2MSEC(gethrtime() - ta->trim_start_time) + 1));
465 * Issues a physical TRIM and takes care of rate limiting (bytes/sec)
466 * and number of concurrent TRIM I/Os.
469 vdev_trim_range(trim_args_t *ta, uint64_t start, uint64_t size)
471 vdev_t *vd = ta->trim_vdev;
472 spa_t *spa = vd->vdev_spa;
475 mutex_enter(&vd->vdev_trim_io_lock);
478 * Limit manual TRIM I/Os to the requested rate. This does not
479 * apply to automatic TRIM since no per vdev rate can be specified.
481 if (ta->trim_type == TRIM_TYPE_MANUAL) {
482 while (vd->vdev_trim_rate != 0 && !vdev_trim_should_stop(vd) &&
483 vdev_trim_calculate_rate(ta) > vd->vdev_trim_rate) {
484 cv_timedwait_idle(&vd->vdev_trim_io_cv,
485 &vd->vdev_trim_io_lock, ddi_get_lbolt() +
489 ta->trim_bytes_done += size;
491 /* Limit in flight trimming I/Os */
492 while (vd->vdev_trim_inflight[0] + vd->vdev_trim_inflight[1] +
493 vd->vdev_trim_inflight[2] >= zfs_trim_queue_limit) {
494 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
496 vd->vdev_trim_inflight[ta->trim_type]++;
497 mutex_exit(&vd->vdev_trim_io_lock);
499 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir);
500 VERIFY0(dmu_tx_assign(tx, TXG_WAIT));
501 uint64_t txg = dmu_tx_get_txg(tx);
503 spa_config_enter(spa, SCL_STATE_ALL, vd, RW_READER);
504 mutex_enter(&vd->vdev_trim_lock);
506 if (ta->trim_type == TRIM_TYPE_MANUAL &&
507 vd->vdev_trim_offset[txg & TXG_MASK] == 0) {
508 uint64_t *guid = kmem_zalloc(sizeof (uint64_t), KM_SLEEP);
509 *guid = vd->vdev_guid;
511 /* This is the first write of this txg. */
512 dsl_sync_task_nowait(spa_get_dsl(spa),
513 vdev_trim_zap_update_sync, guid, tx);
517 * We know the vdev_t will still be around since all consumers of
518 * vdev_free must stop the trimming first.
520 if ((ta->trim_type == TRIM_TYPE_MANUAL &&
521 vdev_trim_should_stop(vd)) ||
522 (ta->trim_type == TRIM_TYPE_AUTO &&
523 vdev_autotrim_should_stop(vd->vdev_top))) {
524 mutex_enter(&vd->vdev_trim_io_lock);
525 vd->vdev_trim_inflight[ta->trim_type]--;
526 mutex_exit(&vd->vdev_trim_io_lock);
527 spa_config_exit(vd->vdev_spa, SCL_STATE_ALL, vd);
528 mutex_exit(&vd->vdev_trim_lock);
530 return (SET_ERROR(EINTR));
532 mutex_exit(&vd->vdev_trim_lock);
534 if (ta->trim_type == TRIM_TYPE_MANUAL)
535 vd->vdev_trim_offset[txg & TXG_MASK] = start + size;
537 if (ta->trim_type == TRIM_TYPE_MANUAL) {
539 } else if (ta->trim_type == TRIM_TYPE_AUTO) {
540 cb = vdev_autotrim_cb;
542 cb = vdev_trim_simple_cb;
545 zio_nowait(zio_trim(spa->spa_txg_zio[txg & TXG_MASK], vd,
546 start, size, cb, NULL, ZIO_PRIORITY_TRIM, ZIO_FLAG_CANFAIL,
548 /* vdev_trim_cb and vdev_autotrim_cb release SCL_STATE_ALL */
556 * Issues TRIM I/Os for all ranges in the provided ta->trim_tree range tree.
557 * Additional parameters describing how the TRIM should be performed must
558 * be set in the trim_args structure. See the trim_args definition for
559 * additional information.
562 vdev_trim_ranges(trim_args_t *ta)
564 vdev_t *vd = ta->trim_vdev;
565 zfs_btree_t *t = &ta->trim_tree->rt_root;
566 zfs_btree_index_t idx;
567 uint64_t extent_bytes_max = ta->trim_extent_bytes_max;
568 uint64_t extent_bytes_min = ta->trim_extent_bytes_min;
569 spa_t *spa = vd->vdev_spa;
571 ta->trim_start_time = gethrtime();
572 ta->trim_bytes_done = 0;
574 for (range_seg_t *rs = zfs_btree_first(t, &idx); rs != NULL;
575 rs = zfs_btree_next(t, &idx, &idx)) {
576 uint64_t size = rs_get_end(rs, ta->trim_tree) - rs_get_start(rs,
579 if (extent_bytes_min && size < extent_bytes_min) {
580 spa_iostats_trim_add(spa, ta->trim_type,
581 0, 0, 1, size, 0, 0);
585 /* Split range into legally-sized physical chunks */
586 uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1;
588 for (uint64_t w = 0; w < writes_required; w++) {
591 error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE +
592 rs_get_start(rs, ta->trim_tree) +
593 (w *extent_bytes_max), MIN(size -
594 (w * extent_bytes_max), extent_bytes_max));
605 * Calculates the completion percentage of a manual TRIM.
608 vdev_trim_calculate_progress(vdev_t *vd)
610 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
611 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
612 ASSERT(vd->vdev_leaf_zap != 0);
614 vd->vdev_trim_bytes_est = 0;
615 vd->vdev_trim_bytes_done = 0;
617 for (uint64_t i = 0; i < vd->vdev_top->vdev_ms_count; i++) {
618 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
619 mutex_enter(&msp->ms_lock);
621 uint64_t ms_free = msp->ms_size -
622 metaslab_allocated_space(msp);
624 if (vd->vdev_top->vdev_ops == &vdev_raidz_ops)
625 ms_free /= vd->vdev_top->vdev_children;
628 * Convert the metaslab range to a physical range
629 * on our vdev. We use this to determine if we are
630 * in the middle of this metaslab range.
632 range_seg64_t logical_rs, physical_rs;
633 logical_rs.rs_start = msp->ms_start;
634 logical_rs.rs_end = msp->ms_start + msp->ms_size;
635 vdev_xlate(vd, &logical_rs, &physical_rs);
637 if (vd->vdev_trim_last_offset <= physical_rs.rs_start) {
638 vd->vdev_trim_bytes_est += ms_free;
639 mutex_exit(&msp->ms_lock);
641 } else if (vd->vdev_trim_last_offset > physical_rs.rs_end) {
642 vd->vdev_trim_bytes_done += ms_free;
643 vd->vdev_trim_bytes_est += ms_free;
644 mutex_exit(&msp->ms_lock);
649 * If we get here, we're in the middle of trimming this
650 * metaslab. Load it and walk the free tree for more
651 * accurate progress estimation.
653 VERIFY0(metaslab_load(msp));
655 range_tree_t *rt = msp->ms_allocatable;
656 zfs_btree_t *bt = &rt->rt_root;
657 zfs_btree_index_t idx;
658 for (range_seg_t *rs = zfs_btree_first(bt, &idx);
659 rs != NULL; rs = zfs_btree_next(bt, &idx, &idx)) {
660 logical_rs.rs_start = rs_get_start(rs, rt);
661 logical_rs.rs_end = rs_get_end(rs, rt);
662 vdev_xlate(vd, &logical_rs, &physical_rs);
664 uint64_t size = physical_rs.rs_end -
665 physical_rs.rs_start;
666 vd->vdev_trim_bytes_est += size;
667 if (vd->vdev_trim_last_offset >= physical_rs.rs_end) {
668 vd->vdev_trim_bytes_done += size;
669 } else if (vd->vdev_trim_last_offset >
670 physical_rs.rs_start &&
671 vd->vdev_trim_last_offset <=
672 physical_rs.rs_end) {
673 vd->vdev_trim_bytes_done +=
674 vd->vdev_trim_last_offset -
675 physical_rs.rs_start;
678 mutex_exit(&msp->ms_lock);
683 * Load from disk the vdev's manual TRIM information. This includes the
684 * state, progress, and options provided when initiating the manual TRIM.
687 vdev_trim_load(vdev_t *vd)
690 ASSERT(spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_READER) ||
691 spa_config_held(vd->vdev_spa, SCL_CONFIG, RW_WRITER));
692 ASSERT(vd->vdev_leaf_zap != 0);
694 if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE ||
695 vd->vdev_trim_state == VDEV_TRIM_SUSPENDED) {
696 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
697 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_LAST_OFFSET,
698 sizeof (vd->vdev_trim_last_offset), 1,
699 &vd->vdev_trim_last_offset);
701 vd->vdev_trim_last_offset = 0;
706 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
707 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_RATE,
708 sizeof (vd->vdev_trim_rate), 1,
709 &vd->vdev_trim_rate);
711 vd->vdev_trim_rate = 0;
717 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
718 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_PARTIAL,
719 sizeof (vd->vdev_trim_partial), 1,
720 &vd->vdev_trim_partial);
722 vd->vdev_trim_partial = 0;
728 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
729 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_SECURE,
730 sizeof (vd->vdev_trim_secure), 1,
731 &vd->vdev_trim_secure);
733 vd->vdev_trim_secure = 0;
739 vdev_trim_calculate_progress(vd);
745 * Convert the logical range into a physical range and add it to the
746 * range tree passed in the trim_args_t.
749 vdev_trim_range_add(void *arg, uint64_t start, uint64_t size)
751 trim_args_t *ta = arg;
752 vdev_t *vd = ta->trim_vdev;
753 range_seg64_t logical_rs, physical_rs;
754 logical_rs.rs_start = start;
755 logical_rs.rs_end = start + size;
758 * Every range to be trimmed must be part of ms_allocatable.
759 * When ZFS_DEBUG_TRIM is set load the metaslab to verify this
760 * is always the case.
762 if (zfs_flags & ZFS_DEBUG_TRIM) {
763 metaslab_t *msp = ta->trim_msp;
764 VERIFY0(metaslab_load(msp));
765 VERIFY3B(msp->ms_loaded, ==, B_TRUE);
766 VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
769 ASSERT(vd->vdev_ops->vdev_op_leaf);
770 vdev_xlate(vd, &logical_rs, &physical_rs);
772 IMPLY(vd->vdev_top == vd,
773 logical_rs.rs_start == physical_rs.rs_start);
774 IMPLY(vd->vdev_top == vd,
775 logical_rs.rs_end == physical_rs.rs_end);
778 * Only a manual trim will be traversing the vdev sequentially.
779 * For an auto trim all valid ranges should be added.
781 if (ta->trim_type == TRIM_TYPE_MANUAL) {
783 /* Only add segments that we have not visited yet */
784 if (physical_rs.rs_end <= vd->vdev_trim_last_offset)
787 /* Pick up where we left off mid-range. */
788 if (vd->vdev_trim_last_offset > physical_rs.rs_start) {
789 ASSERT3U(physical_rs.rs_end, >,
790 vd->vdev_trim_last_offset);
791 physical_rs.rs_start = vd->vdev_trim_last_offset;
795 ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
798 * With raidz, it's possible that the logical range does not live on
799 * this leaf vdev. We only add the physical range to this vdev's if it
800 * has a length greater than 0.
802 if (physical_rs.rs_end > physical_rs.rs_start) {
803 range_tree_add(ta->trim_tree, physical_rs.rs_start,
804 physical_rs.rs_end - physical_rs.rs_start);
806 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
811 * Each manual TRIM thread is responsible for trimming the unallocated
812 * space for each leaf vdev. This is accomplished by sequentially iterating
813 * over its top-level metaslabs and issuing TRIM I/O for the space described
814 * by its ms_allocatable. While a metaslab is undergoing trimming it is
815 * not eligible for new allocations.
818 vdev_trim_thread(void *arg)
821 spa_t *spa = vd->vdev_spa;
826 * The VDEV_LEAF_ZAP_TRIM_* entries may have been updated by
827 * vdev_trim(). Wait for the updated values to be reflected
828 * in the zap in order to start with the requested settings.
830 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
832 ASSERT(vdev_is_concrete(vd));
833 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
835 vd->vdev_trim_last_offset = 0;
836 vd->vdev_trim_rate = 0;
837 vd->vdev_trim_partial = 0;
838 vd->vdev_trim_secure = 0;
840 VERIFY0(vdev_trim_load(vd));
843 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
844 ta.trim_extent_bytes_min = zfs_trim_extent_bytes_min;
845 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
846 ta.trim_type = TRIM_TYPE_MANUAL;
850 * When a secure TRIM has been requested infer that the intent
851 * is that everything must be trimmed. Override the default
852 * minimum TRIM size to prevent ranges from being skipped.
854 if (vd->vdev_trim_secure) {
855 ta.trim_flags |= ZIO_TRIM_SECURE;
856 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
859 uint64_t ms_count = 0;
860 for (uint64_t i = 0; !vd->vdev_detached &&
861 i < vd->vdev_top->vdev_ms_count; i++) {
862 metaslab_t *msp = vd->vdev_top->vdev_ms[i];
865 * If we've expanded the top-level vdev or it's our
866 * first pass, calculate our progress.
868 if (vd->vdev_top->vdev_ms_count != ms_count) {
869 vdev_trim_calculate_progress(vd);
870 ms_count = vd->vdev_top->vdev_ms_count;
873 spa_config_exit(spa, SCL_CONFIG, FTAG);
874 metaslab_disable(msp);
875 mutex_enter(&msp->ms_lock);
876 VERIFY0(metaslab_load(msp));
879 * If a partial TRIM was requested skip metaslabs which have
880 * never been initialized and thus have never been written.
882 if (msp->ms_sm == NULL && vd->vdev_trim_partial) {
883 mutex_exit(&msp->ms_lock);
884 metaslab_enable(msp, B_FALSE, B_FALSE);
885 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
886 vdev_trim_calculate_progress(vd);
891 range_tree_walk(msp->ms_allocatable, vdev_trim_range_add, &ta);
892 range_tree_vacate(msp->ms_trim, NULL, NULL);
893 mutex_exit(&msp->ms_lock);
895 error = vdev_trim_ranges(&ta);
896 metaslab_enable(msp, B_TRUE, B_FALSE);
897 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
899 range_tree_vacate(ta.trim_tree, NULL, NULL);
904 spa_config_exit(spa, SCL_CONFIG, FTAG);
905 mutex_enter(&vd->vdev_trim_io_lock);
906 while (vd->vdev_trim_inflight[0] > 0) {
907 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
909 mutex_exit(&vd->vdev_trim_io_lock);
911 range_tree_destroy(ta.trim_tree);
913 mutex_enter(&vd->vdev_trim_lock);
914 if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
915 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
916 vd->vdev_trim_rate, vd->vdev_trim_partial,
917 vd->vdev_trim_secure);
919 ASSERT(vd->vdev_trim_thread != NULL || vd->vdev_trim_inflight[0] == 0);
922 * Drop the vdev_trim_lock while we sync out the txg since it's
923 * possible that a device might be trying to come online and must
924 * check to see if it needs to restart a trim. That thread will be
925 * holding the spa_config_lock which would prevent the txg_wait_synced
928 mutex_exit(&vd->vdev_trim_lock);
929 txg_wait_synced(spa_get_dsl(spa), 0);
930 mutex_enter(&vd->vdev_trim_lock);
932 vd->vdev_trim_thread = NULL;
933 cv_broadcast(&vd->vdev_trim_cv);
934 mutex_exit(&vd->vdev_trim_lock);
940 * Initiates a manual TRIM for the vdev_t. Callers must hold vdev_trim_lock,
941 * the vdev_t must be a leaf and cannot already be manually trimming.
944 vdev_trim(vdev_t *vd, uint64_t rate, boolean_t partial, boolean_t secure)
946 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
947 ASSERT(vd->vdev_ops->vdev_op_leaf);
948 ASSERT(vdev_is_concrete(vd));
949 ASSERT3P(vd->vdev_trim_thread, ==, NULL);
950 ASSERT(!vd->vdev_detached);
951 ASSERT(!vd->vdev_trim_exit_wanted);
952 ASSERT(!vd->vdev_top->vdev_removing);
954 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, rate, partial, secure);
955 vd->vdev_trim_thread = thread_create(NULL, 0,
956 vdev_trim_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
960 * Wait for the trimming thread to be terminated (canceled or stopped).
963 vdev_trim_stop_wait_impl(vdev_t *vd)
965 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
967 while (vd->vdev_trim_thread != NULL)
968 cv_wait(&vd->vdev_trim_cv, &vd->vdev_trim_lock);
970 ASSERT3P(vd->vdev_trim_thread, ==, NULL);
971 vd->vdev_trim_exit_wanted = B_FALSE;
975 * Wait for vdev trim threads which were listed to cleanly exit.
978 vdev_trim_stop_wait(spa_t *spa, list_t *vd_list)
982 ASSERT(MUTEX_HELD(&spa_namespace_lock));
984 while ((vd = list_remove_head(vd_list)) != NULL) {
985 mutex_enter(&vd->vdev_trim_lock);
986 vdev_trim_stop_wait_impl(vd);
987 mutex_exit(&vd->vdev_trim_lock);
992 * Stop trimming a device, with the resultant trimming state being tgt_state.
993 * For blocking behavior pass NULL for vd_list. Otherwise, when a list_t is
994 * provided the stopping vdev is inserted in to the list. Callers are then
995 * required to call vdev_trim_stop_wait() to block for all the trim threads
996 * to exit. The caller must hold vdev_trim_lock and must not be writing to
997 * the spa config, as the trimming thread may try to enter the config as a
998 * reader before exiting.
1001 vdev_trim_stop(vdev_t *vd, vdev_trim_state_t tgt_state, list_t *vd_list)
1003 ASSERT(!spa_config_held(vd->vdev_spa, SCL_CONFIG|SCL_STATE, RW_WRITER));
1004 ASSERT(MUTEX_HELD(&vd->vdev_trim_lock));
1005 ASSERT(vd->vdev_ops->vdev_op_leaf);
1006 ASSERT(vdev_is_concrete(vd));
1009 * Allow cancel requests to proceed even if the trim thread has
1012 if (vd->vdev_trim_thread == NULL && tgt_state != VDEV_TRIM_CANCELED)
1015 vdev_trim_change_state(vd, tgt_state, 0, 0, 0);
1016 vd->vdev_trim_exit_wanted = B_TRUE;
1018 if (vd_list == NULL) {
1019 vdev_trim_stop_wait_impl(vd);
1021 ASSERT(MUTEX_HELD(&spa_namespace_lock));
1022 list_insert_tail(vd_list, vd);
1027 * Requests that all listed vdevs stop trimming.
1030 vdev_trim_stop_all_impl(vdev_t *vd, vdev_trim_state_t tgt_state,
1033 if (vd->vdev_ops->vdev_op_leaf && vdev_is_concrete(vd)) {
1034 mutex_enter(&vd->vdev_trim_lock);
1035 vdev_trim_stop(vd, tgt_state, vd_list);
1036 mutex_exit(&vd->vdev_trim_lock);
1040 for (uint64_t i = 0; i < vd->vdev_children; i++) {
1041 vdev_trim_stop_all_impl(vd->vdev_child[i], tgt_state,
1047 * Convenience function to stop trimming of a vdev tree and set all trim
1048 * thread pointers to NULL.
1051 vdev_trim_stop_all(vdev_t *vd, vdev_trim_state_t tgt_state)
1053 spa_t *spa = vd->vdev_spa;
1057 ASSERT(MUTEX_HELD(&spa_namespace_lock));
1059 list_create(&vd_list, sizeof (vdev_t),
1060 offsetof(vdev_t, vdev_trim_node));
1062 vdev_trim_stop_all_impl(vd, tgt_state, &vd_list);
1065 * Iterate over cache devices and request stop trimming the
1066 * whole device in case we export the pool or remove the cache
1067 * device prematurely.
1069 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1070 vd_l2cache = spa->spa_l2cache.sav_vdevs[i];
1071 vdev_trim_stop_all_impl(vd_l2cache, tgt_state, &vd_list);
1074 vdev_trim_stop_wait(spa, &vd_list);
1076 if (vd->vdev_spa->spa_sync_on) {
1077 /* Make sure that our state has been synced to disk */
1078 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1081 list_destroy(&vd_list);
1085 * Conditionally restarts a manual TRIM given its on-disk state.
1088 vdev_trim_restart(vdev_t *vd)
1090 ASSERT(MUTEX_HELD(&spa_namespace_lock));
1091 ASSERT(!spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER));
1093 if (vd->vdev_leaf_zap != 0) {
1094 mutex_enter(&vd->vdev_trim_lock);
1095 uint64_t trim_state = VDEV_TRIM_NONE;
1096 int err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1097 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_STATE,
1098 sizeof (trim_state), 1, &trim_state);
1099 ASSERT(err == 0 || err == ENOENT);
1100 vd->vdev_trim_state = trim_state;
1102 uint64_t timestamp = 0;
1103 err = zap_lookup(vd->vdev_spa->spa_meta_objset,
1104 vd->vdev_leaf_zap, VDEV_LEAF_ZAP_TRIM_ACTION_TIME,
1105 sizeof (timestamp), 1, ×tamp);
1106 ASSERT(err == 0 || err == ENOENT);
1107 vd->vdev_trim_action_time = timestamp;
1109 if (vd->vdev_trim_state == VDEV_TRIM_SUSPENDED ||
1111 /* load progress for reporting, but don't resume */
1112 VERIFY0(vdev_trim_load(vd));
1113 } else if (vd->vdev_trim_state == VDEV_TRIM_ACTIVE &&
1114 vdev_writeable(vd) && !vd->vdev_top->vdev_removing &&
1115 vd->vdev_trim_thread == NULL) {
1116 VERIFY0(vdev_trim_load(vd));
1117 vdev_trim(vd, vd->vdev_trim_rate,
1118 vd->vdev_trim_partial, vd->vdev_trim_secure);
1121 mutex_exit(&vd->vdev_trim_lock);
1124 for (uint64_t i = 0; i < vd->vdev_children; i++) {
1125 vdev_trim_restart(vd->vdev_child[i]);
1130 * Used by the automatic TRIM when ZFS_DEBUG_TRIM is set to verify that
1131 * every TRIM range is contained within ms_allocatable.
1134 vdev_trim_range_verify(void *arg, uint64_t start, uint64_t size)
1136 trim_args_t *ta = arg;
1137 metaslab_t *msp = ta->trim_msp;
1139 VERIFY3B(msp->ms_loaded, ==, B_TRUE);
1140 VERIFY3U(msp->ms_disabled, >, 0);
1141 VERIFY(range_tree_contains(msp->ms_allocatable, start, size));
1145 * Each automatic TRIM thread is responsible for managing the trimming of a
1146 * top-level vdev in the pool. No automatic TRIM state is maintained on-disk.
1148 * N.B. This behavior is different from a manual TRIM where a thread
1149 * is created for each leaf vdev, instead of each top-level vdev.
1152 vdev_autotrim_thread(void *arg)
1155 spa_t *spa = vd->vdev_spa;
1158 mutex_enter(&vd->vdev_autotrim_lock);
1159 ASSERT3P(vd->vdev_top, ==, vd);
1160 ASSERT3P(vd->vdev_autotrim_thread, !=, NULL);
1161 mutex_exit(&vd->vdev_autotrim_lock);
1162 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1164 uint64_t extent_bytes_max = zfs_trim_extent_bytes_max;
1165 uint64_t extent_bytes_min = zfs_trim_extent_bytes_min;
1167 while (!vdev_autotrim_should_stop(vd)) {
1168 int txgs_per_trim = MAX(zfs_trim_txg_batch, 1);
1169 boolean_t issued_trim = B_FALSE;
1172 * All of the metaslabs are divided in to groups of size
1173 * num_metaslabs / zfs_trim_txg_batch. Each of these groups
1174 * is composed of metaslabs which are spread evenly over the
1177 * For example, when zfs_trim_txg_batch = 32 (default) then
1178 * group 0 will contain metaslabs 0, 32, 64, ...;
1179 * group 1 will contain metaslabs 1, 33, 65, ...;
1180 * group 2 will contain metaslabs 2, 34, 66, ...; and so on.
1182 * On each pass through the while() loop one of these groups
1183 * is selected. This is accomplished by using a shift value
1184 * to select the starting metaslab, then striding over the
1185 * metaslabs using the zfs_trim_txg_batch size. This is
1186 * done to accomplish two things.
1188 * 1) By dividing the metaslabs in to groups, and making sure
1189 * that each group takes a minimum of one txg to process.
1190 * Then zfs_trim_txg_batch controls the minimum number of
1191 * txgs which must occur before a metaslab is revisited.
1193 * 2) Selecting non-consecutive metaslabs distributes the
1194 * TRIM commands for a group evenly over the entire device.
1195 * This can be advantageous for certain types of devices.
1197 for (uint64_t i = shift % txgs_per_trim; i < vd->vdev_ms_count;
1198 i += txgs_per_trim) {
1199 metaslab_t *msp = vd->vdev_ms[i];
1200 range_tree_t *trim_tree;
1202 spa_config_exit(spa, SCL_CONFIG, FTAG);
1203 metaslab_disable(msp);
1204 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1206 mutex_enter(&msp->ms_lock);
1209 * Skip the metaslab when it has never been allocated
1210 * or when there are no recent frees to trim.
1212 if (msp->ms_sm == NULL ||
1213 range_tree_is_empty(msp->ms_trim)) {
1214 mutex_exit(&msp->ms_lock);
1215 metaslab_enable(msp, B_FALSE, B_FALSE);
1220 * Skip the metaslab when it has already been disabled.
1221 * This may happen when a manual TRIM or initialize
1222 * operation is running concurrently. In the case
1223 * of a manual TRIM, the ms_trim tree will have been
1224 * vacated. Only ranges added after the manual TRIM
1225 * disabled the metaslab will be included in the tree.
1226 * These will be processed when the automatic TRIM
1227 * next revisits this metaslab.
1229 if (msp->ms_disabled > 1) {
1230 mutex_exit(&msp->ms_lock);
1231 metaslab_enable(msp, B_FALSE, B_FALSE);
1236 * Allocate an empty range tree which is swapped in
1237 * for the existing ms_trim tree while it is processed.
1239 trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL,
1241 range_tree_swap(&msp->ms_trim, &trim_tree);
1242 ASSERT(range_tree_is_empty(msp->ms_trim));
1245 * There are two cases when constructing the per-vdev
1246 * trim trees for a metaslab. If the top-level vdev
1247 * has no children then it is also a leaf and should
1248 * be trimmed. Otherwise our children are the leaves
1249 * and a trim tree should be constructed for each.
1252 uint64_t children = vd->vdev_children;
1253 if (children == 0) {
1255 tap = kmem_zalloc(sizeof (trim_args_t) *
1256 children, KM_SLEEP);
1257 tap[0].trim_vdev = vd;
1259 tap = kmem_zalloc(sizeof (trim_args_t) *
1260 children, KM_SLEEP);
1262 for (uint64_t c = 0; c < children; c++) {
1263 tap[c].trim_vdev = vd->vdev_child[c];
1267 for (uint64_t c = 0; c < children; c++) {
1268 trim_args_t *ta = &tap[c];
1269 vdev_t *cvd = ta->trim_vdev;
1272 ta->trim_extent_bytes_max = extent_bytes_max;
1273 ta->trim_extent_bytes_min = extent_bytes_min;
1274 ta->trim_type = TRIM_TYPE_AUTO;
1277 if (cvd->vdev_detached ||
1278 !vdev_writeable(cvd) ||
1279 !cvd->vdev_has_trim ||
1280 cvd->vdev_trim_thread != NULL) {
1285 * When a device has an attached hot spare, or
1286 * is being replaced it will not be trimmed.
1287 * This is done to avoid adding additional
1288 * stress to a potentially unhealthy device,
1289 * and to minimize the required rebuild time.
1291 if (!cvd->vdev_ops->vdev_op_leaf)
1294 ta->trim_tree = range_tree_create(NULL,
1295 RANGE_SEG64, NULL, 0, 0);
1296 range_tree_walk(trim_tree,
1297 vdev_trim_range_add, ta);
1300 mutex_exit(&msp->ms_lock);
1301 spa_config_exit(spa, SCL_CONFIG, FTAG);
1304 * Issue the TRIM I/Os for all ranges covered by the
1305 * TRIM trees. These ranges are safe to TRIM because
1306 * no new allocations will be performed until the call
1307 * to metaslab_enabled() below.
1309 for (uint64_t c = 0; c < children; c++) {
1310 trim_args_t *ta = &tap[c];
1313 * Always yield to a manual TRIM if one has
1314 * been started for the child vdev.
1316 if (ta->trim_tree == NULL ||
1317 ta->trim_vdev->vdev_trim_thread != NULL) {
1322 * After this point metaslab_enable() must be
1323 * called with the sync flag set. This is done
1324 * here because vdev_trim_ranges() is allowed
1325 * to be interrupted (EINTR) before issuing all
1326 * of the required TRIM I/Os.
1328 issued_trim = B_TRUE;
1330 int error = vdev_trim_ranges(ta);
1336 * Verify every range which was trimmed is still
1337 * contained within the ms_allocatable tree.
1339 if (zfs_flags & ZFS_DEBUG_TRIM) {
1340 mutex_enter(&msp->ms_lock);
1341 VERIFY0(metaslab_load(msp));
1342 VERIFY3P(tap[0].trim_msp, ==, msp);
1343 range_tree_walk(trim_tree,
1344 vdev_trim_range_verify, &tap[0]);
1345 mutex_exit(&msp->ms_lock);
1348 range_tree_vacate(trim_tree, NULL, NULL);
1349 range_tree_destroy(trim_tree);
1351 metaslab_enable(msp, issued_trim, B_FALSE);
1352 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1354 for (uint64_t c = 0; c < children; c++) {
1355 trim_args_t *ta = &tap[c];
1357 if (ta->trim_tree == NULL)
1360 range_tree_vacate(ta->trim_tree, NULL, NULL);
1361 range_tree_destroy(ta->trim_tree);
1364 kmem_free(tap, sizeof (trim_args_t) * children);
1367 spa_config_exit(spa, SCL_CONFIG, FTAG);
1370 * After completing the group of metaslabs wait for the next
1371 * open txg. This is done to make sure that a minimum of
1372 * zfs_trim_txg_batch txgs will occur before these metaslabs
1373 * are trimmed again.
1375 txg_wait_open(spa_get_dsl(spa), 0, issued_trim);
1378 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1381 for (uint64_t c = 0; c < vd->vdev_children; c++) {
1382 vdev_t *cvd = vd->vdev_child[c];
1383 mutex_enter(&cvd->vdev_trim_io_lock);
1385 while (cvd->vdev_trim_inflight[1] > 0) {
1386 cv_wait(&cvd->vdev_trim_io_cv,
1387 &cvd->vdev_trim_io_lock);
1389 mutex_exit(&cvd->vdev_trim_io_lock);
1392 spa_config_exit(spa, SCL_CONFIG, FTAG);
1395 * When exiting because the autotrim property was set to off, then
1396 * abandon any unprocessed ms_trim ranges to reclaim the memory.
1398 if (spa_get_autotrim(spa) == SPA_AUTOTRIM_OFF) {
1399 for (uint64_t i = 0; i < vd->vdev_ms_count; i++) {
1400 metaslab_t *msp = vd->vdev_ms[i];
1402 mutex_enter(&msp->ms_lock);
1403 range_tree_vacate(msp->ms_trim, NULL, NULL);
1404 mutex_exit(&msp->ms_lock);
1408 mutex_enter(&vd->vdev_autotrim_lock);
1409 ASSERT(vd->vdev_autotrim_thread != NULL);
1410 vd->vdev_autotrim_thread = NULL;
1411 cv_broadcast(&vd->vdev_autotrim_cv);
1412 mutex_exit(&vd->vdev_autotrim_lock);
1418 * Starts an autotrim thread, if needed, for each top-level vdev which can be
1419 * trimmed. A top-level vdev which has been evacuated will never be trimmed.
1422 vdev_autotrim(spa_t *spa)
1424 vdev_t *root_vd = spa->spa_root_vdev;
1426 for (uint64_t i = 0; i < root_vd->vdev_children; i++) {
1427 vdev_t *tvd = root_vd->vdev_child[i];
1429 mutex_enter(&tvd->vdev_autotrim_lock);
1430 if (vdev_writeable(tvd) && !tvd->vdev_removing &&
1431 tvd->vdev_autotrim_thread == NULL) {
1432 ASSERT3P(tvd->vdev_top, ==, tvd);
1434 tvd->vdev_autotrim_thread = thread_create(NULL, 0,
1435 vdev_autotrim_thread, tvd, 0, &p0, TS_RUN,
1437 ASSERT(tvd->vdev_autotrim_thread != NULL);
1439 mutex_exit(&tvd->vdev_autotrim_lock);
1444 * Wait for the vdev_autotrim_thread associated with the passed top-level
1445 * vdev to be terminated (canceled or stopped).
1448 vdev_autotrim_stop_wait(vdev_t *tvd)
1450 mutex_enter(&tvd->vdev_autotrim_lock);
1451 if (tvd->vdev_autotrim_thread != NULL) {
1452 tvd->vdev_autotrim_exit_wanted = B_TRUE;
1454 while (tvd->vdev_autotrim_thread != NULL) {
1455 cv_wait(&tvd->vdev_autotrim_cv,
1456 &tvd->vdev_autotrim_lock);
1459 ASSERT3P(tvd->vdev_autotrim_thread, ==, NULL);
1460 tvd->vdev_autotrim_exit_wanted = B_FALSE;
1462 mutex_exit(&tvd->vdev_autotrim_lock);
1466 * Wait for all of the vdev_autotrim_thread associated with the pool to
1467 * be terminated (canceled or stopped).
1470 vdev_autotrim_stop_all(spa_t *spa)
1472 vdev_t *root_vd = spa->spa_root_vdev;
1474 for (uint64_t i = 0; i < root_vd->vdev_children; i++)
1475 vdev_autotrim_stop_wait(root_vd->vdev_child[i]);
1479 * Conditionally restart all of the vdev_autotrim_thread's for the pool.
1482 vdev_autotrim_restart(spa_t *spa)
1484 ASSERT(MUTEX_HELD(&spa_namespace_lock));
1486 if (spa->spa_autotrim)
1491 vdev_trim_l2arc_thread(void *arg)
1494 spa_t *spa = vd->vdev_spa;
1495 l2arc_dev_t *dev = l2arc_vdev_get(vd);
1497 range_seg64_t physical_rs;
1499 ASSERT(vdev_is_concrete(vd));
1500 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
1502 vd->vdev_trim_last_offset = 0;
1503 vd->vdev_trim_rate = 0;
1504 vd->vdev_trim_partial = 0;
1505 vd->vdev_trim_secure = 0;
1507 bzero(&ta, sizeof (ta));
1509 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
1510 ta.trim_type = TRIM_TYPE_MANUAL;
1511 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
1512 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
1515 physical_rs.rs_start = vd->vdev_trim_bytes_done = 0;
1516 physical_rs.rs_end = vd->vdev_trim_bytes_est =
1517 vdev_get_min_asize(vd);
1519 range_tree_add(ta.trim_tree, physical_rs.rs_start,
1520 physical_rs.rs_end - physical_rs.rs_start);
1522 mutex_enter(&vd->vdev_trim_lock);
1523 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);
1524 mutex_exit(&vd->vdev_trim_lock);
1526 (void) vdev_trim_ranges(&ta);
1528 spa_config_exit(spa, SCL_CONFIG, FTAG);
1529 mutex_enter(&vd->vdev_trim_io_lock);
1530 while (vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] > 0) {
1531 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
1533 mutex_exit(&vd->vdev_trim_io_lock);
1535 range_tree_vacate(ta.trim_tree, NULL, NULL);
1536 range_tree_destroy(ta.trim_tree);
1538 mutex_enter(&vd->vdev_trim_lock);
1539 if (!vd->vdev_trim_exit_wanted && vdev_writeable(vd)) {
1540 vdev_trim_change_state(vd, VDEV_TRIM_COMPLETE,
1541 vd->vdev_trim_rate, vd->vdev_trim_partial,
1542 vd->vdev_trim_secure);
1544 ASSERT(vd->vdev_trim_thread != NULL ||
1545 vd->vdev_trim_inflight[TRIM_TYPE_MANUAL] == 0);
1548 * Drop the vdev_trim_lock while we sync out the txg since it's
1549 * possible that a device might be trying to come online and
1550 * must check to see if it needs to restart a trim. That thread
1551 * will be holding the spa_config_lock which would prevent the
1552 * txg_wait_synced from completing. Same strategy as in
1553 * vdev_trim_thread().
1555 mutex_exit(&vd->vdev_trim_lock);
1556 txg_wait_synced(spa_get_dsl(vd->vdev_spa), 0);
1557 mutex_enter(&vd->vdev_trim_lock);
1560 * Update the header of the cache device here, before
1561 * broadcasting vdev_trim_cv which may lead to the removal
1562 * of the device. The same applies for setting l2ad_trim_all to
1565 spa_config_enter(vd->vdev_spa, SCL_L2ARC, vd,
1567 bzero(dev->l2ad_dev_hdr, dev->l2ad_dev_hdr_asize);
1568 l2arc_dev_hdr_update(dev);
1569 spa_config_exit(vd->vdev_spa, SCL_L2ARC, vd);
1571 vd->vdev_trim_thread = NULL;
1572 if (vd->vdev_trim_state == VDEV_TRIM_COMPLETE)
1573 dev->l2ad_trim_all = B_FALSE;
1575 cv_broadcast(&vd->vdev_trim_cv);
1576 mutex_exit(&vd->vdev_trim_lock);
1582 * Punches out TRIM threads for the L2ARC devices in a spa and assigns them
1583 * to vd->vdev_trim_thread variable. This facilitates the management of
1584 * trimming the whole cache device using TRIM_TYPE_MANUAL upon addition
1585 * to a pool or pool creation or when the header of the device is invalid.
1588 vdev_trim_l2arc(spa_t *spa)
1590 ASSERT(MUTEX_HELD(&spa_namespace_lock));
1593 * Locate the spa's l2arc devices and kick off TRIM threads.
1595 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
1596 vdev_t *vd = spa->spa_l2cache.sav_vdevs[i];
1597 l2arc_dev_t *dev = l2arc_vdev_get(vd);
1599 if (dev == NULL || !dev->l2ad_trim_all) {
1601 * Don't attempt TRIM if the vdev is UNAVAIL or if the
1602 * cache device was not marked for whole device TRIM
1603 * (ie l2arc_trim_ahead = 0, or the L2ARC device header
1604 * is valid with trim_state = VDEV_TRIM_COMPLETE and
1605 * l2ad_log_entries > 0).
1610 mutex_enter(&vd->vdev_trim_lock);
1611 ASSERT(vd->vdev_ops->vdev_op_leaf);
1612 ASSERT(vdev_is_concrete(vd));
1613 ASSERT3P(vd->vdev_trim_thread, ==, NULL);
1614 ASSERT(!vd->vdev_detached);
1615 ASSERT(!vd->vdev_trim_exit_wanted);
1616 ASSERT(!vd->vdev_top->vdev_removing);
1617 vdev_trim_change_state(vd, VDEV_TRIM_ACTIVE, 0, 0, 0);
1618 vd->vdev_trim_thread = thread_create(NULL, 0,
1619 vdev_trim_l2arc_thread, vd, 0, &p0, TS_RUN, maxclsyspri);
1620 mutex_exit(&vd->vdev_trim_lock);
1625 * A wrapper which calls vdev_trim_ranges(). It is intended to be called
1629 vdev_trim_simple(vdev_t *vd, uint64_t start, uint64_t size)
1632 range_seg64_t physical_rs;
1634 physical_rs.rs_start = start;
1635 physical_rs.rs_end = start + size;
1637 ASSERT(vdev_is_concrete(vd));
1638 ASSERT(vd->vdev_ops->vdev_op_leaf);
1639 ASSERT(!vd->vdev_detached);
1640 ASSERT(!vd->vdev_top->vdev_removing);
1642 bzero(&ta, sizeof (ta));
1644 ta.trim_tree = range_tree_create(NULL, RANGE_SEG64, NULL, 0, 0);
1645 ta.trim_type = TRIM_TYPE_SIMPLE;
1646 ta.trim_extent_bytes_max = zfs_trim_extent_bytes_max;
1647 ta.trim_extent_bytes_min = SPA_MINBLOCKSIZE;
1650 ASSERT3U(physical_rs.rs_end, >=, physical_rs.rs_start);
1652 if (physical_rs.rs_end > physical_rs.rs_start) {
1653 range_tree_add(ta.trim_tree, physical_rs.rs_start,
1654 physical_rs.rs_end - physical_rs.rs_start);
1656 ASSERT3U(physical_rs.rs_end, ==, physical_rs.rs_start);
1659 error = vdev_trim_ranges(&ta);
1661 mutex_enter(&vd->vdev_trim_io_lock);
1662 while (vd->vdev_trim_inflight[TRIM_TYPE_SIMPLE] > 0) {
1663 cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock);
1665 mutex_exit(&vd->vdev_trim_io_lock);
1667 range_tree_vacate(ta.trim_tree, NULL, NULL);
1668 range_tree_destroy(ta.trim_tree);
1673 EXPORT_SYMBOL(vdev_trim);
1674 EXPORT_SYMBOL(vdev_trim_stop);
1675 EXPORT_SYMBOL(vdev_trim_stop_all);
1676 EXPORT_SYMBOL(vdev_trim_stop_wait);
1677 EXPORT_SYMBOL(vdev_trim_restart);
1678 EXPORT_SYMBOL(vdev_autotrim);
1679 EXPORT_SYMBOL(vdev_autotrim_stop_all);
1680 EXPORT_SYMBOL(vdev_autotrim_stop_wait);
1681 EXPORT_SYMBOL(vdev_autotrim_restart);
1682 EXPORT_SYMBOL(vdev_trim_l2arc);
1683 EXPORT_SYMBOL(vdev_trim_simple);
1686 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_max, UINT, ZMOD_RW,
1687 "Max size of TRIM commands, larger will be split");
1689 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, extent_bytes_min, UINT, ZMOD_RW,
1690 "Min size of TRIM commands, smaller will be skipped");
1692 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, metaslab_skip, UINT, ZMOD_RW,
1693 "Skip metaslabs which have never been initialized");
1695 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, txg_batch, UINT, ZMOD_RW,
1696 "Min number of txgs to aggregate frees before issuing TRIM");
1698 ZFS_MODULE_PARAM(zfs_trim, zfs_trim_, queue_limit, UINT, ZMOD_RW,
1699 "Max queued TRIMs outstanding per leaf vdev");