sys/contrib/openzfs/cmd/zpool/zpool_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  30  * Copyright (c) 2017 Datto Inc.
  31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  32  * Copyright (c) 2017, Intel Corporation.
  33  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  34  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
  35  * Copyright (c) 2021, Klara Inc.
  36  * Copyright [2021] Hewlett Packard Enterprise Development LP
  37  */
  38
  39 #include <assert.h>
  40 #include <ctype.h>
  41 #include <dirent.h>
  42 #include <errno.h>
  43 #include <fcntl.h>
  44 #include <getopt.h>
  45 #include <libgen.h>
  46 #include <libintl.h>
  47 #include <libuutil.h>
  48 #include <locale.h>
  49 #include <pthread.h>
  50 #include <stdio.h>
  51 #include <stdlib.h>
  52 #include <string.h>
  53 #include <time.h>
  54 #include <unistd.h>
  55 #include <pwd.h>
  56 #include <zone.h>
  57 #include <sys/wait.h>
  58 #include <zfs_prop.h>
  59 #include <sys/fs/zfs.h>
  60 #include <sys/stat.h>
  61 #include <sys/systeminfo.h>
  62 #include <sys/fm/fs/zfs.h>
  63 #include <sys/fm/util.h>
  64 #include <sys/fm/protocol.h>
  65 #include <sys/zfs_ioctl.h>
  66 #include <sys/mount.h>
  67 #include <sys/sysmacros.h>
  68
  69 #include <math.h>
  70
  71 #include <libzfs.h>
  72 #include <libzutil.h>
  73
  74 #include "zpool_util.h"
  75 #include "zfs_comutil.h"
  76 #include "zfeature_common.h"
  77
  78 #include "statcommon.h"
  79
  80 libzfs_handle_t *g_zfs;
  81
  82 static int zpool_do_create(int, char **);
  83 static int zpool_do_destroy(int, char **);
  84
  85 static int zpool_do_add(int, char **);
  86 static int zpool_do_remove(int, char **);
  87 static int zpool_do_labelclear(int, char **);
  88
  89 static int zpool_do_checkpoint(int, char **);
  90
  91 static int zpool_do_list(int, char **);
  92 static int zpool_do_iostat(int, char **);
  93 static int zpool_do_status(int, char **);
  94
  95 static int zpool_do_online(int, char **);
  96 static int zpool_do_offline(int, char **);
  97 static int zpool_do_clear(int, char **);
  98 static int zpool_do_reopen(int, char **);
  99
 100 static int zpool_do_reguid(int, char **);
 101
 102 static int zpool_do_attach(int, char **);
 103 static int zpool_do_detach(int, char **);
 104 static int zpool_do_replace(int, char **);
 105 static int zpool_do_split(int, char **);
 106
 107 static int zpool_do_initialize(int, char **);
 108 static int zpool_do_scrub(int, char **);
 109 static int zpool_do_resilver(int, char **);
 110 static int zpool_do_trim(int, char **);
 111
 112 static int zpool_do_import(int, char **);
 113 static int zpool_do_export(int, char **);
 114
 115 static int zpool_do_upgrade(int, char **);
 116
 117 static int zpool_do_history(int, char **);
 118 static int zpool_do_events(int, char **);
 119
 120 static int zpool_do_get(int, char **);
 121 static int zpool_do_set(int, char **);
 122
 123 static int zpool_do_sync(int, char **);
 124
 125 static int zpool_do_version(int, char **);
 126
 127 static int zpool_do_wait(int, char **);
 128
 129 static int zpool_do_help(int argc, char **argv);
 130
 131 static zpool_compat_status_t zpool_do_load_compat(
 132     const char *, boolean_t *);
 133
 134 /*
 135  * These libumem hooks provide a reasonable set of defaults for the allocator's
 136  * debugging facilities.
 137  */
 138
 139 #ifdef DEBUG
 140 const char *
 141 _umem_debug_init(void)
 142 {
 143         return ("default,verbose"); /* $UMEM_DEBUG setting */
 144 }
 145
 146 const char *
 147 _umem_logging_init(void)
 148 {
 149         return ("fail,contents"); /* $UMEM_LOGGING setting */
 150 }
 151 #endif
 152
 153 typedef enum {
 154         HELP_ADD,
 155         HELP_ATTACH,
 156         HELP_CLEAR,
 157         HELP_CREATE,
 158         HELP_CHECKPOINT,
 159         HELP_DESTROY,
 160         HELP_DETACH,
 161         HELP_EXPORT,
 162         HELP_HISTORY,
 163         HELP_IMPORT,
 164         HELP_IOSTAT,
 165         HELP_LABELCLEAR,
 166         HELP_LIST,
 167         HELP_OFFLINE,
 168         HELP_ONLINE,
 169         HELP_REPLACE,
 170         HELP_REMOVE,
 171         HELP_INITIALIZE,
 172         HELP_SCRUB,
 173         HELP_RESILVER,
 174         HELP_TRIM,
 175         HELP_STATUS,
 176         HELP_UPGRADE,
 177         HELP_EVENTS,
 178         HELP_GET,
 179         HELP_SET,
 180         HELP_SPLIT,
 181         HELP_SYNC,
 182         HELP_REGUID,
 183         HELP_REOPEN,
 184         HELP_VERSION,
 185         HELP_WAIT
 186 } zpool_help_t;
 187
 188
 189 /*
 190  * Flags for stats to display with "zpool iostats"
 191  */
 192 enum iostat_type {
 193         IOS_DEFAULT = 0,
 194         IOS_LATENCY = 1,
 195         IOS_QUEUES = 2,
 196         IOS_L_HISTO = 3,
 197         IOS_RQ_HISTO = 4,
 198         IOS_COUNT,      /* always last element */
 199 };
 200
 201 /* iostat_type entries as bitmasks */
 202 #define IOS_DEFAULT_M   (1ULL << IOS_DEFAULT)
 203 #define IOS_LATENCY_M   (1ULL << IOS_LATENCY)
 204 #define IOS_QUEUES_M    (1ULL << IOS_QUEUES)
 205 #define IOS_L_HISTO_M   (1ULL << IOS_L_HISTO)
 206 #define IOS_RQ_HISTO_M  (1ULL << IOS_RQ_HISTO)
 207
 208 /* Mask of all the histo bits */
 209 #define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
 210
 211 /*
 212  * Lookup table for iostat flags to nvlist names.  Basically a list
 213  * of all the nvlists a flag requires.  Also specifies the order in
 214  * which data gets printed in zpool iostat.
 215  */
 216 static const char *vsx_type_to_nvlist[IOS_COUNT][15] = {
 217         [IOS_L_HISTO] = {
 218             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 219             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 220             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 221             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 222             ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
 223             ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
 224             ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
 225             ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
 226             ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
 227             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 228             ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
 229             NULL},
 230         [IOS_LATENCY] = {
 231             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 232             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 233             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 234             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 235             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 236             ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
 237             NULL},
 238         [IOS_QUEUES] = {
 239             ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
 240             ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
 241             ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
 242             ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
 243             ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
 244             ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
 245             ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
 246             NULL},
 247         [IOS_RQ_HISTO] = {
 248             ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
 249             ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
 250             ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
 251             ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
 252             ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
 253             ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
 254             ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
 255             ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
 256             ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
 257             ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
 258             ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
 259             ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
 260             ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO,
 261             ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO,
 262             NULL},
 263 };
 264
 265
 266 /*
 267  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
 268  * Right now, only one histo bit is ever set at one time, so we can
 269  * just do a highbit64(a)
 270  */
 271 #define IOS_HISTO_IDX(a)        (highbit64(a & IOS_ANYHISTO_M) - 1)
 272
 273 typedef struct zpool_command {
 274         const char      *name;
 275         int             (*func)(int, char **);
 276         zpool_help_t    usage;
 277 } zpool_command_t;
 278
 279 /*
 280  * Master command table.  Each ZFS command has a name, associated function, and
 281  * usage message.  The usage messages need to be internationalized, so we have
 282  * to have a function to return the usage message based on a command index.
 283  *
 284  * These commands are organized according to how they are displayed in the usage
 285  * message.  An empty command (one with a NULL name) indicates an empty line in
 286  * the generic usage message.
 287  */
 288 static zpool_command_t command_table[] = {
 289         { "version",    zpool_do_version,       HELP_VERSION            },
 290         { NULL },
 291         { "create",     zpool_do_create,        HELP_CREATE             },
 292         { "destroy",    zpool_do_destroy,       HELP_DESTROY            },
 293         { NULL },
 294         { "add",        zpool_do_add,           HELP_ADD                },
 295         { "remove",     zpool_do_remove,        HELP_REMOVE             },
 296         { NULL },
 297         { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
 298         { NULL },
 299         { "checkpoint", zpool_do_checkpoint,    HELP_CHECKPOINT         },
 300         { NULL },
 301         { "list",       zpool_do_list,          HELP_LIST               },
 302         { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
 303         { "status",     zpool_do_status,        HELP_STATUS             },
 304         { NULL },
 305         { "online",     zpool_do_online,        HELP_ONLINE             },
 306         { "offline",    zpool_do_offline,       HELP_OFFLINE            },
 307         { "clear",      zpool_do_clear,         HELP_CLEAR              },
 308         { "reopen",     zpool_do_reopen,        HELP_REOPEN             },
 309         { NULL },
 310         { "attach",     zpool_do_attach,        HELP_ATTACH             },
 311         { "detach",     zpool_do_detach,        HELP_DETACH             },
 312         { "replace",    zpool_do_replace,       HELP_REPLACE            },
 313         { "split",      zpool_do_split,         HELP_SPLIT              },
 314         { NULL },
 315         { "initialize", zpool_do_initialize,    HELP_INITIALIZE         },
 316         { "resilver",   zpool_do_resilver,      HELP_RESILVER           },
 317         { "scrub",      zpool_do_scrub,         HELP_SCRUB              },
 318         { "trim",       zpool_do_trim,          HELP_TRIM               },
 319         { NULL },
 320         { "import",     zpool_do_import,        HELP_IMPORT             },
 321         { "export",     zpool_do_export,        HELP_EXPORT             },
 322         { "upgrade",    zpool_do_upgrade,       HELP_UPGRADE            },
 323         { "reguid",     zpool_do_reguid,        HELP_REGUID             },
 324         { NULL },
 325         { "history",    zpool_do_history,       HELP_HISTORY            },
 326         { "events",     zpool_do_events,        HELP_EVENTS             },
 327         { NULL },
 328         { "get",        zpool_do_get,           HELP_GET                },
 329         { "set",        zpool_do_set,           HELP_SET                },
 330         { "sync",       zpool_do_sync,          HELP_SYNC               },
 331         { NULL },
 332         { "wait",       zpool_do_wait,          HELP_WAIT               },
 333 };
 334
 335 #define NCOMMAND        (ARRAY_SIZE(command_table))
 336
 337 #define VDEV_ALLOC_CLASS_LOGS   "logs"
 338
 339 static zpool_command_t *current_command;
 340 static zfs_type_t current_prop_type = (ZFS_TYPE_POOL | ZFS_TYPE_VDEV);
 341 static char history_str[HIS_MAX_RECORD_LEN];
 342 static boolean_t log_history = B_TRUE;
 343 static uint_t timestamp_fmt = NODATE;
 344
 345 static const char *
 346 get_usage(zpool_help_t idx)
 347 {
 348         switch (idx) {
 349         case HELP_ADD:
 350                 return (gettext("\tadd [-fgLnP] [-o property=value] "
 351                     "<pool> <vdev> ...\n"));
 352         case HELP_ATTACH:
 353                 return (gettext("\tattach [-fsw] [-o property=value] "
 354                     "<pool> <device> <new-device>\n"));
 355         case HELP_CLEAR:
 356                 return (gettext("\tclear [[--power]|[-nF]] <pool> [device]\n"));
 357         case HELP_CREATE:
 358                 return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 359                     "\t    [-O file-system-property=value] ... \n"
 360                     "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 361         case HELP_CHECKPOINT:
 362                 return (gettext("\tcheckpoint [-d [-w]] <pool> ...\n"));
 363         case HELP_DESTROY:
 364                 return (gettext("\tdestroy [-f] <pool>\n"));
 365         case HELP_DETACH:
 366                 return (gettext("\tdetach <pool> <device>\n"));
 367         case HELP_EXPORT:
 368                 return (gettext("\texport [-af] <pool> ...\n"));
 369         case HELP_HISTORY:
 370                 return (gettext("\thistory [-il] [<pool>] ...\n"));
 371         case HELP_IMPORT:
 372                 return (gettext("\timport [-d dir] [-D]\n"
 373                     "\timport [-o mntopts] [-o property=value] ... \n"
 374                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 375                     "[-R root] [-F [-n]] -a\n"
 376                     "\timport [-o mntopts] [-o property=value] ... \n"
 377                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 378                     "[-R root] [-F [-n]]\n"
 379                     "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
 380         case HELP_IOSTAT:
 381                 return (gettext("\tiostat [[[-c [script1,script2,...]"
 382                     "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
 383                     "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
 384                     " [[-n] interval [count]]\n"));
 385         case HELP_LABELCLEAR:
 386                 return (gettext("\tlabelclear [-f] <vdev>\n"));
 387         case HELP_LIST:
 388                 return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 389                     "[-T d|u] [pool] ... \n"
 390                     "\t    [interval [count]]\n"));
 391         case HELP_OFFLINE:
 392                 return (gettext("\toffline [--power]|[[-f][-t]] <pool> "
 393                     "<device> ...\n"));
 394         case HELP_ONLINE:
 395                 return (gettext("\tonline [--power][-e] <pool> <device> "
 396                     "...\n"));
 397         case HELP_REPLACE:
 398                 return (gettext("\treplace [-fsw] [-o property=value] "
 399                     "<pool> <device> [new-device]\n"));
 400         case HELP_REMOVE:
 401                 return (gettext("\tremove [-npsw] <pool> <device> ...\n"));
 402         case HELP_REOPEN:
 403                 return (gettext("\treopen [-n] <pool>\n"));
 404         case HELP_INITIALIZE:
 405                 return (gettext("\tinitialize [-c | -s | -u] [-w] <pool> "
 406                     "[<device> ...]\n"));
 407         case HELP_SCRUB:
 408                 return (gettext("\tscrub [-s | -p] [-w] [-e] <pool> ...\n"));
 409         case HELP_RESILVER:
 410                 return (gettext("\tresilver <pool> ...\n"));
 411         case HELP_TRIM:
 412                 return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
 413                     "[<device> ...]\n"));
 414         case HELP_STATUS:
 415                 return (gettext("\tstatus [--power] [-c [script1,script2,...]] "
 416                     "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
 417                     "\t    [interval [count]]\n"));
 418         case HELP_UPGRADE:
 419                 return (gettext("\tupgrade\n"
 420                     "\tupgrade -v\n"
 421                     "\tupgrade [-V version] <-a | pool ...>\n"));
 422         case HELP_EVENTS:
 423                 return (gettext("\tevents [-vHf [pool] | -c]\n"));
 424         case HELP_GET:
 425                 return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
 426                     "<\"all\" | property[,...]> <pool> ...\n"));
 427         case HELP_SET:
 428                 return (gettext("\tset <property=value> <pool>\n"
 429                     "\tset <vdev_property=value> <pool> <vdev>\n"));
 430         case HELP_SPLIT:
 431                 return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
 432                     "\t    [-o property=value] <pool> <newpool> "
 433                     "[<device> ...]\n"));
 434         case HELP_REGUID:
 435                 return (gettext("\treguid <pool>\n"));
 436         case HELP_SYNC:
 437                 return (gettext("\tsync [pool] ...\n"));
 438         case HELP_VERSION:
 439                 return (gettext("\tversion\n"));
 440         case HELP_WAIT:
 441                 return (gettext("\twait [-Hp] [-T d|u] [-t <activity>[,...]] "
 442                     "<pool> [interval]\n"));
 443         default:
 444                 __builtin_unreachable();
 445         }
 446 }
 447
 448 static void
 449 zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
 450 {
 451         uint_t children = 0;
 452         nvlist_t **child;
 453         uint_t i;
 454
 455         (void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
 456             &child, &children);
 457
 458         if (children == 0) {
 459                 char *path = zpool_vdev_name(g_zfs, zhp, nvroot,
 460                     VDEV_NAME_PATH);
 461
 462                 if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 &&
 463                     strcmp(path, VDEV_TYPE_HOLE) != 0)
 464                         fnvlist_add_boolean(res, path);
 465
 466                 free(path);
 467                 return;
 468         }
 469
 470         for (i = 0; i < children; i++) {
 471                 zpool_collect_leaves(zhp, child[i], res);
 472         }
 473 }
 474
 475 /*
 476  * Callback routine that will print out a pool property value.
 477  */
 478 static int
 479 print_pool_prop_cb(int prop, void *cb)
 480 {
 481         FILE *fp = cb;
 482
 483         (void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
 484
 485         if (zpool_prop_readonly(prop))
 486                 (void) fprintf(fp, "  NO   ");
 487         else
 488                 (void) fprintf(fp, " YES   ");
 489
 490         if (zpool_prop_values(prop) == NULL)
 491                 (void) fprintf(fp, "-\n");
 492         else
 493                 (void) fprintf(fp, "%s\n", zpool_prop_values(prop));
 494
 495         return (ZPROP_CONT);
 496 }
 497
 498 /*
 499  * Callback routine that will print out a vdev property value.
 500  */
 501 static int
 502 print_vdev_prop_cb(int prop, void *cb)
 503 {
 504         FILE *fp = cb;
 505
 506         (void) fprintf(fp, "\t%-19s  ", vdev_prop_to_name(prop));
 507
 508         if (vdev_prop_readonly(prop))
 509                 (void) fprintf(fp, "  NO   ");
 510         else
 511                 (void) fprintf(fp, " YES   ");
 512
 513         if (vdev_prop_values(prop) == NULL)
 514                 (void) fprintf(fp, "-\n");
 515         else
 516                 (void) fprintf(fp, "%s\n", vdev_prop_values(prop));
 517
 518         return (ZPROP_CONT);
 519 }
 520
 521 /*
 522  * Given a leaf vdev name like 'L5' return its VDEV_CONFIG_PATH like
 523  * '/dev/disk/by-vdev/L5'.
 524  */
 525 static const char *
 526 vdev_name_to_path(zpool_handle_t *zhp, char *vdev)
 527 {
 528         nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);
 529         if (vdev_nv == NULL) {
 530                 return (NULL);
 531         }
 532         return (fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH));
 533 }
 534
 535 static int
 536 zpool_power_on(zpool_handle_t *zhp, char *vdev)
 537 {
 538         return (zpool_power(zhp, vdev, B_TRUE));
 539 }
 540
 541 static int
 542 zpool_power_on_and_disk_wait(zpool_handle_t *zhp, char *vdev)
 543 {
 544         int rc;
 545
 546         rc = zpool_power_on(zhp, vdev);
 547         if (rc != 0)
 548                 return (rc);
 549
 550         zpool_disk_wait(vdev_name_to_path(zhp, vdev));
 551
 552         return (0);
 553 }
 554
 555 static int
 556 zpool_power_on_pool_and_wait_for_devices(zpool_handle_t *zhp)
 557 {
 558         nvlist_t *nv;
 559         const char *path = NULL;
 560         int rc;
 561
 562         /* Power up all the devices first */
 563         FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
 564                 path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
 565                 if (path != NULL) {
 566                         rc = zpool_power_on(zhp, (char *)path);
 567                         if (rc != 0) {
 568                                 return (rc);
 569                         }
 570                 }
 571         }
 572
 573         /*
 574          * Wait for their devices to show up.  Since we powered them on
 575          * at roughly the same time, they should all come online around
 576          * the same time.
 577          */
 578         FOR_EACH_REAL_LEAF_VDEV(zhp, nv) {
 579                 path = fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH);
 580                 zpool_disk_wait(path);
 581         }
 582
 583         return (0);
 584 }
 585
 586 static int
 587 zpool_power_off(zpool_handle_t *zhp, char *vdev)
 588 {
 589         return (zpool_power(zhp, vdev, B_FALSE));
 590 }
 591
 592 /*
 593  * Display usage message.  If we're inside a command, display only the usage for
 594  * that command.  Otherwise, iterate over the entire command table and display
 595  * a complete usage message.
 596  */
 597 static __attribute__((noreturn)) void
 598 usage(boolean_t requested)
 599 {
 600         FILE *fp = requested ? stdout : stderr;
 601
 602         if (current_command == NULL) {
 603                 int i;
 604
 605                 (void) fprintf(fp, gettext("usage: zpool command args ...\n"));
 606                 (void) fprintf(fp,
 607                     gettext("where 'command' is one of the following:\n\n"));
 608
 609                 for (i = 0; i < NCOMMAND; i++) {
 610                         if (command_table[i].name == NULL)
 611                                 (void) fprintf(fp, "\n");
 612                         else
 613                                 (void) fprintf(fp, "%s",
 614                                     get_usage(command_table[i].usage));
 615                 }
 616
 617                 (void) fprintf(fp,
 618                     gettext("\nFor further help on a command or topic, "
 619                     "run: %s\n"), "zpool help [<topic>]");
 620         } else {
 621                 (void) fprintf(fp, gettext("usage:\n"));
 622                 (void) fprintf(fp, "%s", get_usage(current_command->usage));
 623         }
 624
 625         if (current_command != NULL &&
 626             current_prop_type != (ZFS_TYPE_POOL | ZFS_TYPE_VDEV) &&
 627             ((strcmp(current_command->name, "set") == 0) ||
 628             (strcmp(current_command->name, "get") == 0) ||
 629             (strcmp(current_command->name, "list") == 0))) {
 630
 631                 (void) fprintf(fp, "%s",
 632                     gettext("\nthe following properties are supported:\n"));
 633
 634                 (void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
 635                     "PROPERTY", "EDIT", "VALUES");
 636
 637                 /* Iterate over all properties */
 638                 if (current_prop_type == ZFS_TYPE_POOL) {
 639                         (void) zprop_iter(print_pool_prop_cb, fp, B_FALSE,
 640                             B_TRUE, current_prop_type);
 641
 642                         (void) fprintf(fp, "\t%-19s   ", "feature@...");
 643                         (void) fprintf(fp, "YES   "
 644                             "disabled | enabled | active\n");
 645
 646                         (void) fprintf(fp, gettext("\nThe feature@ properties "
 647                             "must be appended with a feature name.\n"
 648                             "See zpool-features(7).\n"));
 649                 } else if (current_prop_type == ZFS_TYPE_VDEV) {
 650                         (void) zprop_iter(print_vdev_prop_cb, fp, B_FALSE,
 651                             B_TRUE, current_prop_type);
 652                 }
 653         }
 654
 655         /*
 656          * See comments at end of main().
 657          */
 658         if (getenv("ZFS_ABORT") != NULL) {
 659                 (void) printf("dumping core by request\n");
 660                 abort();
 661         }
 662
 663         exit(requested ? 0 : 2);
 664 }
 665
 666 /*
 667  * zpool initialize [-c | -s | -u] [-w] <pool> [<vdev> ...]
 668  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
 669  * if none specified.
 670  *
 671  *      -c      Cancel. Ends active initializing.
 672  *      -s      Suspend. Initializing can then be restarted with no flags.
 673  *      -u      Uninitialize. Clears initialization state.
 674  *      -w      Wait. Blocks until initializing has completed.
 675  */
 676 int
 677 zpool_do_initialize(int argc, char **argv)
 678 {
 679         int c;
 680         char *poolname;
 681         zpool_handle_t *zhp;
 682         nvlist_t *vdevs;
 683         int err = 0;
 684         boolean_t wait = B_FALSE;
 685
 686         struct option long_options[] = {
 687                 {"cancel",      no_argument,            NULL, 'c'},
 688                 {"suspend",     no_argument,            NULL, 's'},
 689                 {"uninit",      no_argument,            NULL, 'u'},
 690                 {"wait",        no_argument,            NULL, 'w'},
 691                 {0, 0, 0, 0}
 692         };
 693
 694         pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
 695         while ((c = getopt_long(argc, argv, "csuw", long_options,
 696             NULL)) != -1) {
 697                 switch (c) {
 698                 case 'c':
 699                         if (cmd_type != POOL_INITIALIZE_START &&
 700                             cmd_type != POOL_INITIALIZE_CANCEL) {
 701                                 (void) fprintf(stderr, gettext("-c cannot be "
 702                                     "combined with other options\n"));
 703                                 usage(B_FALSE);
 704                         }
 705                         cmd_type = POOL_INITIALIZE_CANCEL;
 706                         break;
 707                 case 's':
 708                         if (cmd_type != POOL_INITIALIZE_START &&
 709                             cmd_type != POOL_INITIALIZE_SUSPEND) {
 710                                 (void) fprintf(stderr, gettext("-s cannot be "
 711                                     "combined with other options\n"));
 712                                 usage(B_FALSE);
 713                         }
 714                         cmd_type = POOL_INITIALIZE_SUSPEND;
 715                         break;
 716                 case 'u':
 717                         if (cmd_type != POOL_INITIALIZE_START &&
 718                             cmd_type != POOL_INITIALIZE_UNINIT) {
 719                                 (void) fprintf(stderr, gettext("-u cannot be "
 720                                     "combined with other options\n"));
 721                                 usage(B_FALSE);
 722                         }
 723                         cmd_type = POOL_INITIALIZE_UNINIT;
 724                         break;
 725                 case 'w':
 726                         wait = B_TRUE;
 727                         break;
 728                 case '?':
 729                         if (optopt != 0) {
 730                                 (void) fprintf(stderr,
 731                                     gettext("invalid option '%c'\n"), optopt);
 732                         } else {
 733                                 (void) fprintf(stderr,
 734                                     gettext("invalid option '%s'\n"),
 735                                     argv[optind - 1]);
 736                         }
 737                         usage(B_FALSE);
 738                 }
 739         }
 740
 741         argc -= optind;
 742         argv += optind;
 743
 744         if (argc < 1) {
 745                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 746                 usage(B_FALSE);
 747                 return (-1);
 748         }
 749
 750         if (wait && (cmd_type != POOL_INITIALIZE_START)) {
 751                 (void) fprintf(stderr, gettext("-w cannot be used with -c, -s"
 752                     "or -u\n"));
 753                 usage(B_FALSE);
 754         }
 755
 756         poolname = argv[0];
 757         zhp = zpool_open(g_zfs, poolname);
 758         if (zhp == NULL)
 759                 return (-1);
 760
 761         vdevs = fnvlist_alloc();
 762         if (argc == 1) {
 763                 /* no individual leaf vdevs specified, so add them all */
 764                 nvlist_t *config = zpool_get_config(zhp, NULL);
 765                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
 766                     ZPOOL_CONFIG_VDEV_TREE);
 767                 zpool_collect_leaves(zhp, nvroot, vdevs);
 768         } else {
 769                 for (int i = 1; i < argc; i++) {
 770                         fnvlist_add_boolean(vdevs, argv[i]);
 771                 }
 772         }
 773
 774         if (wait)
 775                 err = zpool_initialize_wait(zhp, cmd_type, vdevs);
 776         else
 777                 err = zpool_initialize(zhp, cmd_type, vdevs);
 778
 779         fnvlist_free(vdevs);
 780         zpool_close(zhp);
 781
 782         return (err);
 783 }
 784
 785 /*
 786  * print a pool vdev config for dry runs
 787  */
 788 static void
 789 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 790     const char *match, int name_flags)
 791 {
 792         nvlist_t **child;
 793         uint_t c, children;
 794         char *vname;
 795         boolean_t printed = B_FALSE;
 796
 797         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 798             &child, &children) != 0) {
 799                 if (name != NULL)
 800                         (void) printf("\t%*s%s\n", indent, "", name);
 801                 return;
 802         }
 803
 804         for (c = 0; c < children; c++) {
 805                 uint64_t is_log = B_FALSE, is_hole = B_FALSE;
 806                 const char *class = "";
 807
 808                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
 809                     &is_hole);
 810
 811                 if (is_hole == B_TRUE) {
 812                         continue;
 813                 }
 814
 815                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 816                     &is_log);
 817                 if (is_log)
 818                         class = VDEV_ALLOC_BIAS_LOG;
 819                 (void) nvlist_lookup_string(child[c],
 820                     ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
 821                 if (strcmp(match, class) != 0)
 822                         continue;
 823
 824                 if (!printed && name != NULL) {
 825                         (void) printf("\t%*s%s\n", indent, "", name);
 826                         printed = B_TRUE;
 827                 }
 828                 vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
 829                 print_vdev_tree(zhp, vname, child[c], indent + 2, "",
 830                     name_flags);
 831                 free(vname);
 832         }
 833 }
 834
 835 /*
 836  * Print the list of l2cache devices for dry runs.
 837  */
 838 static void
 839 print_cache_list(nvlist_t *nv, int indent)
 840 {
 841         nvlist_t **child;
 842         uint_t c, children;
 843
 844         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
 845             &child, &children) == 0 && children > 0) {
 846                 (void) printf("\t%*s%s\n", indent, "", "cache");
 847         } else {
 848                 return;
 849         }
 850         for (c = 0; c < children; c++) {
 851                 char *vname;
 852
 853                 vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
 854                 (void) printf("\t%*s%s\n", indent + 2, "", vname);
 855                 free(vname);
 856         }
 857 }
 858
 859 /*
 860  * Print the list of spares for dry runs.
 861  */
 862 static void
 863 print_spare_list(nvlist_t *nv, int indent)
 864 {
 865         nvlist_t **child;
 866         uint_t c, children;
 867
 868         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
 869             &child, &children) == 0 && children > 0) {
 870                 (void) printf("\t%*s%s\n", indent, "", "spares");
 871         } else {
 872                 return;
 873         }
 874         for (c = 0; c < children; c++) {
 875                 char *vname;
 876
 877                 vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
 878                 (void) printf("\t%*s%s\n", indent + 2, "", vname);
 879                 free(vname);
 880         }
 881 }
 882
 883 static boolean_t
 884 prop_list_contains_feature(nvlist_t *proplist)
 885 {
 886         nvpair_t *nvp;
 887         for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
 888             nvp = nvlist_next_nvpair(proplist, nvp)) {
 889                 if (zpool_prop_feature(nvpair_name(nvp)))
 890                         return (B_TRUE);
 891         }
 892         return (B_FALSE);
 893 }
 894
 895 /*
 896  * Add a property pair (name, string-value) into a property nvlist.
 897  */
 898 static int
 899 add_prop_list(const char *propname, const char *propval, nvlist_t **props,
 900     boolean_t poolprop)
 901 {
 902         zpool_prop_t prop = ZPOOL_PROP_INVAL;
 903         nvlist_t *proplist;
 904         const char *normnm;
 905         const char *strval;
 906
 907         if (*props == NULL &&
 908             nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
 909                 (void) fprintf(stderr,
 910                     gettext("internal error: out of memory\n"));
 911                 return (1);
 912         }
 913
 914         proplist = *props;
 915
 916         if (poolprop) {
 917                 const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 918                 const char *cname =
 919                     zpool_prop_to_name(ZPOOL_PROP_COMPATIBILITY);
 920
 921                 if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 922                     (!zpool_prop_feature(propname) &&
 923                     !zpool_prop_vdev(propname))) {
 924                         (void) fprintf(stderr, gettext("property '%s' is "
 925                             "not a valid pool or vdev property\n"), propname);
 926                         return (2);
 927                 }
 928
 929                 /*
 930                  * feature@ properties and version should not be specified
 931                  * at the same time.
 932                  */
 933                 if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
 934                     nvlist_exists(proplist, vname)) ||
 935                     (prop == ZPOOL_PROP_VERSION &&
 936                     prop_list_contains_feature(proplist))) {
 937                         (void) fprintf(stderr, gettext("'feature@' and "
 938                             "'version' properties cannot be specified "
 939                             "together\n"));
 940                         return (2);
 941                 }
 942
 943                 /*
 944                  * if version is specified, only "legacy" compatibility
 945                  * may be requested
 946                  */
 947                 if ((prop == ZPOOL_PROP_COMPATIBILITY &&
 948                     strcmp(propval, ZPOOL_COMPAT_LEGACY) != 0 &&
 949                     nvlist_exists(proplist, vname)) ||
 950                     (prop == ZPOOL_PROP_VERSION &&
 951                     nvlist_exists(proplist, cname) &&
 952                     strcmp(fnvlist_lookup_string(proplist, cname),
 953                     ZPOOL_COMPAT_LEGACY) != 0)) {
 954                         (void) fprintf(stderr, gettext("when 'version' is "
 955                             "specified, the 'compatibility' feature may only "
 956                             "be set to '" ZPOOL_COMPAT_LEGACY "'\n"));
 957                         return (2);
 958                 }
 959
 960                 if (zpool_prop_feature(propname) || zpool_prop_vdev(propname))
 961                         normnm = propname;
 962                 else
 963                         normnm = zpool_prop_to_name(prop);
 964         } else {
 965                 zfs_prop_t fsprop = zfs_name_to_prop(propname);
 966
 967                 if (zfs_prop_valid_for_type(fsprop, ZFS_TYPE_FILESYSTEM,
 968                     B_FALSE)) {
 969                         normnm = zfs_prop_to_name(fsprop);
 970                 } else if (zfs_prop_user(propname) ||
 971                     zfs_prop_userquota(propname)) {
 972                         normnm = propname;
 973                 } else {
 974                         (void) fprintf(stderr, gettext("property '%s' is "
 975                             "not a valid filesystem property\n"), propname);
 976                         return (2);
 977                 }
 978         }
 979
 980         if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
 981             prop != ZPOOL_PROP_CACHEFILE) {
 982                 (void) fprintf(stderr, gettext("property '%s' "
 983                     "specified multiple times\n"), propname);
 984                 return (2);
 985         }
 986
 987         if (nvlist_add_string(proplist, normnm, propval) != 0) {
 988                 (void) fprintf(stderr, gettext("internal "
 989                     "error: out of memory\n"));
 990                 return (1);
 991         }
 992
 993         return (0);
 994 }
 995
 996 /*
 997  * Set a default property pair (name, string-value) in a property nvlist
 998  */
 999 static int
1000 add_prop_list_default(const char *propname, const char *propval,
1001     nvlist_t **props)
1002 {
1003         const char *pval;
1004
1005         if (nvlist_lookup_string(*props, propname, &pval) == 0)
1006                 return (0);
1007
1008         return (add_prop_list(propname, propval, props, B_TRUE));
1009 }
1010
1011 /*
1012  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
1013  *
1014  *      -f      Force addition of devices, even if they appear in use
1015  *      -g      Display guid for individual vdev name.
1016  *      -L      Follow links when resolving vdev path name.
1017  *      -n      Do not add the devices, but display the resulting layout if
1018  *              they were to be added.
1019  *      -o      Set property=value.
1020  *      -P      Display full path for vdev name.
1021  *
1022  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
1023  * handled by make_root_vdev(), which constructs the nvlist needed to pass to
1024  * libzfs.
1025  */
1026 int
1027 zpool_do_add(int argc, char **argv)
1028 {
1029         boolean_t force = B_FALSE;
1030         boolean_t dryrun = B_FALSE;
1031         int name_flags = 0;
1032         int c;
1033         nvlist_t *nvroot;
1034         char *poolname;
1035         int ret;
1036         zpool_handle_t *zhp;
1037         nvlist_t *config;
1038         nvlist_t *props = NULL;
1039         char *propval;
1040
1041         /* check options */
1042         while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
1043                 switch (c) {
1044                 case 'f':
1045                         force = B_TRUE;
1046                         break;
1047                 case 'g':
1048                         name_flags |= VDEV_NAME_GUID;
1049                         break;
1050                 case 'L':
1051                         name_flags |= VDEV_NAME_FOLLOW_LINKS;
1052                         break;
1053                 case 'n':
1054                         dryrun = B_TRUE;
1055                         break;
1056                 case 'o':
1057                         if ((propval = strchr(optarg, '=')) == NULL) {
1058                                 (void) fprintf(stderr, gettext("missing "
1059                                     "'=' for -o option\n"));
1060                                 usage(B_FALSE);
1061                         }
1062                         *propval = '\0';
1063                         propval++;
1064
1065                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
1066                             (add_prop_list(optarg, propval, &props, B_TRUE)))
1067                                 usage(B_FALSE);
1068                         break;
1069                 case 'P':
1070                         name_flags |= VDEV_NAME_PATH;
1071                         break;
1072                 case '?':
1073                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1074                             optopt);
1075                         usage(B_FALSE);
1076                 }
1077         }
1078
1079         argc -= optind;
1080         argv += optind;
1081
1082         /* get pool name and check number of arguments */
1083         if (argc < 1) {
1084                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1085                 usage(B_FALSE);
1086         }
1087         if (argc < 2) {
1088                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1089                 usage(B_FALSE);
1090         }
1091
1092         poolname = argv[0];
1093
1094         argc--;
1095         argv++;
1096
1097         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1098                 return (1);
1099
1100         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
1101                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
1102                     poolname);
1103                 zpool_close(zhp);
1104                 return (1);
1105         }
1106
1107         /* unless manually specified use "ashift" pool property (if set) */
1108         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
1109                 int intval;
1110                 zprop_source_t src;
1111                 char strval[ZPOOL_MAXPROPLEN];
1112
1113                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
1114                 if (src != ZPROP_SRC_DEFAULT) {
1115                         (void) sprintf(strval, "%" PRId32, intval);
1116                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
1117                             &props, B_TRUE) == 0);
1118                 }
1119         }
1120
1121         /* pass off to make_root_vdev for processing */
1122         nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
1123             argc, argv);
1124         if (nvroot == NULL) {
1125                 zpool_close(zhp);
1126                 return (1);
1127         }
1128
1129         if (dryrun) {
1130                 nvlist_t *poolnvroot;
1131                 nvlist_t **l2child, **sparechild;
1132                 uint_t l2children, sparechildren, c;
1133                 char *vname;
1134                 boolean_t hadcache = B_FALSE, hadspare = B_FALSE;
1135
1136                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1137                     &poolnvroot) == 0);
1138
1139                 (void) printf(gettext("would update '%s' to the following "
1140                     "configuration:\n\n"), zpool_get_name(zhp));
1141
1142                 /* print original main pool and new tree */
1143                 print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
1144                     name_flags | VDEV_NAME_TYPE_ID);
1145                 print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
1146
1147                 /* print other classes: 'dedup', 'special', and 'log' */
1148                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_DEDUP)) {
1149                         print_vdev_tree(zhp, "dedup", poolnvroot, 0,
1150                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
1151                         print_vdev_tree(zhp, NULL, nvroot, 0,
1152                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
1153                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_DEDUP)) {
1154                         print_vdev_tree(zhp, "dedup", nvroot, 0,
1155                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
1156                 }
1157
1158                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
1159                         print_vdev_tree(zhp, "special", poolnvroot, 0,
1160                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1161                         print_vdev_tree(zhp, NULL, nvroot, 0,
1162                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1163                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
1164                         print_vdev_tree(zhp, "special", nvroot, 0,
1165                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1166                 }
1167
1168                 if (num_logs(poolnvroot) > 0) {
1169                         print_vdev_tree(zhp, "logs", poolnvroot, 0,
1170                             VDEV_ALLOC_BIAS_LOG, name_flags);
1171                         print_vdev_tree(zhp, NULL, nvroot, 0,
1172                             VDEV_ALLOC_BIAS_LOG, name_flags);
1173                 } else if (num_logs(nvroot) > 0) {
1174                         print_vdev_tree(zhp, "logs", nvroot, 0,
1175                             VDEV_ALLOC_BIAS_LOG, name_flags);
1176                 }
1177
1178                 /* Do the same for the caches */
1179                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
1180                     &l2child, &l2children) == 0 && l2children) {
1181                         hadcache = B_TRUE;
1182                         (void) printf(gettext("\tcache\n"));
1183                         for (c = 0; c < l2children; c++) {
1184                                 vname = zpool_vdev_name(g_zfs, NULL,
1185                                     l2child[c], name_flags);
1186                                 (void) printf("\t  %s\n", vname);
1187                                 free(vname);
1188                         }
1189                 }
1190                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
1191                     &l2child, &l2children) == 0 && l2children) {
1192                         if (!hadcache)
1193                                 (void) printf(gettext("\tcache\n"));
1194                         for (c = 0; c < l2children; c++) {
1195                                 vname = zpool_vdev_name(g_zfs, NULL,
1196                                     l2child[c], name_flags);
1197                                 (void) printf("\t  %s\n", vname);
1198                                 free(vname);
1199                         }
1200                 }
1201                 /* And finally the spares */
1202                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_SPARES,
1203                     &sparechild, &sparechildren) == 0 && sparechildren > 0) {
1204                         hadspare = B_TRUE;
1205                         (void) printf(gettext("\tspares\n"));
1206                         for (c = 0; c < sparechildren; c++) {
1207                                 vname = zpool_vdev_name(g_zfs, NULL,
1208                                     sparechild[c], name_flags);
1209                                 (void) printf("\t  %s\n", vname);
1210                                 free(vname);
1211                         }
1212                 }
1213                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1214                     &sparechild, &sparechildren) == 0 && sparechildren > 0) {
1215                         if (!hadspare)
1216                                 (void) printf(gettext("\tspares\n"));
1217                         for (c = 0; c < sparechildren; c++) {
1218                                 vname = zpool_vdev_name(g_zfs, NULL,
1219                                     sparechild[c], name_flags);
1220                                 (void) printf("\t  %s\n", vname);
1221                                 free(vname);
1222                         }
1223                 }
1224
1225                 ret = 0;
1226         } else {
1227                 ret = (zpool_add(zhp, nvroot) != 0);
1228         }
1229
1230         nvlist_free(props);
1231         nvlist_free(nvroot);
1232         zpool_close(zhp);
1233
1234         return (ret);
1235 }
1236
1237 /*
1238  * zpool remove [-npsw] <pool> <vdev> ...
1239  *
1240  * Removes the given vdev from the pool.
1241  */
1242 int
1243 zpool_do_remove(int argc, char **argv)
1244 {
1245         char *poolname;
1246         int i, ret = 0;
1247         zpool_handle_t *zhp = NULL;
1248         boolean_t stop = B_FALSE;
1249         int c;
1250         boolean_t noop = B_FALSE;
1251         boolean_t parsable = B_FALSE;
1252         boolean_t wait = B_FALSE;
1253
1254         /* check options */
1255         while ((c = getopt(argc, argv, "npsw")) != -1) {
1256                 switch (c) {
1257                 case 'n':
1258                         noop = B_TRUE;
1259                         break;
1260                 case 'p':
1261                         parsable = B_TRUE;
1262                         break;
1263                 case 's':
1264                         stop = B_TRUE;
1265                         break;
1266                 case 'w':
1267                         wait = B_TRUE;
1268                         break;
1269                 case '?':
1270                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1271                             optopt);
1272                         usage(B_FALSE);
1273                 }
1274         }
1275
1276         argc -= optind;
1277         argv += optind;
1278
1279         /* get pool name and check number of arguments */
1280         if (argc < 1) {
1281                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1282                 usage(B_FALSE);
1283         }
1284
1285         poolname = argv[0];
1286
1287         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1288                 return (1);
1289
1290         if (stop && noop) {
1291                 zpool_close(zhp);
1292                 (void) fprintf(stderr, gettext("stop request ignored\n"));
1293                 return (0);
1294         }
1295
1296         if (stop) {
1297                 if (argc > 1) {
1298                         (void) fprintf(stderr, gettext("too many arguments\n"));
1299                         usage(B_FALSE);
1300                 }
1301                 if (zpool_vdev_remove_cancel(zhp) != 0)
1302                         ret = 1;
1303                 if (wait) {
1304                         (void) fprintf(stderr, gettext("invalid option "
1305                             "combination: -w cannot be used with -s\n"));
1306                         usage(B_FALSE);
1307                 }
1308         } else {
1309                 if (argc < 2) {
1310                         (void) fprintf(stderr, gettext("missing device\n"));
1311                         usage(B_FALSE);
1312                 }
1313
1314                 for (i = 1; i < argc; i++) {
1315                         if (noop) {
1316                                 uint64_t size;
1317
1318                                 if (zpool_vdev_indirect_size(zhp, argv[i],
1319                                     &size) != 0) {
1320                                         ret = 1;
1321                                         break;
1322                                 }
1323                                 if (parsable) {
1324                                         (void) printf("%s %llu\n",
1325                                             argv[i], (unsigned long long)size);
1326                                 } else {
1327                                         char valstr[32];
1328                                         zfs_nicenum(size, valstr,
1329                                             sizeof (valstr));
1330                                         (void) printf("Memory that will be "
1331                                             "used after removing %s: %s\n",
1332                                             argv[i], valstr);
1333                                 }
1334                         } else {
1335                                 if (zpool_vdev_remove(zhp, argv[i]) != 0)
1336                                         ret = 1;
1337                         }
1338                 }
1339
1340                 if (ret == 0 && wait)
1341                         ret = zpool_wait(zhp, ZPOOL_WAIT_REMOVE);
1342         }
1343         zpool_close(zhp);
1344
1345         return (ret);
1346 }
1347
1348 /*
1349  * Return 1 if a vdev is active (being used in a pool)
1350  * Return 0 if a vdev is inactive (offlined or faulted, or not in active pool)
1351  *
1352  * This is useful for checking if a disk in an active pool is offlined or
1353  * faulted.
1354  */
1355 static int
1356 vdev_is_active(char *vdev_path)
1357 {
1358         int fd;
1359         fd = open(vdev_path, O_EXCL);
1360         if (fd < 0) {
1361                 return (1);   /* cant open O_EXCL - disk is active */
1362         }
1363
1364         close(fd);
1365         return (0);   /* disk is inactive in the pool */
1366 }
1367
1368 /*
1369  * zpool labelclear [-f] <vdev>
1370  *
1371  *      -f      Force clearing the label for the vdevs which are members of
1372  *              the exported or foreign pools.
1373  *
1374  * Verifies that the vdev is not active and zeros out the label information
1375  * on the device.
1376  */
1377 int
1378 zpool_do_labelclear(int argc, char **argv)
1379 {
1380         char vdev[MAXPATHLEN];
1381         char *name = NULL;
1382         int c, fd = -1, ret = 0;
1383         nvlist_t *config;
1384         pool_state_t state;
1385         boolean_t inuse = B_FALSE;
1386         boolean_t force = B_FALSE;
1387
1388         /* check options */
1389         while ((c = getopt(argc, argv, "f")) != -1) {
1390                 switch (c) {
1391                 case 'f':
1392                         force = B_TRUE;
1393                         break;
1394                 default:
1395                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1396                             optopt);
1397                         usage(B_FALSE);
1398                 }
1399         }
1400
1401         argc -= optind;
1402         argv += optind;
1403
1404         /* get vdev name */
1405         if (argc < 1) {
1406                 (void) fprintf(stderr, gettext("missing vdev name\n"));
1407                 usage(B_FALSE);
1408         }
1409         if (argc > 1) {
1410                 (void) fprintf(stderr, gettext("too many arguments\n"));
1411                 usage(B_FALSE);
1412         }
1413
1414         (void) strlcpy(vdev, argv[0], sizeof (vdev));
1415
1416         /*
1417          * If we cannot open an absolute path, we quit.
1418          * Otherwise if the provided vdev name doesn't point to a file,
1419          * try prepending expected disk paths and partition numbers.
1420          */
1421         if ((fd = open(vdev, O_RDWR)) < 0) {
1422                 int error;
1423                 if (vdev[0] == '/') {
1424                         (void) fprintf(stderr, gettext("failed to open "
1425                             "%s: %s\n"), vdev, strerror(errno));
1426                         return (1);
1427                 }
1428
1429                 error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
1430                 if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
1431                         if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
1432                                 error = ENOENT;
1433                 }
1434
1435                 if (error || ((fd = open(vdev, O_RDWR)) < 0)) {
1436                         if (errno == ENOENT) {
1437                                 (void) fprintf(stderr, gettext(
1438                                     "failed to find device %s, try "
1439                                     "specifying absolute path instead\n"),
1440                                     argv[0]);
1441                                 return (1);
1442                         }
1443
1444                         (void) fprintf(stderr, gettext("failed to open %s:"
1445                             " %s\n"), vdev, strerror(errno));
1446                         return (1);
1447                 }
1448         }
1449
1450         /*
1451          * Flush all dirty pages for the block device.  This should not be
1452          * fatal when the device does not support BLKFLSBUF as would be the
1453          * case for a file vdev.
1454          */
1455         if ((zfs_dev_flush(fd) != 0) && (errno != ENOTTY))
1456                 (void) fprintf(stderr, gettext("failed to invalidate "
1457                     "cache for %s: %s\n"), vdev, strerror(errno));
1458
1459         if (zpool_read_label(fd, &config, NULL) != 0) {
1460                 (void) fprintf(stderr,
1461                     gettext("failed to read label from %s\n"), vdev);
1462                 ret = 1;
1463                 goto errout;
1464         }
1465         nvlist_free(config);
1466
1467         ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
1468         if (ret != 0) {
1469                 (void) fprintf(stderr,
1470                     gettext("failed to check state for %s\n"), vdev);
1471                 ret = 1;
1472                 goto errout;
1473         }
1474
1475         if (!inuse)
1476                 goto wipe_label;
1477
1478         switch (state) {
1479         default:
1480         case POOL_STATE_ACTIVE:
1481         case POOL_STATE_SPARE:
1482         case POOL_STATE_L2CACHE:
1483                 /*
1484                  * We allow the user to call 'zpool offline -f'
1485                  * on an offlined disk in an active pool. We can check if
1486                  * the disk is online by calling vdev_is_active().
1487                  */
1488                 if (force && !vdev_is_active(vdev))
1489                         break;
1490
1491                 (void) fprintf(stderr, gettext(
1492                     "%s is a member (%s) of pool \"%s\""),
1493                     vdev, zpool_pool_state_to_name(state), name);
1494
1495                 if (force) {
1496                         (void) fprintf(stderr, gettext(
1497                             ". Offline the disk first to clear its label."));
1498                 }
1499                 printf("\n");
1500                 ret = 1;
1501                 goto errout;
1502
1503         case POOL_STATE_EXPORTED:
1504                 if (force)
1505                         break;
1506                 (void) fprintf(stderr, gettext(
1507                     "use '-f' to override the following error:\n"
1508                     "%s is a member of exported pool \"%s\"\n"),
1509                     vdev, name);
1510                 ret = 1;
1511                 goto errout;
1512
1513         case POOL_STATE_POTENTIALLY_ACTIVE:
1514                 if (force)
1515                         break;
1516                 (void) fprintf(stderr, gettext(
1517                     "use '-f' to override the following error:\n"
1518                     "%s is a member of potentially active pool \"%s\"\n"),
1519                     vdev, name);
1520                 ret = 1;
1521                 goto errout;
1522
1523         case POOL_STATE_DESTROYED:
1524                 /* inuse should never be set for a destroyed pool */
1525                 assert(0);
1526                 break;
1527         }
1528
1529 wipe_label:
1530         ret = zpool_clear_label(fd);
1531         if (ret != 0) {
1532                 (void) fprintf(stderr,
1533                     gettext("failed to clear label for %s\n"), vdev);
1534         }
1535
1536 errout:
1537         free(name);
1538         (void) close(fd);
1539
1540         return (ret);
1541 }
1542
1543 /*
1544  * zpool create [-fnd] [-o property=value] ...
1545  *              [-O file-system-property=value] ...
1546  *              [-R root] [-m mountpoint] <pool> <dev> ...
1547  *
1548  *      -f      Force creation, even if devices appear in use
1549  *      -n      Do not create the pool, but display the resulting layout if it
1550  *              were to be created.
1551  *      -R      Create a pool under an alternate root
1552  *      -m      Set default mountpoint for the root dataset.  By default it's
1553  *              '/<pool>'
1554  *      -o      Set property=value.
1555  *      -o      Set feature@feature=enabled|disabled.
1556  *      -d      Don't automatically enable all supported pool features
1557  *              (individual features can be enabled with -o).
1558  *      -O      Set fsproperty=value in the pool's root file system
1559  *
1560  * Creates the named pool according to the given vdev specification.  The
1561  * bulk of the vdev processing is done in make_root_vdev() in zpool_vdev.c.
1562  * Once we get the nvlist back from make_root_vdev(), we either print out the
1563  * contents (if '-n' was specified), or pass it to libzfs to do the creation.
1564  */
1565 int
1566 zpool_do_create(int argc, char **argv)
1567 {
1568         boolean_t force = B_FALSE;
1569         boolean_t dryrun = B_FALSE;
1570         boolean_t enable_pool_features = B_TRUE;
1571
1572         int c;
1573         nvlist_t *nvroot = NULL;
1574         char *poolname;
1575         char *tname = NULL;
1576         int ret = 1;
1577         char *altroot = NULL;
1578         char *compat = NULL;
1579         char *mountpoint = NULL;
1580         nvlist_t *fsprops = NULL;
1581         nvlist_t *props = NULL;
1582         char *propval;
1583
1584         /* check options */
1585         while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1586                 switch (c) {
1587                 case 'f':
1588                         force = B_TRUE;
1589                         break;
1590                 case 'n':
1591                         dryrun = B_TRUE;
1592                         break;
1593                 case 'd':
1594                         enable_pool_features = B_FALSE;
1595                         break;
1596                 case 'R':
1597                         altroot = optarg;
1598                         if (add_prop_list(zpool_prop_to_name(
1599                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1600                                 goto errout;
1601                         if (add_prop_list_default(zpool_prop_to_name(
1602                             ZPOOL_PROP_CACHEFILE), "none", &props))
1603                                 goto errout;
1604                         break;
1605                 case 'm':
1606                         /* Equivalent to -O mountpoint=optarg */
1607                         mountpoint = optarg;
1608                         break;
1609                 case 'o':
1610                         if ((propval = strchr(optarg, '=')) == NULL) {
1611                                 (void) fprintf(stderr, gettext("missing "
1612                                     "'=' for -o option\n"));
1613                                 goto errout;
1614                         }
1615                         *propval = '\0';
1616                         propval++;
1617
1618                         if (add_prop_list(optarg, propval, &props, B_TRUE))
1619                                 goto errout;
1620
1621                         /*
1622                          * If the user is creating a pool that doesn't support
1623                          * feature flags, don't enable any features.
1624                          */
1625                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1626                                 char *end;
1627                                 u_longlong_t ver;
1628
1629                                 ver = strtoull(propval, &end, 10);
1630                                 if (*end == '\0' &&
1631                                     ver < SPA_VERSION_FEATURES) {
1632                                         enable_pool_features = B_FALSE;
1633                                 }
1634                         }
1635                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1636                                 altroot = propval;
1637                         if (zpool_name_to_prop(optarg) ==
1638                             ZPOOL_PROP_COMPATIBILITY)
1639                                 compat = propval;
1640                         break;
1641                 case 'O':
1642                         if ((propval = strchr(optarg, '=')) == NULL) {
1643                                 (void) fprintf(stderr, gettext("missing "
1644                                     "'=' for -O option\n"));
1645                                 goto errout;
1646                         }
1647                         *propval = '\0';
1648                         propval++;
1649
1650                         /*
1651                          * Mountpoints are checked and then added later.
1652                          * Uniquely among properties, they can be specified
1653                          * more than once, to avoid conflict with -m.
1654                          */
1655                         if (0 == strcmp(optarg,
1656                             zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1657                                 mountpoint = propval;
1658                         } else if (add_prop_list(optarg, propval, &fsprops,
1659                             B_FALSE)) {
1660                                 goto errout;
1661                         }
1662                         break;
1663                 case 't':
1664                         /*
1665                          * Sanity check temporary pool name.
1666                          */
1667                         if (strchr(optarg, '/') != NULL) {
1668                                 (void) fprintf(stderr, gettext("cannot create "
1669                                     "'%s': invalid character '/' in temporary "
1670                                     "name\n"), optarg);
1671                                 (void) fprintf(stderr, gettext("use 'zfs "
1672                                     "create' to create a dataset\n"));
1673                                 goto errout;
1674                         }
1675
1676                         if (add_prop_list(zpool_prop_to_name(
1677                             ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1678                                 goto errout;
1679                         if (add_prop_list_default(zpool_prop_to_name(
1680                             ZPOOL_PROP_CACHEFILE), "none", &props))
1681                                 goto errout;
1682                         tname = optarg;
1683                         break;
1684                 case ':':
1685                         (void) fprintf(stderr, gettext("missing argument for "
1686                             "'%c' option\n"), optopt);
1687                         goto badusage;
1688                 case '?':
1689                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1690                             optopt);
1691                         goto badusage;
1692                 }
1693         }
1694
1695         argc -= optind;
1696         argv += optind;
1697
1698         /* get pool name and check number of arguments */
1699         if (argc < 1) {
1700                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1701                 goto badusage;
1702         }
1703         if (argc < 2) {
1704                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1705                 goto badusage;
1706         }
1707
1708         poolname = argv[0];
1709
1710         /*
1711          * As a special case, check for use of '/' in the name, and direct the
1712          * user to use 'zfs create' instead.
1713          */
1714         if (strchr(poolname, '/') != NULL) {
1715                 (void) fprintf(stderr, gettext("cannot create '%s': invalid "
1716                     "character '/' in pool name\n"), poolname);
1717                 (void) fprintf(stderr, gettext("use 'zfs create' to "
1718                     "create a dataset\n"));
1719                 goto errout;
1720         }
1721
1722         /* pass off to make_root_vdev for bulk processing */
1723         nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1724             argc - 1, argv + 1);
1725         if (nvroot == NULL)
1726                 goto errout;
1727
1728         /* make_root_vdev() allows 0 toplevel children if there are spares */
1729         if (!zfs_allocatable_devs(nvroot)) {
1730                 (void) fprintf(stderr, gettext("invalid vdev "
1731                     "specification: at least one toplevel vdev must be "
1732                     "specified\n"));
1733                 goto errout;
1734         }
1735
1736         if (altroot != NULL && altroot[0] != '/') {
1737                 (void) fprintf(stderr, gettext("invalid alternate root '%s': "
1738                     "must be an absolute path\n"), altroot);
1739                 goto errout;
1740         }
1741
1742         /*
1743          * Check the validity of the mountpoint and direct the user to use the
1744          * '-m' mountpoint option if it looks like its in use.
1745          */
1746         if (mountpoint == NULL ||
1747             (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1748             strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1749                 char buf[MAXPATHLEN];
1750                 DIR *dirp;
1751
1752                 if (mountpoint && mountpoint[0] != '/') {
1753                         (void) fprintf(stderr, gettext("invalid mountpoint "
1754                             "'%s': must be an absolute path, 'legacy', or "
1755                             "'none'\n"), mountpoint);
1756                         goto errout;
1757                 }
1758
1759                 if (mountpoint == NULL) {
1760                         if (altroot != NULL)
1761                                 (void) snprintf(buf, sizeof (buf), "%s/%s",
1762                                     altroot, poolname);
1763                         else
1764                                 (void) snprintf(buf, sizeof (buf), "/%s",
1765                                     poolname);
1766                 } else {
1767                         if (altroot != NULL)
1768                                 (void) snprintf(buf, sizeof (buf), "%s%s",
1769                                     altroot, mountpoint);
1770                         else
1771                                 (void) snprintf(buf, sizeof (buf), "%s",
1772                                     mountpoint);
1773                 }
1774
1775                 if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1776                         (void) fprintf(stderr, gettext("mountpoint '%s' : "
1777                             "%s\n"), buf, strerror(errno));
1778                         (void) fprintf(stderr, gettext("use '-m' "
1779                             "option to provide a different default\n"));
1780                         goto errout;
1781                 } else if (dirp) {
1782                         int count = 0;
1783
1784                         while (count < 3 && readdir(dirp) != NULL)
1785                                 count++;
1786                         (void) closedir(dirp);
1787
1788                         if (count > 2) {
1789                                 (void) fprintf(stderr, gettext("mountpoint "
1790                                     "'%s' exists and is not empty\n"), buf);
1791                                 (void) fprintf(stderr, gettext("use '-m' "
1792                                     "option to provide a "
1793                                     "different default\n"));
1794                                 goto errout;
1795                         }
1796                 }
1797         }
1798
1799         /*
1800          * Now that the mountpoint's validity has been checked, ensure that
1801          * the property is set appropriately prior to creating the pool.
1802          */
1803         if (mountpoint != NULL) {
1804                 ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1805                     mountpoint, &fsprops, B_FALSE);
1806                 if (ret != 0)
1807                         goto errout;
1808         }
1809
1810         ret = 1;
1811         if (dryrun) {
1812                 /*
1813                  * For a dry run invocation, print out a basic message and run
1814                  * through all the vdevs in the list and print out in an
1815                  * appropriate hierarchy.
1816                  */
1817                 (void) printf(gettext("would create '%s' with the "
1818                     "following layout:\n\n"), poolname);
1819
1820                 print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1821                 print_vdev_tree(NULL, "dedup", nvroot, 0,
1822                     VDEV_ALLOC_BIAS_DEDUP, 0);
1823                 print_vdev_tree(NULL, "special", nvroot, 0,
1824                     VDEV_ALLOC_BIAS_SPECIAL, 0);
1825                 print_vdev_tree(NULL, "logs", nvroot, 0,
1826                     VDEV_ALLOC_BIAS_LOG, 0);
1827                 print_cache_list(nvroot, 0);
1828                 print_spare_list(nvroot, 0);
1829
1830                 ret = 0;
1831         } else {
1832                 /*
1833                  * Load in feature set.
1834                  * Note: if compatibility property not given, we'll have
1835                  * NULL, which means 'all features'.
1836                  */
1837                 boolean_t requested_features[SPA_FEATURES];
1838                 if (zpool_do_load_compat(compat, requested_features) !=
1839                     ZPOOL_COMPATIBILITY_OK)
1840                         goto errout;
1841
1842                 /*
1843                  * props contains list of features to enable.
1844                  * For each feature:
1845                  *  - remove it if feature@name=disabled
1846                  *  - leave it there if feature@name=enabled
1847                  *  - add it if:
1848                  *    - enable_pool_features (ie: no '-d' or '-o version')
1849                  *    - it's supported by the kernel module
1850                  *    - it's in the requested feature set
1851                  *  - warn if it's enabled but not in compat
1852                  */
1853                 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
1854                         char propname[MAXPATHLEN];
1855                         const char *propval;
1856                         zfeature_info_t *feat = &spa_feature_table[i];
1857
1858                         (void) snprintf(propname, sizeof (propname),
1859                             "feature@%s", feat->fi_uname);
1860
1861                         if (!nvlist_lookup_string(props, propname, &propval)) {
1862                                 if (strcmp(propval,
1863                                     ZFS_FEATURE_DISABLED) == 0) {
1864                                         (void) nvlist_remove_all(props,
1865                                             propname);
1866                                 } else if (strcmp(propval,
1867                                     ZFS_FEATURE_ENABLED) == 0 &&
1868                                     !requested_features[i]) {
1869                                         (void) fprintf(stderr, gettext(
1870                                             "Warning: feature \"%s\" enabled "
1871                                             "but is not in specified "
1872                                             "'compatibility' feature set.\n"),
1873                                             feat->fi_uname);
1874                                 }
1875                         } else if (
1876                             enable_pool_features &&
1877                             feat->fi_zfs_mod_supported &&
1878                             requested_features[i]) {
1879                                 ret = add_prop_list(propname,
1880                                     ZFS_FEATURE_ENABLED, &props, B_TRUE);
1881                                 if (ret != 0)
1882                                         goto errout;
1883                         }
1884                 }
1885
1886                 ret = 1;
1887                 if (zpool_create(g_zfs, poolname,
1888                     nvroot, props, fsprops) == 0) {
1889                         zfs_handle_t *pool = zfs_open(g_zfs,
1890                             tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1891                         if (pool != NULL) {
1892                                 if (zfs_mount(pool, NULL, 0) == 0) {
1893                                         ret = zfs_share(pool, NULL);
1894                                         zfs_commit_shares(NULL);
1895                                 }
1896                                 zfs_close(pool);
1897                         }
1898                 } else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1899                         (void) fprintf(stderr, gettext("pool name may have "
1900                             "been omitted\n"));
1901                 }
1902         }
1903
1904 errout:
1905         nvlist_free(nvroot);
1906         nvlist_free(fsprops);
1907         nvlist_free(props);
1908         return (ret);
1909 badusage:
1910         nvlist_free(fsprops);
1911         nvlist_free(props);
1912         usage(B_FALSE);
1913         return (2);
1914 }
1915
1916 /*
1917  * zpool destroy <pool>
1918  *
1919  *      -f      Forcefully unmount any datasets
1920  *
1921  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1922  */
1923 int
1924 zpool_do_destroy(int argc, char **argv)
1925 {
1926         boolean_t force = B_FALSE;
1927         int c;
1928         char *pool;
1929         zpool_handle_t *zhp;
1930         int ret;
1931
1932         /* check options */
1933         while ((c = getopt(argc, argv, "f")) != -1) {
1934                 switch (c) {
1935                 case 'f':
1936                         force = B_TRUE;
1937                         break;
1938                 case '?':
1939                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1940                             optopt);
1941                         usage(B_FALSE);
1942                 }
1943         }
1944
1945         argc -= optind;
1946         argv += optind;
1947
1948         /* check arguments */
1949         if (argc < 1) {
1950                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1951                 usage(B_FALSE);
1952         }
1953         if (argc > 1) {
1954                 (void) fprintf(stderr, gettext("too many arguments\n"));
1955                 usage(B_FALSE);
1956         }
1957
1958         pool = argv[0];
1959
1960         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1961                 /*
1962                  * As a special case, check for use of '/' in the name, and
1963                  * direct the user to use 'zfs destroy' instead.
1964                  */
1965                 if (strchr(pool, '/') != NULL)
1966                         (void) fprintf(stderr, gettext("use 'zfs destroy' to "
1967                             "destroy a dataset\n"));
1968                 return (1);
1969         }
1970
1971         if (zpool_disable_datasets(zhp, force) != 0) {
1972                 (void) fprintf(stderr, gettext("could not destroy '%s': "
1973                     "could not unmount datasets\n"), zpool_get_name(zhp));
1974                 zpool_close(zhp);
1975                 return (1);
1976         }
1977
1978         /* The history must be logged as part of the export */
1979         log_history = B_FALSE;
1980
1981         ret = (zpool_destroy(zhp, history_str) != 0);
1982
1983         zpool_close(zhp);
1984
1985         return (ret);
1986 }
1987
1988 typedef struct export_cbdata {
1989         boolean_t force;
1990         boolean_t hardforce;
1991 } export_cbdata_t;
1992
1993 /*
1994  * Export one pool
1995  */
1996 static int
1997 zpool_export_one(zpool_handle_t *zhp, void *data)
1998 {
1999         export_cbdata_t *cb = data;
2000
2001         if (zpool_disable_datasets(zhp, cb->force) != 0)
2002                 return (1);
2003
2004         /* The history must be logged as part of the export */
2005         log_history = B_FALSE;
2006
2007         if (cb->hardforce) {
2008                 if (zpool_export_force(zhp, history_str) != 0)
2009                         return (1);
2010         } else if (zpool_export(zhp, cb->force, history_str) != 0) {
2011                 return (1);
2012         }
2013
2014         return (0);
2015 }
2016
2017 /*
2018  * zpool export [-f] <pool> ...
2019  *
2020  *      -a      Export all pools
2021  *      -f      Forcefully unmount datasets
2022  *
2023  * Export the given pools.  By default, the command will attempt to cleanly
2024  * unmount any active datasets within the pool.  If the '-f' flag is specified,
2025  * then the datasets will be forcefully unmounted.
2026  */
2027 int
2028 zpool_do_export(int argc, char **argv)
2029 {
2030         export_cbdata_t cb;
2031         boolean_t do_all = B_FALSE;
2032         boolean_t force = B_FALSE;
2033         boolean_t hardforce = B_FALSE;
2034         int c, ret;
2035
2036         /* check options */
2037         while ((c = getopt(argc, argv, "afF")) != -1) {
2038                 switch (c) {
2039                 case 'a':
2040                         do_all = B_TRUE;
2041                         break;
2042                 case 'f':
2043                         force = B_TRUE;
2044                         break;
2045                 case 'F':
2046                         hardforce = B_TRUE;
2047                         break;
2048                 case '?':
2049                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
2050                             optopt);
2051                         usage(B_FALSE);
2052                 }
2053         }
2054
2055         cb.force = force;
2056         cb.hardforce = hardforce;
2057         argc -= optind;
2058         argv += optind;
2059
2060         if (do_all) {
2061                 if (argc != 0) {
2062                         (void) fprintf(stderr, gettext("too many arguments\n"));
2063                         usage(B_FALSE);
2064                 }
2065
2066                 return (for_each_pool(argc, argv, B_TRUE, NULL,
2067                     ZFS_TYPE_POOL, B_FALSE, zpool_export_one, &cb));
2068         }
2069
2070         /* check arguments */
2071         if (argc < 1) {
2072                 (void) fprintf(stderr, gettext("missing pool argument\n"));
2073                 usage(B_FALSE);
2074         }
2075
2076         ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
2077             B_FALSE, zpool_export_one, &cb);
2078
2079         return (ret);
2080 }
2081
2082 /*
2083  * Given a vdev configuration, determine the maximum width needed for the device
2084  * name column.
2085  */
2086 static int
2087 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
2088     int name_flags)
2089 {
2090         static const char *const subtypes[] =
2091             {ZPOOL_CONFIG_SPARES, ZPOOL_CONFIG_L2CACHE, ZPOOL_CONFIG_CHILDREN};
2092
2093         char *name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
2094         max = MAX(strlen(name) + depth, max);
2095         free(name);
2096
2097         nvlist_t **child;
2098         uint_t children;
2099         for (size_t i = 0; i < ARRAY_SIZE(subtypes); ++i)
2100                 if (nvlist_lookup_nvlist_array(nv, subtypes[i],
2101                     &child, &children) == 0)
2102                         for (uint_t c = 0; c < children; ++c)
2103                                 max = MAX(max_width(zhp, child[c], depth + 2,
2104                                     max, name_flags), max);
2105
2106         return (max);
2107 }
2108
2109 typedef struct spare_cbdata {
2110         uint64_t        cb_guid;
2111         zpool_handle_t  *cb_zhp;
2112 } spare_cbdata_t;
2113
2114 static boolean_t
2115 find_vdev(nvlist_t *nv, uint64_t search)
2116 {
2117         uint64_t guid;
2118         nvlist_t **child;
2119         uint_t c, children;
2120
2121         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
2122             search == guid)
2123                 return (B_TRUE);
2124
2125         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2126             &child, &children) == 0) {
2127                 for (c = 0; c < children; c++)
2128                         if (find_vdev(child[c], search))
2129                                 return (B_TRUE);
2130         }
2131
2132         return (B_FALSE);
2133 }
2134
2135 static int
2136 find_spare(zpool_handle_t *zhp, void *data)
2137 {
2138         spare_cbdata_t *cbp = data;
2139         nvlist_t *config, *nvroot;
2140
2141         config = zpool_get_config(zhp, NULL);
2142         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2143             &nvroot) == 0);
2144
2145         if (find_vdev(nvroot, cbp->cb_guid)) {
2146                 cbp->cb_zhp = zhp;
2147                 return (1);
2148         }
2149
2150         zpool_close(zhp);
2151         return (0);
2152 }
2153
2154 typedef struct status_cbdata {
2155         int             cb_count;
2156         int             cb_name_flags;
2157         int             cb_namewidth;
2158         boolean_t       cb_allpools;
2159         boolean_t       cb_verbose;
2160         boolean_t       cb_literal;
2161         boolean_t       cb_explain;
2162         boolean_t       cb_first;
2163         boolean_t       cb_dedup_stats;
2164         boolean_t       cb_print_unhealthy;
2165         boolean_t       cb_print_status;
2166         boolean_t       cb_print_slow_ios;
2167         boolean_t       cb_print_vdev_init;
2168         boolean_t       cb_print_vdev_trim;
2169         vdev_cmd_data_list_t    *vcdl;
2170         boolean_t       cb_print_power;
2171 } status_cbdata_t;
2172
2173 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
2174 static boolean_t
2175 is_blank_str(const char *str)
2176 {
2177         for (; str != NULL && *str != '\0'; ++str)
2178                 if (!isblank(*str))
2179                         return (B_FALSE);
2180         return (B_TRUE);
2181 }
2182
2183 /* Print command output lines for specific vdev in a specific pool */
2184 static void
2185 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, const char *path)
2186 {
2187         vdev_cmd_data_t *data;
2188         int i, j;
2189         const char *val;
2190
2191         for (i = 0; i < vcdl->count; i++) {
2192                 if ((strcmp(vcdl->data[i].path, path) != 0) ||
2193                     (strcmp(vcdl->data[i].pool, pool) != 0)) {
2194                         /* Not the vdev we're looking for */
2195                         continue;
2196                 }
2197
2198                 data = &vcdl->data[i];
2199                 /* Print out all the output values for this vdev */
2200                 for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
2201                         val = NULL;
2202                         /* Does this vdev have values for this column? */
2203                         for (int k = 0; k < data->cols_cnt; k++) {
2204                                 if (strcmp(data->cols[k],
2205                                     vcdl->uniq_cols[j]) == 0) {
2206                                         /* yes it does, record the value */
2207                                         val = data->lines[k];
2208                                         break;
2209                                 }
2210                         }
2211                         /*
2212                          * Mark empty values with dashes to make output
2213                          * awk-able.
2214                          */
2215                         if (val == NULL || is_blank_str(val))
2216                                 val = "-";
2217
2218                         printf("%*s", vcdl->uniq_cols_width[j], val);
2219                         if (j < vcdl->uniq_cols_cnt - 1)
2220                                 fputs("  ", stdout);
2221                 }
2222
2223                 /* Print out any values that aren't in a column at the end */
2224                 for (j = data->cols_cnt; j < data->lines_cnt; j++) {
2225                         /* Did we have any columns?  If so print a spacer. */
2226                         if (vcdl->uniq_cols_cnt > 0)
2227                                 fputs("  ", stdout);
2228
2229                         val = data->lines[j];
2230                         fputs(val ?: "", stdout);
2231                 }
2232                 break;
2233         }
2234 }
2235
2236 /*
2237  * Print vdev initialization status for leaves
2238  */
2239 static void
2240 print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
2241 {
2242         if (verbose) {
2243                 if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
2244                     vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
2245                     vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
2246                     !vs->vs_scan_removing) {
2247                         char zbuf[1024];
2248                         char tbuf[256];
2249                         struct tm zaction_ts;
2250
2251                         time_t t = vs->vs_initialize_action_time;
2252                         int initialize_pct = 100;
2253                         if (vs->vs_initialize_state !=
2254                             VDEV_INITIALIZE_COMPLETE) {
2255                                 initialize_pct = (vs->vs_initialize_bytes_done *
2256                                     100 / (vs->vs_initialize_bytes_est + 1));
2257                         }
2258
2259                         (void) localtime_r(&t, &zaction_ts);
2260                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2261
2262                         switch (vs->vs_initialize_state) {
2263                         case VDEV_INITIALIZE_SUSPENDED:
2264                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2265                                     gettext("suspended, started at"), tbuf);
2266                                 break;
2267                         case VDEV_INITIALIZE_ACTIVE:
2268                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2269                                     gettext("started at"), tbuf);
2270                                 break;
2271                         case VDEV_INITIALIZE_COMPLETE:
2272                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2273                                     gettext("completed at"), tbuf);
2274                                 break;
2275                         }
2276
2277                         (void) printf(gettext("  (%d%% initialized%s)"),
2278                             initialize_pct, zbuf);
2279                 } else {
2280                         (void) printf(gettext("  (uninitialized)"));
2281                 }
2282         } else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
2283                 (void) printf(gettext("  (initializing)"));
2284         }
2285 }
2286
2287 /*
2288  * Print vdev TRIM status for leaves
2289  */
2290 static void
2291 print_status_trim(vdev_stat_t *vs, boolean_t verbose)
2292 {
2293         if (verbose) {
2294                 if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
2295                     vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
2296                     vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
2297                     !vs->vs_scan_removing) {
2298                         char zbuf[1024];
2299                         char tbuf[256];
2300                         struct tm zaction_ts;
2301
2302                         time_t t = vs->vs_trim_action_time;
2303                         int trim_pct = 100;
2304                         if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
2305                                 trim_pct = (vs->vs_trim_bytes_done *
2306                                     100 / (vs->vs_trim_bytes_est + 1));
2307                         }
2308
2309                         (void) localtime_r(&t, &zaction_ts);
2310                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2311
2312                         switch (vs->vs_trim_state) {
2313                         case VDEV_TRIM_SUSPENDED:
2314                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2315                                     gettext("suspended, started at"), tbuf);
2316                                 break;
2317                         case VDEV_TRIM_ACTIVE:
2318                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2319                                     gettext("started at"), tbuf);
2320                                 break;
2321                         case VDEV_TRIM_COMPLETE:
2322                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2323                                     gettext("completed at"), tbuf);
2324                                 break;
2325                         }
2326
2327                         (void) printf(gettext("  (%d%% trimmed%s)"),
2328                             trim_pct, zbuf);
2329                 } else if (vs->vs_trim_notsup) {
2330                         (void) printf(gettext("  (trim unsupported)"));
2331                 } else {
2332                         (void) printf(gettext("  (untrimmed)"));
2333                 }
2334         } else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
2335                 (void) printf(gettext("  (trimming)"));
2336         }
2337 }
2338
2339 /*
2340  * Return the color associated with a health string.  This includes returning
2341  * NULL for no color change.
2342  */
2343 static const char *
2344 health_str_to_color(const char *health)
2345 {
2346         if (strcmp(health, gettext("FAULTED")) == 0 ||
2347             strcmp(health, gettext("SUSPENDED")) == 0 ||
2348             strcmp(health, gettext("UNAVAIL")) == 0) {
2349                 return (ANSI_RED);
2350         }
2351
2352         if (strcmp(health, gettext("OFFLINE")) == 0 ||
2353             strcmp(health, gettext("DEGRADED")) == 0 ||
2354             strcmp(health, gettext("REMOVED")) == 0) {
2355                 return (ANSI_YELLOW);
2356         }
2357
2358         return (NULL);
2359 }
2360
2361 /*
2362  * Called for each leaf vdev.  Returns 0 if the vdev is healthy.
2363  * A vdev is unhealthy if any of the following are true:
2364  * 1) there are read, write, or checksum errors,
2365  * 2) its state is not ONLINE, or
2366  * 3) slow IO reporting was requested (-s) and there are slow IOs.
2367  */
2368 static int
2369 vdev_health_check_cb(void *hdl_data, nvlist_t *nv, void *data)
2370 {
2371         status_cbdata_t *cb = data;
2372         vdev_stat_t *vs;
2373         uint_t vsc;
2374         (void) hdl_data;
2375
2376         if (nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2377             (uint64_t **)&vs, &vsc) != 0)
2378                 return (1);
2379
2380         if (vs->vs_checksum_errors || vs->vs_read_errors ||
2381             vs->vs_write_errors || vs->vs_state != VDEV_STATE_HEALTHY)
2382                 return (1);
2383
2384         if (cb->cb_print_slow_ios && vs->vs_slow_ios)
2385                 return (1);
2386
2387         return (0);
2388 }
2389
2390 /*
2391  * Print out configuration state as requested by status_callback.
2392  */
2393 static void
2394 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
2395     nvlist_t *nv, int depth, boolean_t isspare, vdev_rebuild_stat_t *vrs)
2396 {
2397         nvlist_t **child, *root;
2398         uint_t c, i, vsc, children;
2399         pool_scan_stat_t *ps = NULL;
2400         vdev_stat_t *vs;
2401         char rbuf[6], wbuf[6], cbuf[6];
2402         char *vname;
2403         uint64_t notpresent;
2404         spare_cbdata_t spare_cb;
2405         const char *state;
2406         const char *type;
2407         const char *path = NULL;
2408         const char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL,
2409             *scolor = NULL;
2410
2411         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2412             &child, &children) != 0)
2413                 children = 0;
2414
2415         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2416             (uint64_t **)&vs, &vsc) == 0);
2417
2418         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2419
2420         if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2421                 return;
2422
2423         state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2424
2425         if (isspare) {
2426                 /*
2427                  * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
2428                  * online drives.
2429                  */
2430                 if (vs->vs_aux == VDEV_AUX_SPARED)
2431                         state = gettext("INUSE");
2432                 else if (vs->vs_state == VDEV_STATE_HEALTHY)
2433                         state = gettext("AVAIL");
2434         }
2435
2436         /*
2437          * If '-e' is specified then top-level vdevs and their children
2438          * can be pruned if all of their leaves are healthy.
2439          */
2440         if (cb->cb_print_unhealthy && depth > 0 &&
2441             for_each_vdev_in_nvlist(nv, vdev_health_check_cb, cb) == 0) {
2442                 return;
2443         }
2444
2445         printf_color(health_str_to_color(state),
2446             "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
2447             name, state);
2448
2449         if (!isspare) {
2450                 if (vs->vs_read_errors)
2451                         rcolor = ANSI_RED;
2452
2453                 if (vs->vs_write_errors)
2454                         wcolor = ANSI_RED;
2455
2456                 if (vs->vs_checksum_errors)
2457                         ccolor = ANSI_RED;
2458
2459                 if (vs->vs_slow_ios)
2460                         scolor = ANSI_BLUE;
2461
2462                 if (cb->cb_literal) {
2463                         fputc(' ', stdout);
2464                         printf_color(rcolor, "%5llu",
2465                             (u_longlong_t)vs->vs_read_errors);
2466                         fputc(' ', stdout);
2467                         printf_color(wcolor, "%5llu",
2468                             (u_longlong_t)vs->vs_write_errors);
2469                         fputc(' ', stdout);
2470                         printf_color(ccolor, "%5llu",
2471                             (u_longlong_t)vs->vs_checksum_errors);
2472                 } else {
2473                         zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
2474                         zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
2475                         zfs_nicenum(vs->vs_checksum_errors, cbuf,
2476                             sizeof (cbuf));
2477                         fputc(' ', stdout);
2478                         printf_color(rcolor, "%5s", rbuf);
2479                         fputc(' ', stdout);
2480                         printf_color(wcolor, "%5s", wbuf);
2481                         fputc(' ', stdout);
2482                         printf_color(ccolor, "%5s", cbuf);
2483                 }
2484                 if (cb->cb_print_slow_ios) {
2485                         if (children == 0)  {
2486                                 /* Only leafs vdevs have slow IOs */
2487                                 zfs_nicenum(vs->vs_slow_ios, rbuf,
2488                                     sizeof (rbuf));
2489                         } else {
2490                                 snprintf(rbuf, sizeof (rbuf), "-");
2491                         }
2492
2493                         if (cb->cb_literal)
2494                                 printf_color(scolor, " %5llu",
2495                                     (u_longlong_t)vs->vs_slow_ios);
2496                         else
2497                                 printf_color(scolor, " %5s", rbuf);
2498                 }
2499                 if (cb->cb_print_power) {
2500                         if (children == 0)  {
2501                                 /* Only leaf vdevs have physical slots */
2502                                 switch (zpool_power_current_state(zhp, (char *)
2503                                     fnvlist_lookup_string(nv,
2504                                     ZPOOL_CONFIG_PATH))) {
2505                                 case 0:
2506                                         printf_color(ANSI_RED, " %5s",
2507                                             gettext("off"));
2508                                         break;
2509                                 case 1:
2510                                         printf(" %5s", gettext("on"));
2511                                         break;
2512                                 default:
2513                                         printf(" %5s", "-");
2514                                 }
2515                         } else {
2516                                 printf(" %5s", "-");
2517                         }
2518                 }
2519         }
2520
2521         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
2522             &notpresent) == 0) {
2523                 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
2524                 (void) printf("  %s %s", gettext("was"), path);
2525         } else if (vs->vs_aux != 0) {
2526                 (void) printf("  ");
2527                 color_start(ANSI_RED);
2528                 switch (vs->vs_aux) {
2529                 case VDEV_AUX_OPEN_FAILED:
2530                         (void) printf(gettext("cannot open"));
2531                         break;
2532
2533                 case VDEV_AUX_BAD_GUID_SUM:
2534                         (void) printf(gettext("missing device"));
2535                         break;
2536
2537                 case VDEV_AUX_NO_REPLICAS:
2538                         (void) printf(gettext("insufficient replicas"));
2539                         break;
2540
2541                 case VDEV_AUX_VERSION_NEWER:
2542                         (void) printf(gettext("newer version"));
2543                         break;
2544
2545                 case VDEV_AUX_UNSUP_FEAT:
2546                         (void) printf(gettext("unsupported feature(s)"));
2547                         break;
2548
2549                 case VDEV_AUX_ASHIFT_TOO_BIG:
2550                         (void) printf(gettext("unsupported minimum blocksize"));
2551                         break;
2552
2553                 case VDEV_AUX_SPARED:
2554                         verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
2555                             &spare_cb.cb_guid) == 0);
2556                         if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
2557                                 if (strcmp(zpool_get_name(spare_cb.cb_zhp),
2558                                     zpool_get_name(zhp)) == 0)
2559                                         (void) printf(gettext("currently in "
2560                                             "use"));
2561                                 else
2562                                         (void) printf(gettext("in use by "
2563                                             "pool '%s'"),
2564                                             zpool_get_name(spare_cb.cb_zhp));
2565                                 zpool_close(spare_cb.cb_zhp);
2566                         } else {
2567                                 (void) printf(gettext("currently in use"));
2568                         }
2569                         break;
2570
2571                 case VDEV_AUX_ERR_EXCEEDED:
2572                         (void) printf(gettext("too many errors"));
2573                         break;
2574
2575                 case VDEV_AUX_IO_FAILURE:
2576                         (void) printf(gettext("experienced I/O failures"));
2577                         break;
2578
2579                 case VDEV_AUX_BAD_LOG:
2580                         (void) printf(gettext("bad intent log"));
2581                         break;
2582
2583                 case VDEV_AUX_EXTERNAL:
2584                         (void) printf(gettext("external device fault"));
2585                         break;
2586
2587                 case VDEV_AUX_SPLIT_POOL:
2588                         (void) printf(gettext("split into new pool"));
2589                         break;
2590
2591                 case VDEV_AUX_ACTIVE:
2592                         (void) printf(gettext("currently in use"));
2593                         break;
2594
2595                 case VDEV_AUX_CHILDREN_OFFLINE:
2596                         (void) printf(gettext("all children offline"));
2597                         break;
2598
2599                 case VDEV_AUX_BAD_LABEL:
2600                         (void) printf(gettext("invalid label"));
2601                         break;
2602
2603                 default:
2604                         (void) printf(gettext("corrupted data"));
2605                         break;
2606                 }
2607                 color_end();
2608         } else if (children == 0 && !isspare &&
2609             getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
2610             VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
2611             vs->vs_configured_ashift < vs->vs_physical_ashift) {
2612                 (void) printf(
2613                     gettext("  block size: %dB configured, %dB native"),
2614                     1 << vs->vs_configured_ashift, 1 << vs->vs_physical_ashift);
2615         }
2616
2617         if (vs->vs_scan_removing != 0) {
2618                 (void) printf(gettext("  (removing)"));
2619         } else if (VDEV_STAT_VALID(vs_noalloc, vsc) && vs->vs_noalloc != 0) {
2620                 (void) printf(gettext("  (non-allocating)"));
2621         }
2622
2623         /* The root vdev has the scrub/resilver stats */
2624         root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
2625             ZPOOL_CONFIG_VDEV_TREE);
2626         (void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
2627             (uint64_t **)&ps, &c);
2628
2629         /*
2630          * If you force fault a drive that's resilvering, its scan stats can
2631          * get frozen in time, giving the false impression that it's
2632          * being resilvered.  That's why we check the state to see if the vdev
2633          * is healthy before reporting "resilvering" or "repairing".
2634          */
2635         if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0 &&
2636             vs->vs_state == VDEV_STATE_HEALTHY) {
2637                 if (vs->vs_scan_processed != 0) {
2638                         (void) printf(gettext("  (%s)"),
2639                             (ps->pss_func == POOL_SCAN_RESILVER) ?
2640                             "resilvering" : "repairing");
2641                 } else if (vs->vs_resilver_deferred) {
2642                         (void) printf(gettext("  (awaiting resilver)"));
2643                 }
2644         }
2645
2646         /* The top-level vdevs have the rebuild stats */
2647         if (vrs != NULL && vrs->vrs_state == VDEV_REBUILD_ACTIVE &&
2648             children == 0 && vs->vs_state == VDEV_STATE_HEALTHY) {
2649                 if (vs->vs_rebuild_processed != 0) {
2650                         (void) printf(gettext("  (resilvering)"));
2651                 }
2652         }
2653
2654         if (cb->vcdl != NULL) {
2655                 if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
2656                         printf("  ");
2657                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
2658                 }
2659         }
2660
2661         /* Display vdev initialization and trim status for leaves. */
2662         if (children == 0) {
2663                 print_status_initialize(vs, cb->cb_print_vdev_init);
2664                 print_status_trim(vs, cb->cb_print_vdev_trim);
2665         }
2666
2667         (void) printf("\n");
2668
2669         for (c = 0; c < children; c++) {
2670                 uint64_t islog = B_FALSE, ishole = B_FALSE;
2671
2672                 /* Don't print logs or holes here */
2673                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2674                     &islog);
2675                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
2676                     &ishole);
2677                 if (islog || ishole)
2678                         continue;
2679                 /* Only print normal classes here */
2680                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2681                         continue;
2682
2683                 /* Provide vdev_rebuild_stats to children if available */
2684                 if (vrs == NULL) {
2685                         (void) nvlist_lookup_uint64_array(nv,
2686                             ZPOOL_CONFIG_REBUILD_STATS,
2687                             (uint64_t **)&vrs, &i);
2688                 }
2689
2690                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
2691                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2692                 print_status_config(zhp, cb, vname, child[c], depth + 2,
2693                     isspare, vrs);
2694                 free(vname);
2695         }
2696 }
2697
2698 /*
2699  * Print the configuration of an exported pool.  Iterate over all vdevs in the
2700  * pool, printing out the name and status for each one.
2701  */
2702 static void
2703 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
2704     int depth)
2705 {
2706         nvlist_t **child;
2707         uint_t c, children;
2708         vdev_stat_t *vs;
2709         const char *type;
2710         char *vname;
2711
2712         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2713         if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
2714             strcmp(type, VDEV_TYPE_HOLE) == 0)
2715                 return;
2716
2717         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2718             (uint64_t **)&vs, &c) == 0);
2719
2720         (void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
2721         (void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
2722
2723         if (vs->vs_aux != 0) {
2724                 (void) printf("  ");
2725
2726                 switch (vs->vs_aux) {
2727                 case VDEV_AUX_OPEN_FAILED:
2728                         (void) printf(gettext("cannot open"));
2729                         break;
2730
2731                 case VDEV_AUX_BAD_GUID_SUM:
2732                         (void) printf(gettext("missing device"));
2733                         break;
2734
2735                 case VDEV_AUX_NO_REPLICAS:
2736                         (void) printf(gettext("insufficient replicas"));
2737                         break;
2738
2739                 case VDEV_AUX_VERSION_NEWER:
2740                         (void) printf(gettext("newer version"));
2741                         break;
2742
2743                 case VDEV_AUX_UNSUP_FEAT:
2744                         (void) printf(gettext("unsupported feature(s)"));
2745                         break;
2746
2747                 case VDEV_AUX_ERR_EXCEEDED:
2748                         (void) printf(gettext("too many errors"));
2749                         break;
2750
2751                 case VDEV_AUX_ACTIVE:
2752                         (void) printf(gettext("currently in use"));
2753                         break;
2754
2755                 case VDEV_AUX_CHILDREN_OFFLINE:
2756                         (void) printf(gettext("all children offline"));
2757                         break;
2758
2759                 case VDEV_AUX_BAD_LABEL:
2760                         (void) printf(gettext("invalid label"));
2761                         break;
2762
2763                 default:
2764                         (void) printf(gettext("corrupted data"));
2765                         break;
2766                 }
2767         }
2768         (void) printf("\n");
2769
2770         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2771             &child, &children) != 0)
2772                 return;
2773
2774         for (c = 0; c < children; c++) {
2775                 uint64_t is_log = B_FALSE;
2776
2777                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2778                     &is_log);
2779                 if (is_log)
2780                         continue;
2781                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2782                         continue;
2783
2784                 vname = zpool_vdev_name(g_zfs, NULL, child[c],
2785                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2786                 print_import_config(cb, vname, child[c], depth + 2);
2787                 free(vname);
2788         }
2789
2790         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2791             &child, &children) == 0) {
2792                 (void) printf(gettext("\tcache\n"));
2793                 for (c = 0; c < children; c++) {
2794                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2795                             cb->cb_name_flags);
2796                         (void) printf("\t  %s\n", vname);
2797                         free(vname);
2798                 }
2799         }
2800
2801         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2802             &child, &children) == 0) {
2803                 (void) printf(gettext("\tspares\n"));
2804                 for (c = 0; c < children; c++) {
2805                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2806                             cb->cb_name_flags);
2807                         (void) printf("\t  %s\n", vname);
2808                         free(vname);
2809                 }
2810         }
2811 }
2812
2813 /*
2814  * Print specialized class vdevs.
2815  *
2816  * These are recorded as top level vdevs in the main pool child array
2817  * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2818  * print_status_config() or print_import_config() to print the top level
2819  * class vdevs then any of their children (eg mirrored slogs) are printed
2820  * recursively - which works because only the top level vdev is marked.
2821  */
2822 static void
2823 print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2824     const char *class)
2825 {
2826         uint_t c, children;
2827         nvlist_t **child;
2828         boolean_t printed = B_FALSE;
2829
2830         assert(zhp != NULL || !cb->cb_verbose);
2831
2832         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2833             &children) != 0)
2834                 return;
2835
2836         for (c = 0; c < children; c++) {
2837                 uint64_t is_log = B_FALSE;
2838                 const char *bias = NULL;
2839                 const char *type = NULL;
2840
2841                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2842                     &is_log);
2843
2844                 if (is_log) {
2845                         bias = (char *)VDEV_ALLOC_CLASS_LOGS;
2846                 } else {
2847                         (void) nvlist_lookup_string(child[c],
2848                             ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2849                         (void) nvlist_lookup_string(child[c],
2850                             ZPOOL_CONFIG_TYPE, &type);
2851                 }
2852
2853                 if (bias == NULL || strcmp(bias, class) != 0)
2854                         continue;
2855                 if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2856                         continue;
2857
2858                 if (!printed) {
2859                         (void) printf("\t%s\t\n", gettext(class));
2860                         printed = B_TRUE;
2861                 }
2862
2863                 char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2864                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2865                 if (cb->cb_print_status)
2866                         print_status_config(zhp, cb, name, child[c], 2,
2867                             B_FALSE, NULL);
2868                 else
2869                         print_import_config(cb, name, child[c], 2);
2870                 free(name);
2871         }
2872 }
2873
2874 /*
2875  * Display the status for the given pool.
2876  */
2877 static int
2878 show_import(nvlist_t *config, boolean_t report_error)
2879 {
2880         uint64_t pool_state;
2881         vdev_stat_t *vs;
2882         const char *name;
2883         uint64_t guid;
2884         uint64_t hostid = 0;
2885         const char *msgid;
2886         const char *hostname = "unknown";
2887         nvlist_t *nvroot, *nvinfo;
2888         zpool_status_t reason;
2889         zpool_errata_t errata;
2890         const char *health;
2891         uint_t vsc;
2892         const char *comment;
2893         status_cbdata_t cb = { 0 };
2894
2895         verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2896             &name) == 0);
2897         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2898             &guid) == 0);
2899         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2900             &pool_state) == 0);
2901         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2902             &nvroot) == 0);
2903
2904         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2905             (uint64_t **)&vs, &vsc) == 0);
2906         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2907
2908         reason = zpool_import_status(config, &msgid, &errata);
2909
2910         /*
2911          * If we're importing using a cachefile, then we won't report any
2912          * errors unless we are in the scan phase of the import.
2913          */
2914         if (reason != ZPOOL_STATUS_OK && !report_error)
2915                 return (reason);
2916
2917         (void) printf(gettext("   pool: %s\n"), name);
2918         (void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2919         (void) printf(gettext("  state: %s"), health);
2920         if (pool_state == POOL_STATE_DESTROYED)
2921                 (void) printf(gettext(" (DESTROYED)"));
2922         (void) printf("\n");
2923
2924         switch (reason) {
2925         case ZPOOL_STATUS_MISSING_DEV_R:
2926         case ZPOOL_STATUS_MISSING_DEV_NR:
2927         case ZPOOL_STATUS_BAD_GUID_SUM:
2928                 printf_color(ANSI_BOLD, gettext("status: "));
2929                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2930                     "missing from the system.\n"));
2931                 break;
2932
2933         case ZPOOL_STATUS_CORRUPT_LABEL_R:
2934         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2935                 printf_color(ANSI_BOLD, gettext("status: "));
2936                 printf_color(ANSI_YELLOW, gettext("One or more devices contains"
2937                     " corrupted data.\n"));
2938                 break;
2939
2940         case ZPOOL_STATUS_CORRUPT_DATA:
2941                 (void) printf(
2942                     gettext(" status: The pool data is corrupted.\n"));
2943                 break;
2944
2945         case ZPOOL_STATUS_OFFLINE_DEV:
2946                 printf_color(ANSI_BOLD, gettext("status: "));
2947                 printf_color(ANSI_YELLOW, gettext("One or more devices "
2948                     "are offlined.\n"));
2949                 break;
2950
2951         case ZPOOL_STATUS_CORRUPT_POOL:
2952                 printf_color(ANSI_BOLD, gettext("status: "));
2953                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
2954                     "corrupted.\n"));
2955                 break;
2956
2957         case ZPOOL_STATUS_VERSION_OLDER:
2958                 printf_color(ANSI_BOLD, gettext("status: "));
2959                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2960                     "a legacy on-disk version.\n"));
2961                 break;
2962
2963         case ZPOOL_STATUS_VERSION_NEWER:
2964                 printf_color(ANSI_BOLD, gettext("status: "));
2965                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2966                     "an incompatible version.\n"));
2967                 break;
2968
2969         case ZPOOL_STATUS_FEAT_DISABLED:
2970                 printf_color(ANSI_BOLD, gettext("status: "));
2971                 printf_color(ANSI_YELLOW, gettext("Some supported "
2972                     "features are not enabled on the pool.\n\t"
2973                     "(Note that they may be intentionally disabled "
2974                     "if the\n\t'compatibility' property is set.)\n"));
2975                 break;
2976
2977         case ZPOOL_STATUS_COMPATIBILITY_ERR:
2978                 printf_color(ANSI_BOLD, gettext("status: "));
2979                 printf_color(ANSI_YELLOW, gettext("Error reading or parsing "
2980                     "the file(s) indicated by the 'compatibility'\n"
2981                     "property.\n"));
2982                 break;
2983
2984         case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
2985                 printf_color(ANSI_BOLD, gettext("status: "));
2986                 printf_color(ANSI_YELLOW, gettext("One or more features "
2987                     "are enabled on the pool despite not being\n"
2988                     "requested by the 'compatibility' property.\n"));
2989                 break;
2990
2991         case ZPOOL_STATUS_UNSUP_FEAT_READ:
2992                 printf_color(ANSI_BOLD, gettext("status: "));
2993                 printf_color(ANSI_YELLOW, gettext("The pool uses the following "
2994                     "feature(s) not supported on this system:\n"));
2995                 color_start(ANSI_YELLOW);
2996                 zpool_print_unsup_feat(config);
2997                 color_end();
2998                 break;
2999
3000         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
3001                 printf_color(ANSI_BOLD, gettext("status: "));
3002                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
3003                     "accessed in read-only mode on this system. It\n\tcannot be"
3004                     " accessed in read-write mode because it uses the "
3005                     "following\n\tfeature(s) not supported on this system:\n"));
3006                 color_start(ANSI_YELLOW);
3007                 zpool_print_unsup_feat(config);
3008                 color_end();
3009                 break;
3010
3011         case ZPOOL_STATUS_HOSTID_ACTIVE:
3012                 printf_color(ANSI_BOLD, gettext("status: "));
3013                 printf_color(ANSI_YELLOW, gettext("The pool is currently "
3014                     "imported by another system.\n"));
3015                 break;
3016
3017         case ZPOOL_STATUS_HOSTID_REQUIRED:
3018                 printf_color(ANSI_BOLD, gettext("status: "));
3019                 printf_color(ANSI_YELLOW, gettext("The pool has the "
3020                     "multihost property on.  It cannot\n\tbe safely imported "
3021                     "when the system hostid is not set.\n"));
3022                 break;
3023
3024         case ZPOOL_STATUS_HOSTID_MISMATCH:
3025                 printf_color(ANSI_BOLD, gettext("status: "));
3026                 printf_color(ANSI_YELLOW, gettext("The pool was last accessed "
3027                     "by another system.\n"));
3028                 break;
3029
3030         case ZPOOL_STATUS_FAULTED_DEV_R:
3031         case ZPOOL_STATUS_FAULTED_DEV_NR:
3032                 printf_color(ANSI_BOLD, gettext("status: "));
3033                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
3034                     "faulted.\n"));
3035                 break;
3036
3037         case ZPOOL_STATUS_BAD_LOG:
3038                 printf_color(ANSI_BOLD, gettext("status: "));
3039                 printf_color(ANSI_YELLOW, gettext("An intent log record cannot "
3040                     "be read.\n"));
3041                 break;
3042
3043         case ZPOOL_STATUS_RESILVERING:
3044         case ZPOOL_STATUS_REBUILDING:
3045                 printf_color(ANSI_BOLD, gettext("status: "));
3046                 printf_color(ANSI_YELLOW, gettext("One or more devices were "
3047                     "being resilvered.\n"));
3048                 break;
3049
3050         case ZPOOL_STATUS_ERRATA:
3051                 printf_color(ANSI_BOLD, gettext("status: "));
3052                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
3053                     errata);
3054                 break;
3055
3056         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
3057                 printf_color(ANSI_BOLD, gettext("status: "));
3058                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
3059                     "configured to use a non-native block size.\n"
3060                     "\tExpect reduced performance.\n"));
3061                 break;
3062
3063         default:
3064                 /*
3065                  * No other status can be seen when importing pools.
3066                  */
3067                 assert(reason == ZPOOL_STATUS_OK);
3068         }
3069
3070         /*
3071          * Print out an action according to the overall state of the pool.
3072          */
3073         if (vs->vs_state == VDEV_STATE_HEALTHY) {
3074                 if (reason == ZPOOL_STATUS_VERSION_OLDER ||
3075                     reason == ZPOOL_STATUS_FEAT_DISABLED) {
3076                         (void) printf(gettext(" action: The pool can be "
3077                             "imported using its name or numeric identifier, "
3078                             "though\n\tsome features will not be available "
3079                             "without an explicit 'zpool upgrade'.\n"));
3080                 } else if (reason == ZPOOL_STATUS_COMPATIBILITY_ERR) {
3081                         (void) printf(gettext(" action: The pool can be "
3082                             "imported using its name or numeric\n\tidentifier, "
3083                             "though the file(s) indicated by its "
3084                             "'compatibility'\n\tproperty cannot be parsed at "
3085                             "this time.\n"));
3086                 } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
3087                         (void) printf(gettext(" action: The pool can be "
3088                             "imported using its name or numeric "
3089                             "identifier and\n\tthe '-f' flag.\n"));
3090                 } else if (reason == ZPOOL_STATUS_ERRATA) {
3091                         switch (errata) {
3092                         case ZPOOL_ERRATA_NONE:
3093                                 break;
3094
3095                         case ZPOOL_ERRATA_ZOL_2094_SCRUB:
3096                                 (void) printf(gettext(" action: The pool can "
3097                                     "be imported using its name or numeric "
3098                                     "identifier,\n\thowever there is a compat"
3099                                     "ibility issue which should be corrected"
3100                                     "\n\tby running 'zpool scrub'\n"));
3101                                 break;
3102
3103                         case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
3104                                 (void) printf(gettext(" action: The pool can"
3105                                     "not be imported with this version of ZFS "
3106                                     "due to\n\tan active asynchronous destroy. "
3107                                     "Revert to an earlier version\n\tand "
3108                                     "allow the destroy to complete before "
3109                                     "updating.\n"));
3110                                 break;
3111
3112                         case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
3113                                 (void) printf(gettext(" action: Existing "
3114                                     "encrypted datasets contain an on-disk "
3115                                     "incompatibility, which\n\tneeds to be "
3116                                     "corrected. Backup these datasets to new "
3117                                     "encrypted datasets\n\tand destroy the "
3118                                     "old ones.\n"));
3119                                 break;
3120
3121                         case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
3122                                 (void) printf(gettext(" action: Existing "
3123                                     "encrypted snapshots and bookmarks contain "
3124                                     "an on-disk\n\tincompatibility. This may "
3125                                     "cause on-disk corruption if they are used"
3126                                     "\n\twith 'zfs recv'. To correct the "
3127                                     "issue, enable the bookmark_v2 feature.\n\t"
3128                                     "No additional action is needed if there "
3129                                     "are no encrypted snapshots or\n\t"
3130                                     "bookmarks. If preserving the encrypted "
3131                                     "snapshots and bookmarks is\n\trequired, "
3132                                     "use a non-raw send to backup and restore "
3133                                     "them. Alternately,\n\tthey may be removed"
3134                                     " to resolve the incompatibility.\n"));
3135                                 break;
3136                         default:
3137                                 /*
3138                                  * All errata must contain an action message.
3139                                  */
3140                                 assert(0);
3141                         }
3142                 } else {
3143                         (void) printf(gettext(" action: The pool can be "
3144                             "imported using its name or numeric "
3145                             "identifier.\n"));
3146                 }
3147         } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
3148                 (void) printf(gettext(" action: The pool can be imported "
3149                     "despite missing or damaged devices.  The\n\tfault "
3150                     "tolerance of the pool may be compromised if imported.\n"));
3151         } else {
3152                 switch (reason) {
3153                 case ZPOOL_STATUS_VERSION_NEWER:
3154                         (void) printf(gettext(" action: The pool cannot be "
3155                             "imported.  Access the pool on a system running "
3156                             "newer\n\tsoftware, or recreate the pool from "
3157                             "backup.\n"));
3158                         break;
3159                 case ZPOOL_STATUS_UNSUP_FEAT_READ:
3160                         printf_color(ANSI_BOLD, gettext("action: "));
3161                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
3162                             "imported. Access the pool on a system that "
3163                             "supports\n\tthe required feature(s), or recreate "
3164                             "the pool from backup.\n"));
3165                         break;
3166                 case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
3167                         printf_color(ANSI_BOLD, gettext("action: "));
3168                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
3169                             "imported in read-write mode. Import the pool "
3170                             "with\n"
3171                             "\t\"-o readonly=on\", access the pool on a system "
3172                             "that supports the\n\trequired feature(s), or "
3173                             "recreate the pool from backup.\n"));
3174                         break;
3175                 case ZPOOL_STATUS_MISSING_DEV_R:
3176                 case ZPOOL_STATUS_MISSING_DEV_NR:
3177                 case ZPOOL_STATUS_BAD_GUID_SUM:
3178                         (void) printf(gettext(" action: The pool cannot be "
3179                             "imported. Attach the missing\n\tdevices and try "
3180                             "again.\n"));
3181                         break;
3182                 case ZPOOL_STATUS_HOSTID_ACTIVE:
3183                         VERIFY0(nvlist_lookup_nvlist(config,
3184                             ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
3185
3186                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
3187                                 hostname = fnvlist_lookup_string(nvinfo,
3188                                     ZPOOL_CONFIG_MMP_HOSTNAME);
3189
3190                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
3191                                 hostid = fnvlist_lookup_uint64(nvinfo,
3192                                     ZPOOL_CONFIG_MMP_HOSTID);
3193
3194                         (void) printf(gettext(" action: The pool must be "
3195                             "exported from %s (hostid=%"PRIx64")\n\tbefore it "
3196                             "can be safely imported.\n"), hostname, hostid);
3197                         break;
3198                 case ZPOOL_STATUS_HOSTID_REQUIRED:
3199                         (void) printf(gettext(" action: Set a unique system "
3200                             "hostid with the zgenhostid(8) command.\n"));
3201                         break;
3202                 default:
3203                         (void) printf(gettext(" action: The pool cannot be "
3204                             "imported due to damaged devices or data.\n"));
3205                 }
3206         }
3207
3208         /* Print the comment attached to the pool. */
3209         if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
3210                 (void) printf(gettext("comment: %s\n"), comment);
3211
3212         /*
3213          * If the state is "closed" or "can't open", and the aux state
3214          * is "corrupt data":
3215          */
3216         if (((vs->vs_state == VDEV_STATE_CLOSED) ||
3217             (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
3218             (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
3219                 if (pool_state == POOL_STATE_DESTROYED)
3220                         (void) printf(gettext("\tThe pool was destroyed, "
3221                             "but can be imported using the '-Df' flags.\n"));
3222                 else if (pool_state != POOL_STATE_EXPORTED)
3223                         (void) printf(gettext("\tThe pool may be active on "
3224                             "another system, but can be imported using\n\t"
3225                             "the '-f' flag.\n"));
3226         }
3227
3228         if (msgid != NULL) {
3229                 (void) printf(gettext(
3230                     "   see: https://openzfs.github.io/openzfs-docs/msg/%s\n"),
3231                     msgid);
3232         }
3233
3234         (void) printf(gettext(" config:\n\n"));
3235
3236         cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
3237             VDEV_NAME_TYPE_ID);
3238         if (cb.cb_namewidth < 10)
3239                 cb.cb_namewidth = 10;
3240
3241         print_import_config(&cb, name, nvroot, 0);
3242
3243         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
3244         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
3245         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
3246
3247         if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
3248                 (void) printf(gettext("\n\tAdditional devices are known to "
3249                     "be part of this pool, though their\n\texact "
3250                     "configuration cannot be determined.\n"));
3251         }
3252         return (0);
3253 }
3254
3255 static boolean_t
3256 zfs_force_import_required(nvlist_t *config)
3257 {
3258         uint64_t state;
3259         uint64_t hostid = 0;
3260         nvlist_t *nvinfo;
3261
3262         state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
3263         nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3264
3265         /*
3266          * The hostid on LOAD_INFO comes from the MOS label via
3267          * spa_tryimport(). If its not there then we're likely talking to an
3268          * older kernel, so use the top one, which will be from the label
3269          * discovered in zpool_find_import(), or if a cachefile is in use, the
3270          * local hostid.
3271          */
3272         if (nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_HOSTID, &hostid) != 0)
3273                 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID,
3274                     &hostid);
3275
3276         if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
3277                 return (B_TRUE);
3278
3279         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
3280                 mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
3281                     ZPOOL_CONFIG_MMP_STATE);
3282
3283                 if (mmp_state != MMP_STATE_INACTIVE)
3284                         return (B_TRUE);
3285         }
3286
3287         return (B_FALSE);
3288 }
3289
3290 /*
3291  * Perform the import for the given configuration.  This passes the heavy
3292  * lifting off to zpool_import_props(), and then mounts the datasets contained
3293  * within the pool.
3294  */
3295 static int
3296 do_import(nvlist_t *config, const char *newname, const char *mntopts,
3297     nvlist_t *props, int flags)
3298 {
3299         int ret = 0;
3300         int ms_status = 0;
3301         zpool_handle_t *zhp;
3302         const char *name;
3303         uint64_t version;
3304
3305         name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
3306         version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
3307
3308         if (!SPA_VERSION_IS_SUPPORTED(version)) {
3309                 (void) fprintf(stderr, gettext("cannot import '%s': pool "
3310                     "is formatted using an unsupported ZFS version\n"), name);
3311                 return (1);
3312         } else if (zfs_force_import_required(config) &&
3313             !(flags & ZFS_IMPORT_ANY_HOST)) {
3314                 mmp_state_t mmp_state = MMP_STATE_INACTIVE;
3315                 nvlist_t *nvinfo;
3316
3317                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3318                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
3319                         mmp_state = fnvlist_lookup_uint64(nvinfo,
3320                             ZPOOL_CONFIG_MMP_STATE);
3321
3322                 if (mmp_state == MMP_STATE_ACTIVE) {
3323                         const char *hostname = "<unknown>";
3324                         uint64_t hostid = 0;
3325
3326                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
3327                                 hostname = fnvlist_lookup_string(nvinfo,
3328                                     ZPOOL_CONFIG_MMP_HOSTNAME);
3329
3330                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
3331                                 hostid = fnvlist_lookup_uint64(nvinfo,
3332                                     ZPOOL_CONFIG_MMP_HOSTID);
3333
3334                         (void) fprintf(stderr, gettext("cannot import '%s': "
3335                             "pool is imported on %s (hostid: "
3336                             "0x%"PRIx64")\nExport the pool on the other "
3337                             "system, then run 'zpool import'.\n"),
3338                             name, hostname, hostid);
3339                 } else if (mmp_state == MMP_STATE_NO_HOSTID) {
3340                         (void) fprintf(stderr, gettext("Cannot import '%s': "
3341                             "pool has the multihost property on and the\n"
3342                             "system's hostid is not set. Set a unique hostid "
3343                             "with the zgenhostid(8) command.\n"), name);
3344                 } else {
3345                         const char *hostname = "<unknown>";
3346                         time_t timestamp = 0;
3347                         uint64_t hostid = 0;
3348
3349                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTNAME))
3350                                 hostname = fnvlist_lookup_string(nvinfo,
3351                                     ZPOOL_CONFIG_HOSTNAME);
3352                         else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
3353                                 hostname = fnvlist_lookup_string(config,
3354                                     ZPOOL_CONFIG_HOSTNAME);
3355
3356                         if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
3357                                 timestamp = fnvlist_lookup_uint64(config,
3358                                     ZPOOL_CONFIG_TIMESTAMP);
3359
3360                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTID))
3361                                 hostid = fnvlist_lookup_uint64(nvinfo,
3362                                     ZPOOL_CONFIG_HOSTID);
3363                         else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
3364                                 hostid = fnvlist_lookup_uint64(config,
3365                                     ZPOOL_CONFIG_HOSTID);
3366
3367                         (void) fprintf(stderr, gettext("cannot import '%s': "
3368                             "pool was previously in use from another system.\n"
3369                             "Last accessed by %s (hostid=%"PRIx64") at %s"
3370                             "The pool can be imported, use 'zpool import -f' "
3371                             "to import the pool.\n"), name, hostname,
3372                             hostid, ctime(&timestamp));
3373                 }
3374
3375                 return (1);
3376         }
3377
3378         if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
3379                 return (1);
3380
3381         if (newname != NULL)
3382                 name = newname;
3383
3384         if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
3385                 return (1);
3386
3387         /*
3388          * Loading keys is best effort. We don't want to return immediately
3389          * if it fails but we do want to give the error to the caller.
3390          */
3391         if (flags & ZFS_IMPORT_LOAD_KEYS &&
3392             zfs_crypto_attempt_load_keys(g_zfs, name) != 0)
3393                         ret = 1;
3394
3395         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
3396             !(flags & ZFS_IMPORT_ONLY)) {
3397                 ms_status = zpool_enable_datasets(zhp, mntopts, 0);
3398                 if (ms_status == EZFS_SHAREFAILED) {
3399                         (void) fprintf(stderr, gettext("Import was "
3400                             "successful, but unable to share some datasets"));
3401                 } else if (ms_status == EZFS_MOUNTFAILED) {
3402                         (void) fprintf(stderr, gettext("Import was "
3403                             "successful, but unable to mount some datasets"));
3404                 }
3405         }
3406
3407         zpool_close(zhp);
3408         return (ret);
3409 }
3410
3411 static int
3412 import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
3413     char *orig_name, char *new_name,
3414     boolean_t do_destroyed, boolean_t pool_specified, boolean_t do_all,
3415     importargs_t *import)
3416 {
3417         nvlist_t *config = NULL;
3418         nvlist_t *found_config = NULL;
3419         uint64_t pool_state;
3420
3421         /*
3422          * At this point we have a list of import candidate configs. Even if
3423          * we were searching by pool name or guid, we still need to
3424          * post-process the list to deal with pool state and possible
3425          * duplicate names.
3426          */
3427         int err = 0;
3428         nvpair_t *elem = NULL;
3429         boolean_t first = B_TRUE;
3430         while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
3431
3432                 verify(nvpair_value_nvlist(elem, &config) == 0);
3433
3434                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
3435                     &pool_state) == 0);
3436                 if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
3437                         continue;
3438                 if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
3439                         continue;
3440
3441                 verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
3442                     import->policy) == 0);
3443
3444                 if (!pool_specified) {
3445                         if (first)
3446                                 first = B_FALSE;
3447                         else if (!do_all)
3448                                 (void) fputc('\n', stdout);
3449
3450                         if (do_all) {
3451                                 err |= do_import(config, NULL, mntopts,
3452                                     props, flags);
3453                         } else {
3454                                 /*
3455                                  * If we're importing from cachefile, then
3456                                  * we don't want to report errors until we
3457                                  * are in the scan phase of the import. If
3458                                  * we get an error, then we return that error
3459                                  * to invoke the scan phase.
3460                                  */
3461                                 if (import->cachefile && !import->scan)
3462                                         err = show_import(config, B_FALSE);
3463                                 else
3464                                         (void) show_import(config, B_TRUE);
3465                         }
3466                 } else if (import->poolname != NULL) {
3467                         const char *name;
3468
3469                         /*
3470                          * We are searching for a pool based on name.
3471                          */
3472                         verify(nvlist_lookup_string(config,
3473                             ZPOOL_CONFIG_POOL_NAME, &name) == 0);
3474
3475                         if (strcmp(name, import->poolname) == 0) {
3476                                 if (found_config != NULL) {
3477                                         (void) fprintf(stderr, gettext(
3478                                             "cannot import '%s': more than "
3479                                             "one matching pool\n"),
3480                                             import->poolname);
3481                                         (void) fprintf(stderr, gettext(
3482                                             "import by numeric ID instead\n"));
3483                                         err = B_TRUE;
3484                                 }
3485                                 found_config = config;
3486                         }
3487                 } else {
3488                         uint64_t guid;
3489
3490                         /*
3491                          * Search for a pool by guid.
3492                          */
3493                         verify(nvlist_lookup_uint64(config,
3494                             ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3495
3496                         if (guid == import->guid)
3497                                 found_config = config;
3498                 }
3499         }
3500
3501         /*
3502          * If we were searching for a specific pool, verify that we found a
3503          * pool, and then do the import.
3504          */
3505         if (pool_specified && err == 0) {
3506                 if (found_config == NULL) {
3507                         (void) fprintf(stderr, gettext("cannot import '%s': "
3508                             "no such pool available\n"), orig_name);
3509                         err = B_TRUE;
3510                 } else {
3511                         err |= do_import(found_config, new_name,
3512                             mntopts, props, flags);
3513                 }
3514         }
3515
3516         /*
3517          * If we were just looking for pools, report an error if none were
3518          * found.
3519          */
3520         if (!pool_specified && first)
3521                 (void) fprintf(stderr,
3522                     gettext("no pools available to import\n"));
3523         return (err);
3524 }
3525
3526 typedef struct target_exists_args {
3527         const char      *poolname;
3528         uint64_t        poolguid;
3529 } target_exists_args_t;
3530
3531 static int
3532 name_or_guid_exists(zpool_handle_t *zhp, void *data)
3533 {
3534         target_exists_args_t *args = data;
3535         nvlist_t *config = zpool_get_config(zhp, NULL);
3536         int found = 0;
3537
3538         if (config == NULL)
3539                 return (0);
3540
3541         if (args->poolname != NULL) {
3542                 const char *pool_name;
3543
3544                 verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
3545                     &pool_name) == 0);
3546                 if (strcmp(pool_name, args->poolname) == 0)
3547                         found = 1;
3548         } else {
3549                 uint64_t pool_guid;
3550
3551                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3552                     &pool_guid) == 0);
3553                 if (pool_guid == args->poolguid)
3554                         found = 1;
3555         }
3556         zpool_close(zhp);
3557
3558         return (found);
3559 }
3560 /*
3561  * zpool checkpoint <pool>
3562  *       checkpoint --discard <pool>
3563  *
3564  *       -d         Discard the checkpoint from a checkpointed
3565  *       --discard  pool.
3566  *
3567  *       -w         Wait for discarding a checkpoint to complete.
3568  *       --wait
3569  *
3570  * Checkpoints the specified pool, by taking a "snapshot" of its
3571  * current state. A pool can only have one checkpoint at a time.
3572  */
3573 int
3574 zpool_do_checkpoint(int argc, char **argv)
3575 {
3576         boolean_t discard, wait;
3577         char *pool;
3578         zpool_handle_t *zhp;
3579         int c, err;
3580
3581         struct option long_options[] = {
3582                 {"discard", no_argument, NULL, 'd'},
3583                 {"wait", no_argument, NULL, 'w'},
3584                 {0, 0, 0, 0}
3585         };
3586
3587         discard = B_FALSE;
3588         wait = B_FALSE;
3589         while ((c = getopt_long(argc, argv, ":dw", long_options, NULL)) != -1) {
3590                 switch (c) {
3591                 case 'd':
3592                         discard = B_TRUE;
3593                         break;
3594                 case 'w':
3595                         wait = B_TRUE;
3596                         break;
3597                 case '?':
3598                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3599                             optopt);
3600                         usage(B_FALSE);
3601                 }
3602         }
3603
3604         if (wait && !discard) {
3605                 (void) fprintf(stderr, gettext("--wait only valid when "
3606                     "--discard also specified\n"));
3607                 usage(B_FALSE);
3608         }
3609
3610         argc -= optind;
3611         argv += optind;
3612
3613         if (argc < 1) {
3614                 (void) fprintf(stderr, gettext("missing pool argument\n"));
3615                 usage(B_FALSE);
3616         }
3617
3618         if (argc > 1) {
3619                 (void) fprintf(stderr, gettext("too many arguments\n"));
3620                 usage(B_FALSE);
3621         }
3622
3623         pool = argv[0];
3624
3625         if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
3626                 /* As a special case, check for use of '/' in the name */
3627                 if (strchr(pool, '/') != NULL)
3628                         (void) fprintf(stderr, gettext("'zpool checkpoint' "
3629                             "doesn't work on datasets. To save the state "
3630                             "of a dataset from a specific point in time "
3631                             "please use 'zfs snapshot'\n"));
3632                 return (1);
3633         }
3634
3635         if (discard) {
3636                 err = (zpool_discard_checkpoint(zhp) != 0);
3637                 if (err == 0 && wait)
3638                         err = zpool_wait(zhp, ZPOOL_WAIT_CKPT_DISCARD);
3639         } else {
3640                 err = (zpool_checkpoint(zhp) != 0);
3641         }
3642
3643         zpool_close(zhp);
3644
3645         return (err);
3646 }
3647
3648 #define CHECKPOINT_OPT  1024
3649
3650 /*
3651  * zpool import [-d dir] [-D]
3652  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3653  *              [-d dir | -c cachefile | -s] [-f] -a
3654  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3655  *              [-d dir | -c cachefile | -s] [-f] [-n] [-F] <pool | id>
3656  *              [newpool]
3657  *
3658  *      -c      Read pool information from a cachefile instead of searching
3659  *              devices. If importing from a cachefile config fails, then
3660  *              fallback to searching for devices only in the directories that
3661  *              exist in the cachefile.
3662  *
3663  *      -d      Scan in a specific directory, other than /dev/.  More than
3664  *              one directory can be specified using multiple '-d' options.
3665  *
3666  *      -D      Scan for previously destroyed pools or import all or only
3667  *              specified destroyed pools.
3668  *
3669  *      -R      Temporarily import the pool, with all mountpoints relative to
3670  *              the given root.  The pool will remain exported when the machine
3671  *              is rebooted.
3672  *
3673  *      -V      Import even in the presence of faulted vdevs.  This is an
3674  *              intentionally undocumented option for testing purposes, and
3675  *              treats the pool configuration as complete, leaving any bad
3676  *              vdevs in the FAULTED state. In other words, it does verbatim
3677  *              import.
3678  *
3679  *      -f      Force import, even if it appears that the pool is active.
3680  *
3681  *      -F      Attempt rewind if necessary.
3682  *
3683  *      -n      See if rewind would work, but don't actually rewind.
3684  *
3685  *      -N      Import the pool but don't mount datasets.
3686  *
3687  *      -T      Specify a starting txg to use for import. This option is
3688  *              intentionally undocumented option for testing purposes.
3689  *
3690  *      -a      Import all pools found.
3691  *
3692  *      -l      Load encryption keys while importing.
3693  *
3694  *      -o      Set property=value and/or temporary mount options (without '=').
3695  *
3696  *      -s      Scan using the default search path, the libblkid cache will
3697  *              not be consulted.
3698  *
3699  *      --rewind-to-checkpoint
3700  *              Import the pool and revert back to the checkpoint.
3701  *
3702  * The import command scans for pools to import, and import pools based on pool
3703  * name and GUID.  The pool can also be renamed as part of the import process.
3704  */
3705 int
3706 zpool_do_import(int argc, char **argv)
3707 {
3708         char **searchdirs = NULL;
3709         char *env, *envdup = NULL;
3710         int nsearch = 0;
3711         int c;
3712         int err = 0;
3713         nvlist_t *pools = NULL;
3714         boolean_t do_all = B_FALSE;
3715         boolean_t do_destroyed = B_FALSE;
3716         char *mntopts = NULL;
3717         uint64_t searchguid = 0;
3718         char *searchname = NULL;
3719         char *propval;
3720         nvlist_t *policy = NULL;
3721         nvlist_t *props = NULL;
3722         int flags = ZFS_IMPORT_NORMAL;
3723         uint32_t rewind_policy = ZPOOL_NO_REWIND;
3724         boolean_t dryrun = B_FALSE;
3725         boolean_t do_rewind = B_FALSE;
3726         boolean_t xtreme_rewind = B_FALSE;
3727         boolean_t do_scan = B_FALSE;
3728         boolean_t pool_exists = B_FALSE;
3729         boolean_t pool_specified = B_FALSE;
3730         uint64_t txg = -1ULL;
3731         char *cachefile = NULL;
3732         importargs_t idata = { 0 };
3733         char *endptr;
3734
3735         struct option long_options[] = {
3736                 {"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
3737                 {0, 0, 0, 0}
3738         };
3739
3740         /* check options */
3741         while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
3742             long_options, NULL)) != -1) {
3743                 switch (c) {
3744                 case 'a':
3745                         do_all = B_TRUE;
3746                         break;
3747                 case 'c':
3748                         cachefile = optarg;
3749                         break;
3750                 case 'd':
3751                         searchdirs = safe_realloc(searchdirs,
3752                             (nsearch + 1) * sizeof (char *));
3753                         searchdirs[nsearch++] = optarg;
3754                         break;
3755                 case 'D':
3756                         do_destroyed = B_TRUE;
3757                         break;
3758                 case 'f':
3759                         flags |= ZFS_IMPORT_ANY_HOST;
3760                         break;
3761                 case 'F':
3762                         do_rewind = B_TRUE;
3763                         break;
3764                 case 'l':
3765                         flags |= ZFS_IMPORT_LOAD_KEYS;
3766                         break;
3767                 case 'm':
3768                         flags |= ZFS_IMPORT_MISSING_LOG;
3769                         break;
3770                 case 'n':
3771                         dryrun = B_TRUE;
3772                         break;
3773                 case 'N':
3774                         flags |= ZFS_IMPORT_ONLY;
3775                         break;
3776                 case 'o':
3777                         if ((propval = strchr(optarg, '=')) != NULL) {
3778                                 *propval = '\0';
3779                                 propval++;
3780                                 if (add_prop_list(optarg, propval,
3781                                     &props, B_TRUE))
3782                                         goto error;
3783                         } else {
3784                                 mntopts = optarg;
3785                         }
3786                         break;
3787                 case 'R':
3788                         if (add_prop_list(zpool_prop_to_name(
3789                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
3790                                 goto error;
3791                         if (add_prop_list_default(zpool_prop_to_name(
3792                             ZPOOL_PROP_CACHEFILE), "none", &props))
3793                                 goto error;
3794                         break;
3795                 case 's':
3796                         do_scan = B_TRUE;
3797                         break;
3798                 case 't':
3799                         flags |= ZFS_IMPORT_TEMP_NAME;
3800                         if (add_prop_list_default(zpool_prop_to_name(
3801                             ZPOOL_PROP_CACHEFILE), "none", &props))
3802                                 goto error;
3803                         break;
3804
3805                 case 'T':
3806                         errno = 0;
3807                         txg = strtoull(optarg, &endptr, 0);
3808                         if (errno != 0 || *endptr != '\0') {
3809                                 (void) fprintf(stderr,
3810                                     gettext("invalid txg value\n"));
3811                                 usage(B_FALSE);
3812                         }
3813                         rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
3814                         break;
3815                 case 'V':
3816                         flags |= ZFS_IMPORT_VERBATIM;
3817                         break;
3818                 case 'X':
3819                         xtreme_rewind = B_TRUE;
3820                         break;
3821                 case CHECKPOINT_OPT:
3822                         flags |= ZFS_IMPORT_CHECKPOINT;
3823                         break;
3824                 case ':':
3825                         (void) fprintf(stderr, gettext("missing argument for "
3826                             "'%c' option\n"), optopt);
3827                         usage(B_FALSE);
3828                         break;
3829                 case '?':
3830                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3831                             optopt);
3832                         usage(B_FALSE);
3833                 }
3834         }
3835
3836         argc -= optind;
3837         argv += optind;
3838
3839         if (cachefile && nsearch != 0) {
3840                 (void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
3841                 usage(B_FALSE);
3842         }
3843
3844         if (cachefile && do_scan) {
3845                 (void) fprintf(stderr, gettext("-c is incompatible with -s\n"));
3846                 usage(B_FALSE);
3847         }
3848
3849         if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
3850                 (void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
3851                 usage(B_FALSE);
3852         }
3853
3854         if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
3855                 (void) fprintf(stderr, gettext("-l is only meaningful during "
3856                     "an import\n"));
3857                 usage(B_FALSE);
3858         }
3859
3860         if ((dryrun || xtreme_rewind) && !do_rewind) {
3861                 (void) fprintf(stderr,
3862                     gettext("-n or -X only meaningful with -F\n"));
3863                 usage(B_FALSE);
3864         }
3865         if (dryrun)
3866                 rewind_policy = ZPOOL_TRY_REWIND;
3867         else if (do_rewind)
3868                 rewind_policy = ZPOOL_DO_REWIND;
3869         if (xtreme_rewind)
3870                 rewind_policy |= ZPOOL_EXTREME_REWIND;
3871
3872         /* In the future, we can capture further policy and include it here */
3873         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
3874             nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
3875             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
3876             rewind_policy) != 0)
3877                 goto error;
3878
3879         /* check argument count */
3880         if (do_all) {
3881                 if (argc != 0) {
3882                         (void) fprintf(stderr, gettext("too many arguments\n"));
3883                         usage(B_FALSE);
3884                 }
3885         } else {
3886                 if (argc > 2) {
3887                         (void) fprintf(stderr, gettext("too many arguments\n"));
3888                         usage(B_FALSE);
3889                 }
3890         }
3891
3892         /*
3893          * Check for the effective uid.  We do this explicitly here because
3894          * otherwise any attempt to discover pools will silently fail.
3895          */
3896         if (argc == 0 && geteuid() != 0) {
3897                 (void) fprintf(stderr, gettext("cannot "
3898                     "discover pools: permission denied\n"));
3899
3900                 free(searchdirs);
3901                 nvlist_free(props);
3902                 nvlist_free(policy);
3903                 return (1);
3904         }
3905
3906         /*
3907          * Depending on the arguments given, we do one of the following:
3908          *
3909          *      <none>  Iterate through all pools and display information about
3910          *              each one.
3911          *
3912          *      -a      Iterate through all pools and try to import each one.
3913          *
3914          *      <id>    Find the pool that corresponds to the given GUID/pool
3915          *              name and import that one.
3916          *
3917          *      -D      Above options applies only to destroyed pools.
3918          */
3919         if (argc != 0) {
3920                 char *endptr;
3921
3922                 errno = 0;
3923                 searchguid = strtoull(argv[0], &endptr, 10);
3924                 if (errno != 0 || *endptr != '\0') {
3925                         searchname = argv[0];
3926                         searchguid = 0;
3927                 }
3928                 pool_specified = B_TRUE;
3929
3930                 /*
3931                  * User specified a name or guid.  Ensure it's unique.
3932                  */
3933                 target_exists_args_t search = {searchname, searchguid};
3934                 pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
3935         }
3936
3937         /*
3938          * Check the environment for the preferred search path.
3939          */
3940         if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
3941                 char *dir, *tmp = NULL;
3942
3943                 envdup = strdup(env);
3944
3945                 for (dir = strtok_r(envdup, ":", &tmp);
3946                     dir != NULL;
3947                     dir = strtok_r(NULL, ":", &tmp)) {
3948                         searchdirs = safe_realloc(searchdirs,
3949                             (nsearch + 1) * sizeof (char *));
3950                         searchdirs[nsearch++] = dir;
3951                 }
3952         }
3953
3954         idata.path = searchdirs;
3955         idata.paths = nsearch;
3956         idata.poolname = searchname;
3957         idata.guid = searchguid;
3958         idata.cachefile = cachefile;
3959         idata.scan = do_scan;
3960         idata.policy = policy;
3961
3962         libpc_handle_t lpch = {
3963                 .lpc_lib_handle = g_zfs,
3964                 .lpc_ops = &libzfs_config_ops,
3965                 .lpc_printerr = B_TRUE
3966         };
3967         pools = zpool_search_import(&lpch, &idata);
3968
3969         if (pools != NULL && pool_exists &&
3970             (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
3971                 (void) fprintf(stderr, gettext("cannot import '%s': "
3972                     "a pool with that name already exists\n"),
3973                     argv[0]);
3974                 (void) fprintf(stderr, gettext("use the form '%s "
3975                     "<pool | id> <newpool>' to give it a new name\n"),
3976                     "zpool import");
3977                 err = 1;
3978         } else if (pools == NULL && pool_exists) {
3979                 (void) fprintf(stderr, gettext("cannot import '%s': "
3980                     "a pool with that name is already created/imported,\n"),
3981                     argv[0]);
3982                 (void) fprintf(stderr, gettext("and no additional pools "
3983                     "with that name were found\n"));
3984                 err = 1;
3985         } else if (pools == NULL) {
3986                 if (argc != 0) {
3987                         (void) fprintf(stderr, gettext("cannot import '%s': "
3988                             "no such pool available\n"), argv[0]);
3989                 }
3990                 err = 1;
3991         }
3992
3993         if (err == 1) {
3994                 free(searchdirs);
3995                 free(envdup);
3996                 nvlist_free(policy);
3997                 nvlist_free(pools);
3998                 nvlist_free(props);
3999                 return (1);
4000         }
4001
4002         err = import_pools(pools, props, mntopts, flags,
4003             argc >= 1 ? argv[0] : NULL,
4004             argc >= 2 ? argv[1] : NULL,
4005             do_destroyed, pool_specified, do_all, &idata);
4006
4007         /*
4008          * If we're using the cachefile and we failed to import, then
4009          * fallback to scanning the directory for pools that match
4010          * those in the cachefile.
4011          */
4012         if (err != 0 && cachefile != NULL) {
4013                 (void) printf(gettext("cachefile import failed, retrying\n"));
4014
4015                 /*
4016                  * We use the scan flag to gather the directories that exist
4017                  * in the cachefile. If we need to fallback to searching for
4018                  * the pool config, we will only search devices in these
4019                  * directories.
4020                  */
4021                 idata.scan = B_TRUE;
4022                 nvlist_free(pools);
4023                 pools = zpool_search_import(&lpch, &idata);
4024
4025                 err = import_pools(pools, props, mntopts, flags,
4026                     argc >= 1 ? argv[0] : NULL,
4027                     argc >= 2 ? argv[1] : NULL,
4028                     do_destroyed, pool_specified, do_all, &idata);
4029         }
4030
4031 error:
4032         nvlist_free(props);
4033         nvlist_free(pools);
4034         nvlist_free(policy);
4035         free(searchdirs);
4036         free(envdup);
4037
4038         return (err ? 1 : 0);
4039 }
4040
4041 /*
4042  * zpool sync [-f] [pool] ...
4043  *
4044  * -f (undocumented) force uberblock (and config including zpool cache file)
4045  *    update.
4046  *
4047  * Sync the specified pool(s).
4048  * Without arguments "zpool sync" will sync all pools.
4049  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
4050  *
4051  */
4052 static int
4053 zpool_do_sync(int argc, char **argv)
4054 {
4055         int ret;
4056         boolean_t force = B_FALSE;
4057
4058         /* check options */
4059         while ((ret  = getopt(argc, argv, "f")) != -1) {
4060                 switch (ret) {
4061                 case 'f':
4062                         force = B_TRUE;
4063                         break;
4064                 case '?':
4065                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
4066                             optopt);
4067                         usage(B_FALSE);
4068                 }
4069         }
4070
4071         argc -= optind;
4072         argv += optind;
4073
4074         /* if argc == 0 we will execute zpool_sync_one on all pools */
4075         ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
4076             B_FALSE, zpool_sync_one, &force);
4077
4078         return (ret);
4079 }
4080
4081 typedef struct iostat_cbdata {
4082         uint64_t cb_flags;
4083         int cb_namewidth;
4084         int cb_iteration;
4085         boolean_t cb_verbose;
4086         boolean_t cb_literal;
4087         boolean_t cb_scripted;
4088         zpool_list_t *cb_list;
4089         vdev_cmd_data_list_t *vcdl;
4090         vdev_cbdata_t cb_vdevs;
4091 } iostat_cbdata_t;
4092
4093 /*  iostat labels */
4094 typedef struct name_and_columns {
4095         const char *name;       /* Column name */
4096         unsigned int columns;   /* Center name to this number of columns */
4097 } name_and_columns_t;
4098
4099 #define IOSTAT_MAX_LABELS       15      /* Max number of labels on one line */
4100
4101 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
4102 {
4103         [IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
4104             {NULL}},
4105         [IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
4106             {"asyncq_wait", 2}, {"scrub", 1}, {"trim", 1}, {"rebuild", 1},
4107             {NULL}},
4108         [IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
4109             {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
4110             {"trimq_write", 2}, {"rebuildq_write", 2}, {NULL}},
4111         [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
4112             {"asyncq_wait", 2}, {NULL}},
4113         [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
4114             {"async_read", 2}, {"async_write", 2}, {"scrub", 2},
4115             {"trim", 2}, {"rebuild", 2}, {NULL}},
4116 };
4117
4118 /* Shorthand - if "columns" field not set, default to 1 column */
4119 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
4120 {
4121         [IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
4122             {"write"}, {NULL}},
4123         [IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
4124             {"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {"wait"},
4125             {NULL}},
4126         [IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
4127             {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
4128             {"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}},
4129         [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
4130             {"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {"rebuild"},
4131             {NULL}},
4132         [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
4133             {"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
4134             {"ind"}, {"agg"}, {NULL}},
4135 };
4136
4137 static const char *histo_to_title[] = {
4138         [IOS_L_HISTO] = "latency",
4139         [IOS_RQ_HISTO] = "req_size",
4140 };
4141
4142 /*
4143  * Return the number of labels in a null-terminated name_and_columns_t
4144  * array.
4145  *
4146  */
4147 static unsigned int
4148 label_array_len(const name_and_columns_t *labels)
4149 {
4150         int i = 0;
4151
4152         while (labels[i].name)
4153                 i++;
4154
4155         return (i);
4156 }
4157
4158 /*
4159  * Return the number of strings in a null-terminated string array.
4160  * For example:
4161  *
4162  *     const char foo[] = {"bar", "baz", NULL}
4163  *
4164  * returns 2
4165  */
4166 static uint64_t
4167 str_array_len(const char *array[])
4168 {
4169         uint64_t i = 0;
4170         while (array[i])
4171                 i++;
4172
4173         return (i);
4174 }
4175
4176
4177 /*
4178  * Return a default column width for default/latency/queue columns. This does
4179  * not include histograms, which have their columns autosized.
4180  */
4181 static unsigned int
4182 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
4183 {
4184         unsigned long column_width = 5; /* Normal niceprint */
4185         static unsigned long widths[] = {
4186                 /*
4187                  * Choose some sane default column sizes for printing the
4188                  * raw numbers.
4189                  */
4190                 [IOS_DEFAULT] = 15, /* 1PB capacity */
4191                 [IOS_LATENCY] = 10, /* 1B ns = 10sec */
4192                 [IOS_QUEUES] = 6,   /* 1M queue entries */
4193                 [IOS_L_HISTO] = 10, /* 1B ns = 10sec */
4194                 [IOS_RQ_HISTO] = 6, /* 1M queue entries */
4195         };
4196
4197         if (cb->cb_literal)
4198                 column_width = widths[type];
4199
4200         return (column_width);
4201 }
4202
4203 /*
4204  * Print the column labels, i.e:
4205  *
4206  *   capacity     operations     bandwidth
4207  * alloc   free   read  write   read  write  ...
4208  *
4209  * If force_column_width is set, use it for the column width.  If not set, use
4210  * the default column width.
4211  */
4212 static void
4213 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
4214     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
4215 {
4216         int i, idx, s;
4217         int text_start, rw_column_width, spaces_to_end;
4218         uint64_t flags = cb->cb_flags;
4219         uint64_t f;
4220         unsigned int column_width = force_column_width;
4221
4222         /* For each bit set in flags */
4223         for (f = flags; f; f &= ~(1ULL << idx)) {
4224                 idx = lowbit64(f) - 1;
4225                 if (!force_column_width)
4226                         column_width = default_column_width(cb, idx);
4227                 /* Print our top labels centered over "read  write" label. */
4228                 for (i = 0; i < label_array_len(labels[idx]); i++) {
4229                         const char *name = labels[idx][i].name;
4230                         /*
4231                          * We treat labels[][].columns == 0 as shorthand
4232                          * for one column.  It makes writing out the label
4233                          * tables more concise.
4234                          */
4235                         unsigned int columns = MAX(1, labels[idx][i].columns);
4236                         unsigned int slen = strlen(name);
4237
4238                         rw_column_width = (column_width * columns) +
4239                             (2 * (columns - 1));
4240
4241                         text_start = (int)((rw_column_width) / columns -
4242                             slen / columns);
4243                         if (text_start < 0)
4244                                 text_start = 0;
4245
4246                         printf("  ");   /* Two spaces between columns */
4247
4248                         /* Space from beginning of column to label */
4249                         for (s = 0; s < text_start; s++)
4250                                 printf(" ");
4251
4252                         printf("%s", name);
4253
4254                         /* Print space after label to end of column */
4255                         spaces_to_end = rw_column_width - text_start - slen;
4256                         if (spaces_to_end < 0)
4257                                 spaces_to_end = 0;
4258
4259                         for (s = 0; s < spaces_to_end; s++)
4260                                 printf(" ");
4261                 }
4262         }
4263 }
4264
4265
4266 /*
4267  * print_cmd_columns - Print custom column titles from -c
4268  *
4269  * If the user specified the "zpool status|iostat -c" then print their custom
4270  * column titles in the header.  For example, print_cmd_columns() would print
4271  * the "  col1  col2" part of this:
4272  *
4273  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
4274  * ...
4275  *            capacity     operations     bandwidth
4276  * pool        alloc   free   read  write   read  write  col1  col2
4277  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
4278  * mypool       269K  1008M      0      0    107    946
4279  *   mirror     269K  1008M      0      0    107    946
4280  *     sdb         -      -      0      0    102    473  val1  val2
4281  *     sdc         -      -      0      0      5    473  val1  val2
4282  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
4283  */
4284 static void
4285 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
4286 {
4287         int i, j;
4288         vdev_cmd_data_t *data = &vcdl->data[0];
4289
4290         if (vcdl->count == 0 || data == NULL)
4291                 return;
4292
4293         /*
4294          * Each vdev cmd should have the same column names unless the user did
4295          * something weird with their cmd.  Just take the column names from the
4296          * first vdev and assume it works for all of them.
4297          */
4298         for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
4299                 printf("  ");
4300                 if (use_dashes) {
4301                         for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
4302                                 printf("-");
4303                 } else {
4304                         printf_color(ANSI_BOLD, "%*s", vcdl->uniq_cols_width[i],
4305                             vcdl->uniq_cols[i]);
4306                 }
4307         }
4308 }
4309
4310
4311 /*
4312  * Utility function to print out a line of dashes like:
4313  *
4314  *      --------------------------------  -----  -----  -----  -----  -----
4315  *
4316  * ...or a dashed named-row line like:
4317  *
4318  *      logs                                  -      -      -      -      -
4319  *
4320  * @cb:                         iostat data
4321  *
4322  * @force_column_width          If non-zero, use the value as the column width.
4323  *                              Otherwise use the default column widths.
4324  *
4325  * @name:                       Print a dashed named-row line starting
4326  *                              with @name.  Otherwise, print a regular
4327  *                              dashed line.
4328  */
4329 static void
4330 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
4331     const char *name)
4332 {
4333         int i;
4334         unsigned int namewidth;
4335         uint64_t flags = cb->cb_flags;
4336         uint64_t f;
4337         int idx;
4338         const name_and_columns_t *labels;
4339         const char *title;
4340
4341
4342         if (cb->cb_flags & IOS_ANYHISTO_M) {
4343                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
4344         } else if (cb->cb_vdevs.cb_names_count) {
4345                 title = "vdev";
4346         } else  {
4347                 title = "pool";
4348         }
4349
4350         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
4351             name ? strlen(name) : 0);
4352
4353
4354         if (name) {
4355                 printf("%-*s", namewidth, name);
4356         } else {
4357                 for (i = 0; i < namewidth; i++)
4358                         (void) printf("-");
4359         }
4360
4361         /* For each bit in flags */
4362         for (f = flags; f; f &= ~(1ULL << idx)) {
4363                 unsigned int column_width;
4364                 idx = lowbit64(f) - 1;
4365                 if (force_column_width)
4366                         column_width = force_column_width;
4367                 else
4368                         column_width = default_column_width(cb, idx);
4369
4370                 labels = iostat_bottom_labels[idx];
4371                 for (i = 0; i < label_array_len(labels); i++) {
4372                         if (name)
4373                                 printf("  %*s-", column_width - 1, " ");
4374                         else
4375                                 printf("  %.*s", column_width,
4376                                     "--------------------");
4377                 }
4378         }
4379 }
4380
4381
4382 static void
4383 print_iostat_separator_impl(iostat_cbdata_t *cb,
4384     unsigned int force_column_width)
4385 {
4386         print_iostat_dashes(cb, force_column_width, NULL);
4387 }
4388
4389 static void
4390 print_iostat_separator(iostat_cbdata_t *cb)
4391 {
4392         print_iostat_separator_impl(cb, 0);
4393 }
4394
4395 static void
4396 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
4397     const char *histo_vdev_name)
4398 {
4399         unsigned int namewidth;
4400         const char *title;
4401
4402         color_start(ANSI_BOLD);
4403
4404         if (cb->cb_flags & IOS_ANYHISTO_M) {
4405                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
4406         } else if (cb->cb_vdevs.cb_names_count) {
4407                 title = "vdev";
4408         } else  {
4409                 title = "pool";
4410         }
4411
4412         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
4413             histo_vdev_name ? strlen(histo_vdev_name) : 0);
4414
4415         if (histo_vdev_name)
4416                 printf("%-*s", namewidth, histo_vdev_name);
4417         else
4418                 printf("%*s", namewidth, "");
4419
4420
4421         print_iostat_labels(cb, force_column_width, iostat_top_labels);
4422         printf("\n");
4423
4424         printf("%-*s", namewidth, title);
4425
4426         print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
4427         if (cb->vcdl != NULL)
4428                 print_cmd_columns(cb->vcdl, 0);
4429
4430         printf("\n");
4431
4432         print_iostat_separator_impl(cb, force_column_width);
4433
4434         if (cb->vcdl != NULL)
4435                 print_cmd_columns(cb->vcdl, 1);
4436
4437         color_end();
4438
4439         printf("\n");
4440 }
4441
4442 static void
4443 print_iostat_header(iostat_cbdata_t *cb)
4444 {
4445         print_iostat_header_impl(cb, 0, NULL);
4446 }
4447
4448 /*
4449  * Prints a size string (i.e. 120M) with the suffix ("M") colored
4450  * by order of magnitude. Uses column_size to add padding.
4451  */
4452 static void
4453 print_stat_color(const char *statbuf, unsigned int column_size)
4454 {
4455         fputs("  ", stdout);
4456         size_t len = strlen(statbuf);
4457         while (len < column_size) {
4458                 fputc(' ', stdout);
4459                 column_size--;
4460         }
4461         if (*statbuf == '0') {
4462                 color_start(ANSI_GRAY);
4463                 fputc('0', stdout);
4464         } else {
4465                 for (; *statbuf; statbuf++) {
4466                         if (*statbuf == 'K') color_start(ANSI_GREEN);
4467                         else if (*statbuf == 'M') color_start(ANSI_YELLOW);
4468                         else if (*statbuf == 'G') color_start(ANSI_RED);
4469                         else if (*statbuf == 'T') color_start(ANSI_BOLD_BLUE);
4470                         else if (*statbuf == 'P') color_start(ANSI_MAGENTA);
4471                         else if (*statbuf == 'E') color_start(ANSI_CYAN);
4472                         fputc(*statbuf, stdout);
4473                         if (--column_size <= 0)
4474                                 break;
4475                 }
4476         }
4477         color_end();
4478 }
4479
4480 /*
4481  * Display a single statistic.
4482  */
4483 static void
4484 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
4485     unsigned int column_size, boolean_t scripted)
4486 {
4487         char buf[64];
4488
4489         zfs_nicenum_format(value, buf, sizeof (buf), format);
4490
4491         if (scripted)
4492                 printf("\t%s", buf);
4493         else
4494                 print_stat_color(buf, column_size);
4495 }
4496
4497 /*
4498  * Calculate the default vdev stats
4499  *
4500  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
4501  * stats into calcvs.
4502  */
4503 static void
4504 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
4505     vdev_stat_t *calcvs)
4506 {
4507         int i;
4508
4509         memcpy(calcvs, newvs, sizeof (*calcvs));
4510         for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
4511                 calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
4512
4513         for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
4514                 calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
4515 }
4516
4517 /*
4518  * Internal representation of the extended iostats data.
4519  *
4520  * The extended iostat stats are exported in nvlists as either uint64_t arrays
4521  * or single uint64_t's.  We make both look like arrays to make them easier
4522  * to process.  In order to make single uint64_t's look like arrays, we set
4523  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
4524  * we can just use *data and count.
4525  */
4526 struct stat_array {
4527         uint64_t *data;
4528         uint_t count;   /* Number of entries in data[] */
4529         uint64_t __data; /* Only used when data is a single uint64_t */
4530 };
4531
4532 static uint64_t
4533 stat_histo_max(struct stat_array *nva, unsigned int len)
4534 {
4535         uint64_t max = 0;
4536         int i;
4537         for (i = 0; i < len; i++)
4538                 max = MAX(max, array64_max(nva[i].data, nva[i].count));
4539
4540         return (max);
4541 }
4542
4543 /*
4544  * Helper function to lookup a uint64_t array or uint64_t value and store its
4545  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
4546  * it look like a one element array to make it easier to process.
4547  */
4548 static int
4549 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
4550     struct stat_array *nva)
4551 {
4552         nvpair_t *tmp;
4553         int ret;
4554
4555         verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
4556         switch (nvpair_type(tmp)) {
4557         case DATA_TYPE_UINT64_ARRAY:
4558                 ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
4559                 break;
4560         case DATA_TYPE_UINT64:
4561                 ret = nvpair_value_uint64(tmp, &nva->__data);
4562                 nva->data = &nva->__data;
4563                 nva->count = 1;
4564                 break;
4565         default:
4566                 /* Not a uint64_t */
4567                 ret = EINVAL;
4568                 break;
4569         }
4570
4571         return (ret);
4572 }
4573
4574 /*
4575  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
4576  * subtract them, and return the results in a newly allocated stat_array.
4577  * You must free the returned array after you are done with it with
4578  * free_calc_stats().
4579  *
4580  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
4581  * values.
4582  */
4583 static struct stat_array *
4584 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
4585     nvlist_t *newnv)
4586 {
4587         nvlist_t *oldnvx = NULL, *newnvx;
4588         struct stat_array *oldnva, *newnva, *calcnva;
4589         int i, j;
4590         unsigned int alloc_size = (sizeof (struct stat_array)) * len;
4591
4592         /* Extract our extended stats nvlist from the main list */
4593         verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4594             &newnvx) == 0);
4595         if (oldnv) {
4596                 verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4597                     &oldnvx) == 0);
4598         }
4599
4600         newnva = safe_malloc(alloc_size);
4601         oldnva = safe_malloc(alloc_size);
4602         calcnva = safe_malloc(alloc_size);
4603
4604         for (j = 0; j < len; j++) {
4605                 verify(nvpair64_to_stat_array(newnvx, names[j],
4606                     &newnva[j]) == 0);
4607                 calcnva[j].count = newnva[j].count;
4608                 alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
4609                 calcnva[j].data = safe_malloc(alloc_size);
4610                 memcpy(calcnva[j].data, newnva[j].data, alloc_size);
4611
4612                 if (oldnvx) {
4613                         verify(nvpair64_to_stat_array(oldnvx, names[j],
4614                             &oldnva[j]) == 0);
4615                         for (i = 0; i < oldnva[j].count; i++)
4616                                 calcnva[j].data[i] -= oldnva[j].data[i];
4617                 }
4618         }
4619         free(newnva);
4620         free(oldnva);
4621         return (calcnva);
4622 }
4623
4624 static void
4625 free_calc_stats(struct stat_array *nva, unsigned int len)
4626 {
4627         int i;
4628         for (i = 0; i < len; i++)
4629                 free(nva[i].data);
4630
4631         free(nva);
4632 }
4633
4634 static void
4635 print_iostat_histo(struct stat_array *nva, unsigned int len,
4636     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
4637     double scale)
4638 {
4639         int i, j;
4640         char buf[6];
4641         uint64_t val;
4642         enum zfs_nicenum_format format;
4643         unsigned int buckets;
4644         unsigned int start_bucket;
4645
4646         if (cb->cb_literal)
4647                 format = ZFS_NICENUM_RAW;
4648         else
4649                 format = ZFS_NICENUM_1024;
4650
4651         /* All these histos are the same size, so just use nva[0].count */
4652         buckets = nva[0].count;
4653
4654         if (cb->cb_flags & IOS_RQ_HISTO_M) {
4655                 /* Start at 512 - req size should never be lower than this */
4656                 start_bucket = 9;
4657         } else {
4658                 start_bucket = 0;
4659         }
4660
4661         for (j = start_bucket; j < buckets; j++) {
4662                 /* Print histogram bucket label */
4663                 if (cb->cb_flags & IOS_L_HISTO_M) {
4664                         /* Ending range of this bucket */
4665                         val = (1UL << (j + 1)) - 1;
4666                         zfs_nicetime(val, buf, sizeof (buf));
4667                 } else {
4668                         /* Request size (starting range of bucket) */
4669                         val = (1UL << j);
4670                         zfs_nicenum(val, buf, sizeof (buf));
4671                 }
4672
4673                 if (cb->cb_scripted)
4674                         printf("%llu", (u_longlong_t)val);
4675                 else
4676                         printf("%-*s", namewidth, buf);
4677
4678                 /* Print the values on the line */
4679                 for (i = 0; i < len; i++) {
4680                         print_one_stat(nva[i].data[j] * scale, format,
4681                             column_width, cb->cb_scripted);
4682                 }
4683                 printf("\n");
4684         }
4685 }
4686
4687 static void
4688 print_solid_separator(unsigned int length)
4689 {
4690         while (length--)
4691                 printf("-");
4692         printf("\n");
4693 }
4694
4695 static void
4696 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
4697     nvlist_t *newnv, double scale, const char *name)
4698 {
4699         unsigned int column_width;
4700         unsigned int namewidth;
4701         unsigned int entire_width;
4702         enum iostat_type type;
4703         struct stat_array *nva;
4704         const char **names;
4705         unsigned int names_len;
4706
4707         /* What type of histo are we? */
4708         type = IOS_HISTO_IDX(cb->cb_flags);
4709
4710         /* Get NULL-terminated array of nvlist names for our histo */
4711         names = vsx_type_to_nvlist[type];
4712         names_len = str_array_len(names); /* num of names */
4713
4714         nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
4715
4716         if (cb->cb_literal) {
4717                 column_width = MAX(5,
4718                     (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
4719         } else {
4720                 column_width = 5;
4721         }
4722
4723         namewidth = MAX(cb->cb_namewidth,
4724             strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
4725
4726         /*
4727          * Calculate the entire line width of what we're printing.  The
4728          * +2 is for the two spaces between columns:
4729          */
4730         /*       read  write                            */
4731         /*      -----  -----                            */
4732         /*      |___|  <---------- column_width         */
4733         /*                                              */
4734         /*      |__________|  <--- entire_width         */
4735         /*                                              */
4736         entire_width = namewidth + (column_width + 2) *
4737             label_array_len(iostat_bottom_labels[type]);
4738
4739         if (cb->cb_scripted)
4740                 printf("%s\n", name);
4741         else
4742                 print_iostat_header_impl(cb, column_width, name);
4743
4744         print_iostat_histo(nva, names_len, cb, column_width,
4745             namewidth, scale);
4746
4747         free_calc_stats(nva, names_len);
4748         if (!cb->cb_scripted)
4749                 print_solid_separator(entire_width);
4750 }
4751
4752 /*
4753  * Calculate the average latency of a power-of-two latency histogram
4754  */
4755 static uint64_t
4756 single_histo_average(uint64_t *histo, unsigned int buckets)
4757 {
4758         int i;
4759         uint64_t count = 0, total = 0;
4760
4761         for (i = 0; i < buckets; i++) {
4762                 /*
4763                  * Our buckets are power-of-two latency ranges.  Use the
4764                  * midpoint latency of each bucket to calculate the average.
4765                  * For example:
4766                  *
4767                  * Bucket          Midpoint
4768                  * 8ns-15ns:       12ns
4769                  * 16ns-31ns:      24ns
4770                  * ...
4771                  */
4772                 if (histo[i] != 0) {
4773                         total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
4774                         count += histo[i];
4775                 }
4776         }
4777
4778         /* Prevent divide by zero */
4779         return (count == 0 ? 0 : total / count);
4780 }
4781
4782 static void
4783 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *newnv)
4784 {
4785         const char *names[] = {
4786                 ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
4787                 ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
4788                 ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
4789                 ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
4790                 ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
4791                 ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
4792                 ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
4793                 ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
4794                 ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
4795                 ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
4796                 ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
4797                 ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
4798                 ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE,
4799                 ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
4800         };
4801
4802         struct stat_array *nva;
4803
4804         unsigned int column_width = default_column_width(cb, IOS_QUEUES);
4805         enum zfs_nicenum_format format;
4806
4807         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
4808
4809         if (cb->cb_literal)
4810                 format = ZFS_NICENUM_RAW;
4811         else
4812                 format = ZFS_NICENUM_1024;
4813
4814         for (int i = 0; i < ARRAY_SIZE(names); i++) {
4815                 uint64_t val = nva[i].data[0];
4816                 print_one_stat(val, format, column_width, cb->cb_scripted);
4817         }
4818
4819         free_calc_stats(nva, ARRAY_SIZE(names));
4820 }
4821
4822 static void
4823 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
4824     nvlist_t *newnv)
4825 {
4826         int i;
4827         uint64_t val;
4828         const char *names[] = {
4829                 ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
4830                 ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
4831                 ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
4832                 ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
4833                 ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
4834                 ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
4835                 ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
4836                 ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
4837                 ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
4838                 ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
4839                 ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
4840         };
4841         struct stat_array *nva;
4842
4843         unsigned int column_width = default_column_width(cb, IOS_LATENCY);
4844         enum zfs_nicenum_format format;
4845
4846         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
4847
4848         if (cb->cb_literal)
4849                 format = ZFS_NICENUM_RAWTIME;
4850         else
4851                 format = ZFS_NICENUM_TIME;
4852
4853         /* Print our avg latencies on the line */
4854         for (i = 0; i < ARRAY_SIZE(names); i++) {
4855                 /* Compute average latency for a latency histo */
4856                 val = single_histo_average(nva[i].data, nva[i].count);
4857                 print_one_stat(val, format, column_width, cb->cb_scripted);
4858         }
4859         free_calc_stats(nva, ARRAY_SIZE(names));
4860 }
4861
4862 /*
4863  * Print default statistics (capacity/operations/bandwidth)
4864  */
4865 static void
4866 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
4867 {
4868         unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
4869         enum zfs_nicenum_format format;
4870         char na;        /* char to print for "not applicable" values */
4871
4872         if (cb->cb_literal) {
4873                 format = ZFS_NICENUM_RAW;
4874                 na = '0';
4875         } else {
4876                 format = ZFS_NICENUM_1024;
4877                 na = '-';
4878         }
4879
4880         /* only toplevel vdevs have capacity stats */
4881         if (vs->vs_space == 0) {
4882                 if (cb->cb_scripted)
4883                         printf("\t%c\t%c", na, na);
4884                 else
4885                         printf("  %*c  %*c", column_width, na, column_width,
4886                             na);
4887         } else {
4888                 print_one_stat(vs->vs_alloc, format, column_width,
4889                     cb->cb_scripted);
4890                 print_one_stat(vs->vs_space - vs->vs_alloc, format,
4891                     column_width, cb->cb_scripted);
4892         }
4893
4894         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
4895             format, column_width, cb->cb_scripted);
4896         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
4897             format, column_width, cb->cb_scripted);
4898         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
4899             format, column_width, cb->cb_scripted);
4900         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
4901             format, column_width, cb->cb_scripted);
4902 }
4903
4904 static const char *const class_name[] = {
4905         VDEV_ALLOC_BIAS_DEDUP,
4906         VDEV_ALLOC_BIAS_SPECIAL,
4907         VDEV_ALLOC_CLASS_LOGS
4908 };
4909
4910 /*
4911  * Print out all the statistics for the given vdev.  This can either be the
4912  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
4913  * is a verbose output, and we don't want to display the toplevel pool stats.
4914  *
4915  * Returns the number of stat lines printed.
4916  */
4917 static unsigned int
4918 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
4919     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
4920 {
4921         nvlist_t **oldchild, **newchild;
4922         uint_t c, children, oldchildren;
4923         vdev_stat_t *oldvs, *newvs, *calcvs;
4924         vdev_stat_t zerovs = { 0 };
4925         char *vname;
4926         int i;
4927         int ret = 0;
4928         uint64_t tdelta;
4929         double scale;
4930
4931         if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
4932                 return (ret);
4933
4934         calcvs = safe_malloc(sizeof (*calcvs));
4935
4936         if (oldnv != NULL) {
4937                 verify(nvlist_lookup_uint64_array(oldnv,
4938                     ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
4939         } else {
4940                 oldvs = &zerovs;
4941         }
4942
4943         /* Do we only want to see a specific vdev? */
4944         for (i = 0; i < cb->cb_vdevs.cb_names_count; i++) {
4945                 /* Yes we do.  Is this the vdev? */
4946                 if (strcmp(name, cb->cb_vdevs.cb_names[i]) == 0) {
4947                         /*
4948                          * This is our vdev.  Since it is the only vdev we
4949                          * will be displaying, make depth = 0 so that it
4950                          * doesn't get indented.
4951                          */
4952                         depth = 0;
4953                         break;
4954                 }
4955         }
4956
4957         if (cb->cb_vdevs.cb_names_count && (i == cb->cb_vdevs.cb_names_count)) {
4958                 /* Couldn't match the name */
4959                 goto children;
4960         }
4961
4962
4963         verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
4964             (uint64_t **)&newvs, &c) == 0);
4965
4966         /*
4967          * Print the vdev name unless it's is a histogram.  Histograms
4968          * display the vdev name in the header itself.
4969          */
4970         if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
4971                 if (cb->cb_scripted) {
4972                         printf("%s", name);
4973                 } else {
4974                         if (strlen(name) + depth > cb->cb_namewidth)
4975                                 (void) printf("%*s%s", depth, "", name);
4976                         else
4977                                 (void) printf("%*s%s%*s", depth, "", name,
4978                                     (int)(cb->cb_namewidth - strlen(name) -
4979                                     depth), "");
4980                 }
4981         }
4982
4983         /* Calculate our scaling factor */
4984         tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
4985         if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
4986                 /*
4987                  * If we specify printing histograms with no time interval, then
4988                  * print the histogram numbers over the entire lifetime of the
4989                  * vdev.
4990                  */
4991                 scale = 1;
4992         } else {
4993                 if (tdelta == 0)
4994                         scale = 1.0;
4995                 else
4996                         scale = (double)NANOSEC / tdelta;
4997         }
4998
4999         if (cb->cb_flags & IOS_DEFAULT_M) {
5000                 calc_default_iostats(oldvs, newvs, calcvs);
5001                 print_iostat_default(calcvs, cb, scale);
5002         }
5003         if (cb->cb_flags & IOS_LATENCY_M)
5004                 print_iostat_latency(cb, oldnv, newnv);
5005         if (cb->cb_flags & IOS_QUEUES_M)
5006                 print_iostat_queues(cb, newnv);
5007         if (cb->cb_flags & IOS_ANYHISTO_M) {
5008                 printf("\n");
5009                 print_iostat_histos(cb, oldnv, newnv, scale, name);
5010         }
5011
5012         if (cb->vcdl != NULL) {
5013                 const char *path;
5014                 if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
5015                     &path) == 0) {
5016                         printf("  ");
5017                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
5018                 }
5019         }
5020
5021         if (!(cb->cb_flags & IOS_ANYHISTO_M))
5022                 printf("\n");
5023
5024         ret++;
5025
5026 children:
5027
5028         free(calcvs);
5029
5030         if (!cb->cb_verbose)
5031                 return (ret);
5032
5033         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
5034             &newchild, &children) != 0)
5035                 return (ret);
5036
5037         if (oldnv) {
5038                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
5039                     &oldchild, &oldchildren) != 0)
5040                         return (ret);
5041
5042                 children = MIN(oldchildren, children);
5043         }
5044
5045         /*
5046          * print normal top-level devices
5047          */
5048         for (c = 0; c < children; c++) {
5049                 uint64_t ishole = B_FALSE, islog = B_FALSE;
5050
5051                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
5052                     &ishole);
5053
5054                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
5055                     &islog);
5056
5057                 if (ishole || islog)
5058                         continue;
5059
5060                 if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
5061                         continue;
5062
5063                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
5064                     cb->cb_vdevs.cb_name_flags | VDEV_NAME_TYPE_ID);
5065                 ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
5066                     newchild[c], cb, depth + 2);
5067                 free(vname);
5068         }
5069
5070         /*
5071          * print all other top-level devices
5072          */
5073         for (uint_t n = 0; n < ARRAY_SIZE(class_name); n++) {
5074                 boolean_t printed = B_FALSE;
5075
5076                 for (c = 0; c < children; c++) {
5077                         uint64_t islog = B_FALSE;
5078                         const char *bias = NULL;
5079                         const char *type = NULL;
5080
5081                         (void) nvlist_lookup_uint64(newchild[c],
5082                             ZPOOL_CONFIG_IS_LOG, &islog);
5083                         if (islog) {
5084                                 bias = VDEV_ALLOC_CLASS_LOGS;
5085                         } else {
5086                                 (void) nvlist_lookup_string(newchild[c],
5087                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
5088                                 (void) nvlist_lookup_string(newchild[c],
5089                                     ZPOOL_CONFIG_TYPE, &type);
5090                         }
5091                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
5092                                 continue;
5093                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
5094                                 continue;
5095
5096                         if (!printed) {
5097                                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
5098                                     !cb->cb_scripted &&
5099                                     !cb->cb_vdevs.cb_names) {
5100                                         print_iostat_dashes(cb, 0,
5101                                             class_name[n]);
5102                                 }
5103                                 printf("\n");
5104                                 printed = B_TRUE;
5105                         }
5106
5107                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
5108                             cb->cb_vdevs.cb_name_flags | VDEV_NAME_TYPE_ID);
5109                         ret += print_vdev_stats(zhp, vname, oldnv ?
5110                             oldchild[c] : NULL, newchild[c], cb, depth + 2);
5111                         free(vname);
5112                 }
5113         }
5114
5115         /*
5116          * Include level 2 ARC devices in iostat output
5117          */
5118         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
5119             &newchild, &children) != 0)
5120                 return (ret);
5121
5122         if (oldnv) {
5123                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
5124                     &oldchild, &oldchildren) != 0)
5125                         return (ret);
5126
5127                 children = MIN(oldchildren, children);
5128         }
5129
5130         if (children > 0) {
5131                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
5132                     !cb->cb_vdevs.cb_names) {
5133                         print_iostat_dashes(cb, 0, "cache");
5134                 }
5135                 printf("\n");
5136
5137                 for (c = 0; c < children; c++) {
5138                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
5139                             cb->cb_vdevs.cb_name_flags);
5140                         ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
5141                             : NULL, newchild[c], cb, depth + 2);
5142                         free(vname);
5143                 }
5144         }
5145
5146         return (ret);
5147 }
5148
5149 static int
5150 refresh_iostat(zpool_handle_t *zhp, void *data)
5151 {
5152         iostat_cbdata_t *cb = data;
5153         boolean_t missing;
5154
5155         /*
5156          * If the pool has disappeared, remove it from the list and continue.
5157          */
5158         if (zpool_refresh_stats(zhp, &missing) != 0)
5159                 return (-1);
5160
5161         if (missing)
5162                 pool_list_remove(cb->cb_list, zhp);
5163
5164         return (0);
5165 }
5166
5167 /*
5168  * Callback to print out the iostats for the given pool.
5169  */
5170 static int
5171 print_iostat(zpool_handle_t *zhp, void *data)
5172 {
5173         iostat_cbdata_t *cb = data;
5174         nvlist_t *oldconfig, *newconfig;
5175         nvlist_t *oldnvroot, *newnvroot;
5176         int ret;
5177
5178         newconfig = zpool_get_config(zhp, &oldconfig);
5179
5180         if (cb->cb_iteration == 1)
5181                 oldconfig = NULL;
5182
5183         verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
5184             &newnvroot) == 0);
5185
5186         if (oldconfig == NULL)
5187                 oldnvroot = NULL;
5188         else
5189                 verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
5190                     &oldnvroot) == 0);
5191
5192         ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
5193             cb, 0);
5194         if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
5195             !cb->cb_scripted && cb->cb_verbose &&
5196             !cb->cb_vdevs.cb_names_count) {
5197                 print_iostat_separator(cb);
5198                 if (cb->vcdl != NULL) {
5199                         print_cmd_columns(cb->vcdl, 1);
5200                 }
5201                 printf("\n");
5202         }
5203
5204         return (ret);
5205 }
5206
5207 static int
5208 get_columns(void)
5209 {
5210         struct winsize ws;
5211         int columns = 80;
5212         int error;
5213
5214         if (isatty(STDOUT_FILENO)) {
5215                 error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
5216                 if (error == 0)
5217                         columns = ws.ws_col;
5218         } else {
5219                 columns = 999;
5220         }
5221
5222         return (columns);
5223 }
5224
5225 /*
5226  * Return the required length of the pool/vdev name column.  The minimum
5227  * allowed width and output formatting flags must be provided.
5228  */
5229 static int
5230 get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
5231 {
5232         nvlist_t *config, *nvroot;
5233         int width = min_width;
5234
5235         if ((config = zpool_get_config(zhp, NULL)) != NULL) {
5236                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5237                     &nvroot) == 0);
5238                 size_t poolname_len = strlen(zpool_get_name(zhp));
5239                 if (verbose == B_FALSE) {
5240                         width = MAX(poolname_len, min_width);
5241                 } else {
5242                         width = MAX(poolname_len,
5243                             max_width(zhp, nvroot, 0, min_width, flags));
5244                 }
5245         }
5246
5247         return (width);
5248 }
5249
5250 /*
5251  * Parse the input string, get the 'interval' and 'count' value if there is one.
5252  */
5253 static void
5254 get_interval_count(int *argcp, char **argv, float *iv,
5255     unsigned long *cnt)
5256 {
5257         float interval = 0;
5258         unsigned long count = 0;
5259         int argc = *argcp;
5260
5261         /*
5262          * Determine if the last argument is an integer or a pool name
5263          */
5264         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
5265                 char *end;
5266
5267                 errno = 0;
5268                 interval = strtof(argv[argc - 1], &end);
5269
5270                 if (*end == '\0' && errno == 0) {
5271                         if (interval == 0) {
5272                                 (void) fprintf(stderr, gettext(
5273                                     "interval cannot be zero\n"));
5274                                 usage(B_FALSE);
5275                         }
5276                         /*
5277                          * Ignore the last parameter
5278                          */
5279                         argc--;
5280                 } else {
5281                         /*
5282                          * If this is not a valid number, just plow on.  The
5283                          * user will get a more informative error message later
5284                          * on.
5285                          */
5286                         interval = 0;
5287                 }
5288         }
5289
5290         /*
5291          * If the last argument is also an integer, then we have both a count
5292          * and an interval.
5293          */
5294         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
5295                 char *end;
5296
5297                 errno = 0;
5298                 count = interval;
5299                 interval = strtof(argv[argc - 1], &end);
5300
5301                 if (*end == '\0' && errno == 0) {
5302                         if (interval == 0) {
5303                                 (void) fprintf(stderr, gettext(
5304                                     "interval cannot be zero\n"));
5305                                 usage(B_FALSE);
5306                         }
5307
5308                         /*
5309                          * Ignore the last parameter
5310                          */
5311                         argc--;
5312                 } else {
5313                         interval = 0;
5314                 }
5315         }
5316
5317         *iv = interval;
5318         *cnt = count;
5319         *argcp = argc;
5320 }
5321
5322 static void
5323 get_timestamp_arg(char c)
5324 {
5325         if (c == 'u')
5326                 timestamp_fmt = UDATE;
5327         else if (c == 'd')
5328                 timestamp_fmt = DDATE;
5329         else
5330                 usage(B_FALSE);
5331 }
5332
5333 /*
5334  * Return stat flags that are supported by all pools by both the module and
5335  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
5336  * It will get ANDed down until only the flags that are supported on all pools
5337  * remain.
5338  */
5339 static int
5340 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
5341 {
5342         uint64_t *mask = data;
5343         nvlist_t *config, *nvroot, *nvx;
5344         uint64_t flags = 0;
5345         int i, j;
5346
5347         config = zpool_get_config(zhp, NULL);
5348         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5349             &nvroot) == 0);
5350
5351         /* Default stats are always supported, but for completeness.. */
5352         if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
5353                 flags |= IOS_DEFAULT_M;
5354
5355         /* Get our extended stats nvlist from the main list */
5356         if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
5357             &nvx) != 0) {
5358                 /*
5359                  * No extended stats; they're probably running an older
5360                  * module.  No big deal, we support that too.
5361                  */
5362                 goto end;
5363         }
5364
5365         /* For each extended stat, make sure all its nvpairs are supported */
5366         for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
5367                 if (!vsx_type_to_nvlist[j][0])
5368                         continue;
5369
5370                 /* Start off by assuming the flag is supported, then check */
5371                 flags |= (1ULL << j);
5372                 for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
5373                         if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
5374                                 /* flag isn't supported */
5375                                 flags = flags & ~(1ULL  << j);
5376                                 break;
5377                         }
5378                 }
5379         }
5380 end:
5381         *mask = *mask & flags;
5382         return (0);
5383 }
5384
5385 /*
5386  * Return a bitmask of stats that are supported on all pools by both the module
5387  * and zpool iostat.
5388  */
5389 static uint64_t
5390 get_stat_flags(zpool_list_t *list)
5391 {
5392         uint64_t mask = -1;
5393
5394         /*
5395          * get_stat_flags_cb() will lop off bits from "mask" until only the
5396          * flags that are supported on all pools remain.
5397          */
5398         pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
5399         return (mask);
5400 }
5401
5402 /*
5403  * Return 1 if cb_data->cb_names[0] is this vdev's name, 0 otherwise.
5404  */
5405 static int
5406 is_vdev_cb(void *zhp_data, nvlist_t *nv, void *cb_data)
5407 {
5408         uint64_t guid;
5409         vdev_cbdata_t *cb = cb_data;
5410         zpool_handle_t *zhp = zhp_data;
5411
5412         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
5413                 return (0);
5414
5415         return (guid == zpool_vdev_path_to_guid(zhp, cb->cb_names[0]));
5416 }
5417
5418 /*
5419  * Returns 1 if cb_data->cb_names[0] is a vdev name, 0 otherwise.
5420  */
5421 static int
5422 is_vdev(zpool_handle_t *zhp, void *cb_data)
5423 {
5424         return (for_each_vdev(zhp, is_vdev_cb, cb_data));
5425 }
5426
5427 /*
5428  * Check if vdevs are in a pool
5429  *
5430  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
5431  * return 0.  If pool_name is NULL, then search all pools.
5432  */
5433 static int
5434 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
5435     vdev_cbdata_t *cb)
5436 {
5437         char **tmp_name;
5438         int ret = 0;
5439         int i;
5440         int pool_count = 0;
5441
5442         if ((argc == 0) || !*argv)
5443                 return (0);
5444
5445         if (pool_name)
5446                 pool_count = 1;
5447
5448         /* Temporarily hijack cb_names for a second... */
5449         tmp_name = cb->cb_names;
5450
5451         /* Go though our list of prospective vdev names */
5452         for (i = 0; i < argc; i++) {
5453                 cb->cb_names = argv + i;
5454
5455                 /* Is this name a vdev in our pools? */
5456                 ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
5457                     ZFS_TYPE_POOL, B_FALSE, is_vdev, cb);
5458                 if (!ret) {
5459                         /* No match */
5460                         break;
5461                 }
5462         }
5463
5464         cb->cb_names = tmp_name;
5465
5466         return (ret);
5467 }
5468
5469 static int
5470 is_pool_cb(zpool_handle_t *zhp, void *data)
5471 {
5472         char *name = data;
5473         if (strcmp(name, zpool_get_name(zhp)) == 0)
5474                 return (1);
5475
5476         return (0);
5477 }
5478
5479 /*
5480  * Do we have a pool named *name?  If so, return 1, otherwise 0.
5481  */
5482 static int
5483 is_pool(char *name)
5484 {
5485         return (for_each_pool(0, NULL, B_TRUE, NULL, ZFS_TYPE_POOL, B_FALSE,
5486             is_pool_cb, name));
5487 }
5488
5489 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
5490 static int
5491 are_all_pools(int argc, char **argv)
5492 {
5493         if ((argc == 0) || !*argv)
5494                 return (0);
5495
5496         while (--argc >= 0)
5497                 if (!is_pool(argv[argc]))
5498                         return (0);
5499
5500         return (1);
5501 }
5502
5503 /*
5504  * Helper function to print out vdev/pool names we can't resolve.  Used for an
5505  * error message.
5506  */
5507 static void
5508 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
5509     vdev_cbdata_t *cb)
5510 {
5511         int i;
5512         char *name;
5513         char *str;
5514         for (i = 0; i < argc; i++) {
5515                 name = argv[i];
5516
5517                 if (is_pool(name))
5518                         str = gettext("pool");
5519                 else if (are_vdevs_in_pool(1, &name, pool_name, cb))
5520                         str = gettext("vdev in this pool");
5521                 else if (are_vdevs_in_pool(1, &name, NULL, cb))
5522                         str = gettext("vdev in another pool");
5523                 else
5524                         str = gettext("unknown");
5525
5526                 fprintf(stderr, "\t%s (%s)\n", name, str);
5527         }
5528 }
5529
5530 /*
5531  * Same as get_interval_count(), but with additional checks to not misinterpret
5532  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
5533  * cb.cb_vdevs.cb_name_flags.
5534  */
5535 static void
5536 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
5537     unsigned long *count, iostat_cbdata_t *cb)
5538 {
5539         char **tmpargv = argv;
5540         int argc_for_interval = 0;
5541
5542         /* Is the last arg an interval value?  Or a guid? */
5543         if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL,
5544             &cb->cb_vdevs)) {
5545                 /*
5546                  * The last arg is not a guid, so it's probably an
5547                  * interval value.
5548                  */
5549                 argc_for_interval++;
5550
5551                 if (*argc >= 2 &&
5552                     !are_vdevs_in_pool(1, &argv[*argc - 2], NULL,
5553                     &cb->cb_vdevs)) {
5554                         /*
5555                          * The 2nd to last arg is not a guid, so it's probably
5556                          * an interval value.
5557                          */
5558                         argc_for_interval++;
5559                 }
5560         }
5561
5562         /* Point to our list of possible intervals */
5563         tmpargv = &argv[*argc - argc_for_interval];
5564
5565         *argc = *argc - argc_for_interval;
5566         get_interval_count(&argc_for_interval, tmpargv,
5567             interval, count);
5568 }
5569
5570 /*
5571  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
5572  * if we were unable to determine its size.
5573  */
5574 static int
5575 terminal_height(void)
5576 {
5577         struct winsize win;
5578
5579         if (isatty(STDOUT_FILENO) == 0)
5580                 return (-1);
5581
5582         if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) != -1 && win.ws_row > 0)
5583                 return (win.ws_row);
5584
5585         return (-1);
5586 }
5587
5588 /*
5589  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
5590  * print the result.
5591  *
5592  * name:        Short name of the script ('iostat').
5593  * path:        Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
5594  */
5595 static void
5596 print_zpool_script_help(char *name, char *path)
5597 {
5598         char *argv[] = {path, (char *)"-h", NULL};
5599         char **lines = NULL;
5600         int lines_cnt = 0;
5601         int rc;
5602
5603         rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
5604             &lines_cnt);
5605         if (rc != 0 || lines == NULL || lines_cnt <= 0) {
5606                 if (lines != NULL)
5607                         libzfs_free_str_array(lines, lines_cnt);
5608                 return;
5609         }
5610
5611         for (int i = 0; i < lines_cnt; i++)
5612                 if (!is_blank_str(lines[i]))
5613                         printf("  %-14s  %s\n", name, lines[i]);
5614
5615         libzfs_free_str_array(lines, lines_cnt);
5616 }
5617
5618 /*
5619  * Go though the zpool status/iostat -c scripts in the user's path, run their
5620  * help option (-h), and print out the results.
5621  */
5622 static void
5623 print_zpool_dir_scripts(char *dirpath)
5624 {
5625         DIR *dir;
5626         struct dirent *ent;
5627         char fullpath[MAXPATHLEN];
5628         struct stat dir_stat;
5629
5630         if ((dir = opendir(dirpath)) != NULL) {
5631                 /* print all the files and directories within directory */
5632                 while ((ent = readdir(dir)) != NULL) {
5633                         if (snprintf(fullpath, sizeof (fullpath), "%s/%s",
5634                             dirpath, ent->d_name) >= sizeof (fullpath)) {
5635                                 (void) fprintf(stderr,
5636                                     gettext("internal error: "
5637                                     "ZPOOL_SCRIPTS_PATH too large.\n"));
5638                                 exit(1);
5639                         }
5640
5641                         /* Print the scripts */
5642                         if (stat(fullpath, &dir_stat) == 0)
5643                                 if (dir_stat.st_mode & S_IXUSR &&
5644                                     S_ISREG(dir_stat.st_mode))
5645                                         print_zpool_script_help(ent->d_name,
5646                                             fullpath);
5647                 }
5648                 closedir(dir);
5649         }
5650 }
5651
5652 /*
5653  * Print out help text for all zpool status/iostat -c scripts.
5654  */
5655 static void
5656 print_zpool_script_list(const char *subcommand)
5657 {
5658         char *dir, *sp, *tmp;
5659
5660         printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
5661
5662         sp = zpool_get_cmd_search_path();
5663         if (sp == NULL)
5664                 return;
5665
5666         for (dir = strtok_r(sp, ":", &tmp);
5667             dir != NULL;
5668             dir = strtok_r(NULL, ":", &tmp))
5669                 print_zpool_dir_scripts(dir);
5670
5671         free(sp);
5672 }
5673
5674 /*
5675  * Set the minimum pool/vdev name column width.  The width must be at least 10,
5676  * but may be as large as the column width - 42 so it still fits on one line.
5677  * NOTE: 42 is the width of the default capacity/operations/bandwidth output
5678  */
5679 static int
5680 get_namewidth_iostat(zpool_handle_t *zhp, void *data)
5681 {
5682         iostat_cbdata_t *cb = data;
5683         int width, available_width;
5684
5685         /*
5686          * get_namewidth() returns the maximum width of any name in that column
5687          * for any pool/vdev/device line that will be output.
5688          */
5689         width = get_namewidth(zhp, cb->cb_namewidth,
5690             cb->cb_vdevs.cb_name_flags | VDEV_NAME_TYPE_ID, cb->cb_verbose);
5691
5692         /*
5693          * The width we are calculating is the width of the header and also the
5694          * padding width for names that are less than maximum width.  The stats
5695          * take up 42 characters, so the width available for names is:
5696          */
5697         available_width = get_columns() - 42;
5698
5699         /*
5700          * If the maximum width fits on a screen, then great!  Make everything
5701          * line up by justifying all lines to the same width.  If that max
5702          * width is larger than what's available, the name plus stats won't fit
5703          * on one line, and justifying to that width would cause every line to
5704          * wrap on the screen.  We only want lines with long names to wrap.
5705          * Limit the padding to what won't wrap.
5706          */
5707         if (width > available_width)
5708                 width = available_width;
5709
5710         /*
5711          * And regardless of whatever the screen width is (get_columns can
5712          * return 0 if the width is not known or less than 42 for a narrow
5713          * terminal) have the width be a minimum of 10.
5714          */
5715         if (width < 10)
5716                 width = 10;
5717
5718         /* Save the calculated width */
5719         cb->cb_namewidth = width;
5720
5721         return (0);
5722 }
5723
5724 /*
5725  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
5726  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
5727  *              [interval [count]]
5728  *
5729  *      -c CMD  For each vdev, run command CMD
5730  *      -g      Display guid for individual vdev name.
5731  *      -L      Follow links when resolving vdev path name.
5732  *      -P      Display full path for vdev name.
5733  *      -v      Display statistics for individual vdevs
5734  *      -h      Display help
5735  *      -p      Display values in parsable (exact) format.
5736  *      -H      Scripted mode.  Don't display headers, and separate properties
5737  *              by a single tab.
5738  *      -l      Display average latency
5739  *      -q      Display queue depths
5740  *      -w      Display latency histograms
5741  *      -r      Display request size histogram
5742  *      -T      Display a timestamp in date(1) or Unix format
5743  *      -n      Only print headers once
5744  *
5745  * This command can be tricky because we want to be able to deal with pool
5746  * creation/destruction as well as vdev configuration changes.  The bulk of this
5747  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
5748  * on pool_list_update() to detect the addition of new pools.  Configuration
5749  * changes are all handled within libzfs.
5750  */
5751 int
5752 zpool_do_iostat(int argc, char **argv)
5753 {
5754         int c;
5755         int ret;
5756         int npools;
5757         float interval = 0;
5758         unsigned long count = 0;
5759         int winheight = 24;
5760         zpool_list_t *list;
5761         boolean_t verbose = B_FALSE;
5762         boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
5763         boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
5764         boolean_t omit_since_boot = B_FALSE;
5765         boolean_t guid = B_FALSE;
5766         boolean_t follow_links = B_FALSE;
5767         boolean_t full_name = B_FALSE;
5768         boolean_t headers_once = B_FALSE;
5769         iostat_cbdata_t cb = { 0 };
5770         char *cmd = NULL;
5771
5772         /* Used for printing error message */
5773         const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
5774             [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
5775
5776         uint64_t unsupported_flags;
5777
5778         /* check options */
5779         while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwnH")) != -1) {
5780                 switch (c) {
5781                 case 'c':
5782                         if (cmd != NULL) {
5783                                 fprintf(stderr,
5784                                     gettext("Can't set -c flag twice\n"));
5785                                 exit(1);
5786                         }
5787
5788                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
5789                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
5790                                 fprintf(stderr, gettext(
5791                                     "Can't run -c, disabled by "
5792                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
5793                                 exit(1);
5794                         }
5795
5796                         if ((getuid() <= 0 || geteuid() <= 0) &&
5797                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
5798                                 fprintf(stderr, gettext(
5799                                     "Can't run -c with root privileges "
5800                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
5801                                 exit(1);
5802                         }
5803                         cmd = optarg;
5804                         verbose = B_TRUE;
5805                         break;
5806                 case 'g':
5807                         guid = B_TRUE;
5808                         break;
5809                 case 'L':
5810                         follow_links = B_TRUE;
5811                         break;
5812                 case 'P':
5813                         full_name = B_TRUE;
5814                         break;
5815                 case 'T':
5816                         get_timestamp_arg(*optarg);
5817                         break;
5818                 case 'v':
5819                         verbose = B_TRUE;
5820                         break;
5821                 case 'p':
5822                         parsable = B_TRUE;
5823                         break;
5824                 case 'l':
5825                         latency = B_TRUE;
5826                         break;
5827                 case 'q':
5828                         queues = B_TRUE;
5829                         break;
5830                 case 'H':
5831                         scripted = B_TRUE;
5832                         break;
5833                 case 'w':
5834                         l_histo = B_TRUE;
5835                         break;
5836                 case 'r':
5837                         rq_histo = B_TRUE;
5838                         break;
5839                 case 'y':
5840                         omit_since_boot = B_TRUE;
5841                         break;
5842                 case 'n':
5843                         headers_once = B_TRUE;
5844                         break;
5845                 case 'h':
5846                         usage(B_FALSE);
5847                         break;
5848                 case '?':
5849                         if (optopt == 'c') {
5850                                 print_zpool_script_list("iostat");
5851                                 exit(0);
5852                         } else {
5853                                 fprintf(stderr,
5854                                     gettext("invalid option '%c'\n"), optopt);
5855                         }
5856                         usage(B_FALSE);
5857                 }
5858         }
5859
5860         argc -= optind;
5861         argv += optind;
5862
5863         cb.cb_literal = parsable;
5864         cb.cb_scripted = scripted;
5865
5866         if (guid)
5867                 cb.cb_vdevs.cb_name_flags |= VDEV_NAME_GUID;
5868         if (follow_links)
5869                 cb.cb_vdevs.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5870         if (full_name)
5871                 cb.cb_vdevs.cb_name_flags |= VDEV_NAME_PATH;
5872         cb.cb_iteration = 0;
5873         cb.cb_namewidth = 0;
5874         cb.cb_verbose = verbose;
5875
5876         /* Get our interval and count values (if any) */
5877         if (guid) {
5878                 get_interval_count_filter_guids(&argc, argv, &interval,
5879                     &count, &cb);
5880         } else {
5881                 get_interval_count(&argc, argv, &interval, &count);
5882         }
5883
5884         if (argc == 0) {
5885                 /* No args, so just print the defaults. */
5886         } else if (are_all_pools(argc, argv)) {
5887                 /* All the args are pool names */
5888         } else if (are_vdevs_in_pool(argc, argv, NULL, &cb.cb_vdevs)) {
5889                 /* All the args are vdevs */
5890                 cb.cb_vdevs.cb_names = argv;
5891                 cb.cb_vdevs.cb_names_count = argc;
5892                 argc = 0; /* No pools to process */
5893         } else if (are_all_pools(1, argv)) {
5894                 /* The first arg is a pool name */
5895                 if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
5896                     &cb.cb_vdevs)) {
5897                         /* ...and the rest are vdev names */
5898                         cb.cb_vdevs.cb_names = argv + 1;
5899                         cb.cb_vdevs.cb_names_count = argc - 1;
5900                         argc = 1; /* One pool to process */
5901                 } else {
5902                         fprintf(stderr, gettext("Expected either a list of "));
5903                         fprintf(stderr, gettext("pools, or list of vdevs in"));
5904                         fprintf(stderr, " \"%s\", ", argv[0]);
5905                         fprintf(stderr, gettext("but got:\n"));
5906                         error_list_unresolved_vdevs(argc - 1, argv + 1,
5907                             argv[0], &cb.cb_vdevs);
5908                         fprintf(stderr, "\n");
5909                         usage(B_FALSE);
5910                         return (1);
5911                 }
5912         } else {
5913                 /*
5914                  * The args don't make sense. The first arg isn't a pool name,
5915                  * nor are all the args vdevs.
5916                  */
5917                 fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
5918                 fprintf(stderr, "\n");
5919                 return (1);
5920         }
5921
5922         if (cb.cb_vdevs.cb_names_count != 0) {
5923                 /*
5924                  * If user specified vdevs, it implies verbose.
5925                  */
5926                 cb.cb_verbose = B_TRUE;
5927         }
5928
5929         /*
5930          * Construct the list of all interesting pools.
5931          */
5932         ret = 0;
5933         if ((list = pool_list_get(argc, argv, NULL, ZFS_TYPE_POOL, parsable,
5934             &ret)) == NULL)
5935                 return (1);
5936
5937         if (pool_list_count(list) == 0 && argc != 0) {
5938                 pool_list_free(list);
5939                 return (1);
5940         }
5941
5942         if (pool_list_count(list) == 0 && interval == 0) {
5943                 pool_list_free(list);
5944                 (void) fprintf(stderr, gettext("no pools available\n"));
5945                 return (1);
5946         }
5947
5948         if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
5949                 pool_list_free(list);
5950                 (void) fprintf(stderr,
5951                     gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
5952                 usage(B_FALSE);
5953                 return (1);
5954         }
5955
5956         if (l_histo && rq_histo) {
5957                 pool_list_free(list);
5958                 (void) fprintf(stderr,
5959                     gettext("Only one of [-r|-w] can be passed at a time\n"));
5960                 usage(B_FALSE);
5961                 return (1);
5962         }
5963
5964         /*
5965          * Enter the main iostat loop.
5966          */
5967         cb.cb_list = list;
5968
5969         if (l_histo) {
5970                 /*
5971                  * Histograms tables look out of place when you try to display
5972                  * them with the other stats, so make a rule that you can only
5973                  * print histograms by themselves.
5974                  */
5975                 cb.cb_flags = IOS_L_HISTO_M;
5976         } else if (rq_histo) {
5977                 cb.cb_flags = IOS_RQ_HISTO_M;
5978         } else {
5979                 cb.cb_flags = IOS_DEFAULT_M;
5980                 if (latency)
5981                         cb.cb_flags |= IOS_LATENCY_M;
5982                 if (queues)
5983                         cb.cb_flags |= IOS_QUEUES_M;
5984         }
5985
5986         /*
5987          * See if the module supports all the stats we want to display.
5988          */
5989         unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
5990         if (unsupported_flags) {
5991                 uint64_t f;
5992                 int idx;
5993                 fprintf(stderr,
5994                     gettext("The loaded zfs module doesn't support:"));
5995
5996                 /* for each bit set in unsupported_flags */
5997                 for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
5998                         idx = lowbit64(f) - 1;
5999                         fprintf(stderr, " -%c", flag_to_arg[idx]);
6000                 }
6001
6002                 fprintf(stderr, ".  Try running a newer module.\n");
6003                 pool_list_free(list);
6004
6005                 return (1);
6006         }
6007
6008         for (;;) {
6009                 if ((npools = pool_list_count(list)) == 0)
6010                         (void) fprintf(stderr, gettext("no pools available\n"));
6011                 else {
6012                         /*
6013                          * If this is the first iteration and -y was supplied
6014                          * we skip any printing.
6015                          */
6016                         boolean_t skip = (omit_since_boot &&
6017                             cb.cb_iteration == 0);
6018
6019                         /*
6020                          * Refresh all statistics.  This is done as an
6021                          * explicit step before calculating the maximum name
6022                          * width, so that any * configuration changes are
6023                          * properly accounted for.
6024                          */
6025                         (void) pool_list_iter(list, B_FALSE, refresh_iostat,
6026                             &cb);
6027
6028                         /*
6029                          * Iterate over all pools to determine the maximum width
6030                          * for the pool / device name column across all pools.
6031                          */
6032                         cb.cb_namewidth = 0;
6033                         (void) pool_list_iter(list, B_FALSE,
6034                             get_namewidth_iostat, &cb);
6035
6036                         if (timestamp_fmt != NODATE)
6037                                 print_timestamp(timestamp_fmt);
6038
6039                         if (cmd != NULL && cb.cb_verbose &&
6040                             !(cb.cb_flags & IOS_ANYHISTO_M)) {
6041                                 cb.vcdl = all_pools_for_each_vdev_run(argc,
6042                                     argv, cmd, g_zfs, cb.cb_vdevs.cb_names,
6043                                     cb.cb_vdevs.cb_names_count,
6044                                     cb.cb_vdevs.cb_name_flags);
6045                         } else {
6046                                 cb.vcdl = NULL;
6047                         }
6048
6049
6050                         /*
6051                          * Check terminal size so we can print headers
6052                          * even when terminal window has its height
6053                          * changed.
6054                          */
6055                         winheight = terminal_height();
6056                         /*
6057                          * Are we connected to TTY? If not, headers_once
6058                          * should be true, to avoid breaking scripts.
6059                          */
6060                         if (winheight < 0)
6061                                 headers_once = B_TRUE;
6062
6063                         /*
6064                          * If it's the first time and we're not skipping it,
6065                          * or either skip or verbose mode, print the header.
6066                          *
6067                          * The histogram code explicitly prints its header on
6068                          * every vdev, so skip this for histograms.
6069                          */
6070                         if (((++cb.cb_iteration == 1 && !skip) ||
6071                             (skip != verbose) ||
6072                             (!headers_once &&
6073                             (cb.cb_iteration % winheight) == 0)) &&
6074                             (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
6075                             !cb.cb_scripted)
6076                                 print_iostat_header(&cb);
6077
6078                         if (skip) {
6079                                 (void) fflush(stdout);
6080                                 (void) fsleep(interval);
6081                                 continue;
6082                         }
6083
6084                         pool_list_iter(list, B_FALSE, print_iostat, &cb);
6085
6086                         /*
6087                          * If there's more than one pool, and we're not in
6088                          * verbose mode (which prints a separator for us),
6089                          * then print a separator.
6090                          *
6091                          * In addition, if we're printing specific vdevs then
6092                          * we also want an ending separator.
6093                          */
6094                         if (((npools > 1 && !verbose &&
6095                             !(cb.cb_flags & IOS_ANYHISTO_M)) ||
6096                             (!(cb.cb_flags & IOS_ANYHISTO_M) &&
6097                             cb.cb_vdevs.cb_names_count)) &&
6098                             !cb.cb_scripted) {
6099                                 print_iostat_separator(&cb);
6100                                 if (cb.vcdl != NULL)
6101                                         print_cmd_columns(cb.vcdl, 1);
6102                                 printf("\n");
6103                         }
6104
6105                         if (cb.vcdl != NULL)
6106                                 free_vdev_cmd_data_list(cb.vcdl);
6107
6108                 }
6109
6110                 if (interval == 0)
6111                         break;
6112
6113                 if (count != 0 && --count == 0)
6114                         break;
6115
6116                 (void) fflush(stdout);
6117                 (void) fsleep(interval);
6118         }
6119
6120         pool_list_free(list);
6121
6122         return (ret);
6123 }
6124
6125 typedef struct list_cbdata {
6126         boolean_t       cb_verbose;
6127         int             cb_name_flags;
6128         int             cb_namewidth;
6129         boolean_t       cb_scripted;
6130         zprop_list_t    *cb_proplist;
6131         boolean_t       cb_literal;
6132 } list_cbdata_t;
6133
6134
6135 /*
6136  * Given a list of columns to display, output appropriate headers for each one.
6137  */
6138 static void
6139 print_header(list_cbdata_t *cb)
6140 {
6141         zprop_list_t *pl = cb->cb_proplist;
6142         char headerbuf[ZPOOL_MAXPROPLEN];
6143         const char *header;
6144         boolean_t first = B_TRUE;
6145         boolean_t right_justify;
6146         size_t width = 0;
6147
6148         for (; pl != NULL; pl = pl->pl_next) {
6149                 width = pl->pl_width;
6150                 if (first && cb->cb_verbose) {
6151                         /*
6152                          * Reset the width to accommodate the verbose listing
6153                          * of devices.
6154                          */
6155                         width = cb->cb_namewidth;
6156                 }
6157
6158                 if (!first)
6159                         (void) fputs("  ", stdout);
6160                 else
6161                         first = B_FALSE;
6162
6163                 right_justify = B_FALSE;
6164                 if (pl->pl_prop != ZPROP_USERPROP) {
6165                         header = zpool_prop_column_name(pl->pl_prop);
6166                         right_justify = zpool_prop_align_right(pl->pl_prop);
6167                 } else {
6168                         int i;
6169
6170                         for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
6171                                 headerbuf[i] = toupper(pl->pl_user_prop[i]);
6172                         headerbuf[i] = '\0';
6173                         header = headerbuf;
6174                 }
6175
6176                 if (pl->pl_next == NULL && !right_justify)
6177                         (void) fputs(header, stdout);
6178                 else if (right_justify)
6179                         (void) printf("%*s", (int)width, header);
6180                 else
6181                         (void) printf("%-*s", (int)width, header);
6182         }
6183
6184         (void) fputc('\n', stdout);
6185 }
6186
6187 /*
6188  * Given a pool and a list of properties, print out all the properties according
6189  * to the described layout. Used by zpool_do_list().
6190  */
6191 static void
6192 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
6193 {
6194         zprop_list_t *pl = cb->cb_proplist;
6195         boolean_t first = B_TRUE;
6196         char property[ZPOOL_MAXPROPLEN];
6197         const char *propstr;
6198         boolean_t right_justify;
6199         size_t width;
6200
6201         for (; pl != NULL; pl = pl->pl_next) {
6202
6203                 width = pl->pl_width;
6204                 if (first && cb->cb_verbose) {
6205                         /*
6206                          * Reset the width to accommodate the verbose listing
6207                          * of devices.
6208                          */
6209                         width = cb->cb_namewidth;
6210                 }
6211
6212                 if (!first) {
6213                         if (cb->cb_scripted)
6214                                 (void) fputc('\t', stdout);
6215                         else
6216                                 (void) fputs("  ", stdout);
6217                 } else {
6218                         first = B_FALSE;
6219                 }
6220
6221                 right_justify = B_FALSE;
6222                 if (pl->pl_prop != ZPROP_USERPROP) {
6223                         if (zpool_get_prop(zhp, pl->pl_prop, property,
6224                             sizeof (property), NULL, cb->cb_literal) != 0)
6225                                 propstr = "-";
6226                         else
6227                                 propstr = property;
6228
6229                         right_justify = zpool_prop_align_right(pl->pl_prop);
6230                 } else if ((zpool_prop_feature(pl->pl_user_prop) ||
6231                     zpool_prop_unsupported(pl->pl_user_prop)) &&
6232                     zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
6233                     sizeof (property)) == 0) {
6234                         propstr = property;
6235                 } else if (zfs_prop_user(pl->pl_user_prop) &&
6236                     zpool_get_userprop(zhp, pl->pl_user_prop, property,
6237                     sizeof (property), NULL) == 0) {
6238                         propstr = property;
6239                 } else {
6240                         propstr = "-";
6241                 }
6242
6243                 /*
6244                  * If this is being called in scripted mode, or if this is the
6245                  * last column and it is left-justified, don't include a width
6246                  * format specifier.
6247                  */
6248                 if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
6249                         (void) fputs(propstr, stdout);
6250                 else if (right_justify)
6251                         (void) printf("%*s", (int)width, propstr);
6252                 else
6253                         (void) printf("%-*s", (int)width, propstr);
6254         }
6255
6256         (void) fputc('\n', stdout);
6257 }
6258
6259 static void
6260 print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
6261     boolean_t scripted, boolean_t valid, enum zfs_nicenum_format format)
6262 {
6263         char propval[64];
6264         boolean_t fixed;
6265         size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
6266
6267         switch (prop) {
6268         case ZPOOL_PROP_SIZE:
6269         case ZPOOL_PROP_EXPANDSZ:
6270         case ZPOOL_PROP_CHECKPOINT:
6271         case ZPOOL_PROP_DEDUPRATIO:
6272                 if (value == 0)
6273                         (void) strlcpy(propval, "-", sizeof (propval));
6274                 else
6275                         zfs_nicenum_format(value, propval, sizeof (propval),
6276                             format);
6277                 break;
6278         case ZPOOL_PROP_FRAGMENTATION:
6279                 if (value == ZFS_FRAG_INVALID) {
6280                         (void) strlcpy(propval, "-", sizeof (propval));
6281                 } else if (format == ZFS_NICENUM_RAW) {
6282                         (void) snprintf(propval, sizeof (propval), "%llu",
6283                             (unsigned long long)value);
6284                 } else {
6285                         (void) snprintf(propval, sizeof (propval), "%llu%%",
6286                             (unsigned long long)value);
6287                 }
6288                 break;
6289         case ZPOOL_PROP_CAPACITY:
6290                 /* capacity value is in parts-per-10,000 (aka permyriad) */
6291                 if (format == ZFS_NICENUM_RAW)
6292                         (void) snprintf(propval, sizeof (propval), "%llu",
6293                             (unsigned long long)value / 100);
6294                 else
6295                         (void) snprintf(propval, sizeof (propval),
6296                             value < 1000 ? "%1.2f%%" : value < 10000 ?
6297                             "%2.1f%%" : "%3.0f%%", value / 100.0);
6298                 break;
6299         case ZPOOL_PROP_HEALTH:
6300                 width = 8;
6301                 (void) strlcpy(propval, str, sizeof (propval));
6302                 break;
6303         default:
6304                 zfs_nicenum_format(value, propval, sizeof (propval), format);
6305         }
6306
6307         if (!valid)
6308                 (void) strlcpy(propval, "-", sizeof (propval));
6309
6310         if (scripted)
6311                 (void) printf("\t%s", propval);
6312         else
6313                 (void) printf("  %*s", (int)width, propval);
6314 }
6315
6316 /*
6317  * print static default line per vdev
6318  * not compatible with '-o' <proplist> option
6319  */
6320 static void
6321 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
6322     list_cbdata_t *cb, int depth, boolean_t isspare)
6323 {
6324         nvlist_t **child;
6325         vdev_stat_t *vs;
6326         uint_t c, children;
6327         char *vname;
6328         boolean_t scripted = cb->cb_scripted;
6329         uint64_t islog = B_FALSE;
6330         const char *dashes = "%-*s      -      -      -        -         "
6331             "-      -      -      -         -\n";
6332
6333         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
6334             (uint64_t **)&vs, &c) == 0);
6335
6336         if (name != NULL) {
6337                 boolean_t toplevel = (vs->vs_space != 0);
6338                 uint64_t cap;
6339                 enum zfs_nicenum_format format;
6340                 const char *state;
6341
6342                 if (cb->cb_literal)
6343                         format = ZFS_NICENUM_RAW;
6344                 else
6345                         format = ZFS_NICENUM_1024;
6346
6347                 if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
6348                         return;
6349
6350                 if (scripted)
6351                         (void) printf("\t%s", name);
6352                 else if (strlen(name) + depth > cb->cb_namewidth)
6353                         (void) printf("%*s%s", depth, "", name);
6354                 else
6355                         (void) printf("%*s%s%*s", depth, "", name,
6356                             (int)(cb->cb_namewidth - strlen(name) - depth), "");
6357
6358                 /*
6359                  * Print the properties for the individual vdevs. Some
6360                  * properties are only applicable to toplevel vdevs. The
6361                  * 'toplevel' boolean value is passed to the print_one_column()
6362                  * to indicate that the value is valid.
6363                  */
6364                 if (VDEV_STAT_VALID(vs_pspace, c) && vs->vs_pspace)
6365                         print_one_column(ZPOOL_PROP_SIZE, vs->vs_pspace, NULL,
6366                             scripted, B_TRUE, format);
6367                 else
6368                         print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, NULL,
6369                             scripted, toplevel, format);
6370                 print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
6371                     scripted, toplevel, format);
6372                 print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
6373                     NULL, scripted, toplevel, format);
6374                 print_one_column(ZPOOL_PROP_CHECKPOINT,
6375                     vs->vs_checkpoint_space, NULL, scripted, toplevel, format);
6376                 print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
6377                     scripted, B_TRUE, format);
6378                 print_one_column(ZPOOL_PROP_FRAGMENTATION,
6379                     vs->vs_fragmentation, NULL, scripted,
6380                     (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
6381                     format);
6382                 cap = (vs->vs_space == 0) ? 0 :
6383                     (vs->vs_alloc * 10000 / vs->vs_space);
6384                 print_one_column(ZPOOL_PROP_CAPACITY, cap, NULL,
6385                     scripted, toplevel, format);
6386                 print_one_column(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
6387                     scripted, toplevel, format);
6388                 state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
6389                 if (isspare) {
6390                         if (vs->vs_aux == VDEV_AUX_SPARED)
6391                                 state = "INUSE";
6392                         else if (vs->vs_state == VDEV_STATE_HEALTHY)
6393                                 state = "AVAIL";
6394                 }
6395                 print_one_column(ZPOOL_PROP_HEALTH, 0, state, scripted,
6396                     B_TRUE, format);
6397                 (void) fputc('\n', stdout);
6398         }
6399
6400         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
6401             &child, &children) != 0)
6402                 return;
6403
6404         /* list the normal vdevs first */
6405         for (c = 0; c < children; c++) {
6406                 uint64_t ishole = B_FALSE;
6407
6408                 if (nvlist_lookup_uint64(child[c],
6409                     ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
6410                         continue;
6411
6412                 if (nvlist_lookup_uint64(child[c],
6413                     ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
6414                         continue;
6415
6416                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
6417                         continue;
6418
6419                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
6420                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
6421                 print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
6422                 free(vname);
6423         }
6424
6425         /* list the classes: 'logs', 'dedup', and 'special' */
6426         for (uint_t n = 0; n < ARRAY_SIZE(class_name); n++) {
6427                 boolean_t printed = B_FALSE;
6428
6429                 for (c = 0; c < children; c++) {
6430                         const char *bias = NULL;
6431                         const char *type = NULL;
6432
6433                         if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
6434                             &islog) == 0 && islog) {
6435                                 bias = VDEV_ALLOC_CLASS_LOGS;
6436                         } else {
6437                                 (void) nvlist_lookup_string(child[c],
6438                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
6439                                 (void) nvlist_lookup_string(child[c],
6440                                     ZPOOL_CONFIG_TYPE, &type);
6441                         }
6442                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
6443                                 continue;
6444                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
6445                                 continue;
6446
6447                         if (!printed) {
6448                                 /* LINTED E_SEC_PRINTF_VAR_FMT */
6449                                 (void) printf(dashes, cb->cb_namewidth,
6450                                     class_name[n]);
6451                                 printed = B_TRUE;
6452                         }
6453                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
6454                             cb->cb_name_flags | VDEV_NAME_TYPE_ID);
6455                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
6456                             B_FALSE);
6457                         free(vname);
6458                 }
6459         }
6460
6461         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
6462             &child, &children) == 0 && children > 0) {
6463                 /* LINTED E_SEC_PRINTF_VAR_FMT */
6464                 (void) printf(dashes, cb->cb_namewidth, "cache");
6465                 for (c = 0; c < children; c++) {
6466                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
6467                             cb->cb_name_flags);
6468                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
6469                             B_FALSE);
6470                         free(vname);
6471                 }
6472         }
6473
6474         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
6475             &children) == 0 && children > 0) {
6476                 /* LINTED E_SEC_PRINTF_VAR_FMT */
6477                 (void) printf(dashes, cb->cb_namewidth, "spare");
6478                 for (c = 0; c < children; c++) {
6479                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
6480                             cb->cb_name_flags);
6481                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
6482                             B_TRUE);
6483                         free(vname);
6484                 }
6485         }
6486 }
6487
6488 /*
6489  * Generic callback function to list a pool.
6490  */
6491 static int
6492 list_callback(zpool_handle_t *zhp, void *data)
6493 {
6494         list_cbdata_t *cbp = data;
6495
6496         print_pool(zhp, cbp);
6497
6498         if (cbp->cb_verbose) {
6499                 nvlist_t *config, *nvroot;
6500
6501                 config = zpool_get_config(zhp, NULL);
6502                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
6503                     &nvroot) == 0);
6504                 print_list_stats(zhp, NULL, nvroot, cbp, 0, B_FALSE);
6505         }
6506
6507         return (0);
6508 }
6509
6510 /*
6511  * Set the minimum pool/vdev name column width.  The width must be at least 9,
6512  * but may be as large as needed.
6513  */
6514 static int
6515 get_namewidth_list(zpool_handle_t *zhp, void *data)
6516 {
6517         list_cbdata_t *cb = data;
6518         int width;
6519
6520         width = get_namewidth(zhp, cb->cb_namewidth,
6521             cb->cb_name_flags | VDEV_NAME_TYPE_ID, cb->cb_verbose);
6522
6523         if (width < 9)
6524                 width = 9;
6525
6526         cb->cb_namewidth = width;
6527
6528         return (0);
6529 }
6530
6531 /*
6532  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
6533  *
6534  *      -g      Display guid for individual vdev name.
6535  *      -H      Scripted mode.  Don't display headers, and separate properties
6536  *              by a single tab.
6537  *      -L      Follow links when resolving vdev path name.
6538  *      -o      List of properties to display.  Defaults to
6539  *              "name,size,allocated,free,expandsize,fragmentation,capacity,"
6540  *              "dedupratio,health,altroot"
6541  *      -p      Display values in parsable (exact) format.
6542  *      -P      Display full path for vdev name.
6543  *      -T      Display a timestamp in date(1) or Unix format
6544  *
6545  * List all pools in the system, whether or not they're healthy.  Output space
6546  * statistics for each one, as well as health status summary.
6547  */
6548 int
6549 zpool_do_list(int argc, char **argv)
6550 {
6551         int c;
6552         int ret = 0;
6553         list_cbdata_t cb = { 0 };
6554         static char default_props[] =
6555             "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
6556             "capacity,dedupratio,health,altroot";
6557         char *props = default_props;
6558         float interval = 0;
6559         unsigned long count = 0;
6560         zpool_list_t *list;
6561         boolean_t first = B_TRUE;
6562         current_prop_type = ZFS_TYPE_POOL;
6563
6564         /* check options */
6565         while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
6566                 switch (c) {
6567                 case 'g':
6568                         cb.cb_name_flags |= VDEV_NAME_GUID;
6569                         break;
6570                 case 'H':
6571                         cb.cb_scripted = B_TRUE;
6572                         break;
6573                 case 'L':
6574                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6575                         break;
6576                 case 'o':
6577                         props = optarg;
6578                         break;
6579                 case 'P':
6580                         cb.cb_name_flags |= VDEV_NAME_PATH;
6581                         break;
6582                 case 'p':
6583                         cb.cb_literal = B_TRUE;
6584                         break;
6585                 case 'T':
6586                         get_timestamp_arg(*optarg);
6587                         break;
6588                 case 'v':
6589                         cb.cb_verbose = B_TRUE;
6590                         cb.cb_namewidth = 8;    /* 8 until precalc is avail */
6591                         break;
6592                 case ':':
6593                         (void) fprintf(stderr, gettext("missing argument for "
6594                             "'%c' option\n"), optopt);
6595                         usage(B_FALSE);
6596                         break;
6597                 case '?':
6598                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6599                             optopt);
6600                         usage(B_FALSE);
6601                 }
6602         }
6603
6604         argc -= optind;
6605         argv += optind;
6606
6607         get_interval_count(&argc, argv, &interval, &count);
6608
6609         if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
6610                 usage(B_FALSE);
6611
6612         for (;;) {
6613                 if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
6614                     ZFS_TYPE_POOL, cb.cb_literal, &ret)) == NULL)
6615                         return (1);
6616
6617                 if (pool_list_count(list) == 0)
6618                         break;
6619
6620                 cb.cb_namewidth = 0;
6621                 (void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);
6622
6623                 if (timestamp_fmt != NODATE)
6624                         print_timestamp(timestamp_fmt);
6625
6626                 if (!cb.cb_scripted && (first || cb.cb_verbose)) {
6627                         print_header(&cb);
6628                         first = B_FALSE;
6629                 }
6630                 ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
6631
6632                 if (interval == 0)
6633                         break;
6634
6635                 if (count != 0 && --count == 0)
6636                         break;
6637
6638                 pool_list_free(list);
6639
6640                 (void) fflush(stdout);
6641                 (void) fsleep(interval);
6642         }
6643
6644         if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
6645                 (void) printf(gettext("no pools available\n"));
6646                 ret = 0;
6647         }
6648
6649         pool_list_free(list);
6650         zprop_free_list(cb.cb_proplist);
6651         return (ret);
6652 }
6653
6654 static int
6655 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
6656 {
6657         boolean_t force = B_FALSE;
6658         boolean_t rebuild = B_FALSE;
6659         boolean_t wait = B_FALSE;
6660         int c;
6661         nvlist_t *nvroot;
6662         char *poolname, *old_disk, *new_disk;
6663         zpool_handle_t *zhp;
6664         nvlist_t *props = NULL;
6665         char *propval;
6666         int ret;
6667
6668         /* check options */
6669         while ((c = getopt(argc, argv, "fo:sw")) != -1) {
6670                 switch (c) {
6671                 case 'f':
6672                         force = B_TRUE;
6673                         break;
6674                 case 'o':
6675                         if ((propval = strchr(optarg, '=')) == NULL) {
6676                                 (void) fprintf(stderr, gettext("missing "
6677                                     "'=' for -o option\n"));
6678                                 usage(B_FALSE);
6679                         }
6680                         *propval = '\0';
6681                         propval++;
6682
6683                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
6684                             (add_prop_list(optarg, propval, &props, B_TRUE)))
6685                                 usage(B_FALSE);
6686                         break;
6687                 case 's':
6688                         rebuild = B_TRUE;
6689                         break;
6690                 case 'w':
6691                         wait = B_TRUE;
6692                         break;
6693                 case '?':
6694                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6695                             optopt);
6696                         usage(B_FALSE);
6697                 }
6698         }
6699
6700         argc -= optind;
6701         argv += optind;
6702
6703         /* get pool name and check number of arguments */
6704         if (argc < 1) {
6705                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6706                 usage(B_FALSE);
6707         }
6708
6709         poolname = argv[0];
6710
6711         if (argc < 2) {
6712                 (void) fprintf(stderr,
6713                     gettext("missing <device> specification\n"));
6714                 usage(B_FALSE);
6715         }
6716
6717         old_disk = argv[1];
6718
6719         if (argc < 3) {
6720                 if (!replacing) {
6721                         (void) fprintf(stderr,
6722                             gettext("missing <new_device> specification\n"));
6723                         usage(B_FALSE);
6724                 }
6725                 new_disk = old_disk;
6726                 argc -= 1;
6727                 argv += 1;
6728         } else {
6729                 new_disk = argv[2];
6730                 argc -= 2;
6731                 argv += 2;
6732         }
6733
6734         if (argc > 1) {
6735                 (void) fprintf(stderr, gettext("too many arguments\n"));
6736                 usage(B_FALSE);
6737         }
6738
6739         if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
6740                 nvlist_free(props);
6741                 return (1);
6742         }
6743
6744         if (zpool_get_config(zhp, NULL) == NULL) {
6745                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
6746                     poolname);
6747                 zpool_close(zhp);
6748                 nvlist_free(props);
6749                 return (1);
6750         }
6751
6752         /* unless manually specified use "ashift" pool property (if set) */
6753         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
6754                 int intval;
6755                 zprop_source_t src;
6756                 char strval[ZPOOL_MAXPROPLEN];
6757
6758                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
6759                 if (src != ZPROP_SRC_DEFAULT) {
6760                         (void) sprintf(strval, "%" PRId32, intval);
6761                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
6762                             &props, B_TRUE) == 0);
6763                 }
6764         }
6765
6766         nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
6767             argc, argv);
6768         if (nvroot == NULL) {
6769                 zpool_close(zhp);
6770                 nvlist_free(props);
6771                 return (1);
6772         }
6773
6774         ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
6775             rebuild);
6776
6777         if (ret == 0 && wait) {
6778                 zpool_wait_activity_t activity = ZPOOL_WAIT_RESILVER;
6779                 char raidz_prefix[] = "raidz";
6780                 if (replacing) {
6781                         activity = ZPOOL_WAIT_REPLACE;
6782                 } else if (strncmp(old_disk,
6783                     raidz_prefix, strlen(raidz_prefix)) == 0) {
6784                         activity = ZPOOL_WAIT_RAIDZ_EXPAND;
6785                 }
6786                 ret = zpool_wait(zhp, activity);
6787         }
6788
6789         nvlist_free(props);
6790         nvlist_free(nvroot);
6791         zpool_close(zhp);
6792
6793         return (ret);
6794 }
6795
6796 /*
6797  * zpool replace [-fsw] [-o property=value] <pool> <device> <new_device>
6798  *
6799  *      -f      Force attach, even if <new_device> appears to be in use.
6800  *      -s      Use sequential instead of healing reconstruction for resilver.
6801  *      -o      Set property=value.
6802  *      -w      Wait for replacing to complete before returning
6803  *
6804  * Replace <device> with <new_device>.
6805  */
6806 int
6807 zpool_do_replace(int argc, char **argv)
6808 {
6809         return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
6810 }
6811
6812 /*
6813  * zpool attach [-fsw] [-o property=value] <pool> <device>|<vdev> <new_device>
6814  *
6815  *      -f      Force attach, even if <new_device> appears to be in use.
6816  *      -s      Use sequential instead of healing reconstruction for resilver.
6817  *      -o      Set property=value.
6818  *      -w      Wait for resilvering (mirror) or expansion (raidz) to complete
6819  *              before returning.
6820  *
6821  * Attach <new_device> to a <device> or <vdev>, where the vdev can be of type
6822  * mirror or raidz. If <device> is not part of a mirror, then <device> will
6823  * be transformed into a mirror of <device> and <new_device>. When a mirror
6824  * is involved, <new_device> will begin life with a DTL of [0, now], and will
6825  * immediately begin to resilver itself. For the raidz case, a expansion will
6826  * commence and reflow the raidz data across all the disks including the
6827  * <new_device>.
6828  */
6829 int
6830 zpool_do_attach(int argc, char **argv)
6831 {
6832         return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
6833 }
6834
6835 /*
6836  * zpool detach [-f] <pool> <device>
6837  *
6838  *      -f      Force detach of <device>, even if DTLs argue against it
6839  *              (not supported yet)
6840  *
6841  * Detach a device from a mirror.  The operation will be refused if <device>
6842  * is the last device in the mirror, or if the DTLs indicate that this device
6843  * has the only valid copy of some data.
6844  */
6845 int
6846 zpool_do_detach(int argc, char **argv)
6847 {
6848         int c;
6849         char *poolname, *path;
6850         zpool_handle_t *zhp;
6851         int ret;
6852
6853         /* check options */
6854         while ((c = getopt(argc, argv, "")) != -1) {
6855                 switch (c) {
6856                 case '?':
6857                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6858                             optopt);
6859                         usage(B_FALSE);
6860                 }
6861         }
6862
6863         argc -= optind;
6864         argv += optind;
6865
6866         /* get pool name and check number of arguments */
6867         if (argc < 1) {
6868                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6869                 usage(B_FALSE);
6870         }
6871
6872         if (argc < 2) {
6873                 (void) fprintf(stderr,
6874                     gettext("missing <device> specification\n"));
6875                 usage(B_FALSE);
6876         }
6877
6878         poolname = argv[0];
6879         path = argv[1];
6880
6881         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6882                 return (1);
6883
6884         ret = zpool_vdev_detach(zhp, path);
6885
6886         zpool_close(zhp);
6887
6888         return (ret);
6889 }
6890
6891 /*
6892  * zpool split [-gLnP] [-o prop=val] ...
6893  *              [-o mntopt] ...
6894  *              [-R altroot] <pool> <newpool> [<device> ...]
6895  *
6896  *      -g      Display guid for individual vdev name.
6897  *      -L      Follow links when resolving vdev path name.
6898  *      -n      Do not split the pool, but display the resulting layout if
6899  *              it were to be split.
6900  *      -o      Set property=value, or set mount options.
6901  *      -P      Display full path for vdev name.
6902  *      -R      Mount the split-off pool under an alternate root.
6903  *      -l      Load encryption keys while importing.
6904  *
6905  * Splits the named pool and gives it the new pool name.  Devices to be split
6906  * off may be listed, provided that no more than one device is specified
6907  * per top-level vdev mirror.  The newly split pool is left in an exported
6908  * state unless -R is specified.
6909  *
6910  * Restrictions: the top-level of the pool pool must only be made up of
6911  * mirrors; all devices in the pool must be healthy; no device may be
6912  * undergoing a resilvering operation.
6913  */
6914 int
6915 zpool_do_split(int argc, char **argv)
6916 {
6917         char *srcpool, *newpool, *propval;
6918         char *mntopts = NULL;
6919         splitflags_t flags;
6920         int c, ret = 0;
6921         int ms_status = 0;
6922         boolean_t loadkeys = B_FALSE;
6923         zpool_handle_t *zhp;
6924         nvlist_t *config, *props = NULL;
6925
6926         flags.dryrun = B_FALSE;
6927         flags.import = B_FALSE;
6928         flags.name_flags = 0;
6929
6930         /* check options */
6931         while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
6932                 switch (c) {
6933                 case 'g':
6934                         flags.name_flags |= VDEV_NAME_GUID;
6935                         break;
6936                 case 'L':
6937                         flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
6938                         break;
6939                 case 'R':
6940                         flags.import = B_TRUE;
6941                         if (add_prop_list(
6942                             zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
6943                             &props, B_TRUE) != 0) {
6944                                 nvlist_free(props);
6945                                 usage(B_FALSE);
6946                         }
6947                         break;
6948                 case 'l':
6949                         loadkeys = B_TRUE;
6950                         break;
6951                 case 'n':
6952                         flags.dryrun = B_TRUE;
6953                         break;
6954                 case 'o':
6955                         if ((propval = strchr(optarg, '=')) != NULL) {
6956                                 *propval = '\0';
6957                                 propval++;
6958                                 if (add_prop_list(optarg, propval,
6959                                     &props, B_TRUE) != 0) {
6960                                         nvlist_free(props);
6961                                         usage(B_FALSE);
6962                                 }
6963                         } else {
6964                                 mntopts = optarg;
6965                         }
6966                         break;
6967                 case 'P':
6968                         flags.name_flags |= VDEV_NAME_PATH;
6969                         break;
6970                 case ':':
6971                         (void) fprintf(stderr, gettext("missing argument for "
6972                             "'%c' option\n"), optopt);
6973                         usage(B_FALSE);
6974                         break;
6975                 case '?':
6976                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6977                             optopt);
6978                         usage(B_FALSE);
6979                         break;
6980                 }
6981         }
6982
6983         if (!flags.import && mntopts != NULL) {
6984                 (void) fprintf(stderr, gettext("setting mntopts is only "
6985                     "valid when importing the pool\n"));
6986                 usage(B_FALSE);
6987         }
6988
6989         if (!flags.import && loadkeys) {
6990                 (void) fprintf(stderr, gettext("loading keys is only "
6991                     "valid when importing the pool\n"));
6992                 usage(B_FALSE);
6993         }
6994
6995         argc -= optind;
6996         argv += optind;
6997
6998         if (argc < 1) {
6999                 (void) fprintf(stderr, gettext("Missing pool name\n"));
7000                 usage(B_FALSE);
7001         }
7002         if (argc < 2) {
7003                 (void) fprintf(stderr, gettext("Missing new pool name\n"));
7004                 usage(B_FALSE);
7005         }
7006
7007         srcpool = argv[0];
7008         newpool = argv[1];
7009
7010         argc -= 2;
7011         argv += 2;
7012
7013         if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
7014                 nvlist_free(props);
7015                 return (1);
7016         }
7017
7018         config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
7019         if (config == NULL) {
7020                 ret = 1;
7021         } else {
7022                 if (flags.dryrun) {
7023                         (void) printf(gettext("would create '%s' with the "
7024                             "following layout:\n\n"), newpool);
7025                         print_vdev_tree(NULL, newpool, config, 0, "",
7026                             flags.name_flags);
7027                         print_vdev_tree(NULL, "dedup", config, 0,
7028                             VDEV_ALLOC_BIAS_DEDUP, 0);
7029                         print_vdev_tree(NULL, "special", config, 0,
7030                             VDEV_ALLOC_BIAS_SPECIAL, 0);
7031                 }
7032         }
7033
7034         zpool_close(zhp);
7035
7036         if (ret != 0 || flags.dryrun || !flags.import) {
7037                 nvlist_free(config);
7038                 nvlist_free(props);
7039                 return (ret);
7040         }
7041
7042         /*
7043          * The split was successful. Now we need to open the new
7044          * pool and import it.
7045          */
7046         if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
7047                 nvlist_free(config);
7048                 nvlist_free(props);
7049                 return (1);
7050         }
7051
7052         if (loadkeys) {
7053                 ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
7054                 if (ret != 0)
7055                         ret = 1;
7056         }
7057
7058         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL) {
7059                 ms_status = zpool_enable_datasets(zhp, mntopts, 0);
7060                 if (ms_status == EZFS_SHAREFAILED) {
7061                         (void) fprintf(stderr, gettext("Split was successful, "
7062                             "datasets are mounted but sharing of some datasets "
7063                             "has failed\n"));
7064                 } else if (ms_status == EZFS_MOUNTFAILED) {
7065                         (void) fprintf(stderr, gettext("Split was successful"
7066                             ", but some datasets could not be mounted\n"));
7067                         (void) fprintf(stderr, gettext("Try doing '%s' with a "
7068                             "different altroot\n"), "zpool import");
7069                 }
7070         }
7071         zpool_close(zhp);
7072         nvlist_free(config);
7073         nvlist_free(props);
7074
7075         return (ret);
7076 }
7077
7078 #define POWER_OPT 1024
7079
7080 /*
7081  * zpool online [--power] <pool> <device> ...
7082  *
7083  * --power: Power on the enclosure slot to the drive (if possible)
7084  */
7085 int
7086 zpool_do_online(int argc, char **argv)
7087 {
7088         int c, i;
7089         char *poolname;
7090         zpool_handle_t *zhp;
7091         int ret = 0;
7092         vdev_state_t newstate;
7093         int flags = 0;
7094         boolean_t is_power_on = B_FALSE;
7095         struct option long_options[] = {
7096                 {"power", no_argument, NULL, POWER_OPT},
7097                 {0, 0, 0, 0}
7098         };
7099
7100         /* check options */
7101         while ((c = getopt_long(argc, argv, "e", long_options, NULL)) != -1) {
7102                 switch (c) {
7103                 case 'e':
7104                         flags |= ZFS_ONLINE_EXPAND;
7105                         break;
7106                 case POWER_OPT:
7107                         is_power_on = B_TRUE;
7108                         break;
7109                 case '?':
7110                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7111                             optopt);
7112                         usage(B_FALSE);
7113                 }
7114         }
7115
7116         if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
7117                 is_power_on = B_TRUE;
7118
7119         argc -= optind;
7120         argv += optind;
7121
7122         /* get pool name and check number of arguments */
7123         if (argc < 1) {
7124                 (void) fprintf(stderr, gettext("missing pool name\n"));
7125                 usage(B_FALSE);
7126         }
7127         if (argc < 2) {
7128                 (void) fprintf(stderr, gettext("missing device name\n"));
7129                 usage(B_FALSE);
7130         }
7131
7132         poolname = argv[0];
7133
7134         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
7135                 return (1);
7136
7137         for (i = 1; i < argc; i++) {
7138                 vdev_state_t oldstate;
7139                 boolean_t avail_spare, l2cache;
7140                 int rc;
7141
7142                 if (is_power_on) {
7143                         rc = zpool_power_on_and_disk_wait(zhp, argv[i]);
7144                         if (rc == ENOTSUP) {
7145                                 (void) fprintf(stderr,
7146                                     gettext("Power control not supported\n"));
7147                         }
7148                         if (rc != 0)
7149                                 return (rc);
7150                 }
7151
7152                 nvlist_t *tgt = zpool_find_vdev(zhp, argv[i], &avail_spare,
7153                     &l2cache, NULL);
7154                 if (tgt == NULL) {
7155                         ret = 1;
7156                         continue;
7157                 }
7158                 uint_t vsc;
7159                 oldstate = ((vdev_stat_t *)fnvlist_lookup_uint64_array(tgt,
7160                     ZPOOL_CONFIG_VDEV_STATS, &vsc))->vs_state;
7161                 if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
7162                         if (newstate != VDEV_STATE_HEALTHY) {
7163                                 (void) printf(gettext("warning: device '%s' "
7164                                     "onlined, but remains in faulted state\n"),
7165                                     argv[i]);
7166                                 if (newstate == VDEV_STATE_FAULTED)
7167                                         (void) printf(gettext("use 'zpool "
7168                                             "clear' to restore a faulted "
7169                                             "device\n"));
7170                                 else
7171                                         (void) printf(gettext("use 'zpool "
7172                                             "replace' to replace devices "
7173                                             "that are no longer present\n"));
7174                                 if ((flags & ZFS_ONLINE_EXPAND)) {
7175                                         (void) printf(gettext("%s: failed "
7176                                             "to expand usable space on "
7177                                             "unhealthy device '%s'\n"),
7178                                             (oldstate >= VDEV_STATE_DEGRADED ?
7179                                             "error" : "warning"), argv[i]);
7180                                         if (oldstate >= VDEV_STATE_DEGRADED) {
7181                                                 ret = 1;
7182                                                 break;
7183                                         }
7184                                 }
7185                         }
7186                 } else {
7187                         ret = 1;
7188                 }
7189         }
7190
7191         zpool_close(zhp);
7192
7193         return (ret);
7194 }
7195
7196 /*
7197  * zpool offline [-ft]|[--power] <pool> <device> ...
7198  *
7199  *
7200  *      -f      Force the device into a faulted state.
7201  *
7202  *      -t      Only take the device off-line temporarily.  The offline/faulted
7203  *              state will not be persistent across reboots.
7204  *
7205  *      --power Power off the enclosure slot to the drive (if possible)
7206  */
7207 int
7208 zpool_do_offline(int argc, char **argv)
7209 {
7210         int c, i;
7211         char *poolname;
7212         zpool_handle_t *zhp;
7213         int ret = 0;
7214         boolean_t istmp = B_FALSE;
7215         boolean_t fault = B_FALSE;
7216         boolean_t is_power_off = B_FALSE;
7217
7218         struct option long_options[] = {
7219                 {"power", no_argument, NULL, POWER_OPT},
7220                 {0, 0, 0, 0}
7221         };
7222
7223         /* check options */
7224         while ((c = getopt_long(argc, argv, "ft", long_options, NULL)) != -1) {
7225                 switch (c) {
7226                 case 'f':
7227                         fault = B_TRUE;
7228                         break;
7229                 case 't':
7230                         istmp = B_TRUE;
7231                         break;
7232                 case POWER_OPT:
7233                         is_power_off = B_TRUE;
7234                         break;
7235                 case '?':
7236                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7237                             optopt);
7238                         usage(B_FALSE);
7239                 }
7240         }
7241
7242         if (is_power_off && fault) {
7243                 (void) fprintf(stderr,
7244                     gettext("-0 and -f cannot be used together\n"));
7245                 usage(B_FALSE);
7246                 return (1);
7247         }
7248
7249         if (is_power_off && istmp) {
7250                 (void) fprintf(stderr,
7251                     gettext("-0 and -t cannot be used together\n"));
7252                 usage(B_FALSE);
7253                 return (1);
7254         }
7255
7256         argc -= optind;
7257         argv += optind;
7258
7259         /* get pool name and check number of arguments */
7260         if (argc < 1) {
7261                 (void) fprintf(stderr, gettext("missing pool name\n"));
7262                 usage(B_FALSE);
7263         }
7264         if (argc < 2) {
7265                 (void) fprintf(stderr, gettext("missing device name\n"));
7266                 usage(B_FALSE);
7267         }
7268
7269         poolname = argv[0];
7270
7271         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
7272                 return (1);
7273
7274         for (i = 1; i < argc; i++) {
7275                 uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
7276                 if (is_power_off) {
7277                         /*
7278                          * Note: we have to power off first, then set REMOVED,
7279                          * or else zpool_vdev_set_removed_state() returns
7280                          * EAGAIN.
7281                          */
7282                         ret = zpool_power_off(zhp, argv[i]);
7283                         if (ret != 0) {
7284                                 (void) fprintf(stderr, "%s %s %d\n",
7285                                     gettext("unable to power off slot for"),
7286                                     argv[i], ret);
7287                         }
7288                         zpool_vdev_set_removed_state(zhp, guid, VDEV_AUX_NONE);
7289
7290                 } else if (fault) {
7291                         vdev_aux_t aux;
7292                         if (istmp == B_FALSE) {
7293                                 /* Force the fault to persist across imports */
7294                                 aux = VDEV_AUX_EXTERNAL_PERSIST;
7295                         } else {
7296                                 aux = VDEV_AUX_EXTERNAL;
7297                         }
7298
7299                         if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
7300                                 ret = 1;
7301                 } else {
7302                         if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
7303                                 ret = 1;
7304                 }
7305         }
7306
7307         zpool_close(zhp);
7308
7309         return (ret);
7310 }
7311
7312 /*
7313  * zpool clear [-nF]|[--power] <pool> [device]
7314  *
7315  * Clear all errors associated with a pool or a particular device.
7316  */
7317 int
7318 zpool_do_clear(int argc, char **argv)
7319 {
7320         int c;
7321         int ret = 0;
7322         boolean_t dryrun = B_FALSE;
7323         boolean_t do_rewind = B_FALSE;
7324         boolean_t xtreme_rewind = B_FALSE;
7325         boolean_t is_power_on = B_FALSE;
7326         uint32_t rewind_policy = ZPOOL_NO_REWIND;
7327         nvlist_t *policy = NULL;
7328         zpool_handle_t *zhp;
7329         char *pool, *device;
7330
7331         struct option long_options[] = {
7332                 {"power", no_argument, NULL, POWER_OPT},
7333                 {0, 0, 0, 0}
7334         };
7335
7336         /* check options */
7337         while ((c = getopt_long(argc, argv, "FnX", long_options,
7338             NULL)) != -1) {
7339                 switch (c) {
7340                 case 'F':
7341                         do_rewind = B_TRUE;
7342                         break;
7343                 case 'n':
7344                         dryrun = B_TRUE;
7345                         break;
7346                 case 'X':
7347                         xtreme_rewind = B_TRUE;
7348                         break;
7349                 case POWER_OPT:
7350                         is_power_on = B_TRUE;
7351                         break;
7352                 case '?':
7353                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7354                             optopt);
7355                         usage(B_FALSE);
7356                 }
7357         }
7358
7359         if (libzfs_envvar_is_set("ZPOOL_AUTO_POWER_ON_SLOT"))
7360                 is_power_on = B_TRUE;
7361
7362         argc -= optind;
7363         argv += optind;
7364
7365         if (argc < 1) {
7366                 (void) fprintf(stderr, gettext("missing pool name\n"));
7367                 usage(B_FALSE);
7368         }
7369
7370         if (argc > 2) {
7371                 (void) fprintf(stderr, gettext("too many arguments\n"));
7372                 usage(B_FALSE);
7373         }
7374
7375         if ((dryrun || xtreme_rewind) && !do_rewind) {
7376                 (void) fprintf(stderr,
7377                     gettext("-n or -X only meaningful with -F\n"));
7378                 usage(B_FALSE);
7379         }
7380         if (dryrun)
7381                 rewind_policy = ZPOOL_TRY_REWIND;
7382         else if (do_rewind)
7383                 rewind_policy = ZPOOL_DO_REWIND;
7384         if (xtreme_rewind)
7385                 rewind_policy |= ZPOOL_EXTREME_REWIND;
7386
7387         /* In future, further rewind policy choices can be passed along here */
7388         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
7389             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
7390             rewind_policy) != 0) {
7391                 return (1);
7392         }
7393
7394         pool = argv[0];
7395         device = argc == 2 ? argv[1] : NULL;
7396
7397         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
7398                 nvlist_free(policy);
7399                 return (1);
7400         }
7401
7402         if (is_power_on) {
7403                 if (device == NULL) {
7404                         zpool_power_on_pool_and_wait_for_devices(zhp);
7405                 } else {
7406                         zpool_power_on_and_disk_wait(zhp, device);
7407                 }
7408         }
7409
7410         if (zpool_clear(zhp, device, policy) != 0)
7411                 ret = 1;
7412
7413         zpool_close(zhp);
7414
7415         nvlist_free(policy);
7416
7417         return (ret);
7418 }
7419
7420 /*
7421  * zpool reguid <pool>
7422  */
7423 int
7424 zpool_do_reguid(int argc, char **argv)
7425 {
7426         int c;
7427         char *poolname;
7428         zpool_handle_t *zhp;
7429         int ret = 0;
7430
7431         /* check options */
7432         while ((c = getopt(argc, argv, "")) != -1) {
7433                 switch (c) {
7434                 case '?':
7435                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7436                             optopt);
7437                         usage(B_FALSE);
7438                 }
7439         }
7440
7441         argc -= optind;
7442         argv += optind;
7443
7444         /* get pool name and check number of arguments */
7445         if (argc < 1) {
7446                 (void) fprintf(stderr, gettext("missing pool name\n"));
7447                 usage(B_FALSE);
7448         }
7449
7450         if (argc > 1) {
7451                 (void) fprintf(stderr, gettext("too many arguments\n"));
7452                 usage(B_FALSE);
7453         }
7454
7455         poolname = argv[0];
7456         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
7457                 return (1);
7458
7459         ret = zpool_reguid(zhp);
7460
7461         zpool_close(zhp);
7462         return (ret);
7463 }
7464
7465
7466 /*
7467  * zpool reopen <pool>
7468  *
7469  * Reopen the pool so that the kernel can update the sizes of all vdevs.
7470  */
7471 int
7472 zpool_do_reopen(int argc, char **argv)
7473 {
7474         int c;
7475         int ret = 0;
7476         boolean_t scrub_restart = B_TRUE;
7477
7478         /* check options */
7479         while ((c = getopt(argc, argv, "n")) != -1) {
7480                 switch (c) {
7481                 case 'n':
7482                         scrub_restart = B_FALSE;
7483                         break;
7484                 case '?':
7485                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7486                             optopt);
7487                         usage(B_FALSE);
7488                 }
7489         }
7490
7491         argc -= optind;
7492         argv += optind;
7493
7494         /* if argc == 0 we will execute zpool_reopen_one on all pools */
7495         ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7496             B_FALSE, zpool_reopen_one, &scrub_restart);
7497
7498         return (ret);
7499 }
7500
7501 typedef struct scrub_cbdata {
7502         int     cb_type;
7503         pool_scrub_cmd_t cb_scrub_cmd;
7504 } scrub_cbdata_t;
7505
7506 static boolean_t
7507 zpool_has_checkpoint(zpool_handle_t *zhp)
7508 {
7509         nvlist_t *config, *nvroot;
7510
7511         config = zpool_get_config(zhp, NULL);
7512
7513         if (config != NULL) {
7514                 pool_checkpoint_stat_t *pcs = NULL;
7515                 uint_t c;
7516
7517                 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
7518                 (void) nvlist_lookup_uint64_array(nvroot,
7519                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7520
7521                 if (pcs == NULL || pcs->pcs_state == CS_NONE)
7522                         return (B_FALSE);
7523
7524                 assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
7525                     pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
7526                 return (B_TRUE);
7527         }
7528
7529         return (B_FALSE);
7530 }
7531
7532 static int
7533 scrub_callback(zpool_handle_t *zhp, void *data)
7534 {
7535         scrub_cbdata_t *cb = data;
7536         int err;
7537
7538         /*
7539          * Ignore faulted pools.
7540          */
7541         if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
7542                 (void) fprintf(stderr, gettext("cannot scan '%s': pool is "
7543                     "currently unavailable\n"), zpool_get_name(zhp));
7544                 return (1);
7545         }
7546
7547         err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
7548
7549         if (err == 0 && zpool_has_checkpoint(zhp) &&
7550             cb->cb_type == POOL_SCAN_SCRUB) {
7551                 (void) printf(gettext("warning: will not scrub state that "
7552                     "belongs to the checkpoint of pool '%s'\n"),
7553                     zpool_get_name(zhp));
7554         }
7555
7556         return (err != 0);
7557 }
7558
7559 static int
7560 wait_callback(zpool_handle_t *zhp, void *data)
7561 {
7562         zpool_wait_activity_t *act = data;
7563         return (zpool_wait(zhp, *act));
7564 }
7565
7566 /*
7567  * zpool scrub [-s | -p] [-w] [-e] <pool> ...
7568  *
7569  *      -e      Only scrub blocks in the error log.
7570  *      -s      Stop.  Stops any in-progress scrub.
7571  *      -p      Pause. Pause in-progress scrub.
7572  *      -w      Wait.  Blocks until scrub has completed.
7573  */
7574 int
7575 zpool_do_scrub(int argc, char **argv)
7576 {
7577         int c;
7578         scrub_cbdata_t cb;
7579         boolean_t wait = B_FALSE;
7580         int error;
7581
7582         cb.cb_type = POOL_SCAN_SCRUB;
7583         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7584
7585         boolean_t is_error_scrub = B_FALSE;
7586         boolean_t is_pause = B_FALSE;
7587         boolean_t is_stop = B_FALSE;
7588
7589         /* check options */
7590         while ((c = getopt(argc, argv, "spwe")) != -1) {
7591                 switch (c) {
7592                 case 'e':
7593                         is_error_scrub = B_TRUE;
7594                         break;
7595                 case 's':
7596                         is_stop = B_TRUE;
7597                         break;
7598                 case 'p':
7599                         is_pause = B_TRUE;
7600                         break;
7601                 case 'w':
7602                         wait = B_TRUE;
7603                         break;
7604                 case '?':
7605                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7606                             optopt);
7607                         usage(B_FALSE);
7608                 }
7609         }
7610
7611         if (is_pause && is_stop) {
7612                 (void) fprintf(stderr, gettext("invalid option "
7613                     "combination :-s and -p are mutually exclusive\n"));
7614                 usage(B_FALSE);
7615         } else {
7616                 if (is_error_scrub)
7617                         cb.cb_type = POOL_SCAN_ERRORSCRUB;
7618
7619                 if (is_pause) {
7620                         cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
7621                 } else if (is_stop) {
7622                         cb.cb_type = POOL_SCAN_NONE;
7623                 } else {
7624                         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7625                 }
7626         }
7627
7628         if (wait && (cb.cb_type == POOL_SCAN_NONE ||
7629             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
7630                 (void) fprintf(stderr, gettext("invalid option combination: "
7631                     "-w cannot be used with -p or -s\n"));
7632                 usage(B_FALSE);
7633         }
7634
7635         argc -= optind;
7636         argv += optind;
7637
7638         if (argc < 1) {
7639                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7640                 usage(B_FALSE);
7641         }
7642
7643         error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7644             B_FALSE, scrub_callback, &cb);
7645
7646         if (wait && !error) {
7647                 zpool_wait_activity_t act = ZPOOL_WAIT_SCRUB;
7648                 error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7649                     B_FALSE, wait_callback, &act);
7650         }
7651
7652         return (error);
7653 }
7654
7655 /*
7656  * zpool resilver <pool> ...
7657  *
7658  *      Restarts any in-progress resilver
7659  */
7660 int
7661 zpool_do_resilver(int argc, char **argv)
7662 {
7663         int c;
7664         scrub_cbdata_t cb;
7665
7666         cb.cb_type = POOL_SCAN_RESILVER;
7667         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7668
7669         /* check options */
7670         while ((c = getopt(argc, argv, "")) != -1) {
7671                 switch (c) {
7672                 case '?':
7673                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7674                             optopt);
7675                         usage(B_FALSE);
7676                 }
7677         }
7678
7679         argc -= optind;
7680         argv += optind;
7681
7682         if (argc < 1) {
7683                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7684                 usage(B_FALSE);
7685         }
7686
7687         return (for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7688             B_FALSE, scrub_callback, &cb));
7689 }
7690
7691 /*
7692  * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
7693  *
7694  *      -c              Cancel. Ends any in-progress trim.
7695  *      -d              Secure trim.  Requires kernel and device support.
7696  *      -r <rate>       Sets the TRIM rate in bytes (per second). Supports
7697  *                      adding a multiplier suffix such as 'k' or 'm'.
7698  *      -s              Suspend. TRIM can then be restarted with no flags.
7699  *      -w              Wait. Blocks until trimming has completed.
7700  */
7701 int
7702 zpool_do_trim(int argc, char **argv)
7703 {
7704         struct option long_options[] = {
7705                 {"cancel",      no_argument,            NULL,   'c'},
7706                 {"secure",      no_argument,            NULL,   'd'},
7707                 {"rate",        required_argument,      NULL,   'r'},
7708                 {"suspend",     no_argument,            NULL,   's'},
7709                 {"wait",        no_argument,            NULL,   'w'},
7710                 {0, 0, 0, 0}
7711         };
7712
7713         pool_trim_func_t cmd_type = POOL_TRIM_START;
7714         uint64_t rate = 0;
7715         boolean_t secure = B_FALSE;
7716         boolean_t wait = B_FALSE;
7717
7718         int c;
7719         while ((c = getopt_long(argc, argv, "cdr:sw", long_options, NULL))
7720             != -1) {
7721                 switch (c) {
7722                 case 'c':
7723                         if (cmd_type != POOL_TRIM_START &&
7724                             cmd_type != POOL_TRIM_CANCEL) {
7725                                 (void) fprintf(stderr, gettext("-c cannot be "
7726                                     "combined with other options\n"));
7727                                 usage(B_FALSE);
7728                         }
7729                         cmd_type = POOL_TRIM_CANCEL;
7730                         break;
7731                 case 'd':
7732                         if (cmd_type != POOL_TRIM_START) {
7733                                 (void) fprintf(stderr, gettext("-d cannot be "
7734                                     "combined with the -c or -s options\n"));
7735                                 usage(B_FALSE);
7736                         }
7737                         secure = B_TRUE;
7738                         break;
7739                 case 'r':
7740                         if (cmd_type != POOL_TRIM_START) {
7741                                 (void) fprintf(stderr, gettext("-r cannot be "
7742                                     "combined with the -c or -s options\n"));
7743                                 usage(B_FALSE);
7744                         }
7745                         if (zfs_nicestrtonum(g_zfs, optarg, &rate) == -1) {
7746                                 (void) fprintf(stderr, "%s: %s\n",
7747                                     gettext("invalid value for rate"),
7748                                     libzfs_error_description(g_zfs));
7749                                 usage(B_FALSE);
7750                         }
7751                         break;
7752                 case 's':
7753                         if (cmd_type != POOL_TRIM_START &&
7754                             cmd_type != POOL_TRIM_SUSPEND) {
7755                                 (void) fprintf(stderr, gettext("-s cannot be "
7756                                     "combined with other options\n"));
7757                                 usage(B_FALSE);
7758                         }
7759                         cmd_type = POOL_TRIM_SUSPEND;
7760                         break;
7761                 case 'w':
7762                         wait = B_TRUE;
7763                         break;
7764                 case '?':
7765                         if (optopt != 0) {
7766                                 (void) fprintf(stderr,
7767                                     gettext("invalid option '%c'\n"), optopt);
7768                         } else {
7769                                 (void) fprintf(stderr,
7770                                     gettext("invalid option '%s'\n"),
7771                                     argv[optind - 1]);
7772                         }
7773                         usage(B_FALSE);
7774                 }
7775         }
7776
7777         argc -= optind;
7778         argv += optind;
7779
7780         if (argc < 1) {
7781                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7782                 usage(B_FALSE);
7783                 return (-1);
7784         }
7785
7786         if (wait && (cmd_type != POOL_TRIM_START)) {
7787                 (void) fprintf(stderr, gettext("-w cannot be used with -c or "
7788                     "-s\n"));
7789                 usage(B_FALSE);
7790         }
7791
7792         char *poolname = argv[0];
7793         zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
7794         if (zhp == NULL)
7795                 return (-1);
7796
7797         trimflags_t trim_flags = {
7798                 .secure = secure,
7799                 .rate = rate,
7800                 .wait = wait,
7801         };
7802
7803         nvlist_t *vdevs = fnvlist_alloc();
7804         if (argc == 1) {
7805                 /* no individual leaf vdevs specified, so add them all */
7806                 nvlist_t *config = zpool_get_config(zhp, NULL);
7807                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
7808                     ZPOOL_CONFIG_VDEV_TREE);
7809                 zpool_collect_leaves(zhp, nvroot, vdevs);
7810                 trim_flags.fullpool = B_TRUE;
7811         } else {
7812                 trim_flags.fullpool = B_FALSE;
7813                 for (int i = 1; i < argc; i++) {
7814                         fnvlist_add_boolean(vdevs, argv[i]);
7815                 }
7816         }
7817
7818         int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
7819
7820         fnvlist_free(vdevs);
7821         zpool_close(zhp);
7822
7823         return (error);
7824 }
7825
7826 /*
7827  * Converts a total number of seconds to a human readable string broken
7828  * down in to days/hours/minutes/seconds.
7829  */
7830 static void
7831 secs_to_dhms(uint64_t total, char *buf)
7832 {
7833         uint64_t days = total / 60 / 60 / 24;
7834         uint64_t hours = (total / 60 / 60) % 24;
7835         uint64_t mins = (total / 60) % 60;
7836         uint64_t secs = (total % 60);
7837
7838         if (days > 0) {
7839                 (void) sprintf(buf, "%llu days %02llu:%02llu:%02llu",
7840                     (u_longlong_t)days, (u_longlong_t)hours,
7841                     (u_longlong_t)mins, (u_longlong_t)secs);
7842         } else {
7843                 (void) sprintf(buf, "%02llu:%02llu:%02llu",
7844                     (u_longlong_t)hours, (u_longlong_t)mins,
7845                     (u_longlong_t)secs);
7846         }
7847 }
7848
7849 /*
7850  * Print out detailed error scrub status.
7851  */
7852 static void
7853 print_err_scrub_status(pool_scan_stat_t *ps)
7854 {
7855         time_t start, end, pause;
7856         uint64_t total_secs_left;
7857         uint64_t secs_left, mins_left, hours_left, days_left;
7858         uint64_t examined, to_be_examined;
7859
7860         if (ps == NULL || ps->pss_error_scrub_func != POOL_SCAN_ERRORSCRUB) {
7861                 return;
7862         }
7863
7864         (void) printf(gettext(" scrub: "));
7865
7866         start = ps->pss_error_scrub_start;
7867         end = ps->pss_error_scrub_end;
7868         pause = ps->pss_pass_error_scrub_pause;
7869         examined = ps->pss_error_scrub_examined;
7870         to_be_examined = ps->pss_error_scrub_to_be_examined;
7871
7872         assert(ps->pss_error_scrub_func == POOL_SCAN_ERRORSCRUB);
7873
7874         if (ps->pss_error_scrub_state == DSS_FINISHED) {
7875                 total_secs_left = end - start;
7876                 days_left = total_secs_left / 60 / 60 / 24;
7877                 hours_left = (total_secs_left / 60 / 60) % 24;
7878                 mins_left = (total_secs_left / 60) % 60;
7879                 secs_left = (total_secs_left % 60);
7880
7881                 (void) printf(gettext("scrubbed %llu error blocks in %llu days "
7882                     "%02llu:%02llu:%02llu on %s"), (u_longlong_t)examined,
7883                     (u_longlong_t)days_left, (u_longlong_t)hours_left,
7884                     (u_longlong_t)mins_left, (u_longlong_t)secs_left,
7885                     ctime(&end));
7886
7887                 return;
7888         } else if (ps->pss_error_scrub_state == DSS_CANCELED) {
7889                 (void) printf(gettext("error scrub canceled on %s"),
7890                     ctime(&end));
7891                 return;
7892         }
7893         assert(ps->pss_error_scrub_state == DSS_ERRORSCRUBBING);
7894
7895         /* Error scrub is in progress. */
7896         if (pause == 0) {
7897                 (void) printf(gettext("error scrub in progress since %s"),
7898                     ctime(&start));
7899         } else {
7900                 (void) printf(gettext("error scrub paused since %s"),
7901                     ctime(&pause));
7902                 (void) printf(gettext("\terror scrub started on %s"),
7903                     ctime(&start));
7904         }
7905
7906         double fraction_done = (double)examined / (to_be_examined + examined);
7907         (void) printf(gettext("\t%.2f%% done, issued I/O for %llu error"
7908             " blocks"), 100 * fraction_done, (u_longlong_t)examined);
7909
7910         (void) printf("\n");
7911 }
7912
7913 /*
7914  * Print out detailed scrub status.
7915  */
7916 static void
7917 print_scan_scrub_resilver_status(pool_scan_stat_t *ps)
7918 {
7919         time_t start, end, pause;
7920         uint64_t pass_scanned, scanned, pass_issued, issued, total_s, total_i;
7921         uint64_t elapsed, scan_rate, issue_rate;
7922         double fraction_done;
7923         char processed_buf[7], scanned_buf[7], issued_buf[7], total_s_buf[7];
7924         char total_i_buf[7], srate_buf[7], irate_buf[7], time_buf[32];
7925
7926         printf("  ");
7927         printf_color(ANSI_BOLD, gettext("scan:"));
7928         printf(" ");
7929
7930         /* If there's never been a scan, there's not much to say. */
7931         if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
7932             ps->pss_func >= POOL_SCAN_FUNCS) {
7933                 (void) printf(gettext("none requested\n"));
7934                 return;
7935         }
7936
7937         start = ps->pss_start_time;
7938         end = ps->pss_end_time;
7939         pause = ps->pss_pass_scrub_pause;
7940
7941         zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
7942
7943         int is_resilver = ps->pss_func == POOL_SCAN_RESILVER;
7944         int is_scrub = ps->pss_func == POOL_SCAN_SCRUB;
7945         assert(is_resilver || is_scrub);
7946
7947         /* Scan is finished or canceled. */
7948         if (ps->pss_state == DSS_FINISHED) {
7949                 secs_to_dhms(end - start, time_buf);
7950
7951                 if (is_scrub) {
7952                         (void) printf(gettext("scrub repaired %s "
7953                             "in %s with %llu errors on %s"), processed_buf,
7954                             time_buf, (u_longlong_t)ps->pss_errors,
7955                             ctime(&end));
7956                 } else if (is_resilver) {
7957                         (void) printf(gettext("resilvered %s "
7958                             "in %s with %llu errors on %s"), processed_buf,
7959                             time_buf, (u_longlong_t)ps->pss_errors,
7960                             ctime(&end));
7961                 }
7962                 return;
7963         } else if (ps->pss_state == DSS_CANCELED) {
7964                 if (is_scrub) {
7965                         (void) printf(gettext("scrub canceled on %s"),
7966                             ctime(&end));
7967                 } else if (is_resilver) {
7968                         (void) printf(gettext("resilver canceled on %s"),
7969                             ctime(&end));
7970                 }
7971                 return;
7972         }
7973
7974         assert(ps->pss_state == DSS_SCANNING);
7975
7976         /* Scan is in progress. Resilvers can't be paused. */
7977         if (is_scrub) {
7978                 if (pause == 0) {
7979                         (void) printf(gettext("scrub in progress since %s"),
7980                             ctime(&start));
7981                 } else {
7982                         (void) printf(gettext("scrub paused since %s"),
7983                             ctime(&pause));
7984                         (void) printf(gettext("\tscrub started on %s"),
7985                             ctime(&start));
7986                 }
7987         } else if (is_resilver) {
7988                 (void) printf(gettext("resilver in progress since %s"),
7989                     ctime(&start));
7990         }
7991
7992         scanned = ps->pss_examined;
7993         pass_scanned = ps->pss_pass_exam;
7994         issued = ps->pss_issued;
7995         pass_issued = ps->pss_pass_issued;
7996         total_s = ps->pss_to_examine;
7997         total_i = ps->pss_to_examine - ps->pss_skipped;
7998
7999         /* we are only done with a block once we have issued the IO for it */
8000         fraction_done = (double)issued / total_i;
8001
8002         /* elapsed time for this pass, rounding up to 1 if it's 0 */
8003         elapsed = time(NULL) - ps->pss_pass_start;
8004         elapsed -= ps->pss_pass_scrub_spent_paused;
8005         elapsed = (elapsed != 0) ? elapsed : 1;
8006
8007         scan_rate = pass_scanned / elapsed;
8008         issue_rate = pass_issued / elapsed;
8009
8010         /* format all of the numbers we will be reporting */
8011         zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
8012         zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
8013         zfs_nicebytes(total_s, total_s_buf, sizeof (total_s_buf));
8014         zfs_nicebytes(total_i, total_i_buf, sizeof (total_i_buf));
8015
8016         /* do not print estimated time if we have a paused scrub */
8017         (void) printf(gettext("\t%s / %s scanned"), scanned_buf, total_s_buf);
8018         if (pause == 0 && scan_rate > 0) {
8019                 zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
8020                 (void) printf(gettext(" at %s/s"), srate_buf);
8021         }
8022         (void) printf(gettext(", %s / %s issued"), issued_buf, total_i_buf);
8023         if (pause == 0 && issue_rate > 0) {
8024                 zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
8025                 (void) printf(gettext(" at %s/s"), irate_buf);
8026         }
8027         (void) printf(gettext("\n"));
8028
8029         if (is_resilver) {
8030                 (void) printf(gettext("\t%s resilvered, %.2f%% done"),
8031                     processed_buf, 100 * fraction_done);
8032         } else if (is_scrub) {
8033                 (void) printf(gettext("\t%s repaired, %.2f%% done"),
8034                     processed_buf, 100 * fraction_done);
8035         }
8036
8037         if (pause == 0) {
8038                 /*
8039                  * Only provide an estimate iff:
8040                  * 1) we haven't yet issued all we expected, and
8041                  * 2) the issue rate exceeds 10 MB/s, and
8042                  * 3) it's either:
8043                  *    a) a resilver which has started repairs, or
8044                  *    b) a scrub which has entered the issue phase.
8045                  */
8046                 if (total_i >= issued && issue_rate >= 10 * 1024 * 1024 &&
8047                     ((is_resilver && ps->pss_processed > 0) ||
8048                     (is_scrub && issued > 0))) {
8049                         secs_to_dhms((total_i - issued) / issue_rate, time_buf);
8050                         (void) printf(gettext(", %s to go\n"), time_buf);
8051                 } else {
8052                         (void) printf(gettext(", no estimated "
8053                             "completion time\n"));
8054                 }
8055         } else {
8056                 (void) printf(gettext("\n"));
8057         }
8058 }
8059
8060 static void
8061 print_rebuild_status_impl(vdev_rebuild_stat_t *vrs, uint_t c, char *vdev_name)
8062 {
8063         if (vrs == NULL || vrs->vrs_state == VDEV_REBUILD_NONE)
8064                 return;
8065
8066         printf("  ");
8067         printf_color(ANSI_BOLD, gettext("scan:"));
8068         printf(" ");
8069
8070         uint64_t bytes_scanned = vrs->vrs_bytes_scanned;
8071         uint64_t bytes_issued = vrs->vrs_bytes_issued;
8072         uint64_t bytes_rebuilt = vrs->vrs_bytes_rebuilt;
8073         uint64_t bytes_est_s = vrs->vrs_bytes_est;
8074         uint64_t bytes_est_i = vrs->vrs_bytes_est;
8075         if (c > offsetof(vdev_rebuild_stat_t, vrs_pass_bytes_skipped) / 8)
8076                 bytes_est_i -= vrs->vrs_pass_bytes_skipped;
8077         uint64_t scan_rate = (vrs->vrs_pass_bytes_scanned /
8078             (vrs->vrs_pass_time_ms + 1)) * 1000;
8079         uint64_t issue_rate = (vrs->vrs_pass_bytes_issued /
8080             (vrs->vrs_pass_time_ms + 1)) * 1000;
8081         double scan_pct = MIN((double)bytes_scanned * 100 /
8082             (bytes_est_s + 1), 100);
8083
8084         /* Format all of the numbers we will be reporting */
8085         char bytes_scanned_buf[7], bytes_issued_buf[7];
8086         char bytes_rebuilt_buf[7], bytes_est_s_buf[7], bytes_est_i_buf[7];
8087         char scan_rate_buf[7], issue_rate_buf[7], time_buf[32];
8088         zfs_nicebytes(bytes_scanned, bytes_scanned_buf,
8089             sizeof (bytes_scanned_buf));
8090         zfs_nicebytes(bytes_issued, bytes_issued_buf,
8091             sizeof (bytes_issued_buf));
8092         zfs_nicebytes(bytes_rebuilt, bytes_rebuilt_buf,
8093             sizeof (bytes_rebuilt_buf));
8094         zfs_nicebytes(bytes_est_s, bytes_est_s_buf, sizeof (bytes_est_s_buf));
8095         zfs_nicebytes(bytes_est_i, bytes_est_i_buf, sizeof (bytes_est_i_buf));
8096
8097         time_t start = vrs->vrs_start_time;
8098         time_t end = vrs->vrs_end_time;
8099
8100         /* Rebuild is finished or canceled. */
8101         if (vrs->vrs_state == VDEV_REBUILD_COMPLETE) {
8102                 secs_to_dhms(vrs->vrs_scan_time_ms / 1000, time_buf);
8103                 (void) printf(gettext("resilvered (%s) %s in %s "
8104                     "with %llu errors on %s"), vdev_name, bytes_rebuilt_buf,
8105                     time_buf, (u_longlong_t)vrs->vrs_errors, ctime(&end));
8106                 return;
8107         } else if (vrs->vrs_state == VDEV_REBUILD_CANCELED) {
8108                 (void) printf(gettext("resilver (%s) canceled on %s"),
8109                     vdev_name, ctime(&end));
8110                 return;
8111         } else if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
8112                 (void) printf(gettext("resilver (%s) in progress since %s"),
8113                     vdev_name, ctime(&start));
8114         }
8115
8116         assert(vrs->vrs_state == VDEV_REBUILD_ACTIVE);
8117
8118         (void) printf(gettext("\t%s / %s scanned"), bytes_scanned_buf,
8119             bytes_est_s_buf);
8120         if (scan_rate > 0) {
8121                 zfs_nicebytes(scan_rate, scan_rate_buf, sizeof (scan_rate_buf));
8122                 (void) printf(gettext(" at %s/s"), scan_rate_buf);
8123         }
8124         (void) printf(gettext(", %s / %s issued"), bytes_issued_buf,
8125             bytes_est_i_buf);
8126         if (issue_rate > 0) {
8127                 zfs_nicebytes(issue_rate, issue_rate_buf,
8128                     sizeof (issue_rate_buf));
8129                 (void) printf(gettext(" at %s/s"), issue_rate_buf);
8130         }
8131         (void) printf(gettext("\n"));
8132
8133         (void) printf(gettext("\t%s resilvered, %.2f%% done"),
8134             bytes_rebuilt_buf, scan_pct);
8135
8136         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
8137                 if (bytes_est_s >= bytes_scanned &&
8138                     scan_rate >= 10 * 1024 * 1024) {
8139                         secs_to_dhms((bytes_est_s - bytes_scanned) / scan_rate,
8140                             time_buf);
8141                         (void) printf(gettext(", %s to go\n"), time_buf);
8142                 } else {
8143                         (void) printf(gettext(", no estimated "
8144                             "completion time\n"));
8145                 }
8146         } else {
8147                 (void) printf(gettext("\n"));
8148         }
8149 }
8150
8151 /*
8152  * Print rebuild status for top-level vdevs.
8153  */
8154 static void
8155 print_rebuild_status(zpool_handle_t *zhp, nvlist_t *nvroot)
8156 {
8157         nvlist_t **child;
8158         uint_t children;
8159
8160         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8161             &child, &children) != 0)
8162                 children = 0;
8163
8164         for (uint_t c = 0; c < children; c++) {
8165                 vdev_rebuild_stat_t *vrs;
8166                 uint_t i;
8167
8168                 if (nvlist_lookup_uint64_array(child[c],
8169                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
8170                         char *name = zpool_vdev_name(g_zfs, zhp,
8171                             child[c], VDEV_NAME_TYPE_ID);
8172                         print_rebuild_status_impl(vrs, i, name);
8173                         free(name);
8174                 }
8175         }
8176 }
8177
8178 /*
8179  * As we don't scrub checkpointed blocks, we want to warn the user that we
8180  * skipped scanning some blocks if a checkpoint exists or existed at any
8181  * time during the scan.  If a sequential instead of healing reconstruction
8182  * was performed then the blocks were reconstructed.  However, their checksums
8183  * have not been verified so we still print the warning.
8184  */
8185 static void
8186 print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
8187 {
8188         if (ps == NULL || pcs == NULL)
8189                 return;
8190
8191         if (pcs->pcs_state == CS_NONE ||
8192             pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
8193                 return;
8194
8195         assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
8196
8197         if (ps->pss_state == DSS_NONE)
8198                 return;
8199
8200         if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
8201             ps->pss_end_time < pcs->pcs_start_time)
8202                 return;
8203
8204         if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
8205                 (void) printf(gettext("    scan warning: skipped blocks "
8206                     "that are only referenced by the checkpoint.\n"));
8207         } else {
8208                 assert(ps->pss_state == DSS_SCANNING);
8209                 (void) printf(gettext("    scan warning: skipping blocks "
8210                     "that are only referenced by the checkpoint.\n"));
8211         }
8212 }
8213
8214 /*
8215  * Returns B_TRUE if there is an active rebuild in progress.  Otherwise,
8216  * B_FALSE is returned and 'rebuild_end_time' is set to the end time for
8217  * the last completed (or cancelled) rebuild.
8218  */
8219 static boolean_t
8220 check_rebuilding(nvlist_t *nvroot, uint64_t *rebuild_end_time)
8221 {
8222         nvlist_t **child;
8223         uint_t children;
8224         boolean_t rebuilding = B_FALSE;
8225         uint64_t end_time = 0;
8226
8227         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8228             &child, &children) != 0)
8229                 children = 0;
8230
8231         for (uint_t c = 0; c < children; c++) {
8232                 vdev_rebuild_stat_t *vrs;
8233                 uint_t i;
8234
8235                 if (nvlist_lookup_uint64_array(child[c],
8236                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
8237
8238                         if (vrs->vrs_end_time > end_time)
8239                                 end_time = vrs->vrs_end_time;
8240
8241                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
8242                                 rebuilding = B_TRUE;
8243                                 end_time = 0;
8244                                 break;
8245                         }
8246                 }
8247         }
8248
8249         if (rebuild_end_time != NULL)
8250                 *rebuild_end_time = end_time;
8251
8252         return (rebuilding);
8253 }
8254
8255 /*
8256  * Print the scan status.
8257  */
8258 static void
8259 print_scan_status(zpool_handle_t *zhp, nvlist_t *nvroot)
8260 {
8261         uint64_t rebuild_end_time = 0, resilver_end_time = 0;
8262         boolean_t have_resilver = B_FALSE, have_scrub = B_FALSE;
8263         boolean_t have_errorscrub = B_FALSE;
8264         boolean_t active_resilver = B_FALSE;
8265         pool_checkpoint_stat_t *pcs = NULL;
8266         pool_scan_stat_t *ps = NULL;
8267         uint_t c;
8268         time_t scrub_start = 0, errorscrub_start = 0;
8269
8270         if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
8271             (uint64_t **)&ps, &c) == 0) {
8272                 if (ps->pss_func == POOL_SCAN_RESILVER) {
8273                         resilver_end_time = ps->pss_end_time;
8274                         active_resilver = (ps->pss_state == DSS_SCANNING);
8275                 }
8276
8277                 have_resilver = (ps->pss_func == POOL_SCAN_RESILVER);
8278                 have_scrub = (ps->pss_func == POOL_SCAN_SCRUB);
8279                 scrub_start = ps->pss_start_time;
8280                 if (c > offsetof(pool_scan_stat_t,
8281                     pss_pass_error_scrub_pause) / 8) {
8282                         have_errorscrub = (ps->pss_error_scrub_func ==
8283                             POOL_SCAN_ERRORSCRUB);
8284                         errorscrub_start = ps->pss_error_scrub_start;
8285                 }
8286         }
8287
8288         boolean_t active_rebuild = check_rebuilding(nvroot, &rebuild_end_time);
8289         boolean_t have_rebuild = (active_rebuild || (rebuild_end_time > 0));
8290
8291         /* Always print the scrub status when available. */
8292         if (have_scrub && scrub_start > errorscrub_start)
8293                 print_scan_scrub_resilver_status(ps);
8294         else if (have_errorscrub && errorscrub_start >= scrub_start)
8295                 print_err_scrub_status(ps);
8296
8297         /*
8298          * When there is an active resilver or rebuild print its status.
8299          * Otherwise print the status of the last resilver or rebuild.
8300          */
8301         if (active_resilver || (!active_rebuild && have_resilver &&
8302             resilver_end_time && resilver_end_time > rebuild_end_time)) {
8303                 print_scan_scrub_resilver_status(ps);
8304         } else if (active_rebuild || (!active_resilver && have_rebuild &&
8305             rebuild_end_time && rebuild_end_time > resilver_end_time)) {
8306                 print_rebuild_status(zhp, nvroot);
8307         }
8308
8309         (void) nvlist_lookup_uint64_array(nvroot,
8310             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
8311         print_checkpoint_scan_warning(ps, pcs);
8312 }
8313
8314 /*
8315  * Print out detailed removal status.
8316  */
8317 static void
8318 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
8319 {
8320         char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
8321         time_t start, end;
8322         nvlist_t *config, *nvroot;
8323         nvlist_t **child;
8324         uint_t children;
8325         char *vdev_name;
8326
8327         if (prs == NULL || prs->prs_state == DSS_NONE)
8328                 return;
8329
8330         /*
8331          * Determine name of vdev.
8332          */
8333         config = zpool_get_config(zhp, NULL);
8334         nvroot = fnvlist_lookup_nvlist(config,
8335             ZPOOL_CONFIG_VDEV_TREE);
8336         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8337             &child, &children) == 0);
8338         assert(prs->prs_removing_vdev < children);
8339         vdev_name = zpool_vdev_name(g_zfs, zhp,
8340             child[prs->prs_removing_vdev], B_TRUE);
8341
8342         printf_color(ANSI_BOLD, gettext("remove: "));
8343
8344         start = prs->prs_start_time;
8345         end = prs->prs_end_time;
8346         zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
8347
8348         /*
8349          * Removal is finished or canceled.
8350          */
8351         if (prs->prs_state == DSS_FINISHED) {
8352                 uint64_t minutes_taken = (end - start) / 60;
8353
8354                 (void) printf(gettext("Removal of vdev %llu copied %s "
8355                     "in %lluh%um, completed on %s"),
8356                     (longlong_t)prs->prs_removing_vdev,
8357                     copied_buf,
8358                     (u_longlong_t)(minutes_taken / 60),
8359                     (uint_t)(minutes_taken % 60),
8360                     ctime((time_t *)&end));
8361         } else if (prs->prs_state == DSS_CANCELED) {
8362                 (void) printf(gettext("Removal of %s canceled on %s"),
8363                     vdev_name, ctime(&end));
8364         } else {
8365                 uint64_t copied, total, elapsed, mins_left, hours_left;
8366                 double fraction_done;
8367                 uint_t rate;
8368
8369                 assert(prs->prs_state == DSS_SCANNING);
8370
8371                 /*
8372                  * Removal is in progress.
8373                  */
8374                 (void) printf(gettext(
8375                     "Evacuation of %s in progress since %s"),
8376                     vdev_name, ctime(&start));
8377
8378                 copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
8379                 total = prs->prs_to_copy;
8380                 fraction_done = (double)copied / total;
8381
8382                 /* elapsed time for this pass */
8383                 elapsed = time(NULL) - prs->prs_start_time;
8384                 elapsed = elapsed > 0 ? elapsed : 1;
8385                 rate = copied / elapsed;
8386                 rate = rate > 0 ? rate : 1;
8387                 mins_left = ((total - copied) / rate) / 60;
8388                 hours_left = mins_left / 60;
8389
8390                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
8391                 zfs_nicenum(total, total_buf, sizeof (total_buf));
8392                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
8393
8394                 /*
8395                  * do not print estimated time if hours_left is more than
8396                  * 30 days
8397                  */
8398                 (void) printf(gettext(
8399                     "\t%s copied out of %s at %s/s, %.2f%% done"),
8400                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
8401                 if (hours_left < (30 * 24)) {
8402                         (void) printf(gettext(", %lluh%um to go\n"),
8403                             (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
8404                 } else {
8405                         (void) printf(gettext(
8406                             ", (copy is slow, no estimated time)\n"));
8407                 }
8408         }
8409         free(vdev_name);
8410
8411         if (prs->prs_mapping_memory > 0) {
8412                 char mem_buf[7];
8413                 zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
8414                 (void) printf(gettext(
8415                     "\t%s memory used for removed device mappings\n"),
8416                     mem_buf);
8417         }
8418 }
8419
8420 /*
8421  * Print out detailed raidz expansion status.
8422  */
8423 static void
8424 print_raidz_expand_status(zpool_handle_t *zhp, pool_raidz_expand_stat_t *pres)
8425 {
8426         char copied_buf[7];
8427
8428         if (pres == NULL || pres->pres_state == DSS_NONE)
8429                 return;
8430
8431         /*
8432          * Determine name of vdev.
8433          */
8434         nvlist_t *config = zpool_get_config(zhp, NULL);
8435         nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
8436             ZPOOL_CONFIG_VDEV_TREE);
8437         nvlist_t **child;
8438         uint_t children;
8439         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8440             &child, &children) == 0);
8441         assert(pres->pres_expanding_vdev < children);
8442
8443         printf_color(ANSI_BOLD, gettext("expand: "));
8444
8445         time_t start = pres->pres_start_time;
8446         time_t end = pres->pres_end_time;
8447         char *vname =
8448             zpool_vdev_name(g_zfs, zhp, child[pres->pres_expanding_vdev], 0);
8449         zfs_nicenum(pres->pres_reflowed, copied_buf, sizeof (copied_buf));
8450
8451         /*
8452          * Expansion is finished or canceled.
8453          */
8454         if (pres->pres_state == DSS_FINISHED) {
8455                 char time_buf[32];
8456                 secs_to_dhms(end - start, time_buf);
8457
8458                 (void) printf(gettext("expanded %s-%u copied %s in %s, "
8459                     "on %s"), vname, (int)pres->pres_expanding_vdev,
8460                     copied_buf, time_buf, ctime((time_t *)&end));
8461         } else {
8462                 char examined_buf[7], total_buf[7], rate_buf[7];
8463                 uint64_t copied, total, elapsed, secs_left;
8464                 double fraction_done;
8465                 uint_t rate;
8466
8467                 assert(pres->pres_state == DSS_SCANNING);
8468
8469                 /*
8470                  * Expansion is in progress.
8471                  */
8472                 (void) printf(gettext(
8473                     "expansion of %s-%u in progress since %s"),
8474                     vname, (int)pres->pres_expanding_vdev, ctime(&start));
8475
8476                 copied = pres->pres_reflowed > 0 ? pres->pres_reflowed : 1;
8477                 total = pres->pres_to_reflow;
8478                 fraction_done = (double)copied / total;
8479
8480                 /* elapsed time for this pass */
8481                 elapsed = time(NULL) - pres->pres_start_time;
8482                 elapsed = elapsed > 0 ? elapsed : 1;
8483                 rate = copied / elapsed;
8484                 rate = rate > 0 ? rate : 1;
8485                 secs_left = (total - copied) / rate;
8486
8487                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
8488                 zfs_nicenum(total, total_buf, sizeof (total_buf));
8489                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
8490
8491                 /*
8492                  * do not print estimated time if hours_left is more than
8493                  * 30 days
8494                  */
8495                 (void) printf(gettext("\t%s / %s copied at %s/s, %.2f%% done"),
8496                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
8497                 if (pres->pres_waiting_for_resilver) {
8498                         (void) printf(gettext(", paused for resilver or "
8499                             "clear\n"));
8500                 } else if (secs_left < (30 * 24 * 3600)) {
8501                         char time_buf[32];
8502                         secs_to_dhms(secs_left, time_buf);
8503                         (void) printf(gettext(", %s to go\n"), time_buf);
8504                 } else {
8505                         (void) printf(gettext(
8506                             ", (copy is slow, no estimated time)\n"));
8507                 }
8508         }
8509         free(vname);
8510 }
8511 static void
8512 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
8513 {
8514         time_t start;
8515         char space_buf[7];
8516
8517         if (pcs == NULL || pcs->pcs_state == CS_NONE)
8518                 return;
8519
8520         (void) printf(gettext("checkpoint: "));
8521
8522         start = pcs->pcs_start_time;
8523         zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
8524
8525         if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
8526                 char *date = ctime(&start);
8527
8528                 /*
8529                  * ctime() adds a newline at the end of the generated
8530                  * string, thus the weird format specifier and the
8531                  * strlen() call used to chop it off from the output.
8532                  */
8533                 (void) printf(gettext("created %.*s, consumes %s\n"),
8534                     (int)(strlen(date) - 1), date, space_buf);
8535                 return;
8536         }
8537
8538         assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
8539
8540         (void) printf(gettext("discarding, %s remaining.\n"),
8541             space_buf);
8542 }
8543
8544 static void
8545 print_error_log(zpool_handle_t *zhp)
8546 {
8547         nvlist_t *nverrlist = NULL;
8548         nvpair_t *elem;
8549         char *pathname;
8550         size_t len = MAXPATHLEN * 2;
8551
8552         if (zpool_get_errlog(zhp, &nverrlist) != 0)
8553                 return;
8554
8555         (void) printf("errors: Permanent errors have been "
8556             "detected in the following files:\n\n");
8557
8558         pathname = safe_malloc(len);
8559         elem = NULL;
8560         while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
8561                 nvlist_t *nv;
8562                 uint64_t dsobj, obj;
8563
8564                 verify(nvpair_value_nvlist(elem, &nv) == 0);
8565                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
8566                     &dsobj) == 0);
8567                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
8568                     &obj) == 0);
8569                 zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
8570                 (void) printf("%7s %s\n", "", pathname);
8571         }
8572         free(pathname);
8573         nvlist_free(nverrlist);
8574 }
8575
8576 static void
8577 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
8578     uint_t nspares)
8579 {
8580         uint_t i;
8581         char *name;
8582
8583         if (nspares == 0)
8584                 return;
8585
8586         (void) printf(gettext("\tspares\n"));
8587
8588         for (i = 0; i < nspares; i++) {
8589                 name = zpool_vdev_name(g_zfs, zhp, spares[i],
8590                     cb->cb_name_flags);
8591                 print_status_config(zhp, cb, name, spares[i], 2, B_TRUE, NULL);
8592                 free(name);
8593         }
8594 }
8595
8596 static void
8597 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
8598     uint_t nl2cache)
8599 {
8600         uint_t i;
8601         char *name;
8602
8603         if (nl2cache == 0)
8604                 return;
8605
8606         (void) printf(gettext("\tcache\n"));
8607
8608         for (i = 0; i < nl2cache; i++) {
8609                 name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
8610                     cb->cb_name_flags);
8611                 print_status_config(zhp, cb, name, l2cache[i], 2,
8612                     B_FALSE, NULL);
8613                 free(name);
8614         }
8615 }
8616
8617 static void
8618 print_dedup_stats(nvlist_t *config)
8619 {
8620         ddt_histogram_t *ddh;
8621         ddt_stat_t *dds;
8622         ddt_object_t *ddo;
8623         uint_t c;
8624         char dspace[6], mspace[6];
8625
8626         /*
8627          * If the pool was faulted then we may not have been able to
8628          * obtain the config. Otherwise, if we have anything in the dedup
8629          * table continue processing the stats.
8630          */
8631         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
8632             (uint64_t **)&ddo, &c) != 0)
8633                 return;
8634
8635         (void) printf("\n");
8636         (void) printf(gettext(" dedup: "));
8637         if (ddo->ddo_count == 0) {
8638                 (void) printf(gettext("no DDT entries\n"));
8639                 return;
8640         }
8641
8642         zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
8643         zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
8644         (void) printf("DDT entries %llu, size %s on disk, %s in core\n",
8645             (u_longlong_t)ddo->ddo_count,
8646             dspace,
8647             mspace);
8648
8649         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
8650             (uint64_t **)&dds, &c) == 0);
8651         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
8652             (uint64_t **)&ddh, &c) == 0);
8653         zpool_dump_ddt(dds, ddh);
8654 }
8655
8656 /*
8657  * Display a summary of pool status.  Displays a summary such as:
8658  *
8659  *        pool: tank
8660  *      status: DEGRADED
8661  *      reason: One or more devices ...
8662  *         see: https://openzfs.github.io/openzfs-docs/msg/ZFS-xxxx-01
8663  *      config:
8664  *              mirror          DEGRADED
8665  *                c1t0d0        OK
8666  *                c2t0d0        UNAVAIL
8667  *
8668  * When given the '-v' option, we print out the complete config.  If the '-e'
8669  * option is specified, then we print out error rate information as well.
8670  */
8671 static int
8672 status_callback(zpool_handle_t *zhp, void *data)
8673 {
8674         status_cbdata_t *cbp = data;
8675         nvlist_t *config, *nvroot;
8676         const char *msgid;
8677         zpool_status_t reason;
8678         zpool_errata_t errata;
8679         const char *health;
8680         uint_t c;
8681         vdev_stat_t *vs;
8682
8683         config = zpool_get_config(zhp, NULL);
8684         reason = zpool_get_status(zhp, &msgid, &errata);
8685
8686         cbp->cb_count++;
8687
8688         /*
8689          * If we were given 'zpool status -x', only report those pools with
8690          * problems.
8691          */
8692         if (cbp->cb_explain &&
8693             (reason == ZPOOL_STATUS_OK ||
8694             reason == ZPOOL_STATUS_VERSION_OLDER ||
8695             reason == ZPOOL_STATUS_FEAT_DISABLED ||
8696             reason == ZPOOL_STATUS_COMPATIBILITY_ERR ||
8697             reason == ZPOOL_STATUS_INCOMPATIBLE_FEAT)) {
8698                 if (!cbp->cb_allpools) {
8699                         (void) printf(gettext("pool '%s' is healthy\n"),
8700                             zpool_get_name(zhp));
8701                         if (cbp->cb_first)
8702                                 cbp->cb_first = B_FALSE;
8703                 }
8704                 return (0);
8705         }
8706
8707         if (cbp->cb_first)
8708                 cbp->cb_first = B_FALSE;
8709         else
8710                 (void) printf("\n");
8711
8712         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
8713         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
8714             (uint64_t **)&vs, &c) == 0);
8715
8716         health = zpool_get_state_str(zhp);
8717
8718         printf("  ");
8719         printf_color(ANSI_BOLD, gettext("pool:"));
8720         printf(" %s\n", zpool_get_name(zhp));
8721         fputc(' ', stdout);
8722         printf_color(ANSI_BOLD, gettext("state: "));
8723
8724         printf_color(health_str_to_color(health), "%s", health);
8725
8726         fputc('\n', stdout);
8727
8728         switch (reason) {
8729         case ZPOOL_STATUS_MISSING_DEV_R:
8730                 printf_color(ANSI_BOLD, gettext("status: "));
8731                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8732                     "not be opened.  Sufficient replicas exist for\n\tthe pool "
8733                     "to continue functioning in a degraded state.\n"));
8734                 printf_color(ANSI_BOLD, gettext("action: "));
8735                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
8736                     "and online it using 'zpool online'.\n"));
8737                 break;
8738
8739         case ZPOOL_STATUS_MISSING_DEV_NR:
8740                 printf_color(ANSI_BOLD, gettext("status: "));
8741                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8742                     "not be opened.  There are insufficient\n\treplicas for the"
8743                     " pool to continue functioning.\n"));
8744                 printf_color(ANSI_BOLD, gettext("action: "));
8745                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
8746                     "and online it using 'zpool online'.\n"));
8747                 break;
8748
8749         case ZPOOL_STATUS_CORRUPT_LABEL_R:
8750                 printf_color(ANSI_BOLD, gettext("status: "));
8751                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8752                     "not be used because the label is missing or\n\tinvalid.  "
8753                     "Sufficient replicas exist for the pool to continue\n\t"
8754                     "functioning in a degraded state.\n"));
8755                 printf_color(ANSI_BOLD, gettext("action: "));
8756                 printf_color(ANSI_YELLOW, gettext("Replace the device using "
8757                     "'zpool replace'.\n"));
8758                 break;
8759
8760         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
8761                 printf_color(ANSI_BOLD, gettext("status: "));
8762                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8763                     "not be used because the label is missing \n\tor invalid.  "
8764                     "There are insufficient replicas for the pool to "
8765                     "continue\n\tfunctioning.\n"));
8766                 zpool_explain_recover(zpool_get_handle(zhp),
8767                     zpool_get_name(zhp), reason, config);
8768                 break;
8769
8770         case ZPOOL_STATUS_FAILING_DEV:
8771                 printf_color(ANSI_BOLD, gettext("status: "));
8772                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8773                     "experienced an unrecoverable error.  An\n\tattempt was "
8774                     "made to correct the error.  Applications are "
8775                     "unaffected.\n"));
8776                 printf_color(ANSI_BOLD, gettext("action: "));
8777                         printf_color(ANSI_YELLOW, gettext("Determine if the "
8778                     "device needs to be replaced, and clear the errors\n\tusing"
8779                     " 'zpool clear' or replace the device with 'zpool "
8780                     "replace'.\n"));
8781                 break;
8782
8783         case ZPOOL_STATUS_OFFLINE_DEV:
8784                 printf_color(ANSI_BOLD, gettext("status: "));
8785                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8786                     "been taken offline by the administrator.\n\tSufficient "
8787                     "replicas exist for the pool to continue functioning in "
8788                     "a\n\tdegraded state.\n"));
8789                 printf_color(ANSI_BOLD, gettext("action: "));
8790                 printf_color(ANSI_YELLOW, gettext("Online the device "
8791                     "using 'zpool online' or replace the device with\n\t'zpool "
8792                     "replace'.\n"));
8793                 break;
8794
8795         case ZPOOL_STATUS_REMOVED_DEV:
8796                 printf_color(ANSI_BOLD, gettext("status: "));
8797                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8798                     "been removed by the administrator.\n\tSufficient "
8799                     "replicas exist for the pool to continue functioning in "
8800                     "a\n\tdegraded state.\n"));
8801                 printf_color(ANSI_BOLD, gettext("action: "));
8802                 printf_color(ANSI_YELLOW, gettext("Online the device "
8803                     "using zpool online' or replace the device with\n\t'zpool "
8804                     "replace'.\n"));
8805                 break;
8806
8807         case ZPOOL_STATUS_RESILVERING:
8808         case ZPOOL_STATUS_REBUILDING:
8809                 printf_color(ANSI_BOLD, gettext("status: "));
8810                 printf_color(ANSI_YELLOW, gettext("One or more devices is "
8811                     "currently being resilvered.  The pool will\n\tcontinue "
8812                     "to function, possibly in a degraded state.\n"));
8813                 printf_color(ANSI_BOLD, gettext("action: "));
8814                 printf_color(ANSI_YELLOW, gettext("Wait for the resilver to "
8815                     "complete.\n"));
8816                 break;
8817
8818         case ZPOOL_STATUS_REBUILD_SCRUB:
8819                 printf_color(ANSI_BOLD, gettext("status: "));
8820                 printf_color(ANSI_YELLOW, gettext("One or more devices have "
8821                     "been sequentially resilvered, scrubbing\n\tthe pool "
8822                     "is recommended.\n"));
8823                 printf_color(ANSI_BOLD, gettext("action: "));
8824                 printf_color(ANSI_YELLOW, gettext("Use 'zpool scrub' to "
8825                     "verify all data checksums.\n"));
8826                 break;
8827
8828         case ZPOOL_STATUS_CORRUPT_DATA:
8829                 printf_color(ANSI_BOLD, gettext("status: "));
8830                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8831                     "experienced an error resulting in data\n\tcorruption.  "
8832                     "Applications may be affected.\n"));
8833                 printf_color(ANSI_BOLD, gettext("action: "));
8834                 printf_color(ANSI_YELLOW, gettext("Restore the file in question"
8835                     " if possible.  Otherwise restore the\n\tentire pool from "
8836                     "backup.\n"));
8837                 break;
8838
8839         case ZPOOL_STATUS_CORRUPT_POOL:
8840                 printf_color(ANSI_BOLD, gettext("status: "));
8841                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
8842                     "corrupted and the pool cannot be opened.\n"));
8843                 zpool_explain_recover(zpool_get_handle(zhp),
8844                     zpool_get_name(zhp), reason, config);
8845                 break;
8846
8847         case ZPOOL_STATUS_VERSION_OLDER:
8848                 printf_color(ANSI_BOLD, gettext("status: "));
8849                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
8850                     "a legacy on-disk format.  The pool can\n\tstill be used, "
8851                     "but some features are unavailable.\n"));
8852                 printf_color(ANSI_BOLD, gettext("action: "));
8853                 printf_color(ANSI_YELLOW, gettext("Upgrade the pool using "
8854                     "'zpool upgrade'.  Once this is done, the\n\tpool will no "
8855                     "longer be accessible on software that does not support\n\t"
8856                     "feature flags.\n"));
8857                 break;
8858
8859         case ZPOOL_STATUS_VERSION_NEWER:
8860                 printf_color(ANSI_BOLD, gettext("status: "));
8861                 printf_color(ANSI_YELLOW, gettext("The pool has been upgraded "
8862                     "to a newer, incompatible on-disk version.\n\tThe pool "
8863                     "cannot be accessed on this system.\n"));
8864                 printf_color(ANSI_BOLD, gettext("action: "));
8865                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8866                     "system running more recent software, or\n\trestore the "
8867                     "pool from backup.\n"));
8868                 break;
8869
8870         case ZPOOL_STATUS_FEAT_DISABLED:
8871                 printf_color(ANSI_BOLD, gettext("status: "));
8872                 printf_color(ANSI_YELLOW, gettext("Some supported and "
8873                     "requested features are not enabled on the pool.\n\t"
8874                     "The pool can still be used, but some features are "
8875                     "unavailable.\n"));
8876                 printf_color(ANSI_BOLD, gettext("action: "));
8877                 printf_color(ANSI_YELLOW, gettext("Enable all features using "
8878                     "'zpool upgrade'. Once this is done,\n\tthe pool may no "
8879                     "longer be accessible by software that does not support\n\t"
8880                     "the features. See zpool-features(7) for details.\n"));
8881                 break;
8882
8883         case ZPOOL_STATUS_COMPATIBILITY_ERR:
8884                 printf_color(ANSI_BOLD, gettext("status: "));
8885                 printf_color(ANSI_YELLOW, gettext("This pool has a "
8886                     "compatibility list specified, but it could not be\n\t"
8887                     "read/parsed at this time. The pool can still be used, "
8888                     "but this\n\tshould be investigated.\n"));
8889                 printf_color(ANSI_BOLD, gettext("action: "));
8890                 printf_color(ANSI_YELLOW, gettext("Check the value of the "
8891                     "'compatibility' property against the\n\t"
8892                     "appropriate file in " ZPOOL_SYSCONF_COMPAT_D " or "
8893                     ZPOOL_DATA_COMPAT_D ".\n"));
8894                 break;
8895
8896         case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
8897                 printf_color(ANSI_BOLD, gettext("status: "));
8898                 printf_color(ANSI_YELLOW, gettext("One or more features "
8899                     "are enabled on the pool despite not being\n\t"
8900                     "requested by the 'compatibility' property.\n"));
8901                 printf_color(ANSI_BOLD, gettext("action: "));
8902                 printf_color(ANSI_YELLOW, gettext("Consider setting "
8903                     "'compatibility' to an appropriate value, or\n\t"
8904                     "adding needed features to the relevant file in\n\t"
8905                     ZPOOL_SYSCONF_COMPAT_D " or " ZPOOL_DATA_COMPAT_D ".\n"));
8906                 break;
8907
8908         case ZPOOL_STATUS_UNSUP_FEAT_READ:
8909                 printf_color(ANSI_BOLD, gettext("status: "));
8910                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8911                     "on this system because it uses the\n\tfollowing feature(s)"
8912                     " not supported on this system:\n"));
8913                 zpool_print_unsup_feat(config);
8914                 (void) printf("\n");
8915                 printf_color(ANSI_BOLD, gettext("action: "));
8916                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8917                     "system that supports the required feature(s),\n\tor "
8918                     "restore the pool from backup.\n"));
8919                 break;
8920
8921         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
8922                 printf_color(ANSI_BOLD, gettext("status: "));
8923                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
8924                     "accessed in read-only mode on this system. It\n\tcannot be"
8925                     " accessed in read-write mode because it uses the "
8926                     "following\n\tfeature(s) not supported on this system:\n"));
8927                 zpool_print_unsup_feat(config);
8928                 (void) printf("\n");
8929                 printf_color(ANSI_BOLD, gettext("action: "));
8930                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8931                     "in read-write mode. Import the pool with\n"
8932                     "\t\"-o readonly=on\", access the pool from a system that "
8933                     "supports the\n\trequired feature(s), or restore the "
8934                     "pool from backup.\n"));
8935                 break;
8936
8937         case ZPOOL_STATUS_FAULTED_DEV_R:
8938                 printf_color(ANSI_BOLD, gettext("status: "));
8939                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8940                     "faulted in response to persistent errors.\n\tSufficient "
8941                     "replicas exist for the pool to continue functioning "
8942                     "in a\n\tdegraded state.\n"));
8943                 printf_color(ANSI_BOLD, gettext("action: "));
8944                 printf_color(ANSI_YELLOW, gettext("Replace the faulted device, "
8945                     "or use 'zpool clear' to mark the device\n\trepaired.\n"));
8946                 break;
8947
8948         case ZPOOL_STATUS_FAULTED_DEV_NR:
8949                 printf_color(ANSI_BOLD, gettext("status: "));
8950                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8951                     "faulted in response to persistent errors.  There are "
8952                     "insufficient replicas for the pool to\n\tcontinue "
8953                     "functioning.\n"));
8954                 printf_color(ANSI_BOLD, gettext("action: "));
8955                 printf_color(ANSI_YELLOW, gettext("Destroy and re-create the "
8956                     "pool from a backup source.  Manually marking the device\n"
8957                     "\trepaired using 'zpool clear' may allow some data "
8958                     "to be recovered.\n"));
8959                 break;
8960
8961         case ZPOOL_STATUS_IO_FAILURE_MMP:
8962                 printf_color(ANSI_BOLD, gettext("status: "));
8963                 printf_color(ANSI_YELLOW, gettext("The pool is suspended "
8964                     "because multihost writes failed or were delayed;\n\t"
8965                     "another system could import the pool undetected.\n"));
8966                 printf_color(ANSI_BOLD, gettext("action: "));
8967                 printf_color(ANSI_YELLOW, gettext("Make sure the pool's devices"
8968                     " are connected, then reboot your system and\n\timport the "
8969                     "pool.\n"));
8970                 break;
8971
8972         case ZPOOL_STATUS_IO_FAILURE_WAIT:
8973         case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
8974                 printf_color(ANSI_BOLD, gettext("status: "));
8975                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8976                     "faulted in response to IO failures.\n"));
8977                 printf_color(ANSI_BOLD, gettext("action: "));
8978                 printf_color(ANSI_YELLOW, gettext("Make sure the affected "
8979                     "devices are connected, then run 'zpool clear'.\n"));
8980                 break;
8981
8982         case ZPOOL_STATUS_BAD_LOG:
8983                 printf_color(ANSI_BOLD, gettext("status: "));
8984                 printf_color(ANSI_YELLOW, gettext("An intent log record "
8985                     "could not be read.\n"
8986                     "\tWaiting for administrator intervention to fix the "
8987                     "faulted pool.\n"));
8988                 printf_color(ANSI_BOLD, gettext("action: "));
8989                 printf_color(ANSI_YELLOW, gettext("Either restore the affected "
8990                     "device(s) and run 'zpool online',\n"
8991                     "\tor ignore the intent log records by running "
8992                     "'zpool clear'.\n"));
8993                 break;
8994
8995         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
8996                 (void) printf(gettext("status: One or more devices are "
8997                     "configured to use a non-native block size.\n"
8998                     "\tExpect reduced performance.\n"));
8999                 (void) printf(gettext("action: Replace affected devices with "
9000                     "devices that support the\n\tconfigured block size, or "
9001                     "migrate data to a properly configured\n\tpool.\n"));
9002                 break;
9003
9004         case ZPOOL_STATUS_HOSTID_MISMATCH:
9005                 printf_color(ANSI_BOLD, gettext("status: "));
9006                 printf_color(ANSI_YELLOW, gettext("Mismatch between pool hostid"
9007                     " and system hostid on imported pool.\n\tThis pool was "
9008                     "previously imported into a system with a different "
9009                     "hostid,\n\tand then was verbatim imported into this "
9010                     "system.\n"));
9011                 printf_color(ANSI_BOLD, gettext("action: "));
9012                 printf_color(ANSI_YELLOW, gettext("Export this pool on all "
9013                     "systems on which it is imported.\n"
9014                     "\tThen import it to correct the mismatch.\n"));
9015                 break;
9016
9017         case ZPOOL_STATUS_ERRATA:
9018                 printf_color(ANSI_BOLD, gettext("status: "));
9019                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
9020                     errata);
9021
9022                 switch (errata) {
9023                 case ZPOOL_ERRATA_NONE:
9024                         break;
9025
9026                 case ZPOOL_ERRATA_ZOL_2094_SCRUB:
9027                         printf_color(ANSI_BOLD, gettext("action: "));
9028                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
9029                             " run 'zpool scrub'.\n"));
9030                         break;
9031
9032                 case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
9033                         (void) printf(gettext("\tExisting encrypted datasets "
9034                             "contain an on-disk incompatibility\n\twhich "
9035                             "needs to be corrected.\n"));
9036                         printf_color(ANSI_BOLD, gettext("action: "));
9037                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
9038                             " backup existing encrypted datasets to new\n\t"
9039                             "encrypted datasets and destroy the old ones. "
9040                             "'zfs mount -o ro' can\n\tbe used to temporarily "
9041                             "mount existing encrypted datasets readonly.\n"));
9042                         break;
9043
9044                 case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
9045                         (void) printf(gettext("\tExisting encrypted snapshots "
9046                             "and bookmarks contain an on-disk\n\tincompat"
9047                             "ibility. This may cause on-disk corruption if "
9048                             "they are used\n\twith 'zfs recv'.\n"));
9049                         printf_color(ANSI_BOLD, gettext("action: "));
9050                         printf_color(ANSI_YELLOW, gettext("To correct the"
9051                             "issue, enable the bookmark_v2 feature. No "
9052                             "additional\n\taction is needed if there are no "
9053                             "encrypted snapshots or bookmarks.\n\tIf preserving"
9054                             "the encrypted snapshots and bookmarks is required,"
9055                             " use\n\ta non-raw send to backup and restore them."
9056                             " Alternately, they may be\n\tremoved to resolve "
9057                             "the incompatibility.\n"));
9058                         break;
9059
9060                 default:
9061                         /*
9062                          * All errata which allow the pool to be imported
9063                          * must contain an action message.
9064                          */
9065                         assert(0);
9066                 }
9067                 break;
9068
9069         default:
9070                 /*
9071                  * The remaining errors can't actually be generated, yet.
9072                  */
9073                 assert(reason == ZPOOL_STATUS_OK);
9074         }
9075
9076         if (msgid != NULL) {
9077                 printf("   ");
9078                 printf_color(ANSI_BOLD, gettext("see:"));
9079                 printf(gettext(
9080                     " https://openzfs.github.io/openzfs-docs/msg/%s\n"),
9081                     msgid);
9082         }
9083
9084         if (config != NULL) {
9085                 uint64_t nerr;
9086                 nvlist_t **spares, **l2cache;
9087                 uint_t nspares, nl2cache;
9088
9089                 print_scan_status(zhp, nvroot);
9090
9091                 pool_removal_stat_t *prs = NULL;
9092                 (void) nvlist_lookup_uint64_array(nvroot,
9093                     ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
9094                 print_removal_status(zhp, prs);
9095
9096                 pool_checkpoint_stat_t *pcs = NULL;
9097                 (void) nvlist_lookup_uint64_array(nvroot,
9098                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
9099                 print_checkpoint_status(pcs);
9100
9101                 pool_raidz_expand_stat_t *pres = NULL;
9102                 (void) nvlist_lookup_uint64_array(nvroot,
9103                     ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t **)&pres, &c);
9104                 print_raidz_expand_status(zhp, pres);
9105
9106                 cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
9107                     cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
9108                 if (cbp->cb_namewidth < 10)
9109                         cbp->cb_namewidth = 10;
9110
9111                 color_start(ANSI_BOLD);
9112                 (void) printf(gettext("config:\n\n"));
9113                 (void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
9114                     cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
9115                     "CKSUM");
9116                 color_end();
9117
9118                 if (cbp->cb_print_slow_ios) {
9119                         printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
9120                 }
9121
9122                 if (cbp->cb_print_power) {
9123                         printf_color(ANSI_BOLD, " %5s", gettext("POWER"));
9124                 }
9125
9126                 if (cbp->vcdl != NULL)
9127                         print_cmd_columns(cbp->vcdl, 0);
9128
9129                 printf("\n");
9130
9131                 print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
9132                     B_FALSE, NULL);
9133
9134                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
9135                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
9136                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
9137
9138                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
9139                     &l2cache, &nl2cache) == 0)
9140                         print_l2cache(zhp, cbp, l2cache, nl2cache);
9141
9142                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
9143                     &spares, &nspares) == 0)
9144                         print_spares(zhp, cbp, spares, nspares);
9145
9146                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
9147                     &nerr) == 0) {
9148                         (void) printf("\n");
9149                         if (nerr == 0) {
9150                                 (void) printf(gettext(
9151                                     "errors: No known data errors\n"));
9152                         } else if (!cbp->cb_verbose) {
9153                                 color_start(ANSI_RED);
9154                                 (void) printf(gettext("errors: %llu data "
9155                                     "errors, use '-v' for a list\n"),
9156                                     (u_longlong_t)nerr);
9157                                 color_end();
9158                         } else {
9159                                 print_error_log(zhp);
9160                         }
9161                 }
9162
9163                 if (cbp->cb_dedup_stats)
9164                         print_dedup_stats(config);
9165         } else {
9166                 (void) printf(gettext("config: The configuration cannot be "
9167                     "determined.\n"));
9168         }
9169
9170         return (0);
9171 }
9172
9173 /*
9174  * zpool status [-c [script1,script2,...]] [-igLpPstvx] [--power] [-T d|u] ...
9175  *              [pool] [interval [count]]
9176  *
9177  *      -c CMD  For each vdev, run command CMD
9178  *      -e      Display only unhealthy vdevs
9179  *      -i      Display vdev initialization status.
9180  *      -g      Display guid for individual vdev name.
9181  *      -L      Follow links when resolving vdev path name.
9182  *      -p      Display values in parsable (exact) format.
9183  *      -P      Display full path for vdev name.
9184  *      -s      Display slow IOs column.
9185  *      -v      Display complete error logs
9186  *      -x      Display only pools with potential problems
9187  *      -D      Display dedup status (undocumented)
9188  *      -t      Display vdev TRIM status.
9189  *      -T      Display a timestamp in date(1) or Unix format
9190  *      --power Display vdev enclosure slot power status
9191  *
9192  * Describes the health status of all pools or some subset.
9193  */
9194 int
9195 zpool_do_status(int argc, char **argv)
9196 {
9197         int c;
9198         int ret;
9199         float interval = 0;
9200         unsigned long count = 0;
9201         status_cbdata_t cb = { 0 };
9202         char *cmd = NULL;
9203
9204         struct option long_options[] = {
9205                 {"power", no_argument, NULL, POWER_OPT},
9206                 {0, 0, 0, 0}
9207         };
9208
9209         /* check options */
9210         while ((c = getopt_long(argc, argv, "c:eigLpPsvxDtT:", long_options,
9211             NULL)) != -1) {
9212                 switch (c) {
9213                 case 'c':
9214                         if (cmd != NULL) {
9215                                 fprintf(stderr,
9216                                     gettext("Can't set -c flag twice\n"));
9217                                 exit(1);
9218                         }
9219
9220                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
9221                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
9222                                 fprintf(stderr, gettext(
9223                                     "Can't run -c, disabled by "
9224                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
9225                                 exit(1);
9226                         }
9227
9228                         if ((getuid() <= 0 || geteuid() <= 0) &&
9229                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
9230                                 fprintf(stderr, gettext(
9231                                     "Can't run -c with root privileges "
9232                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
9233                                 exit(1);
9234                         }
9235                         cmd = optarg;
9236                         break;
9237                 case 'e':
9238                         cb.cb_print_unhealthy = B_TRUE;
9239                         break;
9240                 case 'i':
9241                         cb.cb_print_vdev_init = B_TRUE;
9242                         break;
9243                 case 'g':
9244                         cb.cb_name_flags |= VDEV_NAME_GUID;
9245                         break;
9246                 case 'L':
9247                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
9248                         break;
9249                 case 'p':
9250                         cb.cb_literal = B_TRUE;
9251                         break;
9252                 case 'P':
9253                         cb.cb_name_flags |= VDEV_NAME_PATH;
9254                         break;
9255                 case 's':
9256                         cb.cb_print_slow_ios = B_TRUE;
9257                         break;
9258                 case 'v':
9259                         cb.cb_verbose = B_TRUE;
9260                         break;
9261                 case 'x':
9262                         cb.cb_explain = B_TRUE;
9263                         break;
9264                 case 'D':
9265                         cb.cb_dedup_stats = B_TRUE;
9266                         break;
9267                 case 't':
9268                         cb.cb_print_vdev_trim = B_TRUE;
9269                         break;
9270                 case 'T':
9271                         get_timestamp_arg(*optarg);
9272                         break;
9273                 case POWER_OPT:
9274                         cb.cb_print_power = B_TRUE;
9275                         break;
9276                 case '?':
9277                         if (optopt == 'c') {
9278                                 print_zpool_script_list("status");
9279                                 exit(0);
9280                         } else {
9281                                 fprintf(stderr,
9282                                     gettext("invalid option '%c'\n"), optopt);
9283                         }
9284                         usage(B_FALSE);
9285                 }
9286         }
9287
9288         argc -= optind;
9289         argv += optind;
9290
9291         get_interval_count(&argc, argv, &interval, &count);
9292
9293         if (argc == 0)
9294                 cb.cb_allpools = B_TRUE;
9295
9296         cb.cb_first = B_TRUE;
9297         cb.cb_print_status = B_TRUE;
9298
9299         for (;;) {
9300                 if (timestamp_fmt != NODATE)
9301                         print_timestamp(timestamp_fmt);
9302
9303                 if (cmd != NULL)
9304                         cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
9305                             NULL, NULL, 0, 0);
9306
9307                 ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
9308                     cb.cb_literal, status_callback, &cb);
9309
9310                 if (cb.vcdl != NULL)
9311                         free_vdev_cmd_data_list(cb.vcdl);
9312
9313                 if (argc == 0 && cb.cb_count == 0)
9314                         (void) fprintf(stderr, gettext("no pools available\n"));
9315                 else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
9316                         (void) printf(gettext("all pools are healthy\n"));
9317
9318                 if (ret != 0)
9319                         return (ret);
9320
9321                 if (interval == 0)
9322                         break;
9323
9324                 if (count != 0 && --count == 0)
9325                         break;
9326
9327                 (void) fflush(stdout);
9328                 (void) fsleep(interval);
9329         }
9330
9331         return (0);
9332 }
9333
9334 typedef struct upgrade_cbdata {
9335         int     cb_first;
9336         int     cb_argc;
9337         uint64_t cb_version;
9338         char    **cb_argv;
9339 } upgrade_cbdata_t;
9340
9341 static int
9342 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
9343 {
9344         int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
9345         int *count = (int *)unsupp_fs;
9346
9347         if (zfs_version > ZPL_VERSION) {
9348                 (void) printf(gettext("%s (v%d) is not supported by this "
9349                     "implementation of ZFS.\n"),
9350                     zfs_get_name(zhp), zfs_version);
9351                 (*count)++;
9352         }
9353
9354         zfs_iter_filesystems_v2(zhp, 0, check_unsupp_fs, unsupp_fs);
9355
9356         zfs_close(zhp);
9357
9358         return (0);
9359 }
9360
9361 static int
9362 upgrade_version(zpool_handle_t *zhp, uint64_t version)
9363 {
9364         int ret;
9365         nvlist_t *config;
9366         uint64_t oldversion;
9367         int unsupp_fs = 0;
9368
9369         config = zpool_get_config(zhp, NULL);
9370         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9371             &oldversion) == 0);
9372
9373         char compat[ZFS_MAXPROPLEN];
9374         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
9375             ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
9376                 compat[0] = '\0';
9377
9378         assert(SPA_VERSION_IS_SUPPORTED(oldversion));
9379         assert(oldversion < version);
9380
9381         ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
9382         if (ret != 0)
9383                 return (ret);
9384
9385         if (unsupp_fs) {
9386                 (void) fprintf(stderr, gettext("Upgrade not performed due "
9387                     "to %d unsupported filesystems (max v%d).\n"),
9388                     unsupp_fs, (int)ZPL_VERSION);
9389                 return (1);
9390         }
9391
9392         if (strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) {
9393                 (void) fprintf(stderr, gettext("Upgrade not performed because "
9394                     "'compatibility' property set to '"
9395                     ZPOOL_COMPAT_LEGACY "'.\n"));
9396                 return (1);
9397         }
9398
9399         ret = zpool_upgrade(zhp, version);
9400         if (ret != 0)
9401                 return (ret);
9402
9403         if (version >= SPA_VERSION_FEATURES) {
9404                 (void) printf(gettext("Successfully upgraded "
9405                     "'%s' from version %llu to feature flags.\n"),
9406                     zpool_get_name(zhp), (u_longlong_t)oldversion);
9407         } else {
9408                 (void) printf(gettext("Successfully upgraded "
9409                     "'%s' from version %llu to version %llu.\n"),
9410                     zpool_get_name(zhp), (u_longlong_t)oldversion,
9411                     (u_longlong_t)version);
9412         }
9413
9414         return (0);
9415 }
9416
9417 static int
9418 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
9419 {
9420         int i, ret, count;
9421         boolean_t firstff = B_TRUE;
9422         nvlist_t *enabled = zpool_get_features(zhp);
9423
9424         char compat[ZFS_MAXPROPLEN];
9425         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
9426             ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
9427                 compat[0] = '\0';
9428
9429         boolean_t requested_features[SPA_FEATURES];
9430         if (zpool_do_load_compat(compat, requested_features) !=
9431             ZPOOL_COMPATIBILITY_OK)
9432                 return (-1);
9433
9434         count = 0;
9435         for (i = 0; i < SPA_FEATURES; i++) {
9436                 const char *fname = spa_feature_table[i].fi_uname;
9437                 const char *fguid = spa_feature_table[i].fi_guid;
9438
9439                 if (!spa_feature_table[i].fi_zfs_mod_supported)
9440                         continue;
9441
9442                 if (!nvlist_exists(enabled, fguid) && requested_features[i]) {
9443                         char *propname;
9444                         verify(-1 != asprintf(&propname, "feature@%s", fname));
9445                         ret = zpool_set_prop(zhp, propname,
9446                             ZFS_FEATURE_ENABLED);
9447                         if (ret != 0) {
9448                                 free(propname);
9449                                 return (ret);
9450                         }
9451                         count++;
9452
9453                         if (firstff) {
9454                                 (void) printf(gettext("Enabled the "
9455                                     "following features on '%s':\n"),
9456                                     zpool_get_name(zhp));
9457                                 firstff = B_FALSE;
9458                         }
9459                         (void) printf(gettext("  %s\n"), fname);
9460                         free(propname);
9461                 }
9462         }
9463
9464         if (countp != NULL)
9465                 *countp = count;
9466         return (0);
9467 }
9468
9469 static int
9470 upgrade_cb(zpool_handle_t *zhp, void *arg)
9471 {
9472         upgrade_cbdata_t *cbp = arg;
9473         nvlist_t *config;
9474         uint64_t version;
9475         boolean_t modified_pool = B_FALSE;
9476         int ret;
9477
9478         config = zpool_get_config(zhp, NULL);
9479         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9480             &version) == 0);
9481
9482         assert(SPA_VERSION_IS_SUPPORTED(version));
9483
9484         if (version < cbp->cb_version) {
9485                 cbp->cb_first = B_FALSE;
9486                 ret = upgrade_version(zhp, cbp->cb_version);
9487                 if (ret != 0)
9488                         return (ret);
9489                 modified_pool = B_TRUE;
9490
9491                 /*
9492                  * If they did "zpool upgrade -a", then we could
9493                  * be doing ioctls to different pools.  We need
9494                  * to log this history once to each pool, and bypass
9495                  * the normal history logging that happens in main().
9496                  */
9497                 (void) zpool_log_history(g_zfs, history_str);
9498                 log_history = B_FALSE;
9499         }
9500
9501         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
9502                 int count;
9503                 ret = upgrade_enable_all(zhp, &count);
9504                 if (ret != 0)
9505                         return (ret);
9506
9507                 if (count > 0) {
9508                         cbp->cb_first = B_FALSE;
9509                         modified_pool = B_TRUE;
9510                 }
9511         }
9512
9513         if (modified_pool) {
9514                 (void) printf("\n");
9515                 (void) after_zpool_upgrade(zhp);
9516         }
9517
9518         return (0);
9519 }
9520
9521 static int
9522 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
9523 {
9524         upgrade_cbdata_t *cbp = arg;
9525         nvlist_t *config;
9526         uint64_t version;
9527
9528         config = zpool_get_config(zhp, NULL);
9529         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9530             &version) == 0);
9531
9532         assert(SPA_VERSION_IS_SUPPORTED(version));
9533
9534         if (version < SPA_VERSION_FEATURES) {
9535                 if (cbp->cb_first) {
9536                         (void) printf(gettext("The following pools are "
9537                             "formatted with legacy version numbers and can\n"
9538                             "be upgraded to use feature flags.  After "
9539                             "being upgraded, these pools\nwill no "
9540                             "longer be accessible by software that does not "
9541                             "support feature\nflags.\n\n"
9542                             "Note that setting a pool's 'compatibility' "
9543                             "feature to '" ZPOOL_COMPAT_LEGACY "' will\n"
9544                             "inhibit upgrades.\n\n"));
9545                         (void) printf(gettext("VER  POOL\n"));
9546                         (void) printf(gettext("---  ------------\n"));
9547                         cbp->cb_first = B_FALSE;
9548                 }
9549
9550                 (void) printf("%2llu   %s\n", (u_longlong_t)version,
9551                     zpool_get_name(zhp));
9552         }
9553
9554         return (0);
9555 }
9556
9557 static int
9558 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
9559 {
9560         upgrade_cbdata_t *cbp = arg;
9561         nvlist_t *config;
9562         uint64_t version;
9563
9564         config = zpool_get_config(zhp, NULL);
9565         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9566             &version) == 0);
9567
9568         if (version >= SPA_VERSION_FEATURES) {
9569                 int i;
9570                 boolean_t poolfirst = B_TRUE;
9571                 nvlist_t *enabled = zpool_get_features(zhp);
9572
9573                 for (i = 0; i < SPA_FEATURES; i++) {
9574                         const char *fguid = spa_feature_table[i].fi_guid;
9575                         const char *fname = spa_feature_table[i].fi_uname;
9576
9577                         if (!spa_feature_table[i].fi_zfs_mod_supported)
9578                                 continue;
9579
9580                         if (!nvlist_exists(enabled, fguid)) {
9581                                 if (cbp->cb_first) {
9582                                         (void) printf(gettext("\nSome "
9583                                             "supported features are not "
9584                                             "enabled on the following pools. "
9585                                             "Once a\nfeature is enabled the "
9586                                             "pool may become incompatible with "
9587                                             "software\nthat does not support "
9588                                             "the feature. See "
9589                                             "zpool-features(7) for "
9590                                             "details.\n\n"
9591                                             "Note that the pool "
9592                                             "'compatibility' feature can be "
9593                                             "used to inhibit\nfeature "
9594                                             "upgrades.\n\n"));
9595                                         (void) printf(gettext("POOL  "
9596                                             "FEATURE\n"));
9597                                         (void) printf(gettext("------"
9598                                             "---------\n"));
9599                                         cbp->cb_first = B_FALSE;
9600                                 }
9601
9602                                 if (poolfirst) {
9603                                         (void) printf(gettext("%s\n"),
9604                                             zpool_get_name(zhp));
9605                                         poolfirst = B_FALSE;
9606                                 }
9607
9608                                 (void) printf(gettext("      %s\n"), fname);
9609                         }
9610                         /*
9611                          * If they did "zpool upgrade -a", then we could
9612                          * be doing ioctls to different pools.  We need
9613                          * to log this history once to each pool, and bypass
9614                          * the normal history logging that happens in main().
9615                          */
9616                         (void) zpool_log_history(g_zfs, history_str);
9617                         log_history = B_FALSE;
9618                 }
9619         }
9620
9621         return (0);
9622 }
9623
9624 static int
9625 upgrade_one(zpool_handle_t *zhp, void *data)
9626 {
9627         boolean_t modified_pool = B_FALSE;
9628         upgrade_cbdata_t *cbp = data;
9629         uint64_t cur_version;
9630         int ret;
9631
9632         if (strcmp("log", zpool_get_name(zhp)) == 0) {
9633                 (void) fprintf(stderr, gettext("'log' is now a reserved word\n"
9634                     "Pool 'log' must be renamed using export and import"
9635                     " to upgrade.\n"));
9636                 return (1);
9637         }
9638
9639         cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
9640         if (cur_version > cbp->cb_version) {
9641                 (void) printf(gettext("Pool '%s' is already formatted "
9642                     "using more current version '%llu'.\n\n"),
9643                     zpool_get_name(zhp), (u_longlong_t)cur_version);
9644                 return (0);
9645         }
9646
9647         if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
9648                 (void) printf(gettext("Pool '%s' is already formatted "
9649                     "using version %llu.\n\n"), zpool_get_name(zhp),
9650                     (u_longlong_t)cbp->cb_version);
9651                 return (0);
9652         }
9653
9654         if (cur_version != cbp->cb_version) {
9655                 modified_pool = B_TRUE;
9656                 ret = upgrade_version(zhp, cbp->cb_version);
9657                 if (ret != 0)
9658                         return (ret);
9659         }
9660
9661         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
9662                 int count = 0;
9663                 ret = upgrade_enable_all(zhp, &count);
9664                 if (ret != 0)
9665                         return (ret);
9666
9667                 if (count != 0) {
9668                         modified_pool = B_TRUE;
9669                 } else if (cur_version == SPA_VERSION) {
9670                         (void) printf(gettext("Pool '%s' already has all "
9671                             "supported and requested features enabled.\n"),
9672                             zpool_get_name(zhp));
9673                 }
9674         }
9675
9676         if (modified_pool) {
9677                 (void) printf("\n");
9678                 (void) after_zpool_upgrade(zhp);
9679         }
9680
9681         return (0);
9682 }
9683
9684 /*
9685  * zpool upgrade
9686  * zpool upgrade -v
9687  * zpool upgrade [-V version] <-a | pool ...>
9688  *
9689  * With no arguments, display downrev'd ZFS pool available for upgrade.
9690  * Individual pools can be upgraded by specifying the pool, and '-a' will
9691  * upgrade all pools.
9692  */
9693 int
9694 zpool_do_upgrade(int argc, char **argv)
9695 {
9696         int c;
9697         upgrade_cbdata_t cb = { 0 };
9698         int ret = 0;
9699         boolean_t showversions = B_FALSE;
9700         boolean_t upgradeall = B_FALSE;
9701         char *end;
9702
9703
9704         /* check options */
9705         while ((c = getopt(argc, argv, ":avV:")) != -1) {
9706                 switch (c) {
9707                 case 'a':
9708                         upgradeall = B_TRUE;
9709                         break;
9710                 case 'v':
9711                         showversions = B_TRUE;
9712                         break;
9713                 case 'V':
9714                         cb.cb_version = strtoll(optarg, &end, 10);
9715                         if (*end != '\0' ||
9716                             !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
9717                                 (void) fprintf(stderr,
9718                                     gettext("invalid version '%s'\n"), optarg);
9719                                 usage(B_FALSE);
9720                         }
9721                         break;
9722                 case ':':
9723                         (void) fprintf(stderr, gettext("missing argument for "
9724                             "'%c' option\n"), optopt);
9725                         usage(B_FALSE);
9726                         break;
9727                 case '?':
9728                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9729                             optopt);
9730                         usage(B_FALSE);
9731                 }
9732         }
9733
9734         cb.cb_argc = argc;
9735         cb.cb_argv = argv;
9736         argc -= optind;
9737         argv += optind;
9738
9739         if (cb.cb_version == 0) {
9740                 cb.cb_version = SPA_VERSION;
9741         } else if (!upgradeall && argc == 0) {
9742                 (void) fprintf(stderr, gettext("-V option is "
9743                     "incompatible with other arguments\n"));
9744                 usage(B_FALSE);
9745         }
9746
9747         if (showversions) {
9748                 if (upgradeall || argc != 0) {
9749                         (void) fprintf(stderr, gettext("-v option is "
9750                             "incompatible with other arguments\n"));
9751                         usage(B_FALSE);
9752                 }
9753         } else if (upgradeall) {
9754                 if (argc != 0) {
9755                         (void) fprintf(stderr, gettext("-a option should not "
9756                             "be used along with a pool name\n"));
9757                         usage(B_FALSE);
9758                 }
9759         }
9760
9761         (void) printf("%s", gettext("This system supports ZFS pool feature "
9762             "flags.\n\n"));
9763         if (showversions) {
9764                 int i;
9765
9766                 (void) printf(gettext("The following features are "
9767                     "supported:\n\n"));
9768                 (void) printf(gettext("FEAT DESCRIPTION\n"));
9769                 (void) printf("----------------------------------------------"
9770                     "---------------\n");
9771                 for (i = 0; i < SPA_FEATURES; i++) {
9772                         zfeature_info_t *fi = &spa_feature_table[i];
9773                         if (!fi->fi_zfs_mod_supported)
9774                                 continue;
9775                         const char *ro =
9776                             (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
9777                             " (read-only compatible)" : "";
9778
9779                         (void) printf("%-37s%s\n", fi->fi_uname, ro);
9780                         (void) printf("     %s\n", fi->fi_desc);
9781                 }
9782                 (void) printf("\n");
9783
9784                 (void) printf(gettext("The following legacy versions are also "
9785                     "supported:\n\n"));
9786                 (void) printf(gettext("VER  DESCRIPTION\n"));
9787                 (void) printf("---  -----------------------------------------"
9788                     "---------------\n");
9789                 (void) printf(gettext(" 1   Initial ZFS version\n"));
9790                 (void) printf(gettext(" 2   Ditto blocks "
9791                     "(replicated metadata)\n"));
9792                 (void) printf(gettext(" 3   Hot spares and double parity "
9793                     "RAID-Z\n"));
9794                 (void) printf(gettext(" 4   zpool history\n"));
9795                 (void) printf(gettext(" 5   Compression using the gzip "
9796                     "algorithm\n"));
9797                 (void) printf(gettext(" 6   bootfs pool property\n"));
9798                 (void) printf(gettext(" 7   Separate intent log devices\n"));
9799                 (void) printf(gettext(" 8   Delegated administration\n"));
9800                 (void) printf(gettext(" 9   refquota and refreservation "
9801                     "properties\n"));
9802                 (void) printf(gettext(" 10  Cache devices\n"));
9803                 (void) printf(gettext(" 11  Improved scrub performance\n"));
9804                 (void) printf(gettext(" 12  Snapshot properties\n"));
9805                 (void) printf(gettext(" 13  snapused property\n"));
9806                 (void) printf(gettext(" 14  passthrough-x aclinherit\n"));
9807                 (void) printf(gettext(" 15  user/group space accounting\n"));
9808                 (void) printf(gettext(" 16  stmf property support\n"));
9809                 (void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
9810                 (void) printf(gettext(" 18  Snapshot user holds\n"));
9811                 (void) printf(gettext(" 19  Log device removal\n"));
9812                 (void) printf(gettext(" 20  Compression using zle "
9813                     "(zero-length encoding)\n"));
9814                 (void) printf(gettext(" 21  Deduplication\n"));
9815                 (void) printf(gettext(" 22  Received properties\n"));
9816                 (void) printf(gettext(" 23  Slim ZIL\n"));
9817                 (void) printf(gettext(" 24  System attributes\n"));
9818                 (void) printf(gettext(" 25  Improved scrub stats\n"));
9819                 (void) printf(gettext(" 26  Improved snapshot deletion "
9820                     "performance\n"));
9821                 (void) printf(gettext(" 27  Improved snapshot creation "
9822                     "performance\n"));
9823                 (void) printf(gettext(" 28  Multiple vdev replacements\n"));
9824                 (void) printf(gettext("\nFor more information on a particular "
9825                     "version, including supported releases,\n"));
9826                 (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
9827         } else if (argc == 0 && upgradeall) {
9828                 cb.cb_first = B_TRUE;
9829                 ret = zpool_iter(g_zfs, upgrade_cb, &cb);
9830                 if (ret == 0 && cb.cb_first) {
9831                         if (cb.cb_version == SPA_VERSION) {
9832                                 (void) printf(gettext("All pools are already "
9833                                     "formatted using feature flags.\n\n"));
9834                                 (void) printf(gettext("Every feature flags "
9835                                     "pool already has all supported and "
9836                                     "requested features enabled.\n"));
9837                         } else {
9838                                 (void) printf(gettext("All pools are already "
9839                                     "formatted with version %llu or higher.\n"),
9840                                     (u_longlong_t)cb.cb_version);
9841                         }
9842                 }
9843         } else if (argc == 0) {
9844                 cb.cb_first = B_TRUE;
9845                 ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
9846                 assert(ret == 0);
9847
9848                 if (cb.cb_first) {
9849                         (void) printf(gettext("All pools are formatted "
9850                             "using feature flags.\n\n"));
9851                 } else {
9852                         (void) printf(gettext("\nUse 'zpool upgrade -v' "
9853                             "for a list of available legacy versions.\n"));
9854                 }
9855
9856                 cb.cb_first = B_TRUE;
9857                 ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
9858                 assert(ret == 0);
9859
9860                 if (cb.cb_first) {
9861                         (void) printf(gettext("Every feature flags pool has "
9862                             "all supported and requested features enabled.\n"));
9863                 } else {
9864                         (void) printf(gettext("\n"));
9865                 }
9866         } else {
9867                 ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
9868                     B_FALSE, upgrade_one, &cb);
9869         }
9870
9871         return (ret);
9872 }
9873
9874 typedef struct hist_cbdata {
9875         boolean_t first;
9876         boolean_t longfmt;
9877         boolean_t internal;
9878 } hist_cbdata_t;
9879
9880 static void
9881 print_history_records(nvlist_t *nvhis, hist_cbdata_t *cb)
9882 {
9883         nvlist_t **records;
9884         uint_t numrecords;
9885         int i;
9886
9887         verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
9888             &records, &numrecords) == 0);
9889         for (i = 0; i < numrecords; i++) {
9890                 nvlist_t *rec = records[i];
9891                 char tbuf[64] = "";
9892
9893                 if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
9894                         time_t tsec;
9895                         struct tm t;
9896
9897                         tsec = fnvlist_lookup_uint64(records[i],
9898                             ZPOOL_HIST_TIME);
9899                         (void) localtime_r(&tsec, &t);
9900                         (void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
9901                 }
9902
9903                 if (nvlist_exists(rec, ZPOOL_HIST_ELAPSED_NS)) {
9904                         uint64_t elapsed_ns = fnvlist_lookup_int64(records[i],
9905                             ZPOOL_HIST_ELAPSED_NS);
9906                         (void) snprintf(tbuf + strlen(tbuf),
9907                             sizeof (tbuf) - strlen(tbuf),
9908                             " (%lldms)", (long long)elapsed_ns / 1000 / 1000);
9909                 }
9910
9911                 if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
9912                         (void) printf("%s %s", tbuf,
9913                             fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
9914                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
9915                         int ievent =
9916                             fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
9917                         if (!cb->internal)
9918                                 continue;
9919                         if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
9920                                 (void) printf("%s unrecognized record:\n",
9921                                     tbuf);
9922                                 dump_nvlist(rec, 4);
9923                                 continue;
9924                         }
9925                         (void) printf("%s [internal %s txg:%lld] %s", tbuf,
9926                             zfs_history_event_names[ievent],
9927                             (longlong_t)fnvlist_lookup_uint64(
9928                             rec, ZPOOL_HIST_TXG),
9929                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
9930                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
9931                         if (!cb->internal)
9932                                 continue;
9933                         (void) printf("%s [txg:%lld] %s", tbuf,
9934                             (longlong_t)fnvlist_lookup_uint64(
9935                             rec, ZPOOL_HIST_TXG),
9936                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
9937                         if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
9938                                 (void) printf(" %s (%llu)",
9939                                     fnvlist_lookup_string(rec,
9940                                     ZPOOL_HIST_DSNAME),
9941                                     (u_longlong_t)fnvlist_lookup_uint64(rec,
9942                                     ZPOOL_HIST_DSID));
9943                         }
9944                         (void) printf(" %s", fnvlist_lookup_string(rec,
9945                             ZPOOL_HIST_INT_STR));
9946                 } else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
9947                         if (!cb->internal)
9948                                 continue;
9949                         (void) printf("%s ioctl %s\n", tbuf,
9950                             fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
9951                         if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
9952                                 (void) printf("    input:\n");
9953                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9954                                     ZPOOL_HIST_INPUT_NVL), 8);
9955                         }
9956                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
9957                                 (void) printf("    output:\n");
9958                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9959                                     ZPOOL_HIST_OUTPUT_NVL), 8);
9960                         }
9961                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_SIZE)) {
9962                                 (void) printf("    output nvlist omitted; "
9963                                     "original size: %lldKB\n",
9964                                     (longlong_t)fnvlist_lookup_int64(rec,
9965                                     ZPOOL_HIST_OUTPUT_SIZE) / 1024);
9966                         }
9967                         if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
9968                                 (void) printf("    errno: %lld\n",
9969                                     (longlong_t)fnvlist_lookup_int64(rec,
9970                                     ZPOOL_HIST_ERRNO));
9971                         }
9972                 } else {
9973                         if (!cb->internal)
9974                                 continue;
9975                         (void) printf("%s unrecognized record:\n", tbuf);
9976                         dump_nvlist(rec, 4);
9977                 }
9978
9979                 if (!cb->longfmt) {
9980                         (void) printf("\n");
9981                         continue;
9982                 }
9983                 (void) printf(" [");
9984                 if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
9985                         uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
9986                         struct passwd *pwd = getpwuid(who);
9987                         (void) printf("user %d ", (int)who);
9988                         if (pwd != NULL)
9989                                 (void) printf("(%s) ", pwd->pw_name);
9990                 }
9991                 if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
9992                         (void) printf("on %s",
9993                             fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
9994                 }
9995                 if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
9996                         (void) printf(":%s",
9997                             fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
9998                 }
9999
10000                 (void) printf("]");
10001                 (void) printf("\n");
10002         }
10003 }
10004
10005 /*
10006  * Print out the command history for a specific pool.
10007  */
10008 static int
10009 get_history_one(zpool_handle_t *zhp, void *data)
10010 {
10011         nvlist_t *nvhis;
10012         int ret;
10013         hist_cbdata_t *cb = (hist_cbdata_t *)data;
10014         uint64_t off = 0;
10015         boolean_t eof = B_FALSE;
10016
10017         cb->first = B_FALSE;
10018
10019         (void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
10020
10021         while (!eof) {
10022                 if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
10023                         return (ret);
10024
10025                 print_history_records(nvhis, cb);
10026                 nvlist_free(nvhis);
10027         }
10028         (void) printf("\n");
10029
10030         return (ret);
10031 }
10032
10033 /*
10034  * zpool history <pool>
10035  *
10036  * Displays the history of commands that modified pools.
10037  */
10038 int
10039 zpool_do_history(int argc, char **argv)
10040 {
10041         hist_cbdata_t cbdata = { 0 };
10042         int ret;
10043         int c;
10044
10045         cbdata.first = B_TRUE;
10046         /* check options */
10047         while ((c = getopt(argc, argv, "li")) != -1) {
10048                 switch (c) {
10049                 case 'l':
10050                         cbdata.longfmt = B_TRUE;
10051                         break;
10052                 case 'i':
10053                         cbdata.internal = B_TRUE;
10054                         break;
10055                 case '?':
10056                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10057                             optopt);
10058                         usage(B_FALSE);
10059                 }
10060         }
10061         argc -= optind;
10062         argv += optind;
10063
10064         ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
10065             B_FALSE, get_history_one, &cbdata);
10066
10067         if (argc == 0 && cbdata.first == B_TRUE) {
10068                 (void) fprintf(stderr, gettext("no pools available\n"));
10069                 return (0);
10070         }
10071
10072         return (ret);
10073 }
10074
10075 typedef struct ev_opts {
10076         int verbose;
10077         int scripted;
10078         int follow;
10079         int clear;
10080         char poolname[ZFS_MAX_DATASET_NAME_LEN];
10081 } ev_opts_t;
10082
10083 static void
10084 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
10085 {
10086         char ctime_str[26], str[32];
10087         const char *ptr;
10088         int64_t *tv;
10089         uint_t n;
10090
10091         verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
10092         memset(str, ' ', 32);
10093         (void) ctime_r((const time_t *)&tv[0], ctime_str);
10094         (void) memcpy(str, ctime_str+4,  6);            /* 'Jun 30' */
10095         (void) memcpy(str+7, ctime_str+20, 4);          /* '1993' */
10096         (void) memcpy(str+12, ctime_str+11, 8);         /* '21:49:08' */
10097         (void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
10098         if (opts->scripted)
10099                 (void) printf(gettext("%s\t"), str);
10100         else
10101                 (void) printf(gettext("%s "), str);
10102
10103         verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
10104         (void) printf(gettext("%s\n"), ptr);
10105 }
10106
10107 static void
10108 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
10109 {
10110         nvpair_t *nvp;
10111
10112         for (nvp = nvlist_next_nvpair(nvl, NULL);
10113             nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
10114
10115                 data_type_t type = nvpair_type(nvp);
10116                 const char *name = nvpair_name(nvp);
10117
10118                 boolean_t b;
10119                 uint8_t i8;
10120                 uint16_t i16;
10121                 uint32_t i32;
10122                 uint64_t i64;
10123                 const char *str;
10124                 nvlist_t *cnv;
10125
10126                 printf(gettext("%*s%s = "), depth, "", name);
10127
10128                 switch (type) {
10129                 case DATA_TYPE_BOOLEAN:
10130                         printf(gettext("%s"), "1");
10131                         break;
10132
10133                 case DATA_TYPE_BOOLEAN_VALUE:
10134                         (void) nvpair_value_boolean_value(nvp, &b);
10135                         printf(gettext("%s"), b ? "1" : "0");
10136                         break;
10137
10138                 case DATA_TYPE_BYTE:
10139                         (void) nvpair_value_byte(nvp, &i8);
10140                         printf(gettext("0x%x"), i8);
10141                         break;
10142
10143                 case DATA_TYPE_INT8:
10144                         (void) nvpair_value_int8(nvp, (void *)&i8);
10145                         printf(gettext("0x%x"), i8);
10146                         break;
10147
10148                 case DATA_TYPE_UINT8:
10149                         (void) nvpair_value_uint8(nvp, &i8);
10150                         printf(gettext("0x%x"), i8);
10151                         break;
10152
10153                 case DATA_TYPE_INT16:
10154                         (void) nvpair_value_int16(nvp, (void *)&i16);
10155                         printf(gettext("0x%x"), i16);
10156                         break;
10157
10158                 case DATA_TYPE_UINT16:
10159                         (void) nvpair_value_uint16(nvp, &i16);
10160                         printf(gettext("0x%x"), i16);
10161                         break;
10162
10163                 case DATA_TYPE_INT32:
10164                         (void) nvpair_value_int32(nvp, (void *)&i32);
10165                         printf(gettext("0x%x"), i32);
10166                         break;
10167
10168                 case DATA_TYPE_UINT32:
10169                         (void) nvpair_value_uint32(nvp, &i32);
10170                         printf(gettext("0x%x"), i32);
10171                         break;
10172
10173                 case DATA_TYPE_INT64:
10174                         (void) nvpair_value_int64(nvp, (void *)&i64);
10175                         printf(gettext("0x%llx"), (u_longlong_t)i64);
10176                         break;
10177
10178                 case DATA_TYPE_UINT64:
10179                         (void) nvpair_value_uint64(nvp, &i64);
10180                         /*
10181                          * translate vdev state values to readable
10182                          * strings to aide zpool events consumers
10183                          */
10184                         if (strcmp(name,
10185                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
10186                             strcmp(name,
10187                             FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
10188                                 printf(gettext("\"%s\" (0x%llx)"),
10189                                     zpool_state_to_name(i64, VDEV_AUX_NONE),
10190                                     (u_longlong_t)i64);
10191                         } else {
10192                                 printf(gettext("0x%llx"), (u_longlong_t)i64);
10193                         }
10194                         break;
10195
10196                 case DATA_TYPE_HRTIME:
10197                         (void) nvpair_value_hrtime(nvp, (void *)&i64);
10198                         printf(gettext("0x%llx"), (u_longlong_t)i64);
10199                         break;
10200
10201                 case DATA_TYPE_STRING:
10202                         (void) nvpair_value_string(nvp, &str);
10203                         printf(gettext("\"%s\""), str ? str : "<NULL>");
10204                         break;
10205
10206                 case DATA_TYPE_NVLIST:
10207                         printf(gettext("(embedded nvlist)\n"));
10208                         (void) nvpair_value_nvlist(nvp, &cnv);
10209                         zpool_do_events_nvprint(cnv, depth + 8);
10210                         printf(gettext("%*s(end %s)"), depth, "", name);
10211                         break;
10212
10213                 case DATA_TYPE_NVLIST_ARRAY: {
10214                         nvlist_t **val;
10215                         uint_t i, nelem;
10216
10217                         (void) nvpair_value_nvlist_array(nvp, &val, &nelem);
10218                         printf(gettext("(%d embedded nvlists)\n"), nelem);
10219                         for (i = 0; i < nelem; i++) {
10220                                 printf(gettext("%*s%s[%d] = %s\n"),
10221                                     depth, "", name, i, "(embedded nvlist)");
10222                                 zpool_do_events_nvprint(val[i], depth + 8);
10223                                 printf(gettext("%*s(end %s[%i])\n"),
10224                                     depth, "", name, i);
10225                         }
10226                         printf(gettext("%*s(end %s)\n"), depth, "", name);
10227                         }
10228                         break;
10229
10230                 case DATA_TYPE_INT8_ARRAY: {
10231                         int8_t *val;
10232                         uint_t i, nelem;
10233
10234                         (void) nvpair_value_int8_array(nvp, &val, &nelem);
10235                         for (i = 0; i < nelem; i++)
10236                                 printf(gettext("0x%x "), val[i]);
10237
10238                         break;
10239                         }
10240
10241                 case DATA_TYPE_UINT8_ARRAY: {
10242                         uint8_t *val;
10243                         uint_t i, nelem;
10244
10245                         (void) nvpair_value_uint8_array(nvp, &val, &nelem);
10246                         for (i = 0; i < nelem; i++)
10247                                 printf(gettext("0x%x "), val[i]);
10248
10249                         break;
10250                         }
10251
10252                 case DATA_TYPE_INT16_ARRAY: {
10253                         int16_t *val;
10254                         uint_t i, nelem;
10255
10256                         (void) nvpair_value_int16_array(nvp, &val, &nelem);
10257                         for (i = 0; i < nelem; i++)
10258                                 printf(gettext("0x%x "), val[i]);
10259
10260                         break;
10261                         }
10262
10263                 case DATA_TYPE_UINT16_ARRAY: {
10264                         uint16_t *val;
10265                         uint_t i, nelem;
10266
10267                         (void) nvpair_value_uint16_array(nvp, &val, &nelem);
10268                         for (i = 0; i < nelem; i++)
10269                                 printf(gettext("0x%x "), val[i]);
10270
10271                         break;
10272                         }
10273
10274                 case DATA_TYPE_INT32_ARRAY: {
10275                         int32_t *val;
10276                         uint_t i, nelem;
10277
10278                         (void) nvpair_value_int32_array(nvp, &val, &nelem);
10279                         for (i = 0; i < nelem; i++)
10280                                 printf(gettext("0x%x "), val[i]);
10281
10282                         break;
10283                         }
10284
10285                 case DATA_TYPE_UINT32_ARRAY: {
10286                         uint32_t *val;
10287                         uint_t i, nelem;
10288
10289                         (void) nvpair_value_uint32_array(nvp, &val, &nelem);
10290                         for (i = 0; i < nelem; i++)
10291                                 printf(gettext("0x%x "), val[i]);
10292
10293                         break;
10294                         }
10295
10296                 case DATA_TYPE_INT64_ARRAY: {
10297                         int64_t *val;
10298                         uint_t i, nelem;
10299
10300                         (void) nvpair_value_int64_array(nvp, &val, &nelem);
10301                         for (i = 0; i < nelem; i++)
10302                                 printf(gettext("0x%llx "),
10303                                     (u_longlong_t)val[i]);
10304
10305                         break;
10306                         }
10307
10308                 case DATA_TYPE_UINT64_ARRAY: {
10309                         uint64_t *val;
10310                         uint_t i, nelem;
10311
10312                         (void) nvpair_value_uint64_array(nvp, &val, &nelem);
10313                         for (i = 0; i < nelem; i++)
10314                                 printf(gettext("0x%llx "),
10315                                     (u_longlong_t)val[i]);
10316
10317                         break;
10318                         }
10319
10320                 case DATA_TYPE_STRING_ARRAY: {
10321                         const char **str;
10322                         uint_t i, nelem;
10323
10324                         (void) nvpair_value_string_array(nvp, &str, &nelem);
10325                         for (i = 0; i < nelem; i++)
10326                                 printf(gettext("\"%s\" "),
10327                                     str[i] ? str[i] : "<NULL>");
10328
10329                         break;
10330                         }
10331
10332                 case DATA_TYPE_BOOLEAN_ARRAY:
10333                 case DATA_TYPE_BYTE_ARRAY:
10334                 case DATA_TYPE_DOUBLE:
10335                 case DATA_TYPE_DONTCARE:
10336                 case DATA_TYPE_UNKNOWN:
10337                         printf(gettext("<unknown>"));
10338                         break;
10339                 }
10340
10341                 printf(gettext("\n"));
10342         }
10343 }
10344
10345 static int
10346 zpool_do_events_next(ev_opts_t *opts)
10347 {
10348         nvlist_t *nvl;
10349         int zevent_fd, ret, dropped;
10350         const char *pool;
10351
10352         zevent_fd = open(ZFS_DEV, O_RDWR);
10353         VERIFY(zevent_fd >= 0);
10354
10355         if (!opts->scripted)
10356                 (void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
10357
10358         while (1) {
10359                 ret = zpool_events_next(g_zfs, &nvl, &dropped,
10360                     (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
10361                 if (ret || nvl == NULL)
10362                         break;
10363
10364                 if (dropped > 0)
10365                         (void) printf(gettext("dropped %d events\n"), dropped);
10366
10367                 if (strlen(opts->poolname) > 0 &&
10368                     nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
10369                     strcmp(opts->poolname, pool) != 0)
10370                         continue;
10371
10372                 zpool_do_events_short(nvl, opts);
10373
10374                 if (opts->verbose) {
10375                         zpool_do_events_nvprint(nvl, 8);
10376                         printf(gettext("\n"));
10377                 }
10378                 (void) fflush(stdout);
10379
10380                 nvlist_free(nvl);
10381         }
10382
10383         VERIFY(0 == close(zevent_fd));
10384
10385         return (ret);
10386 }
10387
10388 static int
10389 zpool_do_events_clear(void)
10390 {
10391         int count, ret;
10392
10393         ret = zpool_events_clear(g_zfs, &count);
10394         if (!ret)
10395                 (void) printf(gettext("cleared %d events\n"), count);
10396
10397         return (ret);
10398 }
10399
10400 /*
10401  * zpool events [-vHf [pool] | -c]
10402  *
10403  * Displays events logs by ZFS.
10404  */
10405 int
10406 zpool_do_events(int argc, char **argv)
10407 {
10408         ev_opts_t opts = { 0 };
10409         int ret;
10410         int c;
10411
10412         /* check options */
10413         while ((c = getopt(argc, argv, "vHfc")) != -1) {
10414                 switch (c) {
10415                 case 'v':
10416                         opts.verbose = 1;
10417                         break;
10418                 case 'H':
10419                         opts.scripted = 1;
10420                         break;
10421                 case 'f':
10422                         opts.follow = 1;
10423                         break;
10424                 case 'c':
10425                         opts.clear = 1;
10426                         break;
10427                 case '?':
10428                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10429                             optopt);
10430                         usage(B_FALSE);
10431                 }
10432         }
10433         argc -= optind;
10434         argv += optind;
10435
10436         if (argc > 1) {
10437                 (void) fprintf(stderr, gettext("too many arguments\n"));
10438                 usage(B_FALSE);
10439         } else if (argc == 1) {
10440                 (void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
10441                 if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
10442                         (void) fprintf(stderr,
10443                             gettext("invalid pool name '%s'\n"), opts.poolname);
10444                         usage(B_FALSE);
10445                 }
10446         }
10447
10448         if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
10449             opts.clear) {
10450                 (void) fprintf(stderr,
10451                     gettext("invalid options combined with -c\n"));
10452                 usage(B_FALSE);
10453         }
10454
10455         if (opts.clear)
10456                 ret = zpool_do_events_clear();
10457         else
10458                 ret = zpool_do_events_next(&opts);
10459
10460         return (ret);
10461 }
10462
10463 static int
10464 get_callback_vdev(zpool_handle_t *zhp, char *vdevname, void *data)
10465 {
10466         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
10467         char value[ZFS_MAXPROPLEN];
10468         zprop_source_t srctype;
10469
10470         for (zprop_list_t *pl = cbp->cb_proplist; pl != NULL;
10471             pl = pl->pl_next) {
10472                 char *prop_name;
10473                 /*
10474                  * If the first property is pool name, it is a special
10475                  * placeholder that we can skip. This will also skip
10476                  * over the name property when 'all' is specified.
10477                  */
10478                 if (pl->pl_prop == ZPOOL_PROP_NAME &&
10479                     pl == cbp->cb_proplist)
10480                         continue;
10481
10482                 if (pl->pl_prop == ZPROP_INVAL) {
10483                         prop_name = pl->pl_user_prop;
10484                 } else {
10485                         prop_name = (char *)vdev_prop_to_name(pl->pl_prop);
10486                 }
10487                 if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop,
10488                     prop_name, value, sizeof (value), &srctype,
10489                     cbp->cb_literal) == 0) {
10490                         zprop_print_one_property(vdevname, cbp, prop_name,
10491                             value, srctype, NULL, NULL);
10492                 }
10493         }
10494
10495         return (0);
10496 }
10497
10498 static int
10499 get_callback_vdev_cb(void *zhp_data, nvlist_t *nv, void *data)
10500 {
10501         zpool_handle_t *zhp = zhp_data;
10502         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
10503         char *vdevname;
10504         const char *type;
10505         int ret;
10506
10507         /*
10508          * zpool_vdev_name() transforms the root vdev name (i.e., root-0) to the
10509          * pool name for display purposes, which is not desired. Fallback to
10510          * zpool_vdev_name() when not dealing with the root vdev.
10511          */
10512         type = fnvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE);
10513         if (zhp != NULL && strcmp(type, "root") == 0)
10514                 vdevname = strdup("root-0");
10515         else
10516                 vdevname = zpool_vdev_name(g_zfs, zhp, nv,
10517                     cbp->cb_vdevs.cb_name_flags);
10518
10519         (void) vdev_expand_proplist(zhp, vdevname, &cbp->cb_proplist);
10520
10521         ret = get_callback_vdev(zhp, vdevname, data);
10522
10523         free(vdevname);
10524
10525         return (ret);
10526 }
10527
10528 static int
10529 get_callback(zpool_handle_t *zhp, void *data)
10530 {
10531         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
10532         char value[ZFS_MAXPROPLEN];
10533         zprop_source_t srctype;
10534         zprop_list_t *pl;
10535         int vid;
10536
10537         if (cbp->cb_type == ZFS_TYPE_VDEV) {
10538                 if (strcmp(cbp->cb_vdevs.cb_names[0], "all-vdevs") == 0) {
10539                         for_each_vdev(zhp, get_callback_vdev_cb, data);
10540                 } else {
10541                         /* Adjust column widths for vdev properties */
10542                         for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
10543                             vid++) {
10544                                 vdev_expand_proplist(zhp,
10545                                     cbp->cb_vdevs.cb_names[vid],
10546                                     &cbp->cb_proplist);
10547                         }
10548                         /* Display the properties */
10549                         for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
10550                             vid++) {
10551                                 get_callback_vdev(zhp,
10552                                     cbp->cb_vdevs.cb_names[vid], data);
10553                         }
10554                 }
10555         } else {
10556                 assert(cbp->cb_type == ZFS_TYPE_POOL);
10557                 for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
10558                         /*
10559                          * Skip the special fake placeholder. This will also
10560                          * skip over the name property when 'all' is specified.
10561                          */
10562                         if (pl->pl_prop == ZPOOL_PROP_NAME &&
10563                             pl == cbp->cb_proplist)
10564                                 continue;
10565
10566                         if (pl->pl_prop == ZPROP_INVAL &&
10567                             zfs_prop_user(pl->pl_user_prop)) {
10568                                 srctype = ZPROP_SRC_LOCAL;
10569
10570                                 if (zpool_get_userprop(zhp, pl->pl_user_prop,
10571                                     value, sizeof (value), &srctype) != 0)
10572                                         continue;
10573
10574                                 zprop_print_one_property(zpool_get_name(zhp),
10575                                     cbp, pl->pl_user_prop, value, srctype,
10576                                     NULL, NULL);
10577                         } else if (pl->pl_prop == ZPROP_INVAL &&
10578                             (zpool_prop_feature(pl->pl_user_prop) ||
10579                             zpool_prop_unsupported(pl->pl_user_prop))) {
10580                                 srctype = ZPROP_SRC_LOCAL;
10581
10582                                 if (zpool_prop_get_feature(zhp,
10583                                     pl->pl_user_prop, value,
10584                                     sizeof (value)) == 0) {
10585                                         zprop_print_one_property(
10586                                             zpool_get_name(zhp), cbp,
10587                                             pl->pl_user_prop, value, srctype,
10588                                             NULL, NULL);
10589                                 }
10590                         } else {
10591                                 if (zpool_get_prop(zhp, pl->pl_prop, value,
10592                                     sizeof (value), &srctype,
10593                                     cbp->cb_literal) != 0)
10594                                         continue;
10595
10596                                 zprop_print_one_property(zpool_get_name(zhp),
10597                                     cbp, zpool_prop_to_name(pl->pl_prop),
10598                                     value, srctype, NULL, NULL);
10599                         }
10600                 }
10601         }
10602
10603         return (0);
10604 }
10605
10606 /*
10607  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
10608  *
10609  *      -H      Scripted mode.  Don't display headers, and separate properties
10610  *              by a single tab.
10611  *      -o      List of columns to display.  Defaults to
10612  *              "name,property,value,source".
10613  *      -p      Display values in parsable (exact) format.
10614  *
10615  * Get properties of pools in the system. Output space statistics
10616  * for each one as well as other attributes.
10617  */
10618 int
10619 zpool_do_get(int argc, char **argv)
10620 {
10621         zprop_get_cbdata_t cb = { 0 };
10622         zprop_list_t fake_name = { 0 };
10623         int ret;
10624         int c, i;
10625         char *propstr = NULL;
10626         char *vdev = NULL;
10627
10628         cb.cb_first = B_TRUE;
10629
10630         /*
10631          * Set up default columns and sources.
10632          */
10633         cb.cb_sources = ZPROP_SRC_ALL;
10634         cb.cb_columns[0] = GET_COL_NAME;
10635         cb.cb_columns[1] = GET_COL_PROPERTY;
10636         cb.cb_columns[2] = GET_COL_VALUE;
10637         cb.cb_columns[3] = GET_COL_SOURCE;
10638         cb.cb_type = ZFS_TYPE_POOL;
10639         cb.cb_vdevs.cb_name_flags |= VDEV_NAME_TYPE_ID;
10640         current_prop_type = cb.cb_type;
10641
10642         /* check options */
10643         while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
10644                 switch (c) {
10645                 case 'p':
10646                         cb.cb_literal = B_TRUE;
10647                         break;
10648                 case 'H':
10649                         cb.cb_scripted = B_TRUE;
10650                         break;
10651                 case 'o':
10652                         memset(&cb.cb_columns, 0, sizeof (cb.cb_columns));
10653                         i = 0;
10654
10655                         for (char *tok; (tok = strsep(&optarg, ",")); ) {
10656                                 static const char *const col_opts[] =
10657                                 { "name", "property", "value", "source",
10658                                     "all" };
10659                                 static const zfs_get_column_t col_cols[] =
10660                                 { GET_COL_NAME, GET_COL_PROPERTY, GET_COL_VALUE,
10661                                     GET_COL_SOURCE };
10662
10663                                 if (i == ZFS_GET_NCOLS - 1) {
10664                                         (void) fprintf(stderr, gettext("too "
10665                                         "many fields given to -o "
10666                                         "option\n"));
10667                                         usage(B_FALSE);
10668                                 }
10669
10670                                 for (c = 0; c < ARRAY_SIZE(col_opts); ++c)
10671                                         if (strcmp(tok, col_opts[c]) == 0)
10672                                                 goto found;
10673
10674                                 (void) fprintf(stderr,
10675                                     gettext("invalid column name '%s'\n"), tok);
10676                                 usage(B_FALSE);
10677
10678 found:
10679                                 if (c >= 4) {
10680                                         if (i > 0) {
10681                                                 (void) fprintf(stderr,
10682                                                     gettext("\"all\" conflicts "
10683                                                     "with specific fields "
10684                                                     "given to -o option\n"));
10685                                                 usage(B_FALSE);
10686                                         }
10687
10688                                         memcpy(cb.cb_columns, col_cols,
10689                                             sizeof (col_cols));
10690                                         i = ZFS_GET_NCOLS - 1;
10691                                 } else
10692                                         cb.cb_columns[i++] = col_cols[c];
10693                         }
10694                         break;
10695                 case '?':
10696                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10697                             optopt);
10698                         usage(B_FALSE);
10699                 }
10700         }
10701
10702         argc -= optind;
10703         argv += optind;
10704
10705         if (argc < 1) {
10706                 (void) fprintf(stderr, gettext("missing property "
10707                     "argument\n"));
10708                 usage(B_FALSE);
10709         }
10710
10711         /* Properties list is needed later by zprop_get_list() */
10712         propstr = argv[0];
10713
10714         argc--;
10715         argv++;
10716
10717         if (argc == 0) {
10718                 /* No args, so just print the defaults. */
10719         } else if (are_all_pools(argc, argv)) {
10720                 /* All the args are pool names */
10721         } else if (are_all_pools(1, argv)) {
10722                 /* The first arg is a pool name */
10723                 if ((argc == 2 && strcmp(argv[1], "all-vdevs") == 0) ||
10724                     (argc == 2 && strcmp(argv[1], "root") == 0) ||
10725                     are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
10726                     &cb.cb_vdevs)) {
10727
10728                         if (strcmp(argv[1], "root") == 0)
10729                                 vdev = strdup("root-0");
10730                         else
10731                                 vdev = strdup(argv[1]);
10732
10733                         /* ... and the rest are vdev names */
10734                         cb.cb_vdevs.cb_names = &vdev;
10735                         cb.cb_vdevs.cb_names_count = argc - 1;
10736                         cb.cb_type = ZFS_TYPE_VDEV;
10737                         argc = 1; /* One pool to process */
10738                 } else {
10739                         fprintf(stderr, gettext("Expected a list of vdevs in"
10740                             " \"%s\", but got:\n"), argv[0]);
10741                         error_list_unresolved_vdevs(argc - 1, argv + 1,
10742                             argv[0], &cb.cb_vdevs);
10743                         fprintf(stderr, "\n");
10744                         usage(B_FALSE);
10745                         return (1);
10746                 }
10747         } else {
10748                 /*
10749                  * The first arg isn't a pool name,
10750                  */
10751                 fprintf(stderr, gettext("missing pool name.\n"));
10752                 fprintf(stderr, "\n");
10753                 usage(B_FALSE);
10754                 return (1);
10755         }
10756
10757         if (zprop_get_list(g_zfs, propstr, &cb.cb_proplist,
10758             cb.cb_type) != 0) {
10759                 /* Use correct list of valid properties (pool or vdev) */
10760                 current_prop_type = cb.cb_type;
10761                 usage(B_FALSE);
10762         }
10763
10764         if (cb.cb_proplist != NULL) {
10765                 fake_name.pl_prop = ZPOOL_PROP_NAME;
10766                 fake_name.pl_width = strlen(gettext("NAME"));
10767                 fake_name.pl_next = cb.cb_proplist;
10768                 cb.cb_proplist = &fake_name;
10769         }
10770
10771         ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist, cb.cb_type,
10772             cb.cb_literal, get_callback, &cb);
10773
10774         if (cb.cb_proplist == &fake_name)
10775                 zprop_free_list(fake_name.pl_next);
10776         else
10777                 zprop_free_list(cb.cb_proplist);
10778
10779         if (vdev != NULL)
10780                 free(vdev);
10781
10782         return (ret);
10783 }
10784
10785 typedef struct set_cbdata {
10786         char *cb_propname;
10787         char *cb_value;
10788         zfs_type_t cb_type;
10789         vdev_cbdata_t cb_vdevs;
10790         boolean_t cb_any_successful;
10791 } set_cbdata_t;
10792
10793 static int
10794 set_pool_callback(zpool_handle_t *zhp, set_cbdata_t *cb)
10795 {
10796         int error;
10797
10798         /* Check if we have out-of-bounds features */
10799         if (strcmp(cb->cb_propname, ZPOOL_CONFIG_COMPATIBILITY) == 0) {
10800                 boolean_t features[SPA_FEATURES];
10801                 if (zpool_do_load_compat(cb->cb_value, features) !=
10802                     ZPOOL_COMPATIBILITY_OK)
10803                         return (-1);
10804
10805                 nvlist_t *enabled = zpool_get_features(zhp);
10806                 spa_feature_t i;
10807                 for (i = 0; i < SPA_FEATURES; i++) {
10808                         const char *fguid = spa_feature_table[i].fi_guid;
10809                         if (nvlist_exists(enabled, fguid) && !features[i])
10810                                 break;
10811                 }
10812                 if (i < SPA_FEATURES)
10813                         (void) fprintf(stderr, gettext("Warning: one or "
10814                             "more features already enabled on pool '%s'\n"
10815                             "are not present in this compatibility set.\n"),
10816                             zpool_get_name(zhp));
10817         }
10818
10819         /* if we're setting a feature, check it's in compatibility set */
10820         if (zpool_prop_feature(cb->cb_propname) &&
10821             strcmp(cb->cb_value, ZFS_FEATURE_ENABLED) == 0) {
10822                 char *fname = strchr(cb->cb_propname, '@') + 1;
10823                 spa_feature_t f;
10824
10825                 if (zfeature_lookup_name(fname, &f) == 0) {
10826                         char compat[ZFS_MAXPROPLEN];
10827                         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY,
10828                             compat, ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
10829                                 compat[0] = '\0';
10830
10831                         boolean_t features[SPA_FEATURES];
10832                         if (zpool_do_load_compat(compat, features) !=
10833                             ZPOOL_COMPATIBILITY_OK) {
10834                                 (void) fprintf(stderr, gettext("Error: "
10835                                     "cannot enable feature '%s' on pool '%s'\n"
10836                                     "because the pool's 'compatibility' "
10837                                     "property cannot be parsed.\n"),
10838                                     fname, zpool_get_name(zhp));
10839                                 return (-1);
10840                         }
10841
10842                         if (!features[f]) {
10843                                 (void) fprintf(stderr, gettext("Error: "
10844                                     "cannot enable feature '%s' on pool '%s'\n"
10845                                     "as it is not specified in this pool's "
10846                                     "current compatibility set.\n"
10847                                     "Consider setting 'compatibility' to a "
10848                                     "less restrictive set, or to 'off'.\n"),
10849                                     fname, zpool_get_name(zhp));
10850                                 return (-1);
10851                         }
10852                 }
10853         }
10854
10855         error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
10856
10857         return (error);
10858 }
10859
10860 static int
10861 set_callback(zpool_handle_t *zhp, void *data)
10862 {
10863         int error;
10864         set_cbdata_t *cb = (set_cbdata_t *)data;
10865
10866         if (cb->cb_type == ZFS_TYPE_VDEV) {
10867                 error = zpool_set_vdev_prop(zhp, *cb->cb_vdevs.cb_names,
10868                     cb->cb_propname, cb->cb_value);
10869         } else {
10870                 assert(cb->cb_type == ZFS_TYPE_POOL);
10871                 error = set_pool_callback(zhp, cb);
10872         }
10873
10874         cb->cb_any_successful = !error;
10875         return (error);
10876 }
10877
10878 int
10879 zpool_do_set(int argc, char **argv)
10880 {
10881         set_cbdata_t cb = { 0 };
10882         int error;
10883         char *vdev = NULL;
10884
10885         current_prop_type = ZFS_TYPE_POOL;
10886         if (argc > 1 && argv[1][0] == '-') {
10887                 (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10888                     argv[1][1]);
10889                 usage(B_FALSE);
10890         }
10891
10892         if (argc < 2) {
10893                 (void) fprintf(stderr, gettext("missing property=value "
10894                     "argument\n"));
10895                 usage(B_FALSE);
10896         }
10897
10898         if (argc < 3) {
10899                 (void) fprintf(stderr, gettext("missing pool name\n"));
10900                 usage(B_FALSE);
10901         }
10902
10903         if (argc > 4) {
10904                 (void) fprintf(stderr, gettext("too many pool names\n"));
10905                 usage(B_FALSE);
10906         }
10907
10908         cb.cb_propname = argv[1];
10909         cb.cb_type = ZFS_TYPE_POOL;
10910         cb.cb_vdevs.cb_name_flags |= VDEV_NAME_TYPE_ID;
10911         cb.cb_value = strchr(cb.cb_propname, '=');
10912         if (cb.cb_value == NULL) {
10913                 (void) fprintf(stderr, gettext("missing value in "
10914                     "property=value argument\n"));
10915                 usage(B_FALSE);
10916         }
10917
10918         *(cb.cb_value) = '\0';
10919         cb.cb_value++;
10920         argc -= 2;
10921         argv += 2;
10922
10923         /* argv[0] is pool name */
10924         if (!is_pool(argv[0])) {
10925                 (void) fprintf(stderr,
10926                     gettext("cannot open '%s': is not a pool\n"), argv[0]);
10927                 return (EINVAL);
10928         }
10929
10930         /* argv[1], when supplied, is vdev name */
10931         if (argc == 2) {
10932
10933                 if (strcmp(argv[1], "root") == 0)
10934                         vdev = strdup("root-0");
10935                 else
10936                         vdev = strdup(argv[1]);
10937
10938                 if (!are_vdevs_in_pool(1, &vdev, argv[0], &cb.cb_vdevs)) {
10939                         (void) fprintf(stderr, gettext(
10940                             "cannot find '%s' in '%s': device not in pool\n"),
10941                             vdev, argv[0]);
10942                         free(vdev);
10943                         return (EINVAL);
10944                 }
10945                 cb.cb_vdevs.cb_names = &vdev;
10946                 cb.cb_vdevs.cb_names_count = 1;
10947                 cb.cb_type = ZFS_TYPE_VDEV;
10948         }
10949
10950         error = for_each_pool(1, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
10951             B_FALSE, set_callback, &cb);
10952
10953         if (vdev != NULL)
10954                 free(vdev);
10955
10956         return (error);
10957 }
10958
10959 /* Add up the total number of bytes left to initialize/trim across all vdevs */
10960 static uint64_t
10961 vdev_activity_remaining(nvlist_t *nv, zpool_wait_activity_t activity)
10962 {
10963         uint64_t bytes_remaining;
10964         nvlist_t **child;
10965         uint_t c, children;
10966         vdev_stat_t *vs;
10967
10968         assert(activity == ZPOOL_WAIT_INITIALIZE ||
10969             activity == ZPOOL_WAIT_TRIM);
10970
10971         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
10972             (uint64_t **)&vs, &c) == 0);
10973
10974         if (activity == ZPOOL_WAIT_INITIALIZE &&
10975             vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
10976                 bytes_remaining = vs->vs_initialize_bytes_est -
10977                     vs->vs_initialize_bytes_done;
10978         else if (activity == ZPOOL_WAIT_TRIM &&
10979             vs->vs_trim_state == VDEV_TRIM_ACTIVE)
10980                 bytes_remaining = vs->vs_trim_bytes_est -
10981                     vs->vs_trim_bytes_done;
10982         else
10983                 bytes_remaining = 0;
10984
10985         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10986             &child, &children) != 0)
10987                 children = 0;
10988
10989         for (c = 0; c < children; c++)
10990                 bytes_remaining += vdev_activity_remaining(child[c], activity);
10991
10992         return (bytes_remaining);
10993 }
10994
10995 /* Add up the total number of bytes left to rebuild across top-level vdevs */
10996 static uint64_t
10997 vdev_activity_top_remaining(nvlist_t *nv)
10998 {
10999         uint64_t bytes_remaining = 0;
11000         nvlist_t **child;
11001         uint_t children;
11002         int error;
11003
11004         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
11005             &child, &children) != 0)
11006                 children = 0;
11007
11008         for (uint_t c = 0; c < children; c++) {
11009                 vdev_rebuild_stat_t *vrs;
11010                 uint_t i;
11011
11012                 error = nvlist_lookup_uint64_array(child[c],
11013                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i);
11014                 if (error == 0) {
11015                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
11016                                 bytes_remaining += (vrs->vrs_bytes_est -
11017                                     vrs->vrs_bytes_rebuilt);
11018                         }
11019                 }
11020         }
11021
11022         return (bytes_remaining);
11023 }
11024
11025 /* Whether any vdevs are 'spare' or 'replacing' vdevs */
11026 static boolean_t
11027 vdev_any_spare_replacing(nvlist_t *nv)
11028 {
11029         nvlist_t **child;
11030         uint_t c, children;
11031         const char *vdev_type;
11032
11033         (void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &vdev_type);
11034
11035         if (strcmp(vdev_type, VDEV_TYPE_REPLACING) == 0 ||
11036             strcmp(vdev_type, VDEV_TYPE_SPARE) == 0 ||
11037             strcmp(vdev_type, VDEV_TYPE_DRAID_SPARE) == 0) {
11038                 return (B_TRUE);
11039         }
11040
11041         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
11042             &child, &children) != 0)
11043                 children = 0;
11044
11045         for (c = 0; c < children; c++) {
11046                 if (vdev_any_spare_replacing(child[c]))
11047                         return (B_TRUE);
11048         }
11049
11050         return (B_FALSE);
11051 }
11052
11053 typedef struct wait_data {
11054         char *wd_poolname;
11055         boolean_t wd_scripted;
11056         boolean_t wd_exact;
11057         boolean_t wd_headers_once;
11058         boolean_t wd_should_exit;
11059         /* Which activities to wait for */
11060         boolean_t wd_enabled[ZPOOL_WAIT_NUM_ACTIVITIES];
11061         float wd_interval;
11062         pthread_cond_t wd_cv;
11063         pthread_mutex_t wd_mutex;
11064 } wait_data_t;
11065
11066 /*
11067  * Print to stdout a single line, containing one column for each activity that
11068  * we are waiting for specifying how many bytes of work are left for that
11069  * activity.
11070  */
11071 static void
11072 print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
11073 {
11074         nvlist_t *config, *nvroot;
11075         uint_t c;
11076         int i;
11077         pool_checkpoint_stat_t *pcs = NULL;
11078         pool_scan_stat_t *pss = NULL;
11079         pool_removal_stat_t *prs = NULL;
11080         pool_raidz_expand_stat_t *pres = NULL;
11081         const char *const headers[] = {"DISCARD", "FREE", "INITIALIZE",
11082             "REPLACE", "REMOVE", "RESILVER", "SCRUB", "TRIM", "RAIDZ_EXPAND"};
11083         int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];
11084
11085         /* Calculate the width of each column */
11086         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
11087                 /*
11088                  * Make sure we have enough space in the col for pretty-printed
11089                  * numbers and for the column header, and then leave a couple
11090                  * spaces between cols for readability.
11091                  */
11092                 col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
11093         }
11094
11095         if (timestamp_fmt != NODATE)
11096                 print_timestamp(timestamp_fmt);
11097
11098         /* Print header if appropriate */
11099         int term_height = terminal_height();
11100         boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
11101             row % (term_height-1) == 0);
11102         if (!wd->wd_scripted && (row == 0 || reprint_header)) {
11103                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
11104                         if (wd->wd_enabled[i])
11105                                 (void) printf("%*s", col_widths[i], headers[i]);
11106                 }
11107                 (void) fputc('\n', stdout);
11108         }
11109
11110         /* Bytes of work remaining in each activity */
11111         int64_t bytes_rem[ZPOOL_WAIT_NUM_ACTIVITIES] = {0};
11112
11113         bytes_rem[ZPOOL_WAIT_FREE] =
11114             zpool_get_prop_int(zhp, ZPOOL_PROP_FREEING, NULL);
11115
11116         config = zpool_get_config(zhp, NULL);
11117         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
11118
11119         (void) nvlist_lookup_uint64_array(nvroot,
11120             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
11121         if (pcs != NULL && pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
11122                 bytes_rem[ZPOOL_WAIT_CKPT_DISCARD] = pcs->pcs_space;
11123
11124         (void) nvlist_lookup_uint64_array(nvroot,
11125             ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
11126         if (prs != NULL && prs->prs_state == DSS_SCANNING)
11127                 bytes_rem[ZPOOL_WAIT_REMOVE] = prs->prs_to_copy -
11128                     prs->prs_copied;
11129
11130         (void) nvlist_lookup_uint64_array(nvroot,
11131             ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&pss, &c);
11132         if (pss != NULL && pss->pss_state == DSS_SCANNING &&
11133             pss->pss_pass_scrub_pause == 0) {
11134                 int64_t rem = pss->pss_to_examine - pss->pss_issued;
11135                 if (pss->pss_func == POOL_SCAN_SCRUB)
11136                         bytes_rem[ZPOOL_WAIT_SCRUB] = rem;
11137                 else
11138                         bytes_rem[ZPOOL_WAIT_RESILVER] = rem;
11139         } else if (check_rebuilding(nvroot, NULL)) {
11140                 bytes_rem[ZPOOL_WAIT_RESILVER] =
11141                     vdev_activity_top_remaining(nvroot);
11142         }
11143
11144         (void) nvlist_lookup_uint64_array(nvroot,
11145             ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t **)&pres, &c);
11146         if (pres != NULL && pres->pres_state == DSS_SCANNING) {
11147                 int64_t rem = pres->pres_to_reflow - pres->pres_reflowed;
11148                 bytes_rem[ZPOOL_WAIT_RAIDZ_EXPAND] = rem;
11149         }
11150
11151         bytes_rem[ZPOOL_WAIT_INITIALIZE] =
11152             vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
11153         bytes_rem[ZPOOL_WAIT_TRIM] =
11154             vdev_activity_remaining(nvroot, ZPOOL_WAIT_TRIM);
11155
11156         /*
11157          * A replace finishes after resilvering finishes, so the amount of work
11158          * left for a replace is the same as for resilvering.
11159          *
11160          * It isn't quite correct to say that if we have any 'spare' or
11161          * 'replacing' vdevs and a resilver is happening, then a replace is in
11162          * progress, like we do here. When a hot spare is used, the faulted vdev
11163          * is not removed after the hot spare is resilvered, so parent 'spare'
11164          * vdev is not removed either. So we could have a 'spare' vdev, but be
11165          * resilvering for a different reason. However, we use it as a heuristic
11166          * because we don't have access to the DTLs, which could tell us whether
11167          * or not we have really finished resilvering a hot spare.
11168          */
11169         if (vdev_any_spare_replacing(nvroot))
11170                 bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
11171
11172         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
11173                 char buf[64];
11174                 if (!wd->wd_enabled[i])
11175                         continue;
11176
11177                 if (wd->wd_exact) {
11178                         (void) snprintf(buf, sizeof (buf), "%" PRIi64,
11179                             bytes_rem[i]);
11180                 } else {
11181                         zfs_nicenum(bytes_rem[i], buf, sizeof (buf));
11182                 }
11183
11184                 if (wd->wd_scripted)
11185                         (void) printf(i == 0 ? "%s" : "\t%s", buf);
11186                 else
11187                         (void) printf(" %*s", col_widths[i] - 1, buf);
11188         }
11189         (void) printf("\n");
11190         (void) fflush(stdout);
11191 }
11192
11193 static void *
11194 wait_status_thread(void *arg)
11195 {
11196         wait_data_t *wd = (wait_data_t *)arg;
11197         zpool_handle_t *zhp;
11198
11199         if ((zhp = zpool_open(g_zfs, wd->wd_poolname)) == NULL)
11200                 return (void *)(1);
11201
11202         for (int row = 0; ; row++) {
11203                 boolean_t missing;
11204                 struct timespec timeout;
11205                 int ret = 0;
11206                 (void) clock_gettime(CLOCK_REALTIME, &timeout);
11207
11208                 if (zpool_refresh_stats(zhp, &missing) != 0 || missing ||
11209                     zpool_props_refresh(zhp) != 0) {
11210                         zpool_close(zhp);
11211                         return (void *)(uintptr_t)(missing ? 0 : 1);
11212                 }
11213
11214                 print_wait_status_row(wd, zhp, row);
11215
11216                 timeout.tv_sec += floor(wd->wd_interval);
11217                 long nanos = timeout.tv_nsec +
11218                     (wd->wd_interval - floor(wd->wd_interval)) * NANOSEC;
11219                 if (nanos >= NANOSEC) {
11220                         timeout.tv_sec++;
11221                         timeout.tv_nsec = nanos - NANOSEC;
11222                 } else {
11223                         timeout.tv_nsec = nanos;
11224                 }
11225                 pthread_mutex_lock(&wd->wd_mutex);
11226                 if (!wd->wd_should_exit)
11227                         ret = pthread_cond_timedwait(&wd->wd_cv, &wd->wd_mutex,
11228                             &timeout);
11229                 pthread_mutex_unlock(&wd->wd_mutex);
11230                 if (ret == 0) {
11231                         break; /* signaled by main thread */
11232                 } else if (ret != ETIMEDOUT) {
11233                         (void) fprintf(stderr, gettext("pthread_cond_timedwait "
11234                             "failed: %s\n"), strerror(ret));
11235                         zpool_close(zhp);
11236                         return (void *)(uintptr_t)(1);
11237                 }
11238         }
11239
11240         zpool_close(zhp);
11241         return (void *)(0);
11242 }
11243
11244 int
11245 zpool_do_wait(int argc, char **argv)
11246 {
11247         boolean_t verbose = B_FALSE;
11248         int c, i;
11249         unsigned long count;
11250         pthread_t status_thr;
11251         int error = 0;
11252         zpool_handle_t *zhp;
11253
11254         wait_data_t wd;
11255         wd.wd_scripted = B_FALSE;
11256         wd.wd_exact = B_FALSE;
11257         wd.wd_headers_once = B_FALSE;
11258         wd.wd_should_exit = B_FALSE;
11259
11260         pthread_mutex_init(&wd.wd_mutex, NULL);
11261         pthread_cond_init(&wd.wd_cv, NULL);
11262
11263         /* By default, wait for all types of activity. */
11264         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++)
11265                 wd.wd_enabled[i] = B_TRUE;
11266
11267         while ((c = getopt(argc, argv, "HpT:t:")) != -1) {
11268                 switch (c) {
11269                 case 'H':
11270                         wd.wd_scripted = B_TRUE;
11271                         break;
11272                 case 'n':
11273                         wd.wd_headers_once = B_TRUE;
11274                         break;
11275                 case 'p':
11276                         wd.wd_exact = B_TRUE;
11277                         break;
11278                 case 'T':
11279                         get_timestamp_arg(*optarg);
11280                         break;
11281                 case 't':
11282                         /* Reset activities array */
11283                         memset(&wd.wd_enabled, 0, sizeof (wd.wd_enabled));
11284
11285                         for (char *tok; (tok = strsep(&optarg, ",")); ) {
11286                                 static const char *const col_opts[] = {
11287                                     "discard", "free", "initialize", "replace",
11288                                     "remove", "resilver", "scrub", "trim",
11289                                     "raidz_expand" };
11290
11291                                 for (i = 0; i < ARRAY_SIZE(col_opts); ++i)
11292                                         if (strcmp(tok, col_opts[i]) == 0) {
11293                                                 wd.wd_enabled[i] = B_TRUE;
11294                                                 goto found;
11295                                         }
11296
11297                                 (void) fprintf(stderr,
11298                                     gettext("invalid activity '%s'\n"), tok);
11299                                 usage(B_FALSE);
11300 found:;
11301                         }
11302                         break;
11303                 case '?':
11304                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
11305                             optopt);
11306                         usage(B_FALSE);
11307                 }
11308         }
11309
11310         argc -= optind;
11311         argv += optind;
11312
11313         get_interval_count(&argc, argv, &wd.wd_interval, &count);
11314         if (count != 0) {
11315                 /* This subcmd only accepts an interval, not a count */
11316                 (void) fprintf(stderr, gettext("too many arguments\n"));
11317                 usage(B_FALSE);
11318         }
11319
11320         if (wd.wd_interval != 0)
11321                 verbose = B_TRUE;
11322
11323         if (argc < 1) {
11324                 (void) fprintf(stderr, gettext("missing 'pool' argument\n"));
11325                 usage(B_FALSE);
11326         }
11327         if (argc > 1) {
11328                 (void) fprintf(stderr, gettext("too many arguments\n"));
11329                 usage(B_FALSE);
11330         }
11331
11332         wd.wd_poolname = argv[0];
11333
11334         if ((zhp = zpool_open(g_zfs, wd.wd_poolname)) == NULL)
11335                 return (1);
11336
11337         if (verbose) {
11338                 /*
11339                  * We use a separate thread for printing status updates because
11340                  * the main thread will call lzc_wait(), which blocks as long
11341                  * as an activity is in progress, which can be a long time.
11342                  */
11343                 if (pthread_create(&status_thr, NULL, wait_status_thread, &wd)
11344                     != 0) {
11345                         (void) fprintf(stderr, gettext("failed to create status"
11346                             "thread: %s\n"), strerror(errno));
11347                         zpool_close(zhp);
11348                         return (1);
11349                 }
11350         }
11351
11352         /*
11353          * Loop over all activities that we are supposed to wait for until none
11354          * of them are in progress. Note that this means we can end up waiting
11355          * for more activities to complete than just those that were in progress
11356          * when we began waiting; if an activity we are interested in begins
11357          * while we are waiting for another activity, we will wait for both to
11358          * complete before exiting.
11359          */
11360         for (;;) {
11361                 boolean_t missing = B_FALSE;
11362                 boolean_t any_waited = B_FALSE;
11363
11364                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
11365                         boolean_t waited;
11366
11367                         if (!wd.wd_enabled[i])
11368                                 continue;
11369
11370                         error = zpool_wait_status(zhp, i, &missing, &waited);
11371                         if (error != 0 || missing)
11372                                 break;
11373
11374                         any_waited = (any_waited || waited);
11375                 }
11376
11377                 if (error != 0 || missing || !any_waited)
11378                         break;
11379         }
11380
11381         zpool_close(zhp);
11382
11383         if (verbose) {
11384                 uintptr_t status;
11385                 pthread_mutex_lock(&wd.wd_mutex);
11386                 wd.wd_should_exit = B_TRUE;
11387                 pthread_cond_signal(&wd.wd_cv);
11388                 pthread_mutex_unlock(&wd.wd_mutex);
11389                 (void) pthread_join(status_thr, (void *)&status);
11390                 if (status != 0)
11391                         error = status;
11392         }
11393
11394         pthread_mutex_destroy(&wd.wd_mutex);
11395         pthread_cond_destroy(&wd.wd_cv);
11396         return (error);
11397 }
11398
11399 static int
11400 find_command_idx(const char *command, int *idx)
11401 {
11402         for (int i = 0; i < NCOMMAND; ++i) {
11403                 if (command_table[i].name == NULL)
11404                         continue;
11405
11406                 if (strcmp(command, command_table[i].name) == 0) {
11407                         *idx = i;
11408                         return (0);
11409                 }
11410         }
11411         return (1);
11412 }
11413
11414 /*
11415  * Display version message
11416  */
11417 static int
11418 zpool_do_version(int argc, char **argv)
11419 {
11420         (void) argc, (void) argv;
11421         return (zfs_version_print() != 0);
11422 }
11423
11424 /* Display documentation */
11425 static int
11426 zpool_do_help(int argc, char **argv)
11427 {
11428         char page[MAXNAMELEN];
11429         if (argc < 3 || strcmp(argv[2], "zpool") == 0)
11430                 strcpy(page, "zpool");
11431         else if (strcmp(argv[2], "concepts") == 0 ||
11432             strcmp(argv[2], "props") == 0)
11433                 snprintf(page, sizeof (page), "zpool%s", argv[2]);
11434         else
11435                 snprintf(page, sizeof (page), "zpool-%s", argv[2]);
11436
11437         execlp("man", "man", page, NULL);
11438
11439         fprintf(stderr, "couldn't run man program: %s", strerror(errno));
11440         return (-1);
11441 }
11442
11443 /*
11444  * Do zpool_load_compat() and print error message on failure
11445  */
11446 static zpool_compat_status_t
11447 zpool_do_load_compat(const char *compat, boolean_t *list)
11448 {
11449         char report[1024];
11450
11451         zpool_compat_status_t ret;
11452
11453         ret = zpool_load_compat(compat, list, report, 1024);
11454         switch (ret) {
11455
11456         case ZPOOL_COMPATIBILITY_OK:
11457                 break;
11458
11459         case ZPOOL_COMPATIBILITY_NOFILES:
11460         case ZPOOL_COMPATIBILITY_BADFILE:
11461         case ZPOOL_COMPATIBILITY_BADTOKEN:
11462                 (void) fprintf(stderr, "Error: %s\n", report);
11463                 break;
11464
11465         case ZPOOL_COMPATIBILITY_WARNTOKEN:
11466                 (void) fprintf(stderr, "Warning: %s\n", report);
11467                 ret = ZPOOL_COMPATIBILITY_OK;
11468                 break;
11469         }
11470         return (ret);
11471 }
11472
11473 int
11474 main(int argc, char **argv)
11475 {
11476         int ret = 0;
11477         int i = 0;
11478         char *cmdname;
11479         char **newargv;
11480
11481         (void) setlocale(LC_ALL, "");
11482         (void) setlocale(LC_NUMERIC, "C");
11483         (void) textdomain(TEXT_DOMAIN);
11484         srand(time(NULL));
11485
11486         opterr = 0;
11487
11488         /*
11489          * Make sure the user has specified some command.
11490          */
11491         if (argc < 2) {
11492                 (void) fprintf(stderr, gettext("missing command\n"));
11493                 usage(B_FALSE);
11494         }
11495
11496         cmdname = argv[1];
11497
11498         /*
11499          * Special case '-?'
11500          */
11501         if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
11502                 usage(B_TRUE);
11503
11504         /*
11505          * Special case '-V|--version'
11506          */
11507         if ((strcmp(cmdname, "-V") == 0) || (strcmp(cmdname, "--version") == 0))
11508                 return (zpool_do_version(argc, argv));
11509
11510         /*
11511          * Special case 'help'
11512          */
11513         if (strcmp(cmdname, "help") == 0)
11514                 return (zpool_do_help(argc, argv));
11515
11516         if ((g_zfs = libzfs_init()) == NULL) {
11517                 (void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
11518                 return (1);
11519         }
11520
11521         libzfs_print_on_error(g_zfs, B_TRUE);
11522
11523         zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
11524
11525         /*
11526          * Many commands modify input strings for string parsing reasons.
11527          * We create a copy to protect the original argv.
11528          */
11529         newargv = safe_malloc((argc + 1) * sizeof (newargv[0]));
11530         for (i = 0; i < argc; i++)
11531                 newargv[i] = strdup(argv[i]);
11532         newargv[argc] = NULL;
11533
11534         /*
11535          * Run the appropriate command.
11536          */
11537         if (find_command_idx(cmdname, &i) == 0) {
11538                 current_command = &command_table[i];
11539                 ret = command_table[i].func(argc - 1, newargv + 1);
11540         } else if (strchr(cmdname, '=')) {
11541                 verify(find_command_idx("set", &i) == 0);
11542                 current_command = &command_table[i];
11543                 ret = command_table[i].func(argc, newargv);
11544         } else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
11545                 /*
11546                  * 'freeze' is a vile debugging abomination, so we treat
11547                  * it as such.
11548                  */
11549                 zfs_cmd_t zc = {"\0"};
11550
11551                 (void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
11552                 ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
11553                 if (ret != 0) {
11554                         (void) fprintf(stderr,
11555                         gettext("failed to freeze pool: %d\n"), errno);
11556                         ret = 1;
11557                 }
11558
11559                 log_history = 0;
11560         } else {
11561                 (void) fprintf(stderr, gettext("unrecognized "
11562                     "command '%s'\n"), cmdname);
11563                 usage(B_FALSE);
11564                 ret = 1;
11565         }
11566
11567         for (i = 0; i < argc; i++)
11568                 free(newargv[i]);
11569         free(newargv);
11570
11571         if (ret == 0 && log_history)
11572                 (void) zpool_log_history(g_zfs, history_str);
11573
11574         libzfs_fini(g_zfs);
11575
11576         /*
11577          * The 'ZFS_ABORT' environment variable causes us to dump core on exit
11578          * for the purposes of running ::findleaks.
11579          */
11580         if (getenv("ZFS_ABORT") != NULL) {
11581                 (void) printf("dumping core by request\n");
11582                 abort();
11583         }
11584
11585         return (ret);
11586 }