sys/contrib/openzfs/cmd/zpool/zpool_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  30  * Copyright (c) 2017 Datto Inc.
  31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  32  * Copyright (c) 2017, Intel Corporation.
  33  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  34  */
  35
  36 #include <assert.h>
  37 #include <ctype.h>
  38 #include <dirent.h>
  39 #include <errno.h>
  40 #include <fcntl.h>
  41 #include <getopt.h>
  42 #include <libgen.h>
  43 #include <libintl.h>
  44 #include <libuutil.h>
  45 #include <locale.h>
  46 #include <pthread.h>
  47 #include <stdio.h>
  48 #include <stdlib.h>
  49 #include <string.h>
  50 #include <strings.h>
  51 #include <time.h>
  52 #include <unistd.h>
  53 #include <pwd.h>
  54 #include <zone.h>
  55 #include <sys/wait.h>
  56 #include <zfs_prop.h>
  57 #include <sys/fs/zfs.h>
  58 #include <sys/stat.h>
  59 #include <sys/systeminfo.h>
  60 #include <sys/fm/fs/zfs.h>
  61 #include <sys/fm/util.h>
  62 #include <sys/fm/protocol.h>
  63 #include <sys/zfs_ioctl.h>
  64 #include <sys/mount.h>
  65 #include <sys/sysmacros.h>
  66
  67 #include <math.h>
  68
  69 #include <libzfs.h>
  70 #include <libzutil.h>
  71
  72 #include "zpool_util.h"
  73 #include "zfs_comutil.h"
  74 #include "zfeature_common.h"
  75
  76 #include "statcommon.h"
  77
  78 libzfs_handle_t *g_zfs;
  79
  80 static int zpool_do_create(int, char **);
  81 static int zpool_do_destroy(int, char **);
  82
  83 static int zpool_do_add(int, char **);
  84 static int zpool_do_remove(int, char **);
  85 static int zpool_do_labelclear(int, char **);
  86
  87 static int zpool_do_checkpoint(int, char **);
  88
  89 static int zpool_do_list(int, char **);
  90 static int zpool_do_iostat(int, char **);
  91 static int zpool_do_status(int, char **);
  92
  93 static int zpool_do_online(int, char **);
  94 static int zpool_do_offline(int, char **);
  95 static int zpool_do_clear(int, char **);
  96 static int zpool_do_reopen(int, char **);
  97
  98 static int zpool_do_reguid(int, char **);
  99
 100 static int zpool_do_attach(int, char **);
 101 static int zpool_do_detach(int, char **);
 102 static int zpool_do_replace(int, char **);
 103 static int zpool_do_split(int, char **);
 104
 105 static int zpool_do_initialize(int, char **);
 106 static int zpool_do_scrub(int, char **);
 107 static int zpool_do_resilver(int, char **);
 108 static int zpool_do_trim(int, char **);
 109
 110 static int zpool_do_import(int, char **);
 111 static int zpool_do_export(int, char **);
 112
 113 static int zpool_do_upgrade(int, char **);
 114
 115 static int zpool_do_history(int, char **);
 116 static int zpool_do_events(int, char **);
 117
 118 static int zpool_do_get(int, char **);
 119 static int zpool_do_set(int, char **);
 120
 121 static int zpool_do_sync(int, char **);
 122
 123 static int zpool_do_version(int, char **);
 124
 125 static int zpool_do_wait(int, char **);
 126
 127 /*
 128  * These libumem hooks provide a reasonable set of defaults for the allocator's
 129  * debugging facilities.
 130  */
 131
 132 #ifdef DEBUG
 133 const char *
 134 _umem_debug_init(void)
 135 {
 136         return ("default,verbose"); /* $UMEM_DEBUG setting */
 137 }
 138
 139 const char *
 140 _umem_logging_init(void)
 141 {
 142         return ("fail,contents"); /* $UMEM_LOGGING setting */
 143 }
 144 #endif
 145
 146 typedef enum {
 147         HELP_ADD,
 148         HELP_ATTACH,
 149         HELP_CLEAR,
 150         HELP_CREATE,
 151         HELP_CHECKPOINT,
 152         HELP_DESTROY,
 153         HELP_DETACH,
 154         HELP_EXPORT,
 155         HELP_HISTORY,
 156         HELP_IMPORT,
 157         HELP_IOSTAT,
 158         HELP_LABELCLEAR,
 159         HELP_LIST,
 160         HELP_OFFLINE,
 161         HELP_ONLINE,
 162         HELP_REPLACE,
 163         HELP_REMOVE,
 164         HELP_INITIALIZE,
 165         HELP_SCRUB,
 166         HELP_RESILVER,
 167         HELP_TRIM,
 168         HELP_STATUS,
 169         HELP_UPGRADE,
 170         HELP_EVENTS,
 171         HELP_GET,
 172         HELP_SET,
 173         HELP_SPLIT,
 174         HELP_SYNC,
 175         HELP_REGUID,
 176         HELP_REOPEN,
 177         HELP_VERSION,
 178         HELP_WAIT
 179 } zpool_help_t;
 180
 181
 182 /*
 183  * Flags for stats to display with "zpool iostats"
 184  */
 185 enum iostat_type {
 186         IOS_DEFAULT = 0,
 187         IOS_LATENCY = 1,
 188         IOS_QUEUES = 2,
 189         IOS_L_HISTO = 3,
 190         IOS_RQ_HISTO = 4,
 191         IOS_COUNT,      /* always last element */
 192 };
 193
 194 /* iostat_type entries as bitmasks */
 195 #define IOS_DEFAULT_M   (1ULL << IOS_DEFAULT)
 196 #define IOS_LATENCY_M   (1ULL << IOS_LATENCY)
 197 #define IOS_QUEUES_M    (1ULL << IOS_QUEUES)
 198 #define IOS_L_HISTO_M   (1ULL << IOS_L_HISTO)
 199 #define IOS_RQ_HISTO_M  (1ULL << IOS_RQ_HISTO)
 200
 201 /* Mask of all the histo bits */
 202 #define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
 203
 204 /*
 205  * Lookup table for iostat flags to nvlist names.  Basically a list
 206  * of all the nvlists a flag requires.  Also specifies the order in
 207  * which data gets printed in zpool iostat.
 208  */
 209 static const char *vsx_type_to_nvlist[IOS_COUNT][13] = {
 210         [IOS_L_HISTO] = {
 211             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 212             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 213             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 214             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 215             ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
 216             ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
 217             ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
 218             ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
 219             ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
 220             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 221             NULL},
 222         [IOS_LATENCY] = {
 223             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 224             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 225             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 226             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 227             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 228             NULL},
 229         [IOS_QUEUES] = {
 230             ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
 231             ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
 232             ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
 233             ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
 234             ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
 235             ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
 236             NULL},
 237         [IOS_RQ_HISTO] = {
 238             ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
 239             ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
 240             ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
 241             ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
 242             ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
 243             ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
 244             ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
 245             ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
 246             ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
 247             ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
 248             ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
 249             ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
 250             NULL},
 251 };
 252
 253
 254 /*
 255  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
 256  * Right now, only one histo bit is ever set at one time, so we can
 257  * just do a highbit64(a)
 258  */
 259 #define IOS_HISTO_IDX(a)        (highbit64(a & IOS_ANYHISTO_M) - 1)
 260
 261 typedef struct zpool_command {
 262         const char      *name;
 263         int             (*func)(int, char **);
 264         zpool_help_t    usage;
 265 } zpool_command_t;
 266
 267 /*
 268  * Master command table.  Each ZFS command has a name, associated function, and
 269  * usage message.  The usage messages need to be internationalized, so we have
 270  * to have a function to return the usage message based on a command index.
 271  *
 272  * These commands are organized according to how they are displayed in the usage
 273  * message.  An empty command (one with a NULL name) indicates an empty line in
 274  * the generic usage message.
 275  */
 276 static zpool_command_t command_table[] = {
 277         { "version",    zpool_do_version,       HELP_VERSION            },
 278         { NULL },
 279         { "create",     zpool_do_create,        HELP_CREATE             },
 280         { "destroy",    zpool_do_destroy,       HELP_DESTROY            },
 281         { NULL },
 282         { "add",        zpool_do_add,           HELP_ADD                },
 283         { "remove",     zpool_do_remove,        HELP_REMOVE             },
 284         { NULL },
 285         { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
 286         { NULL },
 287         { "checkpoint", zpool_do_checkpoint,    HELP_CHECKPOINT         },
 288         { NULL },
 289         { "list",       zpool_do_list,          HELP_LIST               },
 290         { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
 291         { "status",     zpool_do_status,        HELP_STATUS             },
 292         { NULL },
 293         { "online",     zpool_do_online,        HELP_ONLINE             },
 294         { "offline",    zpool_do_offline,       HELP_OFFLINE            },
 295         { "clear",      zpool_do_clear,         HELP_CLEAR              },
 296         { "reopen",     zpool_do_reopen,        HELP_REOPEN             },
 297         { NULL },
 298         { "attach",     zpool_do_attach,        HELP_ATTACH             },
 299         { "detach",     zpool_do_detach,        HELP_DETACH             },
 300         { "replace",    zpool_do_replace,       HELP_REPLACE            },
 301         { "split",      zpool_do_split,         HELP_SPLIT              },
 302         { NULL },
 303         { "initialize", zpool_do_initialize,    HELP_INITIALIZE         },
 304         { "resilver",   zpool_do_resilver,      HELP_RESILVER           },
 305         { "scrub",      zpool_do_scrub,         HELP_SCRUB              },
 306         { "trim",       zpool_do_trim,          HELP_TRIM               },
 307         { NULL },
 308         { "import",     zpool_do_import,        HELP_IMPORT             },
 309         { "export",     zpool_do_export,        HELP_EXPORT             },
 310         { "upgrade",    zpool_do_upgrade,       HELP_UPGRADE            },
 311         { "reguid",     zpool_do_reguid,        HELP_REGUID             },
 312         { NULL },
 313         { "history",    zpool_do_history,       HELP_HISTORY            },
 314         { "events",     zpool_do_events,        HELP_EVENTS             },
 315         { NULL },
 316         { "get",        zpool_do_get,           HELP_GET                },
 317         { "set",        zpool_do_set,           HELP_SET                },
 318         { "sync",       zpool_do_sync,          HELP_SYNC               },
 319         { NULL },
 320         { "wait",       zpool_do_wait,          HELP_WAIT               },
 321 };
 322
 323 #define NCOMMAND        (ARRAY_SIZE(command_table))
 324
 325 #define VDEV_ALLOC_CLASS_LOGS   "logs"
 326
 327 static zpool_command_t *current_command;
 328 static char history_str[HIS_MAX_RECORD_LEN];
 329 static boolean_t log_history = B_TRUE;
 330 static uint_t timestamp_fmt = NODATE;
 331
 332 static const char *
 333 get_usage(zpool_help_t idx)
 334 {
 335         switch (idx) {
 336         case HELP_ADD:
 337                 return (gettext("\tadd [-fgLnP] [-o property=value] "
 338                     "<pool> <vdev> ...\n"));
 339         case HELP_ATTACH:
 340                 return (gettext("\tattach [-fsw] [-o property=value] "
 341                     "<pool> <device> <new-device>\n"));
 342         case HELP_CLEAR:
 343                 return (gettext("\tclear [-nF] <pool> [device]\n"));
 344         case HELP_CREATE:
 345                 return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 346                     "\t    [-O file-system-property=value] ... \n"
 347                     "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 348         case HELP_CHECKPOINT:
 349                 return (gettext("\tcheckpoint [-d [-w]] <pool> ...\n"));
 350         case HELP_DESTROY:
 351                 return (gettext("\tdestroy [-f] <pool>\n"));
 352         case HELP_DETACH:
 353                 return (gettext("\tdetach <pool> <device>\n"));
 354         case HELP_EXPORT:
 355                 return (gettext("\texport [-af] <pool> ...\n"));
 356         case HELP_HISTORY:
 357                 return (gettext("\thistory [-il] [<pool>] ...\n"));
 358         case HELP_IMPORT:
 359                 return (gettext("\timport [-d dir] [-D]\n"
 360                     "\timport [-o mntopts] [-o property=value] ... \n"
 361                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 362                     "[-R root] [-F [-n]] -a\n"
 363                     "\timport [-o mntopts] [-o property=value] ... \n"
 364                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 365                     "[-R root] [-F [-n]]\n"
 366                     "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
 367         case HELP_IOSTAT:
 368                 return (gettext("\tiostat [[[-c [script1,script2,...]"
 369                     "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
 370                     "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
 371                     " [[-n] interval [count]]\n"));
 372         case HELP_LABELCLEAR:
 373                 return (gettext("\tlabelclear [-f] <vdev>\n"));
 374         case HELP_LIST:
 375                 return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 376                     "[-T d|u] [pool] ... \n"
 377                     "\t    [interval [count]]\n"));
 378         case HELP_OFFLINE:
 379                 return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
 380         case HELP_ONLINE:
 381                 return (gettext("\tonline [-e] <pool> <device> ...\n"));
 382         case HELP_REPLACE:
 383                 return (gettext("\treplace [-fsw] [-o property=value] "
 384                     "<pool> <device> [new-device]\n"));
 385         case HELP_REMOVE:
 386                 return (gettext("\tremove [-npsw] <pool> <device> ...\n"));
 387         case HELP_REOPEN:
 388                 return (gettext("\treopen [-n] <pool>\n"));
 389         case HELP_INITIALIZE:
 390                 return (gettext("\tinitialize [-c | -s] [-w] <pool> "
 391                     "[<device> ...]\n"));
 392         case HELP_SCRUB:
 393                 return (gettext("\tscrub [-s | -p] [-w] <pool> ...\n"));
 394         case HELP_RESILVER:
 395                 return (gettext("\tresilver <pool> ...\n"));
 396         case HELP_TRIM:
 397                 return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
 398                     "[<device> ...]\n"));
 399         case HELP_STATUS:
 400                 return (gettext("\tstatus [-c [script1,script2,...]] "
 401                     "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
 402                     "\t    [interval [count]]\n"));
 403         case HELP_UPGRADE:
 404                 return (gettext("\tupgrade\n"
 405                     "\tupgrade -v\n"
 406                     "\tupgrade [-V version] <-a | pool ...>\n"));
 407         case HELP_EVENTS:
 408                 return (gettext("\tevents [-vHf [pool] | -c]\n"));
 409         case HELP_GET:
 410                 return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
 411                     "<\"all\" | property[,...]> <pool> ...\n"));
 412         case HELP_SET:
 413                 return (gettext("\tset <property=value> <pool> \n"));
 414         case HELP_SPLIT:
 415                 return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
 416                     "\t    [-o property=value] <pool> <newpool> "
 417                     "[<device> ...]\n"));
 418         case HELP_REGUID:
 419                 return (gettext("\treguid <pool>\n"));
 420         case HELP_SYNC:
 421                 return (gettext("\tsync [pool] ...\n"));
 422         case HELP_VERSION:
 423                 return (gettext("\tversion\n"));
 424         case HELP_WAIT:
 425                 return (gettext("\twait [-Hp] [-T d|u] [-t <activity>[,...]] "
 426                     "<pool> [interval]\n"));
 427         }
 428
 429         abort();
 430         /* NOTREACHED */
 431 }
 432
 433 static void
 434 zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
 435 {
 436         uint_t children = 0;
 437         nvlist_t **child;
 438         uint_t i;
 439
 440         (void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
 441             &child, &children);
 442
 443         if (children == 0) {
 444                 char *path = zpool_vdev_name(g_zfs, zhp, nvroot,
 445                     VDEV_NAME_PATH);
 446
 447                 if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 &&
 448                     strcmp(path, VDEV_TYPE_HOLE) != 0)
 449                         fnvlist_add_boolean(res, path);
 450
 451                 free(path);
 452                 return;
 453         }
 454
 455         for (i = 0; i < children; i++) {
 456                 zpool_collect_leaves(zhp, child[i], res);
 457         }
 458 }
 459
 460 /*
 461  * Callback routine that will print out a pool property value.
 462  */
 463 static int
 464 print_prop_cb(int prop, void *cb)
 465 {
 466         FILE *fp = cb;
 467
 468         (void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
 469
 470         if (zpool_prop_readonly(prop))
 471                 (void) fprintf(fp, "  NO   ");
 472         else
 473                 (void) fprintf(fp, " YES   ");
 474
 475         if (zpool_prop_values(prop) == NULL)
 476                 (void) fprintf(fp, "-\n");
 477         else
 478                 (void) fprintf(fp, "%s\n", zpool_prop_values(prop));
 479
 480         return (ZPROP_CONT);
 481 }
 482
 483 /*
 484  * Display usage message.  If we're inside a command, display only the usage for
 485  * that command.  Otherwise, iterate over the entire command table and display
 486  * a complete usage message.
 487  */
 488 static void
 489 usage(boolean_t requested)
 490 {
 491         FILE *fp = requested ? stdout : stderr;
 492
 493         if (current_command == NULL) {
 494                 int i;
 495
 496                 (void) fprintf(fp, gettext("usage: zpool command args ...\n"));
 497                 (void) fprintf(fp,
 498                     gettext("where 'command' is one of the following:\n\n"));
 499
 500                 for (i = 0; i < NCOMMAND; i++) {
 501                         if (command_table[i].name == NULL)
 502                                 (void) fprintf(fp, "\n");
 503                         else
 504                                 (void) fprintf(fp, "%s",
 505                                     get_usage(command_table[i].usage));
 506                 }
 507         } else {
 508                 (void) fprintf(fp, gettext("usage:\n"));
 509                 (void) fprintf(fp, "%s", get_usage(current_command->usage));
 510         }
 511
 512         if (current_command != NULL &&
 513             ((strcmp(current_command->name, "set") == 0) ||
 514             (strcmp(current_command->name, "get") == 0) ||
 515             (strcmp(current_command->name, "list") == 0))) {
 516
 517                 (void) fprintf(fp,
 518                     gettext("\nthe following properties are supported:\n"));
 519
 520                 (void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
 521                     "PROPERTY", "EDIT", "VALUES");
 522
 523                 /* Iterate over all properties */
 524                 (void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
 525                     ZFS_TYPE_POOL);
 526
 527                 (void) fprintf(fp, "\t%-19s   ", "feature@...");
 528                 (void) fprintf(fp, "YES   disabled | enabled | active\n");
 529
 530                 (void) fprintf(fp, gettext("\nThe feature@ properties must be "
 531                     "appended with a feature name.\nSee zpool-features(5).\n"));
 532         }
 533
 534         /*
 535          * See comments at end of main().
 536          */
 537         if (getenv("ZFS_ABORT") != NULL) {
 538                 (void) printf("dumping core by request\n");
 539                 abort();
 540         }
 541
 542         exit(requested ? 0 : 2);
 543 }
 544
 545 /*
 546  * zpool initialize [-c | -s] [-w] <pool> [<vdev> ...]
 547  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
 548  * if none specified.
 549  *
 550  *      -c      Cancel. Ends active initializing.
 551  *      -s      Suspend. Initializing can then be restarted with no flags.
 552  *      -w      Wait. Blocks until initializing has completed.
 553  */
 554 int
 555 zpool_do_initialize(int argc, char **argv)
 556 {
 557         int c;
 558         char *poolname;
 559         zpool_handle_t *zhp;
 560         nvlist_t *vdevs;
 561         int err = 0;
 562         boolean_t wait = B_FALSE;
 563
 564         struct option long_options[] = {
 565                 {"cancel",      no_argument,            NULL, 'c'},
 566                 {"suspend",     no_argument,            NULL, 's'},
 567                 {"wait",        no_argument,            NULL, 'w'},
 568                 {0, 0, 0, 0}
 569         };
 570
 571         pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
 572         while ((c = getopt_long(argc, argv, "csw", long_options, NULL)) != -1) {
 573                 switch (c) {
 574                 case 'c':
 575                         if (cmd_type != POOL_INITIALIZE_START &&
 576                             cmd_type != POOL_INITIALIZE_CANCEL) {
 577                                 (void) fprintf(stderr, gettext("-c cannot be "
 578                                     "combined with other options\n"));
 579                                 usage(B_FALSE);
 580                         }
 581                         cmd_type = POOL_INITIALIZE_CANCEL;
 582                         break;
 583                 case 's':
 584                         if (cmd_type != POOL_INITIALIZE_START &&
 585                             cmd_type != POOL_INITIALIZE_SUSPEND) {
 586                                 (void) fprintf(stderr, gettext("-s cannot be "
 587                                     "combined with other options\n"));
 588                                 usage(B_FALSE);
 589                         }
 590                         cmd_type = POOL_INITIALIZE_SUSPEND;
 591                         break;
 592                 case 'w':
 593                         wait = B_TRUE;
 594                         break;
 595                 case '?':
 596                         if (optopt != 0) {
 597                                 (void) fprintf(stderr,
 598                                     gettext("invalid option '%c'\n"), optopt);
 599                         } else {
 600                                 (void) fprintf(stderr,
 601                                     gettext("invalid option '%s'\n"),
 602                                     argv[optind - 1]);
 603                         }
 604                         usage(B_FALSE);
 605                 }
 606         }
 607
 608         argc -= optind;
 609         argv += optind;
 610
 611         if (argc < 1) {
 612                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 613                 usage(B_FALSE);
 614                 return (-1);
 615         }
 616
 617         if (wait && (cmd_type != POOL_INITIALIZE_START)) {
 618                 (void) fprintf(stderr, gettext("-w cannot be used with -c or "
 619                     "-s\n"));
 620                 usage(B_FALSE);
 621         }
 622
 623         poolname = argv[0];
 624         zhp = zpool_open(g_zfs, poolname);
 625         if (zhp == NULL)
 626                 return (-1);
 627
 628         vdevs = fnvlist_alloc();
 629         if (argc == 1) {
 630                 /* no individual leaf vdevs specified, so add them all */
 631                 nvlist_t *config = zpool_get_config(zhp, NULL);
 632                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
 633                     ZPOOL_CONFIG_VDEV_TREE);
 634                 zpool_collect_leaves(zhp, nvroot, vdevs);
 635         } else {
 636                 for (int i = 1; i < argc; i++) {
 637                         fnvlist_add_boolean(vdevs, argv[i]);
 638                 }
 639         }
 640
 641         if (wait)
 642                 err = zpool_initialize_wait(zhp, cmd_type, vdevs);
 643         else
 644                 err = zpool_initialize(zhp, cmd_type, vdevs);
 645
 646         fnvlist_free(vdevs);
 647         zpool_close(zhp);
 648
 649         return (err);
 650 }
 651
 652 /*
 653  * print a pool vdev config for dry runs
 654  */
 655 static void
 656 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 657     const char *match, int name_flags)
 658 {
 659         nvlist_t **child;
 660         uint_t c, children;
 661         char *vname;
 662         boolean_t printed = B_FALSE;
 663
 664         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 665             &child, &children) != 0) {
 666                 if (name != NULL)
 667                         (void) printf("\t%*s%s\n", indent, "", name);
 668                 return;
 669         }
 670
 671         for (c = 0; c < children; c++) {
 672                 uint64_t is_log = B_FALSE;
 673                 char *class = "";
 674
 675                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 676                     &is_log);
 677                 if (is_log)
 678                         class = VDEV_ALLOC_BIAS_LOG;
 679                 (void) nvlist_lookup_string(child[c],
 680                     ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
 681                 if (strcmp(match, class) != 0)
 682                         continue;
 683
 684                 if (!printed && name != NULL) {
 685                         (void) printf("\t%*s%s\n", indent, "", name);
 686                         printed = B_TRUE;
 687                 }
 688                 vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
 689                 print_vdev_tree(zhp, vname, child[c], indent + 2, "",
 690                     name_flags);
 691                 free(vname);
 692         }
 693 }
 694
 695 static boolean_t
 696 prop_list_contains_feature(nvlist_t *proplist)
 697 {
 698         nvpair_t *nvp;
 699         for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
 700             nvp = nvlist_next_nvpair(proplist, nvp)) {
 701                 if (zpool_prop_feature(nvpair_name(nvp)))
 702                         return (B_TRUE);
 703         }
 704         return (B_FALSE);
 705 }
 706
 707 /*
 708  * Add a property pair (name, string-value) into a property nvlist.
 709  */
 710 static int
 711 add_prop_list(const char *propname, char *propval, nvlist_t **props,
 712     boolean_t poolprop)
 713 {
 714         zpool_prop_t prop = ZPOOL_PROP_INVAL;
 715         nvlist_t *proplist;
 716         const char *normnm;
 717         char *strval;
 718
 719         if (*props == NULL &&
 720             nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
 721                 (void) fprintf(stderr,
 722                     gettext("internal error: out of memory\n"));
 723                 return (1);
 724         }
 725
 726         proplist = *props;
 727
 728         if (poolprop) {
 729                 const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 730
 731                 if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 732                     !zpool_prop_feature(propname)) {
 733                         (void) fprintf(stderr, gettext("property '%s' is "
 734                             "not a valid pool property\n"), propname);
 735                         return (2);
 736                 }
 737
 738                 /*
 739                  * feature@ properties and version should not be specified
 740                  * at the same time.
 741                  */
 742                 if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
 743                     nvlist_exists(proplist, vname)) ||
 744                     (prop == ZPOOL_PROP_VERSION &&
 745                     prop_list_contains_feature(proplist))) {
 746                         (void) fprintf(stderr, gettext("'feature@' and "
 747                             "'version' properties cannot be specified "
 748                             "together\n"));
 749                         return (2);
 750                 }
 751
 752
 753                 if (zpool_prop_feature(propname))
 754                         normnm = propname;
 755                 else
 756                         normnm = zpool_prop_to_name(prop);
 757         } else {
 758                 zfs_prop_t fsprop = zfs_name_to_prop(propname);
 759
 760                 if (zfs_prop_valid_for_type(fsprop, ZFS_TYPE_FILESYSTEM,
 761                     B_FALSE)) {
 762                         normnm = zfs_prop_to_name(fsprop);
 763                 } else if (zfs_prop_user(propname) ||
 764                     zfs_prop_userquota(propname)) {
 765                         normnm = propname;
 766                 } else {
 767                         (void) fprintf(stderr, gettext("property '%s' is "
 768                             "not a valid filesystem property\n"), propname);
 769                         return (2);
 770                 }
 771         }
 772
 773         if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
 774             prop != ZPOOL_PROP_CACHEFILE) {
 775                 (void) fprintf(stderr, gettext("property '%s' "
 776                     "specified multiple times\n"), propname);
 777                 return (2);
 778         }
 779
 780         if (nvlist_add_string(proplist, normnm, propval) != 0) {
 781                 (void) fprintf(stderr, gettext("internal "
 782                     "error: out of memory\n"));
 783                 return (1);
 784         }
 785
 786         return (0);
 787 }
 788
 789 /*
 790  * Set a default property pair (name, string-value) in a property nvlist
 791  */
 792 static int
 793 add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
 794     boolean_t poolprop)
 795 {
 796         char *pval;
 797
 798         if (nvlist_lookup_string(*props, propname, &pval) == 0)
 799                 return (0);
 800
 801         return (add_prop_list(propname, propval, props, B_TRUE));
 802 }
 803
 804 /*
 805  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
 806  *
 807  *      -f      Force addition of devices, even if they appear in use
 808  *      -g      Display guid for individual vdev name.
 809  *      -L      Follow links when resolving vdev path name.
 810  *      -n      Do not add the devices, but display the resulting layout if
 811  *              they were to be added.
 812  *      -o      Set property=value.
 813  *      -P      Display full path for vdev name.
 814  *
 815  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
 816  * handled by make_root_vdev(), which constructs the nvlist needed to pass to
 817  * libzfs.
 818  */
 819 int
 820 zpool_do_add(int argc, char **argv)
 821 {
 822         boolean_t force = B_FALSE;
 823         boolean_t dryrun = B_FALSE;
 824         int name_flags = 0;
 825         int c;
 826         nvlist_t *nvroot;
 827         char *poolname;
 828         int ret;
 829         zpool_handle_t *zhp;
 830         nvlist_t *config;
 831         nvlist_t *props = NULL;
 832         char *propval;
 833
 834         /* check options */
 835         while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
 836                 switch (c) {
 837                 case 'f':
 838                         force = B_TRUE;
 839                         break;
 840                 case 'g':
 841                         name_flags |= VDEV_NAME_GUID;
 842                         break;
 843                 case 'L':
 844                         name_flags |= VDEV_NAME_FOLLOW_LINKS;
 845                         break;
 846                 case 'n':
 847                         dryrun = B_TRUE;
 848                         break;
 849                 case 'o':
 850                         if ((propval = strchr(optarg, '=')) == NULL) {
 851                                 (void) fprintf(stderr, gettext("missing "
 852                                     "'=' for -o option\n"));
 853                                 usage(B_FALSE);
 854                         }
 855                         *propval = '\0';
 856                         propval++;
 857
 858                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
 859                             (add_prop_list(optarg, propval, &props, B_TRUE)))
 860                                 usage(B_FALSE);
 861                         break;
 862                 case 'P':
 863                         name_flags |= VDEV_NAME_PATH;
 864                         break;
 865                 case '?':
 866                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 867                             optopt);
 868                         usage(B_FALSE);
 869                 }
 870         }
 871
 872         argc -= optind;
 873         argv += optind;
 874
 875         /* get pool name and check number of arguments */
 876         if (argc < 1) {
 877                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 878                 usage(B_FALSE);
 879         }
 880         if (argc < 2) {
 881                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
 882                 usage(B_FALSE);
 883         }
 884
 885         poolname = argv[0];
 886
 887         argc--;
 888         argv++;
 889
 890         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 891                 return (1);
 892
 893         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
 894                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
 895                     poolname);
 896                 zpool_close(zhp);
 897                 return (1);
 898         }
 899
 900         /* unless manually specified use "ashift" pool property (if set) */
 901         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
 902                 int intval;
 903                 zprop_source_t src;
 904                 char strval[ZPOOL_MAXPROPLEN];
 905
 906                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
 907                 if (src != ZPROP_SRC_DEFAULT) {
 908                         (void) sprintf(strval, "%" PRId32, intval);
 909                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
 910                             &props, B_TRUE) == 0);
 911                 }
 912         }
 913
 914         /* pass off to make_root_vdev for processing */
 915         nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
 916             argc, argv);
 917         if (nvroot == NULL) {
 918                 zpool_close(zhp);
 919                 return (1);
 920         }
 921
 922         if (dryrun) {
 923                 nvlist_t *poolnvroot;
 924                 nvlist_t **l2child;
 925                 uint_t l2children, c;
 926                 char *vname;
 927                 boolean_t hadcache = B_FALSE;
 928
 929                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
 930                     &poolnvroot) == 0);
 931
 932                 (void) printf(gettext("would update '%s' to the following "
 933                     "configuration:\n"), zpool_get_name(zhp));
 934
 935                 /* print original main pool and new tree */
 936                 print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
 937                     name_flags | VDEV_NAME_TYPE_ID);
 938                 print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
 939
 940                 /* print other classes: 'dedup', 'special', and 'log' */
 941                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_DEDUP)) {
 942                         print_vdev_tree(zhp, "dedup", poolnvroot, 0,
 943                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
 944                         print_vdev_tree(zhp, NULL, nvroot, 0,
 945                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
 946                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_DEDUP)) {
 947                         print_vdev_tree(zhp, "dedup", nvroot, 0,
 948                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
 949                 }
 950
 951                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
 952                         print_vdev_tree(zhp, "special", poolnvroot, 0,
 953                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
 954                         print_vdev_tree(zhp, NULL, nvroot, 0,
 955                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
 956                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
 957                         print_vdev_tree(zhp, "special", nvroot, 0,
 958                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
 959                 }
 960
 961                 if (num_logs(poolnvroot) > 0) {
 962                         print_vdev_tree(zhp, "logs", poolnvroot, 0,
 963                             VDEV_ALLOC_BIAS_LOG, name_flags);
 964                         print_vdev_tree(zhp, NULL, nvroot, 0,
 965                             VDEV_ALLOC_BIAS_LOG, name_flags);
 966                 } else if (num_logs(nvroot) > 0) {
 967                         print_vdev_tree(zhp, "logs", nvroot, 0,
 968                             VDEV_ALLOC_BIAS_LOG, name_flags);
 969                 }
 970
 971                 /* Do the same for the caches */
 972                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
 973                     &l2child, &l2children) == 0 && l2children) {
 974                         hadcache = B_TRUE;
 975                         (void) printf(gettext("\tcache\n"));
 976                         for (c = 0; c < l2children; c++) {
 977                                 vname = zpool_vdev_name(g_zfs, NULL,
 978                                     l2child[c], name_flags);
 979                                 (void) printf("\t  %s\n", vname);
 980                                 free(vname);
 981                         }
 982                 }
 983                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
 984                     &l2child, &l2children) == 0 && l2children) {
 985                         if (!hadcache)
 986                                 (void) printf(gettext("\tcache\n"));
 987                         for (c = 0; c < l2children; c++) {
 988                                 vname = zpool_vdev_name(g_zfs, NULL,
 989                                     l2child[c], name_flags);
 990                                 (void) printf("\t  %s\n", vname);
 991                                 free(vname);
 992                         }
 993                 }
 994
 995                 ret = 0;
 996         } else {
 997                 ret = (zpool_add(zhp, nvroot) != 0);
 998         }
 999
1000         nvlist_free(props);
1001         nvlist_free(nvroot);
1002         zpool_close(zhp);
1003
1004         return (ret);
1005 }
1006
1007 /*
1008  * zpool remove [-npsw] <pool> <vdev> ...
1009  *
1010  * Removes the given vdev from the pool.
1011  */
1012 int
1013 zpool_do_remove(int argc, char **argv)
1014 {
1015         char *poolname;
1016         int i, ret = 0;
1017         zpool_handle_t *zhp = NULL;
1018         boolean_t stop = B_FALSE;
1019         int c;
1020         boolean_t noop = B_FALSE;
1021         boolean_t parsable = B_FALSE;
1022         boolean_t wait = B_FALSE;
1023
1024         /* check options */
1025         while ((c = getopt(argc, argv, "npsw")) != -1) {
1026                 switch (c) {
1027                 case 'n':
1028                         noop = B_TRUE;
1029                         break;
1030                 case 'p':
1031                         parsable = B_TRUE;
1032                         break;
1033                 case 's':
1034                         stop = B_TRUE;
1035                         break;
1036                 case 'w':
1037                         wait = B_TRUE;
1038                         break;
1039                 case '?':
1040                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1041                             optopt);
1042                         usage(B_FALSE);
1043                 }
1044         }
1045
1046         argc -= optind;
1047         argv += optind;
1048
1049         /* get pool name and check number of arguments */
1050         if (argc < 1) {
1051                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1052                 usage(B_FALSE);
1053         }
1054
1055         poolname = argv[0];
1056
1057         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1058                 return (1);
1059
1060         if (stop && noop) {
1061                 (void) fprintf(stderr, gettext("stop request ignored\n"));
1062                 return (0);
1063         }
1064
1065         if (stop) {
1066                 if (argc > 1) {
1067                         (void) fprintf(stderr, gettext("too many arguments\n"));
1068                         usage(B_FALSE);
1069                 }
1070                 if (zpool_vdev_remove_cancel(zhp) != 0)
1071                         ret = 1;
1072                 if (wait) {
1073                         (void) fprintf(stderr, gettext("invalid option "
1074                             "combination: -w cannot be used with -s\n"));
1075                         usage(B_FALSE);
1076                 }
1077         } else {
1078                 if (argc < 2) {
1079                         (void) fprintf(stderr, gettext("missing device\n"));
1080                         usage(B_FALSE);
1081                 }
1082
1083                 for (i = 1; i < argc; i++) {
1084                         if (noop) {
1085                                 uint64_t size;
1086
1087                                 if (zpool_vdev_indirect_size(zhp, argv[i],
1088                                     &size) != 0) {
1089                                         ret = 1;
1090                                         break;
1091                                 }
1092                                 if (parsable) {
1093                                         (void) printf("%s %llu\n",
1094                                             argv[i], (unsigned long long)size);
1095                                 } else {
1096                                         char valstr[32];
1097                                         zfs_nicenum(size, valstr,
1098                                             sizeof (valstr));
1099                                         (void) printf("Memory that will be "
1100                                             "used after removing %s: %s\n",
1101                                             argv[i], valstr);
1102                                 }
1103                         } else {
1104                                 if (zpool_vdev_remove(zhp, argv[i]) != 0)
1105                                         ret = 1;
1106                         }
1107                 }
1108
1109                 if (ret == 0 && wait)
1110                         ret = zpool_wait(zhp, ZPOOL_WAIT_REMOVE);
1111         }
1112         zpool_close(zhp);
1113
1114         return (ret);
1115 }
1116
1117 /*
1118  * zpool labelclear [-f] <vdev>
1119  *
1120  *      -f      Force clearing the label for the vdevs which are members of
1121  *              the exported or foreign pools.
1122  *
1123  * Verifies that the vdev is not active and zeros out the label information
1124  * on the device.
1125  */
1126 int
1127 zpool_do_labelclear(int argc, char **argv)
1128 {
1129         char vdev[MAXPATHLEN];
1130         char *name = NULL;
1131         struct stat st;
1132         int c, fd = -1, ret = 0;
1133         nvlist_t *config;
1134         pool_state_t state;
1135         boolean_t inuse = B_FALSE;
1136         boolean_t force = B_FALSE;
1137
1138         /* check options */
1139         while ((c = getopt(argc, argv, "f")) != -1) {
1140                 switch (c) {
1141                 case 'f':
1142                         force = B_TRUE;
1143                         break;
1144                 default:
1145                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1146                             optopt);
1147                         usage(B_FALSE);
1148                 }
1149         }
1150
1151         argc -= optind;
1152         argv += optind;
1153
1154         /* get vdev name */
1155         if (argc < 1) {
1156                 (void) fprintf(stderr, gettext("missing vdev name\n"));
1157                 usage(B_FALSE);
1158         }
1159         if (argc > 1) {
1160                 (void) fprintf(stderr, gettext("too many arguments\n"));
1161                 usage(B_FALSE);
1162         }
1163
1164         /*
1165          * Check if we were given absolute path and use it as is.
1166          * Otherwise if the provided vdev name doesn't point to a file,
1167          * try prepending expected disk paths and partition numbers.
1168          */
1169         (void) strlcpy(vdev, argv[0], sizeof (vdev));
1170         if (vdev[0] != '/' && stat(vdev, &st) != 0) {
1171                 int error;
1172
1173                 error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
1174                 if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
1175                         if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
1176                                 error = ENOENT;
1177                 }
1178
1179                 if (error || (stat(vdev, &st) != 0)) {
1180                         (void) fprintf(stderr, gettext(
1181                             "failed to find device %s, try specifying absolute "
1182                             "path instead\n"), argv[0]);
1183                         return (1);
1184                 }
1185         }
1186
1187         if ((fd = open(vdev, O_RDWR)) < 0) {
1188                 (void) fprintf(stderr, gettext("failed to open %s: %s\n"),
1189                     vdev, strerror(errno));
1190                 return (1);
1191         }
1192
1193         /*
1194          * Flush all dirty pages for the block device.  This should not be
1195          * fatal when the device does not support BLKFLSBUF as would be the
1196          * case for a file vdev.
1197          */
1198         if ((zfs_dev_flush(fd) != 0) && (errno != ENOTTY))
1199                 (void) fprintf(stderr, gettext("failed to invalidate "
1200                     "cache for %s: %s\n"), vdev, strerror(errno));
1201
1202         if (zpool_read_label(fd, &config, NULL) != 0) {
1203                 (void) fprintf(stderr,
1204                     gettext("failed to read label from %s\n"), vdev);
1205                 ret = 1;
1206                 goto errout;
1207         }
1208         nvlist_free(config);
1209
1210         ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
1211         if (ret != 0) {
1212                 (void) fprintf(stderr,
1213                     gettext("failed to check state for %s\n"), vdev);
1214                 ret = 1;
1215                 goto errout;
1216         }
1217
1218         if (!inuse)
1219                 goto wipe_label;
1220
1221         switch (state) {
1222         default:
1223         case POOL_STATE_ACTIVE:
1224         case POOL_STATE_SPARE:
1225         case POOL_STATE_L2CACHE:
1226                 (void) fprintf(stderr, gettext(
1227                     "%s is a member (%s) of pool \"%s\"\n"),
1228                     vdev, zpool_pool_state_to_name(state), name);
1229                 ret = 1;
1230                 goto errout;
1231
1232         case POOL_STATE_EXPORTED:
1233                 if (force)
1234                         break;
1235                 (void) fprintf(stderr, gettext(
1236                     "use '-f' to override the following error:\n"
1237                     "%s is a member of exported pool \"%s\"\n"),
1238                     vdev, name);
1239                 ret = 1;
1240                 goto errout;
1241
1242         case POOL_STATE_POTENTIALLY_ACTIVE:
1243                 if (force)
1244                         break;
1245                 (void) fprintf(stderr, gettext(
1246                     "use '-f' to override the following error:\n"
1247                     "%s is a member of potentially active pool \"%s\"\n"),
1248                     vdev, name);
1249                 ret = 1;
1250                 goto errout;
1251
1252         case POOL_STATE_DESTROYED:
1253                 /* inuse should never be set for a destroyed pool */
1254                 assert(0);
1255                 break;
1256         }
1257
1258 wipe_label:
1259         ret = zpool_clear_label(fd);
1260         if (ret != 0) {
1261                 (void) fprintf(stderr,
1262                     gettext("failed to clear label for %s\n"), vdev);
1263         }
1264
1265 errout:
1266         free(name);
1267         (void) close(fd);
1268
1269         return (ret);
1270 }
1271
1272 /*
1273  * zpool create [-fnd] [-o property=value] ...
1274  *              [-O file-system-property=value] ...
1275  *              [-R root] [-m mountpoint] <pool> <dev> ...
1276  *
1277  *      -f      Force creation, even if devices appear in use
1278  *      -n      Do not create the pool, but display the resulting layout if it
1279  *              were to be created.
1280  *      -R      Create a pool under an alternate root
1281  *      -m      Set default mountpoint for the root dataset.  By default it's
1282  *              '/<pool>'
1283  *      -o      Set property=value.
1284  *      -o      Set feature@feature=enabled|disabled.
1285  *      -d      Don't automatically enable all supported pool features
1286  *              (individual features can be enabled with -o).
1287  *      -O      Set fsproperty=value in the pool's root file system
1288  *
1289  * Creates the named pool according to the given vdev specification.  The
1290  * bulk of the vdev processing is done in make_root_vdev() in zpool_vdev.c.
1291  * Once we get the nvlist back from make_root_vdev(), we either print out the
1292  * contents (if '-n' was specified), or pass it to libzfs to do the creation.
1293  */
1294 int
1295 zpool_do_create(int argc, char **argv)
1296 {
1297         boolean_t force = B_FALSE;
1298         boolean_t dryrun = B_FALSE;
1299         boolean_t enable_all_pool_feat = B_TRUE;
1300         int c;
1301         nvlist_t *nvroot = NULL;
1302         char *poolname;
1303         char *tname = NULL;
1304         int ret = 1;
1305         char *altroot = NULL;
1306         char *mountpoint = NULL;
1307         nvlist_t *fsprops = NULL;
1308         nvlist_t *props = NULL;
1309         char *propval;
1310
1311         /* check options */
1312         while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1313                 switch (c) {
1314                 case 'f':
1315                         force = B_TRUE;
1316                         break;
1317                 case 'n':
1318                         dryrun = B_TRUE;
1319                         break;
1320                 case 'd':
1321                         enable_all_pool_feat = B_FALSE;
1322                         break;
1323                 case 'R':
1324                         altroot = optarg;
1325                         if (add_prop_list(zpool_prop_to_name(
1326                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1327                                 goto errout;
1328                         if (add_prop_list_default(zpool_prop_to_name(
1329                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1330                                 goto errout;
1331                         break;
1332                 case 'm':
1333                         /* Equivalent to -O mountpoint=optarg */
1334                         mountpoint = optarg;
1335                         break;
1336                 case 'o':
1337                         if ((propval = strchr(optarg, '=')) == NULL) {
1338                                 (void) fprintf(stderr, gettext("missing "
1339                                     "'=' for -o option\n"));
1340                                 goto errout;
1341                         }
1342                         *propval = '\0';
1343                         propval++;
1344
1345                         if (add_prop_list(optarg, propval, &props, B_TRUE))
1346                                 goto errout;
1347
1348                         /*
1349                          * If the user is creating a pool that doesn't support
1350                          * feature flags, don't enable any features.
1351                          */
1352                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1353                                 char *end;
1354                                 u_longlong_t ver;
1355
1356                                 ver = strtoull(propval, &end, 10);
1357                                 if (*end == '\0' &&
1358                                     ver < SPA_VERSION_FEATURES) {
1359                                         enable_all_pool_feat = B_FALSE;
1360                                 }
1361                         }
1362                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1363                                 altroot = propval;
1364                         break;
1365                 case 'O':
1366                         if ((propval = strchr(optarg, '=')) == NULL) {
1367                                 (void) fprintf(stderr, gettext("missing "
1368                                     "'=' for -O option\n"));
1369                                 goto errout;
1370                         }
1371                         *propval = '\0';
1372                         propval++;
1373
1374                         /*
1375                          * Mountpoints are checked and then added later.
1376                          * Uniquely among properties, they can be specified
1377                          * more than once, to avoid conflict with -m.
1378                          */
1379                         if (0 == strcmp(optarg,
1380                             zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1381                                 mountpoint = propval;
1382                         } else if (add_prop_list(optarg, propval, &fsprops,
1383                             B_FALSE)) {
1384                                 goto errout;
1385                         }
1386                         break;
1387                 case 't':
1388                         /*
1389                          * Sanity check temporary pool name.
1390                          */
1391                         if (strchr(optarg, '/') != NULL) {
1392                                 (void) fprintf(stderr, gettext("cannot create "
1393                                     "'%s': invalid character '/' in temporary "
1394                                     "name\n"), optarg);
1395                                 (void) fprintf(stderr, gettext("use 'zfs "
1396                                     "create' to create a dataset\n"));
1397                                 goto errout;
1398                         }
1399
1400                         if (add_prop_list(zpool_prop_to_name(
1401                             ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1402                                 goto errout;
1403                         if (add_prop_list_default(zpool_prop_to_name(
1404                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1405                                 goto errout;
1406                         tname = optarg;
1407                         break;
1408                 case ':':
1409                         (void) fprintf(stderr, gettext("missing argument for "
1410                             "'%c' option\n"), optopt);
1411                         goto badusage;
1412                 case '?':
1413                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1414                             optopt);
1415                         goto badusage;
1416                 }
1417         }
1418
1419         argc -= optind;
1420         argv += optind;
1421
1422         /* get pool name and check number of arguments */
1423         if (argc < 1) {
1424                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1425                 goto badusage;
1426         }
1427         if (argc < 2) {
1428                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1429                 goto badusage;
1430         }
1431
1432         poolname = argv[0];
1433
1434         /*
1435          * As a special case, check for use of '/' in the name, and direct the
1436          * user to use 'zfs create' instead.
1437          */
1438         if (strchr(poolname, '/') != NULL) {
1439                 (void) fprintf(stderr, gettext("cannot create '%s': invalid "
1440                     "character '/' in pool name\n"), poolname);
1441                 (void) fprintf(stderr, gettext("use 'zfs create' to "
1442                     "create a dataset\n"));
1443                 goto errout;
1444         }
1445
1446         /* pass off to make_root_vdev for bulk processing */
1447         nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1448             argc - 1, argv + 1);
1449         if (nvroot == NULL)
1450                 goto errout;
1451
1452         /* make_root_vdev() allows 0 toplevel children if there are spares */
1453         if (!zfs_allocatable_devs(nvroot)) {
1454                 (void) fprintf(stderr, gettext("invalid vdev "
1455                     "specification: at least one toplevel vdev must be "
1456                     "specified\n"));
1457                 goto errout;
1458         }
1459
1460         if (altroot != NULL && altroot[0] != '/') {
1461                 (void) fprintf(stderr, gettext("invalid alternate root '%s': "
1462                     "must be an absolute path\n"), altroot);
1463                 goto errout;
1464         }
1465
1466         /*
1467          * Check the validity of the mountpoint and direct the user to use the
1468          * '-m' mountpoint option if it looks like its in use.
1469          */
1470         if (mountpoint == NULL ||
1471             (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1472             strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1473                 char buf[MAXPATHLEN];
1474                 DIR *dirp;
1475
1476                 if (mountpoint && mountpoint[0] != '/') {
1477                         (void) fprintf(stderr, gettext("invalid mountpoint "
1478                             "'%s': must be an absolute path, 'legacy', or "
1479                             "'none'\n"), mountpoint);
1480                         goto errout;
1481                 }
1482
1483                 if (mountpoint == NULL) {
1484                         if (altroot != NULL)
1485                                 (void) snprintf(buf, sizeof (buf), "%s/%s",
1486                                     altroot, poolname);
1487                         else
1488                                 (void) snprintf(buf, sizeof (buf), "/%s",
1489                                     poolname);
1490                 } else {
1491                         if (altroot != NULL)
1492                                 (void) snprintf(buf, sizeof (buf), "%s%s",
1493                                     altroot, mountpoint);
1494                         else
1495                                 (void) snprintf(buf, sizeof (buf), "%s",
1496                                     mountpoint);
1497                 }
1498
1499                 if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1500                         (void) fprintf(stderr, gettext("mountpoint '%s' : "
1501                             "%s\n"), buf, strerror(errno));
1502                         (void) fprintf(stderr, gettext("use '-m' "
1503                             "option to provide a different default\n"));
1504                         goto errout;
1505                 } else if (dirp) {
1506                         int count = 0;
1507
1508                         while (count < 3 && readdir(dirp) != NULL)
1509                                 count++;
1510                         (void) closedir(dirp);
1511
1512                         if (count > 2) {
1513                                 (void) fprintf(stderr, gettext("mountpoint "
1514                                     "'%s' exists and is not empty\n"), buf);
1515                                 (void) fprintf(stderr, gettext("use '-m' "
1516                                     "option to provide a "
1517                                     "different default\n"));
1518                                 goto errout;
1519                         }
1520                 }
1521         }
1522
1523         /*
1524          * Now that the mountpoint's validity has been checked, ensure that
1525          * the property is set appropriately prior to creating the pool.
1526          */
1527         if (mountpoint != NULL) {
1528                 ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1529                     mountpoint, &fsprops, B_FALSE);
1530                 if (ret != 0)
1531                         goto errout;
1532         }
1533
1534         ret = 1;
1535         if (dryrun) {
1536                 /*
1537                  * For a dry run invocation, print out a basic message and run
1538                  * through all the vdevs in the list and print out in an
1539                  * appropriate hierarchy.
1540                  */
1541                 (void) printf(gettext("would create '%s' with the "
1542                     "following layout:\n\n"), poolname);
1543
1544                 print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1545                 print_vdev_tree(NULL, "dedup", nvroot, 0,
1546                     VDEV_ALLOC_BIAS_DEDUP, 0);
1547                 print_vdev_tree(NULL, "special", nvroot, 0,
1548                     VDEV_ALLOC_BIAS_SPECIAL, 0);
1549                 print_vdev_tree(NULL, "logs", nvroot, 0,
1550                     VDEV_ALLOC_BIAS_LOG, 0);
1551
1552                 ret = 0;
1553         } else {
1554                 /*
1555                  * Hand off to libzfs.
1556                  */
1557                 spa_feature_t i;
1558                 for (i = 0; i < SPA_FEATURES; i++) {
1559                         char propname[MAXPATHLEN];
1560                         char *propval;
1561                         zfeature_info_t *feat = &spa_feature_table[i];
1562
1563                         (void) snprintf(propname, sizeof (propname),
1564                             "feature@%s", feat->fi_uname);
1565
1566                         /*
1567                          * Only features contained in props will be enabled:
1568                          * remove from the nvlist every ZFS_FEATURE_DISABLED
1569                          * value and add every missing ZFS_FEATURE_ENABLED if
1570                          * enable_all_pool_feat is set.
1571                          */
1572                         if (!nvlist_lookup_string(props, propname, &propval)) {
1573                                 if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
1574                                         (void) nvlist_remove_all(props,
1575                                             propname);
1576                         } else if (enable_all_pool_feat) {
1577                                 ret = add_prop_list(propname,
1578                                     ZFS_FEATURE_ENABLED, &props, B_TRUE);
1579                                 if (ret != 0)
1580                                         goto errout;
1581                         }
1582                 }
1583
1584                 ret = 1;
1585                 if (zpool_create(g_zfs, poolname,
1586                     nvroot, props, fsprops) == 0) {
1587                         zfs_handle_t *pool = zfs_open(g_zfs,
1588                             tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1589                         if (pool != NULL) {
1590                                 if (zfs_mount(pool, NULL, 0) == 0) {
1591                                         ret = zfs_shareall(pool);
1592                                         zfs_commit_all_shares();
1593                                 }
1594                                 zfs_close(pool);
1595                         }
1596                 } else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1597                         (void) fprintf(stderr, gettext("pool name may have "
1598                             "been omitted\n"));
1599                 }
1600         }
1601
1602 errout:
1603         nvlist_free(nvroot);
1604         nvlist_free(fsprops);
1605         nvlist_free(props);
1606         return (ret);
1607 badusage:
1608         nvlist_free(fsprops);
1609         nvlist_free(props);
1610         usage(B_FALSE);
1611         return (2);
1612 }
1613
1614 /*
1615  * zpool destroy <pool>
1616  *
1617  *      -f      Forcefully unmount any datasets
1618  *
1619  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1620  */
1621 int
1622 zpool_do_destroy(int argc, char **argv)
1623 {
1624         boolean_t force = B_FALSE;
1625         int c;
1626         char *pool;
1627         zpool_handle_t *zhp;
1628         int ret;
1629
1630         /* check options */
1631         while ((c = getopt(argc, argv, "f")) != -1) {
1632                 switch (c) {
1633                 case 'f':
1634                         force = B_TRUE;
1635                         break;
1636                 case '?':
1637                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1638                             optopt);
1639                         usage(B_FALSE);
1640                 }
1641         }
1642
1643         argc -= optind;
1644         argv += optind;
1645
1646         /* check arguments */
1647         if (argc < 1) {
1648                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1649                 usage(B_FALSE);
1650         }
1651         if (argc > 1) {
1652                 (void) fprintf(stderr, gettext("too many arguments\n"));
1653                 usage(B_FALSE);
1654         }
1655
1656         pool = argv[0];
1657
1658         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1659                 /*
1660                  * As a special case, check for use of '/' in the name, and
1661                  * direct the user to use 'zfs destroy' instead.
1662                  */
1663                 if (strchr(pool, '/') != NULL)
1664                         (void) fprintf(stderr, gettext("use 'zfs destroy' to "
1665                             "destroy a dataset\n"));
1666                 return (1);
1667         }
1668
1669         if (zpool_disable_datasets(zhp, force) != 0) {
1670                 (void) fprintf(stderr, gettext("could not destroy '%s': "
1671                     "could not unmount datasets\n"), zpool_get_name(zhp));
1672                 zpool_close(zhp);
1673                 return (1);
1674         }
1675
1676         /* The history must be logged as part of the export */
1677         log_history = B_FALSE;
1678
1679         ret = (zpool_destroy(zhp, history_str) != 0);
1680
1681         zpool_close(zhp);
1682
1683         return (ret);
1684 }
1685
1686 typedef struct export_cbdata {
1687         boolean_t force;
1688         boolean_t hardforce;
1689 } export_cbdata_t;
1690
1691 /*
1692  * Export one pool
1693  */
1694 static int
1695 zpool_export_one(zpool_handle_t *zhp, void *data)
1696 {
1697         export_cbdata_t *cb = data;
1698
1699         if (zpool_disable_datasets(zhp, cb->force) != 0)
1700                 return (1);
1701
1702         /* The history must be logged as part of the export */
1703         log_history = B_FALSE;
1704
1705         if (cb->hardforce) {
1706                 if (zpool_export_force(zhp, history_str) != 0)
1707                         return (1);
1708         } else if (zpool_export(zhp, cb->force, history_str) != 0) {
1709                 return (1);
1710         }
1711
1712         return (0);
1713 }
1714
1715 /*
1716  * zpool export [-f] <pool> ...
1717  *
1718  *      -a      Export all pools
1719  *      -f      Forcefully unmount datasets
1720  *
1721  * Export the given pools.  By default, the command will attempt to cleanly
1722  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1723  * then the datasets will be forcefully unmounted.
1724  */
1725 int
1726 zpool_do_export(int argc, char **argv)
1727 {
1728         export_cbdata_t cb;
1729         boolean_t do_all = B_FALSE;
1730         boolean_t force = B_FALSE;
1731         boolean_t hardforce = B_FALSE;
1732         int c, ret;
1733
1734         /* check options */
1735         while ((c = getopt(argc, argv, "afF")) != -1) {
1736                 switch (c) {
1737                 case 'a':
1738                         do_all = B_TRUE;
1739                         break;
1740                 case 'f':
1741                         force = B_TRUE;
1742                         break;
1743                 case 'F':
1744                         hardforce = B_TRUE;
1745                         break;
1746                 case '?':
1747                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1748                             optopt);
1749                         usage(B_FALSE);
1750                 }
1751         }
1752
1753         cb.force = force;
1754         cb.hardforce = hardforce;
1755         argc -= optind;
1756         argv += optind;
1757
1758         if (do_all) {
1759                 if (argc != 0) {
1760                         (void) fprintf(stderr, gettext("too many arguments\n"));
1761                         usage(B_FALSE);
1762                 }
1763
1764                 return (for_each_pool(argc, argv, B_TRUE, NULL,
1765                     zpool_export_one, &cb));
1766         }
1767
1768         /* check arguments */
1769         if (argc < 1) {
1770                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1771                 usage(B_FALSE);
1772         }
1773
1774         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_export_one, &cb);
1775
1776         return (ret);
1777 }
1778
1779 /*
1780  * Given a vdev configuration, determine the maximum width needed for the device
1781  * name column.
1782  */
1783 static int
1784 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1785     int name_flags)
1786 {
1787         char *name;
1788         nvlist_t **child;
1789         uint_t c, children;
1790         int ret;
1791
1792         name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
1793         if (strlen(name) + depth > max)
1794                 max = strlen(name) + depth;
1795
1796         free(name);
1797
1798         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1799             &child, &children) == 0) {
1800                 for (c = 0; c < children; c++)
1801                         if ((ret = max_width(zhp, child[c], depth + 2,
1802                             max, name_flags)) > max)
1803                                 max = ret;
1804         }
1805
1806         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1807             &child, &children) == 0) {
1808                 for (c = 0; c < children; c++)
1809                         if ((ret = max_width(zhp, child[c], depth + 2,
1810                             max, name_flags)) > max)
1811                                 max = ret;
1812         }
1813
1814         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1815             &child, &children) == 0) {
1816                 for (c = 0; c < children; c++)
1817                         if ((ret = max_width(zhp, child[c], depth + 2,
1818                             max, name_flags)) > max)
1819                                 max = ret;
1820         }
1821
1822         return (max);
1823 }
1824
1825 typedef struct spare_cbdata {
1826         uint64_t        cb_guid;
1827         zpool_handle_t  *cb_zhp;
1828 } spare_cbdata_t;
1829
1830 static boolean_t
1831 find_vdev(nvlist_t *nv, uint64_t search)
1832 {
1833         uint64_t guid;
1834         nvlist_t **child;
1835         uint_t c, children;
1836
1837         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1838             search == guid)
1839                 return (B_TRUE);
1840
1841         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1842             &child, &children) == 0) {
1843                 for (c = 0; c < children; c++)
1844                         if (find_vdev(child[c], search))
1845                                 return (B_TRUE);
1846         }
1847
1848         return (B_FALSE);
1849 }
1850
1851 static int
1852 find_spare(zpool_handle_t *zhp, void *data)
1853 {
1854         spare_cbdata_t *cbp = data;
1855         nvlist_t *config, *nvroot;
1856
1857         config = zpool_get_config(zhp, NULL);
1858         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1859             &nvroot) == 0);
1860
1861         if (find_vdev(nvroot, cbp->cb_guid)) {
1862                 cbp->cb_zhp = zhp;
1863                 return (1);
1864         }
1865
1866         zpool_close(zhp);
1867         return (0);
1868 }
1869
1870 typedef struct status_cbdata {
1871         int             cb_count;
1872         int             cb_name_flags;
1873         int             cb_namewidth;
1874         boolean_t       cb_allpools;
1875         boolean_t       cb_verbose;
1876         boolean_t       cb_literal;
1877         boolean_t       cb_explain;
1878         boolean_t       cb_first;
1879         boolean_t       cb_dedup_stats;
1880         boolean_t       cb_print_status;
1881         boolean_t       cb_print_slow_ios;
1882         boolean_t       cb_print_vdev_init;
1883         boolean_t       cb_print_vdev_trim;
1884         vdev_cmd_data_list_t    *vcdl;
1885 } status_cbdata_t;
1886
1887 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
1888 static int
1889 is_blank_str(char *str)
1890 {
1891         while (str != NULL && *str != '\0') {
1892                 if (!isblank(*str))
1893                         return (0);
1894                 str++;
1895         }
1896         return (1);
1897 }
1898
1899 /* Print command output lines for specific vdev in a specific pool */
1900 static void
1901 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
1902 {
1903         vdev_cmd_data_t *data;
1904         int i, j;
1905         char *val;
1906
1907         for (i = 0; i < vcdl->count; i++) {
1908                 if ((strcmp(vcdl->data[i].path, path) != 0) ||
1909                     (strcmp(vcdl->data[i].pool, pool) != 0)) {
1910                         /* Not the vdev we're looking for */
1911                         continue;
1912                 }
1913
1914                 data = &vcdl->data[i];
1915                 /* Print out all the output values for this vdev */
1916                 for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
1917                         val = NULL;
1918                         /* Does this vdev have values for this column? */
1919                         for (int k = 0; k < data->cols_cnt; k++) {
1920                                 if (strcmp(data->cols[k],
1921                                     vcdl->uniq_cols[j]) == 0) {
1922                                         /* yes it does, record the value */
1923                                         val = data->lines[k];
1924                                         break;
1925                                 }
1926                         }
1927                         /*
1928                          * Mark empty values with dashes to make output
1929                          * awk-able.
1930                          */
1931                         if (is_blank_str(val))
1932                                 val = "-";
1933
1934                         printf("%*s", vcdl->uniq_cols_width[j], val);
1935                         if (j < vcdl->uniq_cols_cnt - 1)
1936                                 printf("  ");
1937                 }
1938
1939                 /* Print out any values that aren't in a column at the end */
1940                 for (j = data->cols_cnt; j < data->lines_cnt; j++) {
1941                         /* Did we have any columns?  If so print a spacer. */
1942                         if (vcdl->uniq_cols_cnt > 0)
1943                                 printf("  ");
1944
1945                         val = data->lines[j];
1946                         printf("%s", val ? val : "");
1947                 }
1948                 break;
1949         }
1950 }
1951
1952 /*
1953  * Print vdev initialization status for leaves
1954  */
1955 static void
1956 print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
1957 {
1958         if (verbose) {
1959                 if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
1960                     vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
1961                     vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
1962                     !vs->vs_scan_removing) {
1963                         char zbuf[1024];
1964                         char tbuf[256];
1965                         struct tm zaction_ts;
1966
1967                         time_t t = vs->vs_initialize_action_time;
1968                         int initialize_pct = 100;
1969                         if (vs->vs_initialize_state !=
1970                             VDEV_INITIALIZE_COMPLETE) {
1971                                 initialize_pct = (vs->vs_initialize_bytes_done *
1972                                     100 / (vs->vs_initialize_bytes_est + 1));
1973                         }
1974
1975                         (void) localtime_r(&t, &zaction_ts);
1976                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
1977
1978                         switch (vs->vs_initialize_state) {
1979                         case VDEV_INITIALIZE_SUSPENDED:
1980                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1981                                     gettext("suspended, started at"), tbuf);
1982                                 break;
1983                         case VDEV_INITIALIZE_ACTIVE:
1984                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1985                                     gettext("started at"), tbuf);
1986                                 break;
1987                         case VDEV_INITIALIZE_COMPLETE:
1988                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1989                                     gettext("completed at"), tbuf);
1990                                 break;
1991                         }
1992
1993                         (void) printf(gettext("  (%d%% initialized%s)"),
1994                             initialize_pct, zbuf);
1995                 } else {
1996                         (void) printf(gettext("  (uninitialized)"));
1997                 }
1998         } else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
1999                 (void) printf(gettext("  (initializing)"));
2000         }
2001 }
2002
2003 /*
2004  * Print vdev TRIM status for leaves
2005  */
2006 static void
2007 print_status_trim(vdev_stat_t *vs, boolean_t verbose)
2008 {
2009         if (verbose) {
2010                 if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
2011                     vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
2012                     vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
2013                     !vs->vs_scan_removing) {
2014                         char zbuf[1024];
2015                         char tbuf[256];
2016                         struct tm zaction_ts;
2017
2018                         time_t t = vs->vs_trim_action_time;
2019                         int trim_pct = 100;
2020                         if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
2021                                 trim_pct = (vs->vs_trim_bytes_done *
2022                                     100 / (vs->vs_trim_bytes_est + 1));
2023                         }
2024
2025                         (void) localtime_r(&t, &zaction_ts);
2026                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2027
2028                         switch (vs->vs_trim_state) {
2029                         case VDEV_TRIM_SUSPENDED:
2030                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2031                                     gettext("suspended, started at"), tbuf);
2032                                 break;
2033                         case VDEV_TRIM_ACTIVE:
2034                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2035                                     gettext("started at"), tbuf);
2036                                 break;
2037                         case VDEV_TRIM_COMPLETE:
2038                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2039                                     gettext("completed at"), tbuf);
2040                                 break;
2041                         }
2042
2043                         (void) printf(gettext("  (%d%% trimmed%s)"),
2044                             trim_pct, zbuf);
2045                 } else if (vs->vs_trim_notsup) {
2046                         (void) printf(gettext("  (trim unsupported)"));
2047                 } else {
2048                         (void) printf(gettext("  (untrimmed)"));
2049                 }
2050         } else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
2051                 (void) printf(gettext("  (trimming)"));
2052         }
2053 }
2054
2055 /*
2056  * Return the color associated with a health string.  This includes returning
2057  * NULL for no color change.
2058  */
2059 static char *
2060 health_str_to_color(const char *health)
2061 {
2062         if (strcmp(health, gettext("FAULTED")) == 0 ||
2063             strcmp(health, gettext("SUSPENDED")) == 0 ||
2064             strcmp(health, gettext("UNAVAIL")) == 0) {
2065                 return (ANSI_RED);
2066         }
2067
2068         if (strcmp(health, gettext("OFFLINE")) == 0 ||
2069             strcmp(health, gettext("DEGRADED")) == 0 ||
2070             strcmp(health, gettext("REMOVED")) == 0) {
2071                 return (ANSI_YELLOW);
2072         }
2073
2074         return (NULL);
2075 }
2076
2077 /*
2078  * Print out configuration state as requested by status_callback.
2079  */
2080 static void
2081 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
2082     nvlist_t *nv, int depth, boolean_t isspare, vdev_rebuild_stat_t *vrs)
2083 {
2084         nvlist_t **child, *root;
2085         uint_t c, i, vsc, children;
2086         pool_scan_stat_t *ps = NULL;
2087         vdev_stat_t *vs;
2088         char rbuf[6], wbuf[6], cbuf[6];
2089         char *vname;
2090         uint64_t notpresent;
2091         spare_cbdata_t spare_cb;
2092         const char *state;
2093         char *type;
2094         char *path = NULL;
2095         char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL;
2096
2097         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2098             &child, &children) != 0)
2099                 children = 0;
2100
2101         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2102             (uint64_t **)&vs, &vsc) == 0);
2103
2104         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2105
2106         if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2107                 return;
2108
2109         state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2110
2111         if (isspare) {
2112                 /*
2113                  * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
2114                  * online drives.
2115                  */
2116                 if (vs->vs_aux == VDEV_AUX_SPARED)
2117                         state = gettext("INUSE");
2118                 else if (vs->vs_state == VDEV_STATE_HEALTHY)
2119                         state = gettext("AVAIL");
2120         }
2121
2122         printf_color(health_str_to_color(state),
2123             "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
2124             name, state);
2125
2126         if (!isspare) {
2127                 if (vs->vs_read_errors)
2128                         rcolor = ANSI_RED;
2129
2130                 if (vs->vs_write_errors)
2131                         wcolor = ANSI_RED;
2132
2133                 if (vs->vs_checksum_errors)
2134                         ccolor = ANSI_RED;
2135
2136                 if (cb->cb_literal) {
2137                         printf(" ");
2138                         printf_color(rcolor, "%5llu",
2139                             (u_longlong_t)vs->vs_read_errors);
2140                         printf(" ");
2141                         printf_color(wcolor, "%5llu",
2142                             (u_longlong_t)vs->vs_write_errors);
2143                         printf(" ");
2144                         printf_color(ccolor, "%5llu",
2145                             (u_longlong_t)vs->vs_checksum_errors);
2146                 } else {
2147                         zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
2148                         zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
2149                         zfs_nicenum(vs->vs_checksum_errors, cbuf,
2150                             sizeof (cbuf));
2151                         printf(" ");
2152                         printf_color(rcolor, "%5s", rbuf);
2153                         printf(" ");
2154                         printf_color(wcolor, "%5s", wbuf);
2155                         printf(" ");
2156                         printf_color(ccolor, "%5s", cbuf);
2157                 }
2158                 if (cb->cb_print_slow_ios) {
2159                         if (children == 0)  {
2160                                 /* Only leafs vdevs have slow IOs */
2161                                 zfs_nicenum(vs->vs_slow_ios, rbuf,
2162                                     sizeof (rbuf));
2163                         } else {
2164                                 snprintf(rbuf, sizeof (rbuf), "-");
2165                         }
2166
2167                         if (cb->cb_literal)
2168                                 printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
2169                         else
2170                                 printf(" %5s", rbuf);
2171                 }
2172         }
2173
2174         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
2175             &notpresent) == 0) {
2176                 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
2177                 (void) printf("  %s %s", gettext("was"), path);
2178         } else if (vs->vs_aux != 0) {
2179                 (void) printf("  ");
2180                 color_start(ANSI_RED);
2181                 switch (vs->vs_aux) {
2182                 case VDEV_AUX_OPEN_FAILED:
2183                         (void) printf(gettext("cannot open"));
2184                         break;
2185
2186                 case VDEV_AUX_BAD_GUID_SUM:
2187                         (void) printf(gettext("missing device"));
2188                         break;
2189
2190                 case VDEV_AUX_NO_REPLICAS:
2191                         (void) printf(gettext("insufficient replicas"));
2192                         break;
2193
2194                 case VDEV_AUX_VERSION_NEWER:
2195                         (void) printf(gettext("newer version"));
2196                         break;
2197
2198                 case VDEV_AUX_UNSUP_FEAT:
2199                         (void) printf(gettext("unsupported feature(s)"));
2200                         break;
2201
2202                 case VDEV_AUX_ASHIFT_TOO_BIG:
2203                         (void) printf(gettext("unsupported minimum blocksize"));
2204                         break;
2205
2206                 case VDEV_AUX_SPARED:
2207                         verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
2208                             &spare_cb.cb_guid) == 0);
2209                         if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
2210                                 if (strcmp(zpool_get_name(spare_cb.cb_zhp),
2211                                     zpool_get_name(zhp)) == 0)
2212                                         (void) printf(gettext("currently in "
2213                                             "use"));
2214                                 else
2215                                         (void) printf(gettext("in use by "
2216                                             "pool '%s'"),
2217                                             zpool_get_name(spare_cb.cb_zhp));
2218                                 zpool_close(spare_cb.cb_zhp);
2219                         } else {
2220                                 (void) printf(gettext("currently in use"));
2221                         }
2222                         break;
2223
2224                 case VDEV_AUX_ERR_EXCEEDED:
2225                         (void) printf(gettext("too many errors"));
2226                         break;
2227
2228                 case VDEV_AUX_IO_FAILURE:
2229                         (void) printf(gettext("experienced I/O failures"));
2230                         break;
2231
2232                 case VDEV_AUX_BAD_LOG:
2233                         (void) printf(gettext("bad intent log"));
2234                         break;
2235
2236                 case VDEV_AUX_EXTERNAL:
2237                         (void) printf(gettext("external device fault"));
2238                         break;
2239
2240                 case VDEV_AUX_SPLIT_POOL:
2241                         (void) printf(gettext("split into new pool"));
2242                         break;
2243
2244                 case VDEV_AUX_ACTIVE:
2245                         (void) printf(gettext("currently in use"));
2246                         break;
2247
2248                 case VDEV_AUX_CHILDREN_OFFLINE:
2249                         (void) printf(gettext("all children offline"));
2250                         break;
2251
2252                 default:
2253                         (void) printf(gettext("corrupted data"));
2254                         break;
2255                 }
2256                 color_end();
2257         }
2258
2259         /* The root vdev has the scrub/resilver stats */
2260         root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
2261             ZPOOL_CONFIG_VDEV_TREE);
2262         (void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
2263             (uint64_t **)&ps, &c);
2264
2265         if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0) {
2266                 if (vs->vs_scan_processed != 0) {
2267                         (void) printf(gettext("  (%s)"),
2268                             (ps->pss_func == POOL_SCAN_RESILVER) ?
2269                             "resilvering" : "repairing");
2270                 } else if (vs->vs_resilver_deferred) {
2271                         (void) printf(gettext("  (awaiting resilver)"));
2272                 }
2273         }
2274
2275         /* The top-level vdevs have the rebuild stats */
2276         if (vrs != NULL && vrs->vrs_state == VDEV_REBUILD_ACTIVE &&
2277             children == 0) {
2278                 if (vs->vs_rebuild_processed != 0) {
2279                         (void) printf(gettext("  (resilvering)"));
2280                 }
2281         }
2282
2283         if (cb->vcdl != NULL) {
2284                 if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
2285                         printf("  ");
2286                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
2287                 }
2288         }
2289
2290         /* Display vdev initialization and trim status for leaves */
2291         if (children == 0) {
2292                 print_status_initialize(vs, cb->cb_print_vdev_init);
2293                 print_status_trim(vs, cb->cb_print_vdev_trim);
2294         }
2295
2296         (void) printf("\n");
2297
2298         for (c = 0; c < children; c++) {
2299                 uint64_t islog = B_FALSE, ishole = B_FALSE;
2300
2301                 /* Don't print logs or holes here */
2302                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2303                     &islog);
2304                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
2305                     &ishole);
2306                 if (islog || ishole)
2307                         continue;
2308                 /* Only print normal classes here */
2309                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2310                         continue;
2311
2312                 /* Provide vdev_rebuild_stats to children if available */
2313                 if (vrs == NULL) {
2314                         (void) nvlist_lookup_uint64_array(nv,
2315                             ZPOOL_CONFIG_REBUILD_STATS,
2316                             (uint64_t **)&vrs, &i);
2317                 }
2318
2319                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
2320                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2321                 print_status_config(zhp, cb, vname, child[c], depth + 2,
2322                     isspare, vrs);
2323                 free(vname);
2324         }
2325 }
2326
2327 /*
2328  * Print the configuration of an exported pool.  Iterate over all vdevs in the
2329  * pool, printing out the name and status for each one.
2330  */
2331 static void
2332 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
2333     int depth)
2334 {
2335         nvlist_t **child;
2336         uint_t c, children;
2337         vdev_stat_t *vs;
2338         char *type, *vname;
2339
2340         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2341         if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
2342             strcmp(type, VDEV_TYPE_HOLE) == 0)
2343                 return;
2344
2345         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2346             (uint64_t **)&vs, &c) == 0);
2347
2348         (void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
2349         (void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
2350
2351         if (vs->vs_aux != 0) {
2352                 (void) printf("  ");
2353
2354                 switch (vs->vs_aux) {
2355                 case VDEV_AUX_OPEN_FAILED:
2356                         (void) printf(gettext("cannot open"));
2357                         break;
2358
2359                 case VDEV_AUX_BAD_GUID_SUM:
2360                         (void) printf(gettext("missing device"));
2361                         break;
2362
2363                 case VDEV_AUX_NO_REPLICAS:
2364                         (void) printf(gettext("insufficient replicas"));
2365                         break;
2366
2367                 case VDEV_AUX_VERSION_NEWER:
2368                         (void) printf(gettext("newer version"));
2369                         break;
2370
2371                 case VDEV_AUX_UNSUP_FEAT:
2372                         (void) printf(gettext("unsupported feature(s)"));
2373                         break;
2374
2375                 case VDEV_AUX_ERR_EXCEEDED:
2376                         (void) printf(gettext("too many errors"));
2377                         break;
2378
2379                 case VDEV_AUX_ACTIVE:
2380                         (void) printf(gettext("currently in use"));
2381                         break;
2382
2383                 case VDEV_AUX_CHILDREN_OFFLINE:
2384                         (void) printf(gettext("all children offline"));
2385                         break;
2386
2387                 default:
2388                         (void) printf(gettext("corrupted data"));
2389                         break;
2390                 }
2391         }
2392         (void) printf("\n");
2393
2394         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2395             &child, &children) != 0)
2396                 return;
2397
2398         for (c = 0; c < children; c++) {
2399                 uint64_t is_log = B_FALSE;
2400
2401                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2402                     &is_log);
2403                 if (is_log)
2404                         continue;
2405                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2406                         continue;
2407
2408                 vname = zpool_vdev_name(g_zfs, NULL, child[c],
2409                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2410                 print_import_config(cb, vname, child[c], depth + 2);
2411                 free(vname);
2412         }
2413
2414         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2415             &child, &children) == 0) {
2416                 (void) printf(gettext("\tcache\n"));
2417                 for (c = 0; c < children; c++) {
2418                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2419                             cb->cb_name_flags);
2420                         (void) printf("\t  %s\n", vname);
2421                         free(vname);
2422                 }
2423         }
2424
2425         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2426             &child, &children) == 0) {
2427                 (void) printf(gettext("\tspares\n"));
2428                 for (c = 0; c < children; c++) {
2429                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2430                             cb->cb_name_flags);
2431                         (void) printf("\t  %s\n", vname);
2432                         free(vname);
2433                 }
2434         }
2435 }
2436
2437 /*
2438  * Print specialized class vdevs.
2439  *
2440  * These are recorded as top level vdevs in the main pool child array
2441  * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2442  * print_status_config() or print_import_config() to print the top level
2443  * class vdevs then any of their children (eg mirrored slogs) are printed
2444  * recursively - which works because only the top level vdev is marked.
2445  */
2446 static void
2447 print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2448     const char *class)
2449 {
2450         uint_t c, children;
2451         nvlist_t **child;
2452         boolean_t printed = B_FALSE;
2453
2454         assert(zhp != NULL || !cb->cb_verbose);
2455
2456         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2457             &children) != 0)
2458                 return;
2459
2460         for (c = 0; c < children; c++) {
2461                 uint64_t is_log = B_FALSE;
2462                 char *bias = NULL;
2463                 char *type = NULL;
2464
2465                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2466                     &is_log);
2467
2468                 if (is_log) {
2469                         bias = VDEV_ALLOC_CLASS_LOGS;
2470                 } else {
2471                         (void) nvlist_lookup_string(child[c],
2472                             ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2473                         (void) nvlist_lookup_string(child[c],
2474                             ZPOOL_CONFIG_TYPE, &type);
2475                 }
2476
2477                 if (bias == NULL || strcmp(bias, class) != 0)
2478                         continue;
2479                 if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2480                         continue;
2481
2482                 if (!printed) {
2483                         (void) printf("\t%s\t\n", gettext(class));
2484                         printed = B_TRUE;
2485                 }
2486
2487                 char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2488                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2489                 if (cb->cb_print_status)
2490                         print_status_config(zhp, cb, name, child[c], 2,
2491                             B_FALSE, NULL);
2492                 else
2493                         print_import_config(cb, name, child[c], 2);
2494                 free(name);
2495         }
2496 }
2497
2498 /*
2499  * Display the status for the given pool.
2500  */
2501 static void
2502 show_import(nvlist_t *config)
2503 {
2504         uint64_t pool_state;
2505         vdev_stat_t *vs;
2506         char *name;
2507         uint64_t guid;
2508         uint64_t hostid = 0;
2509         char *msgid;
2510         char *hostname = "unknown";
2511         nvlist_t *nvroot, *nvinfo;
2512         zpool_status_t reason;
2513         zpool_errata_t errata;
2514         const char *health;
2515         uint_t vsc;
2516         char *comment;
2517         status_cbdata_t cb = { 0 };
2518
2519         verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2520             &name) == 0);
2521         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2522             &guid) == 0);
2523         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2524             &pool_state) == 0);
2525         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2526             &nvroot) == 0);
2527
2528         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2529             (uint64_t **)&vs, &vsc) == 0);
2530         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2531
2532         reason = zpool_import_status(config, &msgid, &errata);
2533
2534         (void) printf(gettext("   pool: %s\n"), name);
2535         (void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2536         (void) printf(gettext("  state: %s"), health);
2537         if (pool_state == POOL_STATE_DESTROYED)
2538                 (void) printf(gettext(" (DESTROYED)"));
2539         (void) printf("\n");
2540
2541         switch (reason) {
2542         case ZPOOL_STATUS_MISSING_DEV_R:
2543         case ZPOOL_STATUS_MISSING_DEV_NR:
2544         case ZPOOL_STATUS_BAD_GUID_SUM:
2545                 printf_color(ANSI_BOLD, gettext("status: "));
2546                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2547                     "missing from the system.\n"));
2548                 break;
2549
2550         case ZPOOL_STATUS_CORRUPT_LABEL_R:
2551         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2552                 printf_color(ANSI_BOLD, gettext("status: "));
2553                 printf_color(ANSI_YELLOW, gettext("One or more devices contains"
2554                     " corrupted data.\n"));
2555                 break;
2556
2557         case ZPOOL_STATUS_CORRUPT_DATA:
2558                 (void) printf(
2559                     gettext(" status: The pool data is corrupted.\n"));
2560                 break;
2561
2562         case ZPOOL_STATUS_OFFLINE_DEV:
2563                 printf_color(ANSI_BOLD, gettext("status: "));
2564                 printf_color(ANSI_YELLOW, gettext("One or more devices "
2565                     "are offlined.\n"));
2566                 break;
2567
2568         case ZPOOL_STATUS_CORRUPT_POOL:
2569                 printf_color(ANSI_BOLD, gettext("status: "));
2570                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
2571                     "corrupted.\n"));
2572                 break;
2573
2574         case ZPOOL_STATUS_VERSION_OLDER:
2575                 printf_color(ANSI_BOLD, gettext("status: "));
2576                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2577                     "a legacy on-disk version.\n"));
2578                 break;
2579
2580         case ZPOOL_STATUS_VERSION_NEWER:
2581                 printf_color(ANSI_BOLD, gettext("status: "));
2582                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2583                     "an incompatible version.\n"));
2584                 break;
2585
2586         case ZPOOL_STATUS_FEAT_DISABLED:
2587                 printf_color(ANSI_BOLD, gettext("status: "));
2588                 printf_color(ANSI_YELLOW, gettext("Some supported features are "
2589                     "not enabled on the pool.\n"));
2590                 break;
2591
2592         case ZPOOL_STATUS_UNSUP_FEAT_READ:
2593                 printf_color(ANSI_BOLD, gettext("status: "));
2594                 printf_color(ANSI_YELLOW, gettext("The pool uses the following "
2595                     "feature(s) not supported on this system:\n"));
2596                 color_start(ANSI_YELLOW);
2597                 zpool_print_unsup_feat(config);
2598                 color_end();
2599                 break;
2600
2601         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2602                 printf_color(ANSI_BOLD, gettext("status: "));
2603                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
2604                     "accessed in read-only mode on this system. It\n\tcannot be"
2605                     " accessed in read-write mode because it uses the "
2606                     "following\n\tfeature(s) not supported on this system:\n"));
2607                 color_start(ANSI_YELLOW);
2608                 zpool_print_unsup_feat(config);
2609                 color_end();
2610                 break;
2611
2612         case ZPOOL_STATUS_HOSTID_ACTIVE:
2613                 printf_color(ANSI_BOLD, gettext("status: "));
2614                 printf_color(ANSI_YELLOW, gettext("The pool is currently "
2615                     "imported by another system.\n"));
2616                 break;
2617
2618         case ZPOOL_STATUS_HOSTID_REQUIRED:
2619                 printf_color(ANSI_BOLD, gettext("status: "));
2620                 printf_color(ANSI_YELLOW, gettext("The pool has the "
2621                     "multihost property on.  It cannot\n\tbe safely imported "
2622                     "when the system hostid is not set.\n"));
2623                 break;
2624
2625         case ZPOOL_STATUS_HOSTID_MISMATCH:
2626                 printf_color(ANSI_BOLD, gettext("status: "));
2627                 printf_color(ANSI_YELLOW, gettext("The pool was last accessed "
2628                     "by another system.\n"));
2629                 break;
2630
2631         case ZPOOL_STATUS_FAULTED_DEV_R:
2632         case ZPOOL_STATUS_FAULTED_DEV_NR:
2633                 printf_color(ANSI_BOLD, gettext("status: "));
2634                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2635                     "faulted.\n"));
2636                 break;
2637
2638         case ZPOOL_STATUS_BAD_LOG:
2639                 printf_color(ANSI_BOLD, gettext("status: "));
2640                 printf_color(ANSI_YELLOW, gettext("An intent log record cannot "
2641                     "be read.\n"));
2642                 break;
2643
2644         case ZPOOL_STATUS_RESILVERING:
2645         case ZPOOL_STATUS_REBUILDING:
2646                 printf_color(ANSI_BOLD, gettext("status: "));
2647                 printf_color(ANSI_YELLOW, gettext("One or more devices were "
2648                     "being resilvered.\n"));
2649                 break;
2650
2651         case ZPOOL_STATUS_ERRATA:
2652                 printf_color(ANSI_BOLD, gettext("status: "));
2653                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
2654                     errata);
2655                 break;
2656
2657         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
2658                 printf_color(ANSI_BOLD, gettext("status: "));
2659                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2660                     "configured to use a non-native block size.\n"
2661                     "\tExpect reduced performance.\n"));
2662                 break;
2663
2664         default:
2665                 /*
2666                  * No other status can be seen when importing pools.
2667                  */
2668                 assert(reason == ZPOOL_STATUS_OK);
2669         }
2670
2671         /*
2672          * Print out an action according to the overall state of the pool.
2673          */
2674         if (vs->vs_state == VDEV_STATE_HEALTHY) {
2675                 if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2676                     reason == ZPOOL_STATUS_FEAT_DISABLED) {
2677                         (void) printf(gettext(" action: The pool can be "
2678                             "imported using its name or numeric identifier, "
2679                             "though\n\tsome features will not be available "
2680                             "without an explicit 'zpool upgrade'.\n"));
2681                 } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2682                         (void) printf(gettext(" action: The pool can be "
2683                             "imported using its name or numeric "
2684                             "identifier and\n\tthe '-f' flag.\n"));
2685                 } else if (reason == ZPOOL_STATUS_ERRATA) {
2686                         switch (errata) {
2687                         case ZPOOL_ERRATA_NONE:
2688                                 break;
2689
2690                         case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2691                                 (void) printf(gettext(" action: The pool can "
2692                                     "be imported using its name or numeric "
2693                                     "identifier,\n\thowever there is a compat"
2694                                     "ibility issue which should be corrected"
2695                                     "\n\tby running 'zpool scrub'\n"));
2696                                 break;
2697
2698                         case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2699                                 (void) printf(gettext(" action: The pool can"
2700                                     "not be imported with this version of ZFS "
2701                                     "due to\n\tan active asynchronous destroy. "
2702                                     "Revert to an earlier version\n\tand "
2703                                     "allow the destroy to complete before "
2704                                     "updating.\n"));
2705                                 break;
2706
2707                         case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2708                                 (void) printf(gettext(" action: Existing "
2709                                     "encrypted datasets contain an on-disk "
2710                                     "incompatibility, which\n\tneeds to be "
2711                                     "corrected. Backup these datasets to new "
2712                                     "encrypted datasets\n\tand destroy the "
2713                                     "old ones.\n"));
2714                                 break;
2715
2716                         case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
2717                                 (void) printf(gettext(" action: Existing "
2718                                     "encrypted snapshots and bookmarks contain "
2719                                     "an on-disk\n\tincompatibility. This may "
2720                                     "cause on-disk corruption if they are used"
2721                                     "\n\twith 'zfs recv'. To correct the "
2722                                     "issue, enable the bookmark_v2 feature.\n\t"
2723                                     "No additional action is needed if there "
2724                                     "are no encrypted snapshots or\n\t"
2725                                     "bookmarks. If preserving the encrypted "
2726                                     "snapshots and bookmarks is\n\trequired, "
2727                                     "use a non-raw send to backup and restore "
2728                                     "them. Alternately,\n\tthey may be removed"
2729                                     " to resolve the incompatibility.\n"));
2730                                 break;
2731                         default:
2732                                 /*
2733                                  * All errata must contain an action message.
2734                                  */
2735                                 assert(0);
2736                         }
2737                 } else {
2738                         (void) printf(gettext(" action: The pool can be "
2739                             "imported using its name or numeric "
2740                             "identifier.\n"));
2741                 }
2742         } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2743                 (void) printf(gettext(" action: The pool can be imported "
2744                     "despite missing or damaged devices.  The\n\tfault "
2745                     "tolerance of the pool may be compromised if imported.\n"));
2746         } else {
2747                 switch (reason) {
2748                 case ZPOOL_STATUS_VERSION_NEWER:
2749                         (void) printf(gettext(" action: The pool cannot be "
2750                             "imported.  Access the pool on a system running "
2751                             "newer\n\tsoftware, or recreate the pool from "
2752                             "backup.\n"));
2753                         break;
2754                 case ZPOOL_STATUS_UNSUP_FEAT_READ:
2755                         printf_color(ANSI_BOLD, gettext("action: "));
2756                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
2757                             "imported. Access the pool on a system that "
2758                             "supports\n\tthe required feature(s), or recreate "
2759                             "the pool from backup.\n"));
2760                         break;
2761                 case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2762                         printf_color(ANSI_BOLD, gettext("action: "));
2763                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
2764                             "imported in read-write mode. Import the pool "
2765                             "with\n"
2766                             "\t\"-o readonly=on\", access the pool on a system "
2767                             "that supports the\n\trequired feature(s), or "
2768                             "recreate the pool from backup.\n"));
2769                         break;
2770                 case ZPOOL_STATUS_MISSING_DEV_R:
2771                 case ZPOOL_STATUS_MISSING_DEV_NR:
2772                 case ZPOOL_STATUS_BAD_GUID_SUM:
2773                         (void) printf(gettext(" action: The pool cannot be "
2774                             "imported. Attach the missing\n\tdevices and try "
2775                             "again.\n"));
2776                         break;
2777                 case ZPOOL_STATUS_HOSTID_ACTIVE:
2778                         VERIFY0(nvlist_lookup_nvlist(config,
2779                             ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
2780
2781                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2782                                 hostname = fnvlist_lookup_string(nvinfo,
2783                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2784
2785                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2786                                 hostid = fnvlist_lookup_uint64(nvinfo,
2787                                     ZPOOL_CONFIG_MMP_HOSTID);
2788
2789                         (void) printf(gettext(" action: The pool must be "
2790                             "exported from %s (hostid=%lx)\n\tbefore it "
2791                             "can be safely imported.\n"), hostname,
2792                             (unsigned long) hostid);
2793                         break;
2794                 case ZPOOL_STATUS_HOSTID_REQUIRED:
2795                         (void) printf(gettext(" action: Set a unique system "
2796                             "hostid with the zgenhostid(8) command.\n"));
2797                         break;
2798                 default:
2799                         (void) printf(gettext(" action: The pool cannot be "
2800                             "imported due to damaged devices or data.\n"));
2801                 }
2802         }
2803
2804         /* Print the comment attached to the pool. */
2805         if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2806                 (void) printf(gettext("comment: %s\n"), comment);
2807
2808         /*
2809          * If the state is "closed" or "can't open", and the aux state
2810          * is "corrupt data":
2811          */
2812         if (((vs->vs_state == VDEV_STATE_CLOSED) ||
2813             (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
2814             (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
2815                 if (pool_state == POOL_STATE_DESTROYED)
2816                         (void) printf(gettext("\tThe pool was destroyed, "
2817                             "but can be imported using the '-Df' flags.\n"));
2818                 else if (pool_state != POOL_STATE_EXPORTED)
2819                         (void) printf(gettext("\tThe pool may be active on "
2820                             "another system, but can be imported using\n\t"
2821                             "the '-f' flag.\n"));
2822         }
2823
2824         if (msgid != NULL) {
2825                 (void) printf(gettext(
2826                     "   see: https://openzfs.github.io/openzfs-docs/msg/%s\n"),
2827                     msgid);
2828         }
2829
2830         (void) printf(gettext(" config:\n\n"));
2831
2832         cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
2833             VDEV_NAME_TYPE_ID);
2834         if (cb.cb_namewidth < 10)
2835                 cb.cb_namewidth = 10;
2836
2837         print_import_config(&cb, name, nvroot, 0);
2838
2839         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
2840         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
2841         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
2842
2843         if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
2844                 (void) printf(gettext("\n\tAdditional devices are known to "
2845                     "be part of this pool, though their\n\texact "
2846                     "configuration cannot be determined.\n"));
2847         }
2848 }
2849
2850 static boolean_t
2851 zfs_force_import_required(nvlist_t *config)
2852 {
2853         uint64_t state;
2854         uint64_t hostid = 0;
2855         nvlist_t *nvinfo;
2856
2857         state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
2858         (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
2859
2860         if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
2861                 return (B_TRUE);
2862
2863         nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2864         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
2865                 mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
2866                     ZPOOL_CONFIG_MMP_STATE);
2867
2868                 if (mmp_state != MMP_STATE_INACTIVE)
2869                         return (B_TRUE);
2870         }
2871
2872         return (B_FALSE);
2873 }
2874
2875 /*
2876  * Perform the import for the given configuration.  This passes the heavy
2877  * lifting off to zpool_import_props(), and then mounts the datasets contained
2878  * within the pool.
2879  */
2880 static int
2881 do_import(nvlist_t *config, const char *newname, const char *mntopts,
2882     nvlist_t *props, int flags)
2883 {
2884         int ret = 0;
2885         zpool_handle_t *zhp;
2886         char *name;
2887         uint64_t version;
2888
2889         name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
2890         version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
2891
2892         if (!SPA_VERSION_IS_SUPPORTED(version)) {
2893                 (void) fprintf(stderr, gettext("cannot import '%s': pool "
2894                     "is formatted using an unsupported ZFS version\n"), name);
2895                 return (1);
2896         } else if (zfs_force_import_required(config) &&
2897             !(flags & ZFS_IMPORT_ANY_HOST)) {
2898                 mmp_state_t mmp_state = MMP_STATE_INACTIVE;
2899                 nvlist_t *nvinfo;
2900
2901                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2902                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
2903                         mmp_state = fnvlist_lookup_uint64(nvinfo,
2904                             ZPOOL_CONFIG_MMP_STATE);
2905
2906                 if (mmp_state == MMP_STATE_ACTIVE) {
2907                         char *hostname = "<unknown>";
2908                         uint64_t hostid = 0;
2909
2910                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2911                                 hostname = fnvlist_lookup_string(nvinfo,
2912                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2913
2914                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2915                                 hostid = fnvlist_lookup_uint64(nvinfo,
2916                                     ZPOOL_CONFIG_MMP_HOSTID);
2917
2918                         (void) fprintf(stderr, gettext("cannot import '%s': "
2919                             "pool is imported on %s (hostid: "
2920                             "0x%lx)\nExport the pool on the other system, "
2921                             "then run 'zpool import'.\n"),
2922                             name, hostname, (unsigned long) hostid);
2923                 } else if (mmp_state == MMP_STATE_NO_HOSTID) {
2924                         (void) fprintf(stderr, gettext("Cannot import '%s': "
2925                             "pool has the multihost property on and the\n"
2926                             "system's hostid is not set. Set a unique hostid "
2927                             "with the zgenhostid(8) command.\n"), name);
2928                 } else {
2929                         char *hostname = "<unknown>";
2930                         uint64_t timestamp = 0;
2931                         uint64_t hostid = 0;
2932
2933                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
2934                                 hostname = fnvlist_lookup_string(config,
2935                                     ZPOOL_CONFIG_HOSTNAME);
2936
2937                         if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
2938                                 timestamp = fnvlist_lookup_uint64(config,
2939                                     ZPOOL_CONFIG_TIMESTAMP);
2940
2941                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
2942                                 hostid = fnvlist_lookup_uint64(config,
2943                                     ZPOOL_CONFIG_HOSTID);
2944
2945                         (void) fprintf(stderr, gettext("cannot import '%s': "
2946                             "pool was previously in use from another system.\n"
2947                             "Last accessed by %s (hostid=%lx) at %s"
2948                             "The pool can be imported, use 'zpool import -f' "
2949                             "to import the pool.\n"), name, hostname,
2950                             (unsigned long)hostid, ctime((time_t *)&timestamp));
2951                 }
2952
2953                 return (1);
2954         }
2955
2956         if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2957                 return (1);
2958
2959         if (newname != NULL)
2960                 name = (char *)newname;
2961
2962         if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2963                 return (1);
2964
2965         /*
2966          * Loading keys is best effort. We don't want to return immediately
2967          * if it fails but we do want to give the error to the caller.
2968          */
2969         if (flags & ZFS_IMPORT_LOAD_KEYS) {
2970                 ret = zfs_crypto_attempt_load_keys(g_zfs, name);
2971                 if (ret != 0)
2972                         ret = 1;
2973         }
2974
2975         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2976             !(flags & ZFS_IMPORT_ONLY) &&
2977             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2978                 zpool_close(zhp);
2979                 return (1);
2980         }
2981
2982         zpool_close(zhp);
2983         return (ret);
2984 }
2985
2986 typedef struct target_exists_args {
2987         const char      *poolname;
2988         uint64_t        poolguid;
2989 } target_exists_args_t;
2990
2991 static int
2992 name_or_guid_exists(zpool_handle_t *zhp, void *data)
2993 {
2994         target_exists_args_t *args = data;
2995         nvlist_t *config = zpool_get_config(zhp, NULL);
2996         int found = 0;
2997
2998         if (config == NULL)
2999                 return (0);
3000
3001         if (args->poolname != NULL) {
3002                 char *pool_name;
3003
3004                 verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
3005                     &pool_name) == 0);
3006                 if (strcmp(pool_name, args->poolname) == 0)
3007                         found = 1;
3008         } else {
3009                 uint64_t pool_guid;
3010
3011                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3012                     &pool_guid) == 0);
3013                 if (pool_guid == args->poolguid)
3014                         found = 1;
3015         }
3016         zpool_close(zhp);
3017
3018         return (found);
3019 }
3020 /*
3021  * zpool checkpoint <pool>
3022  *       checkpoint --discard <pool>
3023  *
3024  *       -d         Discard the checkpoint from a checkpointed
3025  *       --discard  pool.
3026  *
3027  *       -w         Wait for discarding a checkpoint to complete.
3028  *       --wait
3029  *
3030  * Checkpoints the specified pool, by taking a "snapshot" of its
3031  * current state. A pool can only have one checkpoint at a time.
3032  */
3033 int
3034 zpool_do_checkpoint(int argc, char **argv)
3035 {
3036         boolean_t discard, wait;
3037         char *pool;
3038         zpool_handle_t *zhp;
3039         int c, err;
3040
3041         struct option long_options[] = {
3042                 {"discard", no_argument, NULL, 'd'},
3043                 {"wait", no_argument, NULL, 'w'},
3044                 {0, 0, 0, 0}
3045         };
3046
3047         discard = B_FALSE;
3048         wait = B_FALSE;
3049         while ((c = getopt_long(argc, argv, ":dw", long_options, NULL)) != -1) {
3050                 switch (c) {
3051                 case 'd':
3052                         discard = B_TRUE;
3053                         break;
3054                 case 'w':
3055                         wait = B_TRUE;
3056                         break;
3057                 case '?':
3058                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3059                             optopt);
3060                         usage(B_FALSE);
3061                 }
3062         }
3063
3064         if (wait && !discard) {
3065                 (void) fprintf(stderr, gettext("--wait only valid when "
3066                     "--discard also specified\n"));
3067                 usage(B_FALSE);
3068         }
3069
3070         argc -= optind;
3071         argv += optind;
3072
3073         if (argc < 1) {
3074                 (void) fprintf(stderr, gettext("missing pool argument\n"));
3075                 usage(B_FALSE);
3076         }
3077
3078         if (argc > 1) {
3079                 (void) fprintf(stderr, gettext("too many arguments\n"));
3080                 usage(B_FALSE);
3081         }
3082
3083         pool = argv[0];
3084
3085         if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
3086                 /* As a special case, check for use of '/' in the name */
3087                 if (strchr(pool, '/') != NULL)
3088                         (void) fprintf(stderr, gettext("'zpool checkpoint' "
3089                             "doesn't work on datasets. To save the state "
3090                             "of a dataset from a specific point in time "
3091                             "please use 'zfs snapshot'\n"));
3092                 return (1);
3093         }
3094
3095         if (discard) {
3096                 err = (zpool_discard_checkpoint(zhp) != 0);
3097                 if (err == 0 && wait)
3098                         err = zpool_wait(zhp, ZPOOL_WAIT_CKPT_DISCARD);
3099         } else {
3100                 err = (zpool_checkpoint(zhp) != 0);
3101         }
3102
3103         zpool_close(zhp);
3104
3105         return (err);
3106 }
3107
3108 #define CHECKPOINT_OPT  1024
3109
3110 /*
3111  * zpool import [-d dir] [-D]
3112  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3113  *              [-d dir | -c cachefile] [-f] -a
3114  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3115  *              [-d dir | -c cachefile] [-f] [-n] [-F] <pool | id> [newpool]
3116  *
3117  *       -c     Read pool information from a cachefile instead of searching
3118  *              devices.
3119  *
3120  *       -d     Scan in a specific directory, other than /dev/.  More than
3121  *              one directory can be specified using multiple '-d' options.
3122  *
3123  *       -D     Scan for previously destroyed pools or import all or only
3124  *              specified destroyed pools.
3125  *
3126  *       -R     Temporarily import the pool, with all mountpoints relative to
3127  *              the given root.  The pool will remain exported when the machine
3128  *              is rebooted.
3129  *
3130  *       -V     Import even in the presence of faulted vdevs.  This is an
3131  *              intentionally undocumented option for testing purposes, and
3132  *              treats the pool configuration as complete, leaving any bad
3133  *              vdevs in the FAULTED state. In other words, it does verbatim
3134  *              import.
3135  *
3136  *       -f     Force import, even if it appears that the pool is active.
3137  *
3138  *       -F     Attempt rewind if necessary.
3139  *
3140  *       -n     See if rewind would work, but don't actually rewind.
3141  *
3142  *       -N     Import the pool but don't mount datasets.
3143  *
3144  *       -T     Specify a starting txg to use for import. This option is
3145  *              intentionally undocumented option for testing purposes.
3146  *
3147  *       -a     Import all pools found.
3148  *
3149  *       -l     Load encryption keys while importing.
3150  *
3151  *       -o     Set property=value and/or temporary mount options (without '=').
3152  *
3153  *       -s     Scan using the default search path, the libblkid cache will
3154  *              not be consulted.
3155  *
3156  *       --rewind-to-checkpoint
3157  *              Import the pool and revert back to the checkpoint.
3158  *
3159  * The import command scans for pools to import, and import pools based on pool
3160  * name and GUID.  The pool can also be renamed as part of the import process.
3161  */
3162 int
3163 zpool_do_import(int argc, char **argv)
3164 {
3165         char **searchdirs = NULL;
3166         char *env, *envdup = NULL;
3167         int nsearch = 0;
3168         int c;
3169         int err = 0;
3170         nvlist_t *pools = NULL;
3171         boolean_t do_all = B_FALSE;
3172         boolean_t do_destroyed = B_FALSE;
3173         char *mntopts = NULL;
3174         nvpair_t *elem;
3175         nvlist_t *config;
3176         uint64_t searchguid = 0;
3177         char *searchname = NULL;
3178         char *propval;
3179         nvlist_t *found_config;
3180         nvlist_t *policy = NULL;
3181         nvlist_t *props = NULL;
3182         boolean_t first;
3183         int flags = ZFS_IMPORT_NORMAL;
3184         uint32_t rewind_policy = ZPOOL_NO_REWIND;
3185         boolean_t dryrun = B_FALSE;
3186         boolean_t do_rewind = B_FALSE;
3187         boolean_t xtreme_rewind = B_FALSE;
3188         boolean_t do_scan = B_FALSE;
3189         boolean_t pool_exists = B_FALSE;
3190         uint64_t pool_state, txg = -1ULL;
3191         char *cachefile = NULL;
3192         importargs_t idata = { 0 };
3193         char *endptr;
3194
3195         struct option long_options[] = {
3196                 {"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
3197                 {0, 0, 0, 0}
3198         };
3199
3200         /* check options */
3201         while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
3202             long_options, NULL)) != -1) {
3203                 switch (c) {
3204                 case 'a':
3205                         do_all = B_TRUE;
3206                         break;
3207                 case 'c':
3208                         cachefile = optarg;
3209                         break;
3210                 case 'd':
3211                         if (searchdirs == NULL) {
3212                                 searchdirs = safe_malloc(sizeof (char *));
3213                         } else {
3214                                 char **tmp = safe_malloc((nsearch + 1) *
3215                                     sizeof (char *));
3216                                 bcopy(searchdirs, tmp, nsearch *
3217                                     sizeof (char *));
3218                                 free(searchdirs);
3219                                 searchdirs = tmp;
3220                         }
3221                         searchdirs[nsearch++] = optarg;
3222                         break;
3223                 case 'D':
3224                         do_destroyed = B_TRUE;
3225                         break;
3226                 case 'f':
3227                         flags |= ZFS_IMPORT_ANY_HOST;
3228                         break;
3229                 case 'F':
3230                         do_rewind = B_TRUE;
3231                         break;
3232                 case 'l':
3233                         flags |= ZFS_IMPORT_LOAD_KEYS;
3234                         break;
3235                 case 'm':
3236                         flags |= ZFS_IMPORT_MISSING_LOG;
3237                         break;
3238                 case 'n':
3239                         dryrun = B_TRUE;
3240                         break;
3241                 case 'N':
3242                         flags |= ZFS_IMPORT_ONLY;
3243                         break;
3244                 case 'o':
3245                         if ((propval = strchr(optarg, '=')) != NULL) {
3246                                 *propval = '\0';
3247                                 propval++;
3248                                 if (add_prop_list(optarg, propval,
3249                                     &props, B_TRUE))
3250                                         goto error;
3251                         } else {
3252                                 mntopts = optarg;
3253                         }
3254                         break;
3255                 case 'R':
3256                         if (add_prop_list(zpool_prop_to_name(
3257                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
3258                                 goto error;
3259                         if (add_prop_list_default(zpool_prop_to_name(
3260                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
3261                                 goto error;
3262                         break;
3263                 case 's':
3264                         do_scan = B_TRUE;
3265                         break;
3266                 case 't':
3267                         flags |= ZFS_IMPORT_TEMP_NAME;
3268                         if (add_prop_list_default(zpool_prop_to_name(
3269                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
3270                                 goto error;
3271                         break;
3272
3273                 case 'T':
3274                         errno = 0;
3275                         txg = strtoull(optarg, &endptr, 0);
3276                         if (errno != 0 || *endptr != '\0') {
3277                                 (void) fprintf(stderr,
3278                                     gettext("invalid txg value\n"));
3279                                 usage(B_FALSE);
3280                         }
3281                         rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
3282                         break;
3283                 case 'V':
3284                         flags |= ZFS_IMPORT_VERBATIM;
3285                         break;
3286                 case 'X':
3287                         xtreme_rewind = B_TRUE;
3288                         break;
3289                 case CHECKPOINT_OPT:
3290                         flags |= ZFS_IMPORT_CHECKPOINT;
3291                         break;
3292                 case ':':
3293                         (void) fprintf(stderr, gettext("missing argument for "
3294                             "'%c' option\n"), optopt);
3295                         usage(B_FALSE);
3296                         break;
3297                 case '?':
3298                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3299                             optopt);
3300                         usage(B_FALSE);
3301                 }
3302         }
3303
3304         argc -= optind;
3305         argv += optind;
3306
3307         if (cachefile && nsearch != 0) {
3308                 (void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
3309                 usage(B_FALSE);
3310         }
3311
3312         if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
3313                 (void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
3314                 usage(B_FALSE);
3315         }
3316
3317         if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
3318                 (void) fprintf(stderr, gettext("-l is only meaningful during "
3319                     "an import\n"));
3320                 usage(B_FALSE);
3321         }
3322
3323         if ((dryrun || xtreme_rewind) && !do_rewind) {
3324                 (void) fprintf(stderr,
3325                     gettext("-n or -X only meaningful with -F\n"));
3326                 usage(B_FALSE);
3327         }
3328         if (dryrun)
3329                 rewind_policy = ZPOOL_TRY_REWIND;
3330         else if (do_rewind)
3331                 rewind_policy = ZPOOL_DO_REWIND;
3332         if (xtreme_rewind)
3333                 rewind_policy |= ZPOOL_EXTREME_REWIND;
3334
3335         /* In the future, we can capture further policy and include it here */
3336         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
3337             nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
3338             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
3339             rewind_policy) != 0)
3340                 goto error;
3341
3342         /* check argument count */
3343         if (do_all) {
3344                 if (argc != 0) {
3345                         (void) fprintf(stderr, gettext("too many arguments\n"));
3346                         usage(B_FALSE);
3347                 }
3348         } else {
3349                 if (argc > 2) {
3350                         (void) fprintf(stderr, gettext("too many arguments\n"));
3351                         usage(B_FALSE);
3352                 }
3353         }
3354
3355         /*
3356          * Check for the effective uid.  We do this explicitly here because
3357          * otherwise any attempt to discover pools will silently fail.
3358          */
3359         if (argc == 0 && geteuid() != 0) {
3360                 (void) fprintf(stderr, gettext("cannot "
3361                     "discover pools: permission denied\n"));
3362                 if (searchdirs != NULL)
3363                         free(searchdirs);
3364
3365                 nvlist_free(props);
3366                 nvlist_free(policy);
3367                 return (1);
3368         }
3369
3370         /*
3371          * Depending on the arguments given, we do one of the following:
3372          *
3373          *      <none>  Iterate through all pools and display information about
3374          *              each one.
3375          *
3376          *      -a      Iterate through all pools and try to import each one.
3377          *
3378          *      <id>    Find the pool that corresponds to the given GUID/pool
3379          *              name and import that one.
3380          *
3381          *      -D      Above options applies only to destroyed pools.
3382          */
3383         if (argc != 0) {
3384                 char *endptr;
3385
3386                 errno = 0;
3387                 searchguid = strtoull(argv[0], &endptr, 10);
3388                 if (errno != 0 || *endptr != '\0') {
3389                         searchname = argv[0];
3390                         searchguid = 0;
3391                 }
3392                 found_config = NULL;
3393
3394                 /*
3395                  * User specified a name or guid.  Ensure it's unique.
3396                  */
3397                 target_exists_args_t search = {searchname, searchguid};
3398                 pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
3399         }
3400
3401         /*
3402          * Check the environment for the preferred search path.
3403          */
3404         if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
3405                 char *dir;
3406
3407                 envdup = strdup(env);
3408
3409                 dir = strtok(envdup, ":");
3410                 while (dir != NULL) {
3411                         if (searchdirs == NULL) {
3412                                 searchdirs = safe_malloc(sizeof (char *));
3413                         } else {
3414                                 char **tmp = safe_malloc((nsearch + 1) *
3415                                     sizeof (char *));
3416                                 bcopy(searchdirs, tmp, nsearch *
3417                                     sizeof (char *));
3418                                 free(searchdirs);
3419                                 searchdirs = tmp;
3420                         }
3421                         searchdirs[nsearch++] = dir;
3422                         dir = strtok(NULL, ":");
3423                 }
3424         }
3425
3426         idata.path = searchdirs;
3427         idata.paths = nsearch;
3428         idata.poolname = searchname;
3429         idata.guid = searchguid;
3430         idata.cachefile = cachefile;
3431         idata.scan = do_scan;
3432         idata.policy = policy;
3433
3434         pools = zpool_search_import(g_zfs, &idata, &libzfs_config_ops);
3435
3436         if (pools != NULL && pool_exists &&
3437             (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
3438                 (void) fprintf(stderr, gettext("cannot import '%s': "
3439                     "a pool with that name already exists\n"),
3440                     argv[0]);
3441                 (void) fprintf(stderr, gettext("use the form '%s "
3442                     "<pool | id> <newpool>' to give it a new name\n"),
3443                     "zpool import");
3444                 err = 1;
3445         } else if (pools == NULL && pool_exists) {
3446                 (void) fprintf(stderr, gettext("cannot import '%s': "
3447                     "a pool with that name is already created/imported,\n"),
3448                     argv[0]);
3449                 (void) fprintf(stderr, gettext("and no additional pools "
3450                     "with that name were found\n"));
3451                 err = 1;
3452         } else if (pools == NULL) {
3453                 if (argc != 0) {
3454                         (void) fprintf(stderr, gettext("cannot import '%s': "
3455                             "no such pool available\n"), argv[0]);
3456                 }
3457                 err = 1;
3458         }
3459
3460         if (err == 1) {
3461                 if (searchdirs != NULL)
3462                         free(searchdirs);
3463                 if (envdup != NULL)
3464                         free(envdup);
3465                 nvlist_free(policy);
3466                 nvlist_free(pools);
3467                 nvlist_free(props);
3468                 return (1);
3469         }
3470
3471         /*
3472          * At this point we have a list of import candidate configs. Even if
3473          * we were searching by pool name or guid, we still need to
3474          * post-process the list to deal with pool state and possible
3475          * duplicate names.
3476          */
3477         err = 0;
3478         elem = NULL;
3479         first = B_TRUE;
3480         while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
3481
3482                 verify(nvpair_value_nvlist(elem, &config) == 0);
3483
3484                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
3485                     &pool_state) == 0);
3486                 if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
3487                         continue;
3488                 if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
3489                         continue;
3490
3491                 verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
3492                     policy) == 0);
3493
3494                 if (argc == 0) {
3495                         if (first)
3496                                 first = B_FALSE;
3497                         else if (!do_all)
3498                                 (void) printf("\n");
3499
3500                         if (do_all) {
3501                                 err |= do_import(config, NULL, mntopts,
3502                                     props, flags);
3503                         } else {
3504                                 show_import(config);
3505                         }
3506                 } else if (searchname != NULL) {
3507                         char *name;
3508
3509                         /*
3510                          * We are searching for a pool based on name.
3511                          */
3512                         verify(nvlist_lookup_string(config,
3513                             ZPOOL_CONFIG_POOL_NAME, &name) == 0);
3514
3515                         if (strcmp(name, searchname) == 0) {
3516                                 if (found_config != NULL) {
3517                                         (void) fprintf(stderr, gettext(
3518                                             "cannot import '%s': more than "
3519                                             "one matching pool\n"), searchname);
3520                                         (void) fprintf(stderr, gettext(
3521                                             "import by numeric ID instead\n"));
3522                                         err = B_TRUE;
3523                                 }
3524                                 found_config = config;
3525                         }
3526                 } else {
3527                         uint64_t guid;
3528
3529                         /*
3530                          * Search for a pool by guid.
3531                          */
3532                         verify(nvlist_lookup_uint64(config,
3533                             ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3534
3535                         if (guid == searchguid)
3536                                 found_config = config;
3537                 }
3538         }
3539
3540         /*
3541          * If we were searching for a specific pool, verify that we found a
3542          * pool, and then do the import.
3543          */
3544         if (argc != 0 && err == 0) {
3545                 if (found_config == NULL) {
3546                         (void) fprintf(stderr, gettext("cannot import '%s': "
3547                             "no such pool available\n"), argv[0]);
3548                         err = B_TRUE;
3549                 } else {
3550                         err |= do_import(found_config, argc == 1 ? NULL :
3551                             argv[1], mntopts, props, flags);
3552                 }
3553         }
3554
3555         /*
3556          * If we were just looking for pools, report an error if none were
3557          * found.
3558          */
3559         if (argc == 0 && first)
3560                 (void) fprintf(stderr,
3561                     gettext("no pools available to import\n"));
3562
3563 error:
3564         nvlist_free(props);
3565         nvlist_free(pools);
3566         nvlist_free(policy);
3567         if (searchdirs != NULL)
3568                 free(searchdirs);
3569         if (envdup != NULL)
3570                 free(envdup);
3571
3572         return (err ? 1 : 0);
3573 }
3574
3575 /*
3576  * zpool sync [-f] [pool] ...
3577  *
3578  * -f (undocumented) force uberblock (and config including zpool cache file)
3579  *    update.
3580  *
3581  * Sync the specified pool(s).
3582  * Without arguments "zpool sync" will sync all pools.
3583  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
3584  *
3585  */
3586 static int
3587 zpool_do_sync(int argc, char **argv)
3588 {
3589         int ret;
3590         boolean_t force = B_FALSE;
3591
3592         /* check options */
3593         while ((ret  = getopt(argc, argv, "f")) != -1) {
3594                 switch (ret) {
3595                 case 'f':
3596                         force = B_TRUE;
3597                         break;
3598                 case '?':
3599                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3600                             optopt);
3601                         usage(B_FALSE);
3602                 }
3603         }
3604
3605         argc -= optind;
3606         argv += optind;
3607
3608         /* if argc == 0 we will execute zpool_sync_one on all pools */
3609         ret = for_each_pool(argc, argv, B_FALSE, NULL, zpool_sync_one, &force);
3610
3611         return (ret);
3612 }
3613
3614 typedef struct iostat_cbdata {
3615         uint64_t cb_flags;
3616         int cb_name_flags;
3617         int cb_namewidth;
3618         int cb_iteration;
3619         char **cb_vdev_names; /* Only show these vdevs */
3620         unsigned int cb_vdev_names_count;
3621         boolean_t cb_verbose;
3622         boolean_t cb_literal;
3623         boolean_t cb_scripted;
3624         zpool_list_t *cb_list;
3625         vdev_cmd_data_list_t *vcdl;
3626 } iostat_cbdata_t;
3627
3628 /*  iostat labels */
3629 typedef struct name_and_columns {
3630         const char *name;       /* Column name */
3631         unsigned int columns;   /* Center name to this number of columns */
3632 } name_and_columns_t;
3633
3634 #define IOSTAT_MAX_LABELS       13      /* Max number of labels on one line */
3635
3636 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
3637 {
3638         [IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
3639             {NULL}},
3640         [IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3641             {"asyncq_wait", 2}, {"scrub", 1}, {"trim", 1}, {NULL}},
3642         [IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
3643             {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
3644             {"trimq_write", 2}, {NULL}},
3645         [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3646             {"asyncq_wait", 2}, {NULL}},
3647         [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
3648             {"async_read", 2}, {"async_write", 2}, {"scrub", 2},
3649             {"trim", 2}, {NULL}},
3650 };
3651
3652 /* Shorthand - if "columns" field not set, default to 1 column */
3653 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
3654 {
3655         [IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
3656             {"write"}, {NULL}},
3657         [IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3658             {"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {NULL}},
3659         [IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
3660             {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
3661             {"pend"}, {"activ"}, {NULL}},
3662         [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3663             {"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {NULL}},
3664         [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3665             {"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"}, {NULL}},
3666 };
3667
3668 static const char *histo_to_title[] = {
3669         [IOS_L_HISTO] = "latency",
3670         [IOS_RQ_HISTO] = "req_size",
3671 };
3672
3673 /*
3674  * Return the number of labels in a null-terminated name_and_columns_t
3675  * array.
3676  *
3677  */
3678 static unsigned int
3679 label_array_len(const name_and_columns_t *labels)
3680 {
3681         int i = 0;
3682
3683         while (labels[i].name)
3684                 i++;
3685
3686         return (i);
3687 }
3688
3689 /*
3690  * Return the number of strings in a null-terminated string array.
3691  * For example:
3692  *
3693  *     const char foo[] = {"bar", "baz", NULL}
3694  *
3695  * returns 2
3696  */
3697 static uint64_t
3698 str_array_len(const char *array[])
3699 {
3700         uint64_t i = 0;
3701         while (array[i])
3702                 i++;
3703
3704         return (i);
3705 }
3706
3707
3708 /*
3709  * Return a default column width for default/latency/queue columns. This does
3710  * not include histograms, which have their columns autosized.
3711  */
3712 static unsigned int
3713 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
3714 {
3715         unsigned long column_width = 5; /* Normal niceprint */
3716         static unsigned long widths[] = {
3717                 /*
3718                  * Choose some sane default column sizes for printing the
3719                  * raw numbers.
3720                  */
3721                 [IOS_DEFAULT] = 15, /* 1PB capacity */
3722                 [IOS_LATENCY] = 10, /* 1B ns = 10sec */
3723                 [IOS_QUEUES] = 6,   /* 1M queue entries */
3724                 [IOS_L_HISTO] = 10, /* 1B ns = 10sec */
3725                 [IOS_RQ_HISTO] = 6, /* 1M queue entries */
3726         };
3727
3728         if (cb->cb_literal)
3729                 column_width = widths[type];
3730
3731         return (column_width);
3732 }
3733
3734 /*
3735  * Print the column labels, i.e:
3736  *
3737  *   capacity     operations     bandwidth
3738  * alloc   free   read  write   read  write  ...
3739  *
3740  * If force_column_width is set, use it for the column width.  If not set, use
3741  * the default column width.
3742  */
3743 static void
3744 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
3745     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
3746 {
3747         int i, idx, s;
3748         int text_start, rw_column_width, spaces_to_end;
3749         uint64_t flags = cb->cb_flags;
3750         uint64_t f;
3751         unsigned int column_width = force_column_width;
3752
3753         /* For each bit set in flags */
3754         for (f = flags; f; f &= ~(1ULL << idx)) {
3755                 idx = lowbit64(f) - 1;
3756                 if (!force_column_width)
3757                         column_width = default_column_width(cb, idx);
3758                 /* Print our top labels centered over "read  write" label. */
3759                 for (i = 0; i < label_array_len(labels[idx]); i++) {
3760                         const char *name = labels[idx][i].name;
3761                         /*
3762                          * We treat labels[][].columns == 0 as shorthand
3763                          * for one column.  It makes writing out the label
3764                          * tables more concise.
3765                          */
3766                         unsigned int columns = MAX(1, labels[idx][i].columns);
3767                         unsigned int slen = strlen(name);
3768
3769                         rw_column_width = (column_width * columns) +
3770                             (2 * (columns - 1));
3771
3772                         text_start = (int)((rw_column_width) / columns -
3773                             slen / columns);
3774                         if (text_start < 0)
3775                                 text_start = 0;
3776
3777                         printf("  ");   /* Two spaces between columns */
3778
3779                         /* Space from beginning of column to label */
3780                         for (s = 0; s < text_start; s++)
3781                                 printf(" ");
3782
3783                         printf("%s", name);
3784
3785                         /* Print space after label to end of column */
3786                         spaces_to_end = rw_column_width - text_start - slen;
3787                         if (spaces_to_end < 0)
3788                                 spaces_to_end = 0;
3789
3790                         for (s = 0; s < spaces_to_end; s++)
3791                                 printf(" ");
3792                 }
3793         }
3794 }
3795
3796
3797 /*
3798  * print_cmd_columns - Print custom column titles from -c
3799  *
3800  * If the user specified the "zpool status|iostat -c" then print their custom
3801  * column titles in the header.  For example, print_cmd_columns() would print
3802  * the "  col1  col2" part of this:
3803  *
3804  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
3805  * ...
3806  *            capacity     operations     bandwidth
3807  * pool        alloc   free   read  write   read  write  col1  col2
3808  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3809  * mypool       269K  1008M      0      0    107    946
3810  *   mirror     269K  1008M      0      0    107    946
3811  *     sdb         -      -      0      0    102    473  val1  val2
3812  *     sdc         -      -      0      0      5    473  val1  val2
3813  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3814  */
3815 static void
3816 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
3817 {
3818         int i, j;
3819         vdev_cmd_data_t *data = &vcdl->data[0];
3820
3821         if (vcdl->count == 0 || data == NULL)
3822                 return;
3823
3824         /*
3825          * Each vdev cmd should have the same column names unless the user did
3826          * something weird with their cmd.  Just take the column names from the
3827          * first vdev and assume it works for all of them.
3828          */
3829         for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
3830                 printf("  ");
3831                 if (use_dashes) {
3832                         for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
3833                                 printf("-");
3834                 } else {
3835                         printf_color(ANSI_BOLD, "%*s", vcdl->uniq_cols_width[i],
3836                             vcdl->uniq_cols[i]);
3837                 }
3838         }
3839 }
3840
3841
3842 /*
3843  * Utility function to print out a line of dashes like:
3844  *
3845  *      --------------------------------  -----  -----  -----  -----  -----
3846  *
3847  * ...or a dashed named-row line like:
3848  *
3849  *      logs                                  -      -      -      -      -
3850  *
3851  * @cb:                         iostat data
3852  *
3853  * @force_column_width          If non-zero, use the value as the column width.
3854  *                              Otherwise use the default column widths.
3855  *
3856  * @name:                       Print a dashed named-row line starting
3857  *                              with @name.  Otherwise, print a regular
3858  *                              dashed line.
3859  */
3860 static void
3861 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
3862     const char *name)
3863 {
3864         int i;
3865         unsigned int namewidth;
3866         uint64_t flags = cb->cb_flags;
3867         uint64_t f;
3868         int idx;
3869         const name_and_columns_t *labels;
3870         const char *title;
3871
3872
3873         if (cb->cb_flags & IOS_ANYHISTO_M) {
3874                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3875         } else if (cb->cb_vdev_names_count) {
3876                 title = "vdev";
3877         } else  {
3878                 title = "pool";
3879         }
3880
3881         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3882             name ? strlen(name) : 0);
3883
3884
3885         if (name) {
3886                 printf("%-*s", namewidth, name);
3887         } else {
3888                 for (i = 0; i < namewidth; i++)
3889                         (void) printf("-");
3890         }
3891
3892         /* For each bit in flags */
3893         for (f = flags; f; f &= ~(1ULL << idx)) {
3894                 unsigned int column_width;
3895                 idx = lowbit64(f) - 1;
3896                 if (force_column_width)
3897                         column_width = force_column_width;
3898                 else
3899                         column_width = default_column_width(cb, idx);
3900
3901                 labels = iostat_bottom_labels[idx];
3902                 for (i = 0; i < label_array_len(labels); i++) {
3903                         if (name)
3904                                 printf("  %*s-", column_width - 1, " ");
3905                         else
3906                                 printf("  %.*s", column_width,
3907                                     "--------------------");
3908                 }
3909         }
3910 }
3911
3912
3913 static void
3914 print_iostat_separator_impl(iostat_cbdata_t *cb,
3915     unsigned int force_column_width)
3916 {
3917         print_iostat_dashes(cb, force_column_width, NULL);
3918 }
3919
3920 static void
3921 print_iostat_separator(iostat_cbdata_t *cb)
3922 {
3923         print_iostat_separator_impl(cb, 0);
3924 }
3925
3926 static void
3927 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
3928     const char *histo_vdev_name)
3929 {
3930         unsigned int namewidth;
3931         const char *title;
3932
3933         if (cb->cb_flags & IOS_ANYHISTO_M) {
3934                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3935         } else if (cb->cb_vdev_names_count) {
3936                 title = "vdev";
3937         } else  {
3938                 title = "pool";
3939         }
3940
3941         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3942             histo_vdev_name ? strlen(histo_vdev_name) : 0);
3943
3944         if (histo_vdev_name)
3945                 printf("%-*s", namewidth, histo_vdev_name);
3946         else
3947                 printf("%*s", namewidth, "");
3948
3949
3950         print_iostat_labels(cb, force_column_width, iostat_top_labels);
3951         printf("\n");
3952
3953         printf("%-*s", namewidth, title);
3954
3955         print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
3956         if (cb->vcdl != NULL)
3957                 print_cmd_columns(cb->vcdl, 0);
3958
3959         printf("\n");
3960
3961         print_iostat_separator_impl(cb, force_column_width);
3962
3963         if (cb->vcdl != NULL)
3964                 print_cmd_columns(cb->vcdl, 1);
3965
3966         printf("\n");
3967 }
3968
3969 static void
3970 print_iostat_header(iostat_cbdata_t *cb)
3971 {
3972         print_iostat_header_impl(cb, 0, NULL);
3973 }
3974
3975
3976 /*
3977  * Display a single statistic.
3978  */
3979 static void
3980 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
3981     unsigned int column_size, boolean_t scripted)
3982 {
3983         char buf[64];
3984
3985         zfs_nicenum_format(value, buf, sizeof (buf), format);
3986
3987         if (scripted)
3988                 printf("\t%s", buf);
3989         else
3990                 printf("  %*s", column_size, buf);
3991 }
3992
3993 /*
3994  * Calculate the default vdev stats
3995  *
3996  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
3997  * stats into calcvs.
3998  */
3999 static void
4000 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
4001     vdev_stat_t *calcvs)
4002 {
4003         int i;
4004
4005         memcpy(calcvs, newvs, sizeof (*calcvs));
4006         for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
4007                 calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
4008
4009         for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
4010                 calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
4011 }
4012
4013 /*
4014  * Internal representation of the extended iostats data.
4015  *
4016  * The extended iostat stats are exported in nvlists as either uint64_t arrays
4017  * or single uint64_t's.  We make both look like arrays to make them easier
4018  * to process.  In order to make single uint64_t's look like arrays, we set
4019  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
4020  * we can just use *data and count.
4021  */
4022 struct stat_array {
4023         uint64_t *data;
4024         uint_t count;   /* Number of entries in data[] */
4025         uint64_t __data; /* Only used when data is a single uint64_t */
4026 };
4027
4028 static uint64_t
4029 stat_histo_max(struct stat_array *nva, unsigned int len)
4030 {
4031         uint64_t max = 0;
4032         int i;
4033         for (i = 0; i < len; i++)
4034                 max = MAX(max, array64_max(nva[i].data, nva[i].count));
4035
4036         return (max);
4037 }
4038
4039 /*
4040  * Helper function to lookup a uint64_t array or uint64_t value and store its
4041  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
4042  * it look like a one element array to make it easier to process.
4043  */
4044 static int
4045 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
4046     struct stat_array *nva)
4047 {
4048         nvpair_t *tmp;
4049         int ret;
4050
4051         verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
4052         switch (nvpair_type(tmp)) {
4053         case DATA_TYPE_UINT64_ARRAY:
4054                 ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
4055                 break;
4056         case DATA_TYPE_UINT64:
4057                 ret = nvpair_value_uint64(tmp, &nva->__data);
4058                 nva->data = &nva->__data;
4059                 nva->count = 1;
4060                 break;
4061         default:
4062                 /* Not a uint64_t */
4063                 ret = EINVAL;
4064                 break;
4065         }
4066
4067         return (ret);
4068 }
4069
4070 /*
4071  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
4072  * subtract them, and return the results in a newly allocated stat_array.
4073  * You must free the returned array after you are done with it with
4074  * free_calc_stats().
4075  *
4076  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
4077  * values.
4078  */
4079 static struct stat_array *
4080 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
4081     nvlist_t *newnv)
4082 {
4083         nvlist_t *oldnvx = NULL, *newnvx;
4084         struct stat_array *oldnva, *newnva, *calcnva;
4085         int i, j;
4086         unsigned int alloc_size = (sizeof (struct stat_array)) * len;
4087
4088         /* Extract our extended stats nvlist from the main list */
4089         verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4090             &newnvx) == 0);
4091         if (oldnv) {
4092                 verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4093                     &oldnvx) == 0);
4094         }
4095
4096         newnva = safe_malloc(alloc_size);
4097         oldnva = safe_malloc(alloc_size);
4098         calcnva = safe_malloc(alloc_size);
4099
4100         for (j = 0; j < len; j++) {
4101                 verify(nvpair64_to_stat_array(newnvx, names[j],
4102                     &newnva[j]) == 0);
4103                 calcnva[j].count = newnva[j].count;
4104                 alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
4105                 calcnva[j].data = safe_malloc(alloc_size);
4106                 memcpy(calcnva[j].data, newnva[j].data, alloc_size);
4107
4108                 if (oldnvx) {
4109                         verify(nvpair64_to_stat_array(oldnvx, names[j],
4110                             &oldnva[j]) == 0);
4111                         for (i = 0; i < oldnva[j].count; i++)
4112                                 calcnva[j].data[i] -= oldnva[j].data[i];
4113                 }
4114         }
4115         free(newnva);
4116         free(oldnva);
4117         return (calcnva);
4118 }
4119
4120 static void
4121 free_calc_stats(struct stat_array *nva, unsigned int len)
4122 {
4123         int i;
4124         for (i = 0; i < len; i++)
4125                 free(nva[i].data);
4126
4127         free(nva);
4128 }
4129
4130 static void
4131 print_iostat_histo(struct stat_array *nva, unsigned int len,
4132     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
4133     double scale)
4134 {
4135         int i, j;
4136         char buf[6];
4137         uint64_t val;
4138         enum zfs_nicenum_format format;
4139         unsigned int buckets;
4140         unsigned int start_bucket;
4141
4142         if (cb->cb_literal)
4143                 format = ZFS_NICENUM_RAW;
4144         else
4145                 format = ZFS_NICENUM_1024;
4146
4147         /* All these histos are the same size, so just use nva[0].count */
4148         buckets = nva[0].count;
4149
4150         if (cb->cb_flags & IOS_RQ_HISTO_M) {
4151                 /* Start at 512 - req size should never be lower than this */
4152                 start_bucket = 9;
4153         } else {
4154                 start_bucket = 0;
4155         }
4156
4157         for (j = start_bucket; j < buckets; j++) {
4158                 /* Print histogram bucket label */
4159                 if (cb->cb_flags & IOS_L_HISTO_M) {
4160                         /* Ending range of this bucket */
4161                         val = (1UL << (j + 1)) - 1;
4162                         zfs_nicetime(val, buf, sizeof (buf));
4163                 } else {
4164                         /* Request size (starting range of bucket) */
4165                         val = (1UL << j);
4166                         zfs_nicenum(val, buf, sizeof (buf));
4167                 }
4168
4169                 if (cb->cb_scripted)
4170                         printf("%llu", (u_longlong_t)val);
4171                 else
4172                         printf("%-*s", namewidth, buf);
4173
4174                 /* Print the values on the line */
4175                 for (i = 0; i < len; i++) {
4176                         print_one_stat(nva[i].data[j] * scale, format,
4177                             column_width, cb->cb_scripted);
4178                 }
4179                 printf("\n");
4180         }
4181 }
4182
4183 static void
4184 print_solid_separator(unsigned int length)
4185 {
4186         while (length--)
4187                 printf("-");
4188         printf("\n");
4189 }
4190
4191 static void
4192 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
4193     nvlist_t *newnv, double scale, const char *name)
4194 {
4195         unsigned int column_width;
4196         unsigned int namewidth;
4197         unsigned int entire_width;
4198         enum iostat_type type;
4199         struct stat_array *nva;
4200         const char **names;
4201         unsigned int names_len;
4202
4203         /* What type of histo are we? */
4204         type = IOS_HISTO_IDX(cb->cb_flags);
4205
4206         /* Get NULL-terminated array of nvlist names for our histo */
4207         names = vsx_type_to_nvlist[type];
4208         names_len = str_array_len(names); /* num of names */
4209
4210         nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
4211
4212         if (cb->cb_literal) {
4213                 column_width = MAX(5,
4214                     (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
4215         } else {
4216                 column_width = 5;
4217         }
4218
4219         namewidth = MAX(cb->cb_namewidth,
4220             strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
4221
4222         /*
4223          * Calculate the entire line width of what we're printing.  The
4224          * +2 is for the two spaces between columns:
4225          */
4226         /*       read  write                            */
4227         /*      -----  -----                            */
4228         /*      |___|  <---------- column_width         */
4229         /*                                              */
4230         /*      |__________|  <--- entire_width         */
4231         /*                                              */
4232         entire_width = namewidth + (column_width + 2) *
4233             label_array_len(iostat_bottom_labels[type]);
4234
4235         if (cb->cb_scripted)
4236                 printf("%s\n", name);
4237         else
4238                 print_iostat_header_impl(cb, column_width, name);
4239
4240         print_iostat_histo(nva, names_len, cb, column_width,
4241             namewidth, scale);
4242
4243         free_calc_stats(nva, names_len);
4244         if (!cb->cb_scripted)
4245                 print_solid_separator(entire_width);
4246 }
4247
4248 /*
4249  * Calculate the average latency of a power-of-two latency histogram
4250  */
4251 static uint64_t
4252 single_histo_average(uint64_t *histo, unsigned int buckets)
4253 {
4254         int i;
4255         uint64_t count = 0, total = 0;
4256
4257         for (i = 0; i < buckets; i++) {
4258                 /*
4259                  * Our buckets are power-of-two latency ranges.  Use the
4260                  * midpoint latency of each bucket to calculate the average.
4261                  * For example:
4262                  *
4263                  * Bucket          Midpoint
4264                  * 8ns-15ns:       12ns
4265                  * 16ns-31ns:      24ns
4266                  * ...
4267                  */
4268                 if (histo[i] != 0) {
4269                         total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
4270                         count += histo[i];
4271                 }
4272         }
4273
4274         /* Prevent divide by zero */
4275         return (count == 0 ? 0 : total / count);
4276 }
4277
4278 static void
4279 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *oldnv,
4280     nvlist_t *newnv)
4281 {
4282         int i;
4283         uint64_t val;
4284         const char *names[] = {
4285                 ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
4286                 ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
4287                 ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
4288                 ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
4289                 ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
4290                 ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
4291                 ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
4292                 ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
4293                 ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
4294                 ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
4295                 ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
4296                 ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
4297         };
4298
4299         struct stat_array *nva;
4300
4301         unsigned int column_width = default_column_width(cb, IOS_QUEUES);
4302         enum zfs_nicenum_format format;
4303
4304         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
4305
4306         if (cb->cb_literal)
4307                 format = ZFS_NICENUM_RAW;
4308         else
4309                 format = ZFS_NICENUM_1024;
4310
4311         for (i = 0; i < ARRAY_SIZE(names); i++) {
4312                 val = nva[i].data[0];
4313                 print_one_stat(val, format, column_width, cb->cb_scripted);
4314         }
4315
4316         free_calc_stats(nva, ARRAY_SIZE(names));
4317 }
4318
4319 static void
4320 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
4321     nvlist_t *newnv)
4322 {
4323         int i;
4324         uint64_t val;
4325         const char *names[] = {
4326                 ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
4327                 ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
4328                 ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
4329                 ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
4330                 ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
4331                 ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
4332                 ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
4333                 ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
4334                 ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
4335                 ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
4336         };
4337         struct stat_array *nva;
4338
4339         unsigned int column_width = default_column_width(cb, IOS_LATENCY);
4340         enum zfs_nicenum_format format;
4341
4342         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
4343
4344         if (cb->cb_literal)
4345                 format = ZFS_NICENUM_RAWTIME;
4346         else
4347                 format = ZFS_NICENUM_TIME;
4348
4349         /* Print our avg latencies on the line */
4350         for (i = 0; i < ARRAY_SIZE(names); i++) {
4351                 /* Compute average latency for a latency histo */
4352                 val = single_histo_average(nva[i].data, nva[i].count);
4353                 print_one_stat(val, format, column_width, cb->cb_scripted);
4354         }
4355         free_calc_stats(nva, ARRAY_SIZE(names));
4356 }
4357
4358 /*
4359  * Print default statistics (capacity/operations/bandwidth)
4360  */
4361 static void
4362 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
4363 {
4364         unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
4365         enum zfs_nicenum_format format;
4366         char na;        /* char to print for "not applicable" values */
4367
4368         if (cb->cb_literal) {
4369                 format = ZFS_NICENUM_RAW;
4370                 na = '0';
4371         } else {
4372                 format = ZFS_NICENUM_1024;
4373                 na = '-';
4374         }
4375
4376         /* only toplevel vdevs have capacity stats */
4377         if (vs->vs_space == 0) {
4378                 if (cb->cb_scripted)
4379                         printf("\t%c\t%c", na, na);
4380                 else
4381                         printf("  %*c  %*c", column_width, na, column_width,
4382                             na);
4383         } else {
4384                 print_one_stat(vs->vs_alloc, format, column_width,
4385                     cb->cb_scripted);
4386                 print_one_stat(vs->vs_space - vs->vs_alloc, format,
4387                     column_width, cb->cb_scripted);
4388         }
4389
4390         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
4391             format, column_width, cb->cb_scripted);
4392         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
4393             format, column_width, cb->cb_scripted);
4394         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
4395             format, column_width, cb->cb_scripted);
4396         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
4397             format, column_width, cb->cb_scripted);
4398 }
4399
4400 static const char *class_name[] = {
4401         VDEV_ALLOC_BIAS_DEDUP,
4402         VDEV_ALLOC_BIAS_SPECIAL,
4403         VDEV_ALLOC_CLASS_LOGS
4404 };
4405
4406 /*
4407  * Print out all the statistics for the given vdev.  This can either be the
4408  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
4409  * is a verbose output, and we don't want to display the toplevel pool stats.
4410  *
4411  * Returns the number of stat lines printed.
4412  */
4413 static unsigned int
4414 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
4415     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
4416 {
4417         nvlist_t **oldchild, **newchild;
4418         uint_t c, children, oldchildren;
4419         vdev_stat_t *oldvs, *newvs, *calcvs;
4420         vdev_stat_t zerovs = { 0 };
4421         char *vname;
4422         int i;
4423         int ret = 0;
4424         uint64_t tdelta;
4425         double scale;
4426
4427         if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
4428                 return (ret);
4429
4430         calcvs = safe_malloc(sizeof (*calcvs));
4431
4432         if (oldnv != NULL) {
4433                 verify(nvlist_lookup_uint64_array(oldnv,
4434                     ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
4435         } else {
4436                 oldvs = &zerovs;
4437         }
4438
4439         /* Do we only want to see a specific vdev? */
4440         for (i = 0; i < cb->cb_vdev_names_count; i++) {
4441                 /* Yes we do.  Is this the vdev? */
4442                 if (strcmp(name, cb->cb_vdev_names[i]) == 0) {
4443                         /*
4444                          * This is our vdev.  Since it is the only vdev we
4445                          * will be displaying, make depth = 0 so that it
4446                          * doesn't get indented.
4447                          */
4448                         depth = 0;
4449                         break;
4450                 }
4451         }
4452
4453         if (cb->cb_vdev_names_count && (i == cb->cb_vdev_names_count)) {
4454                 /* Couldn't match the name */
4455                 goto children;
4456         }
4457
4458
4459         verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
4460             (uint64_t **)&newvs, &c) == 0);
4461
4462         /*
4463          * Print the vdev name unless it's is a histogram.  Histograms
4464          * display the vdev name in the header itself.
4465          */
4466         if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
4467                 if (cb->cb_scripted) {
4468                         printf("%s", name);
4469                 } else {
4470                         if (strlen(name) + depth > cb->cb_namewidth)
4471                                 (void) printf("%*s%s", depth, "", name);
4472                         else
4473                                 (void) printf("%*s%s%*s", depth, "", name,
4474                                     (int)(cb->cb_namewidth - strlen(name) -
4475                                     depth), "");
4476                 }
4477         }
4478
4479         /* Calculate our scaling factor */
4480         tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
4481         if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
4482                 /*
4483                  * If we specify printing histograms with no time interval, then
4484                  * print the histogram numbers over the entire lifetime of the
4485                  * vdev.
4486                  */
4487                 scale = 1;
4488         } else {
4489                 if (tdelta == 0)
4490                         scale = 1.0;
4491                 else
4492                         scale = (double)NANOSEC / tdelta;
4493         }
4494
4495         if (cb->cb_flags & IOS_DEFAULT_M) {
4496                 calc_default_iostats(oldvs, newvs, calcvs);
4497                 print_iostat_default(calcvs, cb, scale);
4498         }
4499         if (cb->cb_flags & IOS_LATENCY_M)
4500                 print_iostat_latency(cb, oldnv, newnv);
4501         if (cb->cb_flags & IOS_QUEUES_M)
4502                 print_iostat_queues(cb, oldnv, newnv);
4503         if (cb->cb_flags & IOS_ANYHISTO_M) {
4504                 printf("\n");
4505                 print_iostat_histos(cb, oldnv, newnv, scale, name);
4506         }
4507
4508         if (cb->vcdl != NULL) {
4509                 char *path;
4510                 if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
4511                     &path) == 0) {
4512                         printf("  ");
4513                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
4514                 }
4515         }
4516
4517         if (!(cb->cb_flags & IOS_ANYHISTO_M))
4518                 printf("\n");
4519
4520         ret++;
4521
4522 children:
4523
4524         free(calcvs);
4525
4526         if (!cb->cb_verbose)
4527                 return (ret);
4528
4529         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
4530             &newchild, &children) != 0)
4531                 return (ret);
4532
4533         if (oldnv) {
4534                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
4535                     &oldchild, &oldchildren) != 0)
4536                         return (ret);
4537
4538                 children = MIN(oldchildren, children);
4539         }
4540
4541         /*
4542          * print normal top-level devices
4543          */
4544         for (c = 0; c < children; c++) {
4545                 uint64_t ishole = B_FALSE, islog = B_FALSE;
4546
4547                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
4548                     &ishole);
4549
4550                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
4551                     &islog);
4552
4553                 if (ishole || islog)
4554                         continue;
4555
4556                 if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
4557                         continue;
4558
4559                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4560                     cb->cb_name_flags);
4561                 ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
4562                     newchild[c], cb, depth + 2);
4563                 free(vname);
4564         }
4565
4566         /*
4567          * print all other top-level devices
4568          */
4569         for (uint_t n = 0; n < 3; n++) {
4570                 boolean_t printed = B_FALSE;
4571
4572                 for (c = 0; c < children; c++) {
4573                         uint64_t islog = B_FALSE;
4574                         char *bias = NULL;
4575                         char *type = NULL;
4576
4577                         (void) nvlist_lookup_uint64(newchild[c],
4578                             ZPOOL_CONFIG_IS_LOG, &islog);
4579                         if (islog) {
4580                                 bias = VDEV_ALLOC_CLASS_LOGS;
4581                         } else {
4582                                 (void) nvlist_lookup_string(newchild[c],
4583                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
4584                                 (void) nvlist_lookup_string(newchild[c],
4585                                     ZPOOL_CONFIG_TYPE, &type);
4586                         }
4587                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
4588                                 continue;
4589                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
4590                                 continue;
4591
4592                         if (!printed) {
4593                                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
4594                                     !cb->cb_scripted && !cb->cb_vdev_names) {
4595                                         print_iostat_dashes(cb, 0,
4596                                             class_name[n]);
4597                                 }
4598                                 printf("\n");
4599                                 printed = B_TRUE;
4600                         }
4601
4602                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4603                             cb->cb_name_flags);
4604                         ret += print_vdev_stats(zhp, vname, oldnv ?
4605                             oldchild[c] : NULL, newchild[c], cb, depth + 2);
4606                         free(vname);
4607                 }
4608         }
4609
4610         /*
4611          * Include level 2 ARC devices in iostat output
4612          */
4613         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
4614             &newchild, &children) != 0)
4615                 return (ret);
4616
4617         if (oldnv) {
4618                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
4619                     &oldchild, &oldchildren) != 0)
4620                         return (ret);
4621
4622                 children = MIN(oldchildren, children);
4623         }
4624
4625         if (children > 0) {
4626                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
4627                     !cb->cb_vdev_names) {
4628                         print_iostat_dashes(cb, 0, "cache");
4629                 }
4630                 printf("\n");
4631
4632                 for (c = 0; c < children; c++) {
4633                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4634                             cb->cb_name_flags);
4635                         ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
4636                             : NULL, newchild[c], cb, depth + 2);
4637                         free(vname);
4638                 }
4639         }
4640
4641         return (ret);
4642 }
4643
4644 static int
4645 refresh_iostat(zpool_handle_t *zhp, void *data)
4646 {
4647         iostat_cbdata_t *cb = data;
4648         boolean_t missing;
4649
4650         /*
4651          * If the pool has disappeared, remove it from the list and continue.
4652          */
4653         if (zpool_refresh_stats(zhp, &missing) != 0)
4654                 return (-1);
4655
4656         if (missing)
4657                 pool_list_remove(cb->cb_list, zhp);
4658
4659         return (0);
4660 }
4661
4662 /*
4663  * Callback to print out the iostats for the given pool.
4664  */
4665 static int
4666 print_iostat(zpool_handle_t *zhp, void *data)
4667 {
4668         iostat_cbdata_t *cb = data;
4669         nvlist_t *oldconfig, *newconfig;
4670         nvlist_t *oldnvroot, *newnvroot;
4671         int ret;
4672
4673         newconfig = zpool_get_config(zhp, &oldconfig);
4674
4675         if (cb->cb_iteration == 1)
4676                 oldconfig = NULL;
4677
4678         verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
4679             &newnvroot) == 0);
4680
4681         if (oldconfig == NULL)
4682                 oldnvroot = NULL;
4683         else
4684                 verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
4685                     &oldnvroot) == 0);
4686
4687         ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
4688             cb, 0);
4689         if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
4690             !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
4691                 print_iostat_separator(cb);
4692                 if (cb->vcdl != NULL) {
4693                         print_cmd_columns(cb->vcdl, 1);
4694                 }
4695                 printf("\n");
4696         }
4697
4698         return (ret);
4699 }
4700
4701 static int
4702 get_columns(void)
4703 {
4704         struct winsize ws;
4705         int columns = 80;
4706         int error;
4707
4708         if (isatty(STDOUT_FILENO)) {
4709                 error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
4710                 if (error == 0)
4711                         columns = ws.ws_col;
4712         } else {
4713                 columns = 999;
4714         }
4715
4716         return (columns);
4717 }
4718
4719 /*
4720  * Return the required length of the pool/vdev name column.  The minimum
4721  * allowed width and output formatting flags must be provided.
4722  */
4723 static int
4724 get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
4725 {
4726         nvlist_t *config, *nvroot;
4727         int width = min_width;
4728
4729         if ((config = zpool_get_config(zhp, NULL)) != NULL) {
4730                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4731                     &nvroot) == 0);
4732                 unsigned int poolname_len = strlen(zpool_get_name(zhp));
4733                 if (verbose == B_FALSE) {
4734                         width = MAX(poolname_len, min_width);
4735                 } else {
4736                         width = MAX(poolname_len,
4737                             max_width(zhp, nvroot, 0, min_width, flags));
4738                 }
4739         }
4740
4741         return (width);
4742 }
4743
4744 /*
4745  * Parse the input string, get the 'interval' and 'count' value if there is one.
4746  */
4747 static void
4748 get_interval_count(int *argcp, char **argv, float *iv,
4749     unsigned long *cnt)
4750 {
4751         float interval = 0;
4752         unsigned long count = 0;
4753         int argc = *argcp;
4754
4755         /*
4756          * Determine if the last argument is an integer or a pool name
4757          */
4758         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
4759                 char *end;
4760
4761                 errno = 0;
4762                 interval = strtof(argv[argc - 1], &end);
4763
4764                 if (*end == '\0' && errno == 0) {
4765                         if (interval == 0) {
4766                                 (void) fprintf(stderr, gettext("interval "
4767                                     "cannot be zero\n"));
4768                                 usage(B_FALSE);
4769                         }
4770                         /*
4771                          * Ignore the last parameter
4772                          */
4773                         argc--;
4774                 } else {
4775                         /*
4776                          * If this is not a valid number, just plow on.  The
4777                          * user will get a more informative error message later
4778                          * on.
4779                          */
4780                         interval = 0;
4781                 }
4782         }
4783
4784         /*
4785          * If the last argument is also an integer, then we have both a count
4786          * and an interval.
4787          */
4788         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
4789                 char *end;
4790
4791                 errno = 0;
4792                 count = interval;
4793                 interval = strtof(argv[argc - 1], &end);
4794
4795                 if (*end == '\0' && errno == 0) {
4796                         if (interval == 0) {
4797                                 (void) fprintf(stderr, gettext("interval "
4798                                     "cannot be zero\n"));
4799                                 usage(B_FALSE);
4800                         }
4801
4802                         /*
4803                          * Ignore the last parameter
4804                          */
4805                         argc--;
4806                 } else {
4807                         interval = 0;
4808                 }
4809         }
4810
4811         *iv = interval;
4812         *cnt = count;
4813         *argcp = argc;
4814 }
4815
4816 static void
4817 get_timestamp_arg(char c)
4818 {
4819         if (c == 'u')
4820                 timestamp_fmt = UDATE;
4821         else if (c == 'd')
4822                 timestamp_fmt = DDATE;
4823         else
4824                 usage(B_FALSE);
4825 }
4826
4827 /*
4828  * Return stat flags that are supported by all pools by both the module and
4829  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
4830  * It will get ANDed down until only the flags that are supported on all pools
4831  * remain.
4832  */
4833 static int
4834 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
4835 {
4836         uint64_t *mask = data;
4837         nvlist_t *config, *nvroot, *nvx;
4838         uint64_t flags = 0;
4839         int i, j;
4840
4841         config = zpool_get_config(zhp, NULL);
4842         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4843             &nvroot) == 0);
4844
4845         /* Default stats are always supported, but for completeness.. */
4846         if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
4847                 flags |= IOS_DEFAULT_M;
4848
4849         /* Get our extended stats nvlist from the main list */
4850         if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
4851             &nvx) != 0) {
4852                 /*
4853                  * No extended stats; they're probably running an older
4854                  * module.  No big deal, we support that too.
4855                  */
4856                 goto end;
4857         }
4858
4859         /* For each extended stat, make sure all its nvpairs are supported */
4860         for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
4861                 if (!vsx_type_to_nvlist[j][0])
4862                         continue;
4863
4864                 /* Start off by assuming the flag is supported, then check */
4865                 flags |= (1ULL << j);
4866                 for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
4867                         if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
4868                                 /* flag isn't supported */
4869                                 flags = flags & ~(1ULL  << j);
4870                                 break;
4871                         }
4872                 }
4873         }
4874 end:
4875         *mask = *mask & flags;
4876         return (0);
4877 }
4878
4879 /*
4880  * Return a bitmask of stats that are supported on all pools by both the module
4881  * and zpool iostat.
4882  */
4883 static uint64_t
4884 get_stat_flags(zpool_list_t *list)
4885 {
4886         uint64_t mask = -1;
4887
4888         /*
4889          * get_stat_flags_cb() will lop off bits from "mask" until only the
4890          * flags that are supported on all pools remain.
4891          */
4892         pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
4893         return (mask);
4894 }
4895
4896 /*
4897  * Return 1 if cb_data->cb_vdev_names[0] is this vdev's name, 0 otherwise.
4898  */
4899 static int
4900 is_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_data)
4901 {
4902         iostat_cbdata_t *cb = cb_data;
4903         char *name = NULL;
4904         int ret = 0;
4905
4906         name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
4907
4908         if (strcmp(name, cb->cb_vdev_names[0]) == 0)
4909                 ret = 1; /* match */
4910         free(name);
4911
4912         return (ret);
4913 }
4914
4915 /*
4916  * Returns 1 if cb_data->cb_vdev_names[0] is a vdev name, 0 otherwise.
4917  */
4918 static int
4919 is_vdev(zpool_handle_t *zhp, void *cb_data)
4920 {
4921         return (for_each_vdev(zhp, is_vdev_cb, cb_data));
4922 }
4923
4924 /*
4925  * Check if vdevs are in a pool
4926  *
4927  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
4928  * return 0.  If pool_name is NULL, then search all pools.
4929  */
4930 static int
4931 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
4932     iostat_cbdata_t *cb)
4933 {
4934         char **tmp_name;
4935         int ret = 0;
4936         int i;
4937         int pool_count = 0;
4938
4939         if ((argc == 0) || !*argv)
4940                 return (0);
4941
4942         if (pool_name)
4943                 pool_count = 1;
4944
4945         /* Temporarily hijack cb_vdev_names for a second... */
4946         tmp_name = cb->cb_vdev_names;
4947
4948         /* Go though our list of prospective vdev names */
4949         for (i = 0; i < argc; i++) {
4950                 cb->cb_vdev_names = argv + i;
4951
4952                 /* Is this name a vdev in our pools? */
4953                 ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
4954                     is_vdev, cb);
4955                 if (!ret) {
4956                         /* No match */
4957                         break;
4958                 }
4959         }
4960
4961         cb->cb_vdev_names = tmp_name;
4962
4963         return (ret);
4964 }
4965
4966 static int
4967 is_pool_cb(zpool_handle_t *zhp, void *data)
4968 {
4969         char *name = data;
4970         if (strcmp(name, zpool_get_name(zhp)) == 0)
4971                 return (1);
4972
4973         return (0);
4974 }
4975
4976 /*
4977  * Do we have a pool named *name?  If so, return 1, otherwise 0.
4978  */
4979 static int
4980 is_pool(char *name)
4981 {
4982         return (for_each_pool(0, NULL, B_TRUE, NULL,  is_pool_cb, name));
4983 }
4984
4985 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
4986 static int
4987 are_all_pools(int argc, char **argv)
4988 {
4989         if ((argc == 0) || !*argv)
4990                 return (0);
4991
4992         while (--argc >= 0)
4993                 if (!is_pool(argv[argc]))
4994                         return (0);
4995
4996         return (1);
4997 }
4998
4999 /*
5000  * Helper function to print out vdev/pool names we can't resolve.  Used for an
5001  * error message.
5002  */
5003 static void
5004 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
5005     iostat_cbdata_t *cb)
5006 {
5007         int i;
5008         char *name;
5009         char *str;
5010         for (i = 0; i < argc; i++) {
5011                 name = argv[i];
5012
5013                 if (is_pool(name))
5014                         str = gettext("pool");
5015                 else if (are_vdevs_in_pool(1, &name, pool_name, cb))
5016                         str = gettext("vdev in this pool");
5017                 else if (are_vdevs_in_pool(1, &name, NULL, cb))
5018                         str = gettext("vdev in another pool");
5019                 else
5020                         str = gettext("unknown");
5021
5022                 fprintf(stderr, "\t%s (%s)\n", name, str);
5023         }
5024 }
5025
5026 /*
5027  * Same as get_interval_count(), but with additional checks to not misinterpret
5028  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
5029  * cb.cb_name_flags.
5030  */
5031 static void
5032 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
5033     unsigned long *count, iostat_cbdata_t *cb)
5034 {
5035         char **tmpargv = argv;
5036         int argc_for_interval = 0;
5037
5038         /* Is the last arg an interval value?  Or a guid? */
5039         if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL, cb)) {
5040                 /*
5041                  * The last arg is not a guid, so it's probably an
5042                  * interval value.
5043                  */
5044                 argc_for_interval++;
5045
5046                 if (*argc >= 2 &&
5047                     !are_vdevs_in_pool(1, &argv[*argc - 2], NULL, cb)) {
5048                         /*
5049                          * The 2nd to last arg is not a guid, so it's probably
5050                          * an interval value.
5051                          */
5052                         argc_for_interval++;
5053                 }
5054         }
5055
5056         /* Point to our list of possible intervals */
5057         tmpargv = &argv[*argc - argc_for_interval];
5058
5059         *argc = *argc - argc_for_interval;
5060         get_interval_count(&argc_for_interval, tmpargv,
5061             interval, count);
5062 }
5063
5064 /*
5065  * Floating point sleep().  Allows you to pass in a floating point value for
5066  * seconds.
5067  */
5068 static void
5069 fsleep(float sec)
5070 {
5071         struct timespec req;
5072         req.tv_sec = floor(sec);
5073         req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
5074         nanosleep(&req, NULL);
5075 }
5076
5077 /*
5078  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
5079  * if we were unable to determine its size.
5080  */
5081 static int
5082 terminal_height(void)
5083 {
5084         struct winsize win;
5085
5086         if (isatty(STDOUT_FILENO) == 0)
5087                 return (-1);
5088
5089         if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) != -1 && win.ws_row > 0)
5090                 return (win.ws_row);
5091
5092         return (-1);
5093 }
5094
5095 /*
5096  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
5097  * print the result.
5098  *
5099  * name:        Short name of the script ('iostat').
5100  * path:        Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
5101  */
5102 static void
5103 print_zpool_script_help(char *name, char *path)
5104 {
5105         char *argv[] = {path, "-h", NULL};
5106         char **lines = NULL;
5107         int lines_cnt = 0;
5108         int rc;
5109
5110         rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
5111             &lines_cnt);
5112         if (rc != 0 || lines == NULL || lines_cnt <= 0) {
5113                 if (lines != NULL)
5114                         libzfs_free_str_array(lines, lines_cnt);
5115                 return;
5116         }
5117
5118         for (int i = 0; i < lines_cnt; i++)
5119                 if (!is_blank_str(lines[i]))
5120                         printf("  %-14s  %s\n", name, lines[i]);
5121
5122         libzfs_free_str_array(lines, lines_cnt);
5123 }
5124
5125 /*
5126  * Go though the zpool status/iostat -c scripts in the user's path, run their
5127  * help option (-h), and print out the results.
5128  */
5129 static void
5130 print_zpool_dir_scripts(char *dirpath)
5131 {
5132         DIR *dir;
5133         struct dirent *ent;
5134         char fullpath[MAXPATHLEN];
5135         struct stat dir_stat;
5136
5137         if ((dir = opendir(dirpath)) != NULL) {
5138                 /* print all the files and directories within directory */
5139                 while ((ent = readdir(dir)) != NULL) {
5140                         sprintf(fullpath, "%s/%s", dirpath, ent->d_name);
5141
5142                         /* Print the scripts */
5143                         if (stat(fullpath, &dir_stat) == 0)
5144                                 if (dir_stat.st_mode & S_IXUSR &&
5145                                     S_ISREG(dir_stat.st_mode))
5146                                         print_zpool_script_help(ent->d_name,
5147                                             fullpath);
5148                 }
5149                 closedir(dir);
5150         }
5151 }
5152
5153 /*
5154  * Print out help text for all zpool status/iostat -c scripts.
5155  */
5156 static void
5157 print_zpool_script_list(char *subcommand)
5158 {
5159         char *dir, *sp;
5160
5161         printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
5162
5163         sp = zpool_get_cmd_search_path();
5164         if (sp == NULL)
5165                 return;
5166
5167         dir = strtok(sp, ":");
5168         while (dir != NULL) {
5169                 print_zpool_dir_scripts(dir);
5170                 dir = strtok(NULL, ":");
5171         }
5172
5173         free(sp);
5174 }
5175
5176 /*
5177  * Set the minimum pool/vdev name column width.  The width must be at least 10,
5178  * but may be as large as the column width - 42 so it still fits on one line.
5179  * NOTE: 42 is the width of the default capacity/operations/bandwidth output
5180  */
5181 static int
5182 get_namewidth_iostat(zpool_handle_t *zhp, void *data)
5183 {
5184         iostat_cbdata_t *cb = data;
5185         int width, available_width;
5186
5187         /*
5188          * get_namewidth() returns the maximum width of any name in that column
5189          * for any pool/vdev/device line that will be output.
5190          */
5191         width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
5192             cb->cb_verbose);
5193
5194         /*
5195          * The width we are calculating is the width of the header and also the
5196          * padding width for names that are less than maximum width.  The stats
5197          * take up 42 characters, so the width available for names is:
5198          */
5199         available_width = get_columns() - 42;
5200
5201         /*
5202          * If the maximum width fits on a screen, then great!  Make everything
5203          * line up by justifying all lines to the same width.  If that max
5204          * width is larger than what's available, the name plus stats won't fit
5205          * on one line, and justifying to that width would cause every line to
5206          * wrap on the screen.  We only want lines with long names to wrap.
5207          * Limit the padding to what won't wrap.
5208          */
5209         if (width > available_width)
5210                 width = available_width;
5211
5212         /*
5213          * And regardless of whatever the screen width is (get_columns can
5214          * return 0 if the width is not known or less than 42 for a narrow
5215          * terminal) have the width be a minimum of 10.
5216          */
5217         if (width < 10)
5218                 width = 10;
5219
5220         /* Save the calculated width */
5221         cb->cb_namewidth = width;
5222
5223         return (0);
5224 }
5225
5226 /*
5227  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
5228  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
5229  *              [interval [count]]
5230  *
5231  *      -c CMD  For each vdev, run command CMD
5232  *      -g      Display guid for individual vdev name.
5233  *      -L      Follow links when resolving vdev path name.
5234  *      -P      Display full path for vdev name.
5235  *      -v      Display statistics for individual vdevs
5236  *      -h      Display help
5237  *      -p      Display values in parsable (exact) format.
5238  *      -H      Scripted mode.  Don't display headers, and separate properties
5239  *              by a single tab.
5240  *      -l      Display average latency
5241  *      -q      Display queue depths
5242  *      -w      Display latency histograms
5243  *      -r      Display request size histogram
5244  *      -T      Display a timestamp in date(1) or Unix format
5245  *      -n      Only print headers once
5246  *
5247  * This command can be tricky because we want to be able to deal with pool
5248  * creation/destruction as well as vdev configuration changes.  The bulk of this
5249  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
5250  * on pool_list_update() to detect the addition of new pools.  Configuration
5251  * changes are all handled within libzfs.
5252  */
5253 int
5254 zpool_do_iostat(int argc, char **argv)
5255 {
5256         int c;
5257         int ret;
5258         int npools;
5259         float interval = 0;
5260         unsigned long count = 0;
5261         int winheight = 24;
5262         zpool_list_t *list;
5263         boolean_t verbose = B_FALSE;
5264         boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
5265         boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
5266         boolean_t omit_since_boot = B_FALSE;
5267         boolean_t guid = B_FALSE;
5268         boolean_t follow_links = B_FALSE;
5269         boolean_t full_name = B_FALSE;
5270         boolean_t headers_once = B_FALSE;
5271         iostat_cbdata_t cb = { 0 };
5272         char *cmd = NULL;
5273
5274         /* Used for printing error message */
5275         const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
5276             [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
5277
5278         uint64_t unsupported_flags;
5279
5280         /* check options */
5281         while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwnH")) != -1) {
5282                 switch (c) {
5283                 case 'c':
5284                         if (cmd != NULL) {
5285                                 fprintf(stderr,
5286                                     gettext("Can't set -c flag twice\n"));
5287                                 exit(1);
5288                         }
5289
5290                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
5291                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
5292                                 fprintf(stderr, gettext(
5293                                     "Can't run -c, disabled by "
5294                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
5295                                 exit(1);
5296                         }
5297
5298                         if ((getuid() <= 0 || geteuid() <= 0) &&
5299                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
5300                                 fprintf(stderr, gettext(
5301                                     "Can't run -c with root privileges "
5302                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
5303                                 exit(1);
5304                         }
5305                         cmd = optarg;
5306                         verbose = B_TRUE;
5307                         break;
5308                 case 'g':
5309                         guid = B_TRUE;
5310                         break;
5311                 case 'L':
5312                         follow_links = B_TRUE;
5313                         break;
5314                 case 'P':
5315                         full_name = B_TRUE;
5316                         break;
5317                 case 'T':
5318                         get_timestamp_arg(*optarg);
5319                         break;
5320                 case 'v':
5321                         verbose = B_TRUE;
5322                         break;
5323                 case 'p':
5324                         parsable = B_TRUE;
5325                         break;
5326                 case 'l':
5327                         latency = B_TRUE;
5328                         break;
5329                 case 'q':
5330                         queues = B_TRUE;
5331                         break;
5332                 case 'H':
5333                         scripted = B_TRUE;
5334                         break;
5335                 case 'w':
5336                         l_histo = B_TRUE;
5337                         break;
5338                 case 'r':
5339                         rq_histo = B_TRUE;
5340                         break;
5341                 case 'y':
5342                         omit_since_boot = B_TRUE;
5343                         break;
5344                 case 'n':
5345                         headers_once = B_TRUE;
5346                         break;
5347                 case 'h':
5348                         usage(B_FALSE);
5349                         break;
5350                 case '?':
5351                         if (optopt == 'c') {
5352                                 print_zpool_script_list("iostat");
5353                                 exit(0);
5354                         } else {
5355                                 fprintf(stderr,
5356                                     gettext("invalid option '%c'\n"), optopt);
5357                         }
5358                         usage(B_FALSE);
5359                 }
5360         }
5361
5362         argc -= optind;
5363         argv += optind;
5364
5365         cb.cb_literal = parsable;
5366         cb.cb_scripted = scripted;
5367
5368         if (guid)
5369                 cb.cb_name_flags |= VDEV_NAME_GUID;
5370         if (follow_links)
5371                 cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5372         if (full_name)
5373                 cb.cb_name_flags |= VDEV_NAME_PATH;
5374         cb.cb_iteration = 0;
5375         cb.cb_namewidth = 0;
5376         cb.cb_verbose = verbose;
5377
5378         /* Get our interval and count values (if any) */
5379         if (guid) {
5380                 get_interval_count_filter_guids(&argc, argv, &interval,
5381                     &count, &cb);
5382         } else {
5383                 get_interval_count(&argc, argv, &interval, &count);
5384         }
5385
5386         if (argc == 0) {
5387                 /* No args, so just print the defaults. */
5388         } else if (are_all_pools(argc, argv)) {
5389                 /* All the args are pool names */
5390         } else if (are_vdevs_in_pool(argc, argv, NULL, &cb)) {
5391                 /* All the args are vdevs */
5392                 cb.cb_vdev_names = argv;
5393                 cb.cb_vdev_names_count = argc;
5394                 argc = 0; /* No pools to process */
5395         } else if (are_all_pools(1, argv)) {
5396                 /* The first arg is a pool name */
5397                 if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0], &cb)) {
5398                         /* ...and the rest are vdev names */
5399                         cb.cb_vdev_names = argv + 1;
5400                         cb.cb_vdev_names_count = argc - 1;
5401                         argc = 1; /* One pool to process */
5402                 } else {
5403                         fprintf(stderr, gettext("Expected either a list of "));
5404                         fprintf(stderr, gettext("pools, or list of vdevs in"));
5405                         fprintf(stderr, " \"%s\", ", argv[0]);
5406                         fprintf(stderr, gettext("but got:\n"));
5407                         error_list_unresolved_vdevs(argc - 1, argv + 1,
5408                             argv[0], &cb);
5409                         fprintf(stderr, "\n");
5410                         usage(B_FALSE);
5411                         return (1);
5412                 }
5413         } else {
5414                 /*
5415                  * The args don't make sense. The first arg isn't a pool name,
5416                  * nor are all the args vdevs.
5417                  */
5418                 fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
5419                 fprintf(stderr, "\n");
5420                 return (1);
5421         }
5422
5423         if (cb.cb_vdev_names_count != 0) {
5424                 /*
5425                  * If user specified vdevs, it implies verbose.
5426                  */
5427                 cb.cb_verbose = B_TRUE;
5428         }
5429
5430         /*
5431          * Construct the list of all interesting pools.
5432          */
5433         ret = 0;
5434         if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
5435                 return (1);
5436
5437         if (pool_list_count(list) == 0 && argc != 0) {
5438                 pool_list_free(list);
5439                 return (1);
5440         }
5441
5442         if (pool_list_count(list) == 0 && interval == 0) {
5443                 pool_list_free(list);
5444                 (void) fprintf(stderr, gettext("no pools available\n"));
5445                 return (1);
5446         }
5447
5448         if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
5449                 pool_list_free(list);
5450                 (void) fprintf(stderr,
5451                     gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
5452                 usage(B_FALSE);
5453                 return (1);
5454         }
5455
5456         if (l_histo && rq_histo) {
5457                 pool_list_free(list);
5458                 (void) fprintf(stderr,
5459                     gettext("Only one of [-r|-w] can be passed at a time\n"));
5460                 usage(B_FALSE);
5461                 return (1);
5462         }
5463
5464         /*
5465          * Enter the main iostat loop.
5466          */
5467         cb.cb_list = list;
5468
5469         if (l_histo) {
5470                 /*
5471                  * Histograms tables look out of place when you try to display
5472                  * them with the other stats, so make a rule that you can only
5473                  * print histograms by themselves.
5474                  */
5475                 cb.cb_flags = IOS_L_HISTO_M;
5476         } else if (rq_histo) {
5477                 cb.cb_flags = IOS_RQ_HISTO_M;
5478         } else {
5479                 cb.cb_flags = IOS_DEFAULT_M;
5480                 if (latency)
5481                         cb.cb_flags |= IOS_LATENCY_M;
5482                 if (queues)
5483                         cb.cb_flags |= IOS_QUEUES_M;
5484         }
5485
5486         /*
5487          * See if the module supports all the stats we want to display.
5488          */
5489         unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
5490         if (unsupported_flags) {
5491                 uint64_t f;
5492                 int idx;
5493                 fprintf(stderr,
5494                     gettext("The loaded zfs module doesn't support:"));
5495
5496                 /* for each bit set in unsupported_flags */
5497                 for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
5498                         idx = lowbit64(f) - 1;
5499                         fprintf(stderr, " -%c", flag_to_arg[idx]);
5500                 }
5501
5502                 fprintf(stderr, ".  Try running a newer module.\n");
5503                 pool_list_free(list);
5504
5505                 return (1);
5506         }
5507
5508         for (;;) {
5509                 if ((npools = pool_list_count(list)) == 0)
5510                         (void) fprintf(stderr, gettext("no pools available\n"));
5511                 else {
5512                         /*
5513                          * If this is the first iteration and -y was supplied
5514                          * we skip any printing.
5515                          */
5516                         boolean_t skip = (omit_since_boot &&
5517                             cb.cb_iteration == 0);
5518
5519                         /*
5520                          * Refresh all statistics.  This is done as an
5521                          * explicit step before calculating the maximum name
5522                          * width, so that any * configuration changes are
5523                          * properly accounted for.
5524                          */
5525                         (void) pool_list_iter(list, B_FALSE, refresh_iostat,
5526                             &cb);
5527
5528                         /*
5529                          * Iterate over all pools to determine the maximum width
5530                          * for the pool / device name column across all pools.
5531                          */
5532                         cb.cb_namewidth = 0;
5533                         (void) pool_list_iter(list, B_FALSE,
5534                             get_namewidth_iostat, &cb);
5535
5536                         if (timestamp_fmt != NODATE)
5537                                 print_timestamp(timestamp_fmt);
5538
5539                         if (cmd != NULL && cb.cb_verbose &&
5540                             !(cb.cb_flags & IOS_ANYHISTO_M)) {
5541                                 cb.vcdl = all_pools_for_each_vdev_run(argc,
5542                                     argv, cmd, g_zfs, cb.cb_vdev_names,
5543                                     cb.cb_vdev_names_count, cb.cb_name_flags);
5544                         } else {
5545                                 cb.vcdl = NULL;
5546                         }
5547
5548
5549                         /*
5550                          * Check terminal size so we can print headers
5551                          * even when terminal window has its height
5552                          * changed.
5553                          */
5554                         winheight = terminal_height();
5555                         /*
5556                          * Are we connected to TTY? If not, headers_once
5557                          * should be true, to avoid breaking scripts.
5558                          */
5559                         if (winheight < 0)
5560                                 headers_once = B_TRUE;
5561
5562                         /*
5563                          * If it's the first time and we're not skipping it,
5564                          * or either skip or verbose mode, print the header.
5565                          *
5566                          * The histogram code explicitly prints its header on
5567                          * every vdev, so skip this for histograms.
5568                          */
5569                         if (((++cb.cb_iteration == 1 && !skip) ||
5570                             (skip != verbose) ||
5571                             (!headers_once &&
5572                             (cb.cb_iteration % winheight) == 0)) &&
5573                             (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
5574                             !cb.cb_scripted)
5575                                 print_iostat_header(&cb);
5576
5577                         if (skip) {
5578                                 (void) fsleep(interval);
5579                                 continue;
5580                         }
5581
5582                         pool_list_iter(list, B_FALSE, print_iostat, &cb);
5583
5584                         /*
5585                          * If there's more than one pool, and we're not in
5586                          * verbose mode (which prints a separator for us),
5587                          * then print a separator.
5588                          *
5589                          * In addition, if we're printing specific vdevs then
5590                          * we also want an ending separator.
5591                          */
5592                         if (((npools > 1 && !verbose &&
5593                             !(cb.cb_flags & IOS_ANYHISTO_M)) ||
5594                             (!(cb.cb_flags & IOS_ANYHISTO_M) &&
5595                             cb.cb_vdev_names_count)) &&
5596                             !cb.cb_scripted) {
5597                                 print_iostat_separator(&cb);
5598                                 if (cb.vcdl != NULL)
5599                                         print_cmd_columns(cb.vcdl, 1);
5600                                 printf("\n");
5601                         }
5602
5603                         if (cb.vcdl != NULL)
5604                                 free_vdev_cmd_data_list(cb.vcdl);
5605
5606                 }
5607
5608                 /*
5609                  * Flush the output so that redirection to a file isn't buffered
5610                  * indefinitely.
5611                  */
5612                 (void) fflush(stdout);
5613
5614                 if (interval == 0)
5615                         break;
5616
5617                 if (count != 0 && --count == 0)
5618                         break;
5619
5620                 (void) fsleep(interval);
5621         }
5622
5623         pool_list_free(list);
5624
5625         return (ret);
5626 }
5627
5628 typedef struct list_cbdata {
5629         boolean_t       cb_verbose;
5630         int             cb_name_flags;
5631         int             cb_namewidth;
5632         boolean_t       cb_scripted;
5633         zprop_list_t    *cb_proplist;
5634         boolean_t       cb_literal;
5635 } list_cbdata_t;
5636
5637
5638 /*
5639  * Given a list of columns to display, output appropriate headers for each one.
5640  */
5641 static void
5642 print_header(list_cbdata_t *cb)
5643 {
5644         zprop_list_t *pl = cb->cb_proplist;
5645         char headerbuf[ZPOOL_MAXPROPLEN];
5646         const char *header;
5647         boolean_t first = B_TRUE;
5648         boolean_t right_justify;
5649         size_t width = 0;
5650
5651         for (; pl != NULL; pl = pl->pl_next) {
5652                 width = pl->pl_width;
5653                 if (first && cb->cb_verbose) {
5654                         /*
5655                          * Reset the width to accommodate the verbose listing
5656                          * of devices.
5657                          */
5658                         width = cb->cb_namewidth;
5659                 }
5660
5661                 if (!first)
5662                         (void) printf("  ");
5663                 else
5664                         first = B_FALSE;
5665
5666                 right_justify = B_FALSE;
5667                 if (pl->pl_prop != ZPROP_INVAL) {
5668                         header = zpool_prop_column_name(pl->pl_prop);
5669                         right_justify = zpool_prop_align_right(pl->pl_prop);
5670                 } else {
5671                         int i;
5672
5673                         for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
5674                                 headerbuf[i] = toupper(pl->pl_user_prop[i]);
5675                         headerbuf[i] = '\0';
5676                         header = headerbuf;
5677                 }
5678
5679                 if (pl->pl_next == NULL && !right_justify)
5680                         (void) printf("%s", header);
5681                 else if (right_justify)
5682                         (void) printf("%*s", (int)width, header);
5683                 else
5684                         (void) printf("%-*s", (int)width, header);
5685         }
5686
5687         (void) printf("\n");
5688 }
5689
5690 /*
5691  * Given a pool and a list of properties, print out all the properties according
5692  * to the described layout. Used by zpool_do_list().
5693  */
5694 static void
5695 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
5696 {
5697         zprop_list_t *pl = cb->cb_proplist;
5698         boolean_t first = B_TRUE;
5699         char property[ZPOOL_MAXPROPLEN];
5700         char *propstr;
5701         boolean_t right_justify;
5702         size_t width;
5703
5704         for (; pl != NULL; pl = pl->pl_next) {
5705
5706                 width = pl->pl_width;
5707                 if (first && cb->cb_verbose) {
5708                         /*
5709                          * Reset the width to accommodate the verbose listing
5710                          * of devices.
5711                          */
5712                         width = cb->cb_namewidth;
5713                 }
5714
5715                 if (!first) {
5716                         if (cb->cb_scripted)
5717                                 (void) printf("\t");
5718                         else
5719                                 (void) printf("  ");
5720                 } else {
5721                         first = B_FALSE;
5722                 }
5723
5724                 right_justify = B_FALSE;
5725                 if (pl->pl_prop != ZPROP_INVAL) {
5726                         if (zpool_get_prop(zhp, pl->pl_prop, property,
5727                             sizeof (property), NULL, cb->cb_literal) != 0)
5728                                 propstr = "-";
5729                         else
5730                                 propstr = property;
5731
5732                         right_justify = zpool_prop_align_right(pl->pl_prop);
5733                 } else if ((zpool_prop_feature(pl->pl_user_prop) ||
5734                     zpool_prop_unsupported(pl->pl_user_prop)) &&
5735                     zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
5736                     sizeof (property)) == 0) {
5737                         propstr = property;
5738                 } else {
5739                         propstr = "-";
5740                 }
5741
5742
5743                 /*
5744                  * If this is being called in scripted mode, or if this is the
5745                  * last column and it is left-justified, don't include a width
5746                  * format specifier.
5747                  */
5748                 if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
5749                         (void) printf("%s", propstr);
5750                 else if (right_justify)
5751                         (void) printf("%*s", (int)width, propstr);
5752                 else
5753                         (void) printf("%-*s", (int)width, propstr);
5754         }
5755
5756         (void) printf("\n");
5757 }
5758
5759 static void
5760 print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
5761     boolean_t scripted, boolean_t valid, enum zfs_nicenum_format format)
5762 {
5763         char propval[64];
5764         boolean_t fixed;
5765         size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
5766
5767         switch (prop) {
5768         case ZPOOL_PROP_EXPANDSZ:
5769         case ZPOOL_PROP_CHECKPOINT:
5770         case ZPOOL_PROP_DEDUPRATIO:
5771                 if (value == 0)
5772                         (void) strlcpy(propval, "-", sizeof (propval));
5773                 else
5774                         zfs_nicenum_format(value, propval, sizeof (propval),
5775                             format);
5776                 break;
5777         case ZPOOL_PROP_FRAGMENTATION:
5778                 if (value == ZFS_FRAG_INVALID) {
5779                         (void) strlcpy(propval, "-", sizeof (propval));
5780                 } else if (format == ZFS_NICENUM_RAW) {
5781                         (void) snprintf(propval, sizeof (propval), "%llu",
5782                             (unsigned long long)value);
5783                 } else {
5784                         (void) snprintf(propval, sizeof (propval), "%llu%%",
5785                             (unsigned long long)value);
5786                 }
5787                 break;
5788         case ZPOOL_PROP_CAPACITY:
5789                 /* capacity value is in parts-per-10,000 (aka permyriad) */
5790                 if (format == ZFS_NICENUM_RAW)
5791                         (void) snprintf(propval, sizeof (propval), "%llu",
5792                             (unsigned long long)value / 100);
5793                 else
5794                         (void) snprintf(propval, sizeof (propval),
5795                             value < 1000 ? "%1.2f%%" : value < 10000 ?
5796                             "%2.1f%%" : "%3.0f%%", value / 100.0);
5797                 break;
5798         case ZPOOL_PROP_HEALTH:
5799                 width = 8;
5800                 snprintf(propval, sizeof (propval), "%-*s", (int)width, str);
5801                 break;
5802         default:
5803                 zfs_nicenum_format(value, propval, sizeof (propval), format);
5804         }
5805
5806         if (!valid)
5807                 (void) strlcpy(propval, "-", sizeof (propval));
5808
5809         if (scripted)
5810                 (void) printf("\t%s", propval);
5811         else
5812                 (void) printf("  %*s", (int)width, propval);
5813 }
5814
5815 /*
5816  * print static default line per vdev
5817  * not compatible with '-o' <proplist> option
5818  */
5819 static void
5820 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
5821     list_cbdata_t *cb, int depth, boolean_t isspare)
5822 {
5823         nvlist_t **child;
5824         vdev_stat_t *vs;
5825         uint_t c, children;
5826         char *vname;
5827         boolean_t scripted = cb->cb_scripted;
5828         uint64_t islog = B_FALSE;
5829         char *dashes = "%-*s      -      -      -        -         "
5830             "-      -      -      -  -\n";
5831
5832         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
5833             (uint64_t **)&vs, &c) == 0);
5834
5835         if (name != NULL) {
5836                 boolean_t toplevel = (vs->vs_space != 0);
5837                 uint64_t cap;
5838                 enum zfs_nicenum_format format;
5839                 const char *state;
5840
5841                 if (cb->cb_literal)
5842                         format = ZFS_NICENUM_RAW;
5843                 else
5844                         format = ZFS_NICENUM_1024;
5845
5846                 if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
5847                         return;
5848
5849                 if (scripted)
5850                         (void) printf("\t%s", name);
5851                 else if (strlen(name) + depth > cb->cb_namewidth)
5852                         (void) printf("%*s%s", depth, "", name);
5853                 else
5854                         (void) printf("%*s%s%*s", depth, "", name,
5855                             (int)(cb->cb_namewidth - strlen(name) - depth), "");
5856
5857                 /*
5858                  * Print the properties for the individual vdevs. Some
5859                  * properties are only applicable to toplevel vdevs. The
5860                  * 'toplevel' boolean value is passed to the print_one_column()
5861                  * to indicate that the value is valid.
5862                  */
5863                 print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, NULL, scripted,
5864                     toplevel, format);
5865                 print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
5866                     scripted, toplevel, format);
5867                 print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
5868                     NULL, scripted, toplevel, format);
5869                 print_one_column(ZPOOL_PROP_CHECKPOINT,
5870                     vs->vs_checkpoint_space, NULL, scripted, toplevel, format);
5871                 print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
5872                     scripted, B_TRUE, format);
5873                 print_one_column(ZPOOL_PROP_FRAGMENTATION,
5874                     vs->vs_fragmentation, NULL, scripted,
5875                     (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
5876                     format);
5877                 cap = (vs->vs_space == 0) ? 0 :
5878                     (vs->vs_alloc * 10000 / vs->vs_space);
5879                 print_one_column(ZPOOL_PROP_CAPACITY, cap, NULL,
5880                     scripted, toplevel, format);
5881                 print_one_column(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
5882                     scripted, toplevel, format);
5883                 state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
5884                 if (isspare) {
5885                         if (vs->vs_aux == VDEV_AUX_SPARED)
5886                                 state = "INUSE";
5887                         else if (vs->vs_state == VDEV_STATE_HEALTHY)
5888                                 state = "AVAIL";
5889                 }
5890                 print_one_column(ZPOOL_PROP_HEALTH, 0, state, scripted,
5891                     B_TRUE, format);
5892                 (void) printf("\n");
5893         }
5894
5895         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
5896             &child, &children) != 0)
5897                 return;
5898
5899         /* list the normal vdevs first */
5900         for (c = 0; c < children; c++) {
5901                 uint64_t ishole = B_FALSE;
5902
5903                 if (nvlist_lookup_uint64(child[c],
5904                     ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
5905                         continue;
5906
5907                 if (nvlist_lookup_uint64(child[c],
5908                     ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
5909                         continue;
5910
5911                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
5912                         continue;
5913
5914                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
5915                     cb->cb_name_flags);
5916                 print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
5917                 free(vname);
5918         }
5919
5920         /* list the classes: 'logs', 'dedup', and 'special' */
5921         for (uint_t n = 0; n < 3; n++) {
5922                 boolean_t printed = B_FALSE;
5923
5924                 for (c = 0; c < children; c++) {
5925                         char *bias = NULL;
5926                         char *type = NULL;
5927
5928                         if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
5929                             &islog) == 0 && islog) {
5930                                 bias = VDEV_ALLOC_CLASS_LOGS;
5931                         } else {
5932                                 (void) nvlist_lookup_string(child[c],
5933                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
5934                                 (void) nvlist_lookup_string(child[c],
5935                                     ZPOOL_CONFIG_TYPE, &type);
5936                         }
5937                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
5938                                 continue;
5939                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
5940                                 continue;
5941
5942                         if (!printed) {
5943                                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5944                                 (void) printf(dashes, cb->cb_namewidth,
5945                                     class_name[n]);
5946                                 printed = B_TRUE;
5947                         }
5948                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5949                             cb->cb_name_flags);
5950                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
5951                             B_FALSE);
5952                         free(vname);
5953                 }
5954         }
5955
5956         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
5957             &child, &children) == 0 && children > 0) {
5958                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5959                 (void) printf(dashes, cb->cb_namewidth, "cache");
5960                 for (c = 0; c < children; c++) {
5961                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5962                             cb->cb_name_flags);
5963                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
5964                             B_FALSE);
5965                         free(vname);
5966                 }
5967         }
5968
5969         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
5970             &children) == 0 && children > 0) {
5971                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5972                 (void) printf(dashes, cb->cb_namewidth, "spare");
5973                 for (c = 0; c < children; c++) {
5974                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5975                             cb->cb_name_flags);
5976                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
5977                             B_TRUE);
5978                         free(vname);
5979                 }
5980         }
5981 }
5982
5983 /*
5984  * Generic callback function to list a pool.
5985  */
5986 static int
5987 list_callback(zpool_handle_t *zhp, void *data)
5988 {
5989         list_cbdata_t *cbp = data;
5990
5991         print_pool(zhp, cbp);
5992
5993         if (cbp->cb_verbose) {
5994                 nvlist_t *config, *nvroot;
5995
5996                 config = zpool_get_config(zhp, NULL);
5997                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5998                     &nvroot) == 0);
5999                 print_list_stats(zhp, NULL, nvroot, cbp, 0, B_FALSE);
6000         }
6001
6002         return (0);
6003 }
6004
6005 /*
6006  * Set the minimum pool/vdev name column width.  The width must be at least 9,
6007  * but may be as large as needed.
6008  */
6009 static int
6010 get_namewidth_list(zpool_handle_t *zhp, void *data)
6011 {
6012         list_cbdata_t *cb = data;
6013         int width;
6014
6015         width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
6016             cb->cb_verbose);
6017
6018         if (width < 9)
6019                 width = 9;
6020
6021         cb->cb_namewidth = width;
6022
6023         return (0);
6024 }
6025
6026 /*
6027  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
6028  *
6029  *      -g      Display guid for individual vdev name.
6030  *      -H      Scripted mode.  Don't display headers, and separate properties
6031  *              by a single tab.
6032  *      -L      Follow links when resolving vdev path name.
6033  *      -o      List of properties to display.  Defaults to
6034  *              "name,size,allocated,free,expandsize,fragmentation,capacity,"
6035  *              "dedupratio,health,altroot"
6036  *      -p      Display values in parsable (exact) format.
6037  *      -P      Display full path for vdev name.
6038  *      -T      Display a timestamp in date(1) or Unix format
6039  *
6040  * List all pools in the system, whether or not they're healthy.  Output space
6041  * statistics for each one, as well as health status summary.
6042  */
6043 int
6044 zpool_do_list(int argc, char **argv)
6045 {
6046         int c;
6047         int ret = 0;
6048         list_cbdata_t cb = { 0 };
6049         static char default_props[] =
6050             "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
6051             "capacity,dedupratio,health,altroot";
6052         char *props = default_props;
6053         float interval = 0;
6054         unsigned long count = 0;
6055         zpool_list_t *list;
6056         boolean_t first = B_TRUE;
6057
6058         /* check options */
6059         while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
6060                 switch (c) {
6061                 case 'g':
6062                         cb.cb_name_flags |= VDEV_NAME_GUID;
6063                         break;
6064                 case 'H':
6065                         cb.cb_scripted = B_TRUE;
6066                         break;
6067                 case 'L':
6068                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6069                         break;
6070                 case 'o':
6071                         props = optarg;
6072                         break;
6073                 case 'P':
6074                         cb.cb_name_flags |= VDEV_NAME_PATH;
6075                         break;
6076                 case 'p':
6077                         cb.cb_literal = B_TRUE;
6078                         break;
6079                 case 'T':
6080                         get_timestamp_arg(*optarg);
6081                         break;
6082                 case 'v':
6083                         cb.cb_verbose = B_TRUE;
6084                         cb.cb_namewidth = 8;    /* 8 until precalc is avail */
6085                         break;
6086                 case ':':
6087                         (void) fprintf(stderr, gettext("missing argument for "
6088                             "'%c' option\n"), optopt);
6089                         usage(B_FALSE);
6090                         break;
6091                 case '?':
6092                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6093                             optopt);
6094                         usage(B_FALSE);
6095                 }
6096         }
6097
6098         argc -= optind;
6099         argv += optind;
6100
6101         get_interval_count(&argc, argv, &interval, &count);
6102
6103         if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
6104                 usage(B_FALSE);
6105
6106         for (;;) {
6107                 if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
6108                     &ret)) == NULL)
6109                         return (1);
6110
6111                 if (pool_list_count(list) == 0)
6112                         break;
6113
6114                 cb.cb_namewidth = 0;
6115                 (void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);
6116
6117                 if (timestamp_fmt != NODATE)
6118                         print_timestamp(timestamp_fmt);
6119
6120                 if (!cb.cb_scripted && (first || cb.cb_verbose)) {
6121                         print_header(&cb);
6122                         first = B_FALSE;
6123                 }
6124                 ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
6125
6126                 if (interval == 0)
6127                         break;
6128
6129                 if (count != 0 && --count == 0)
6130                         break;
6131
6132                 pool_list_free(list);
6133                 (void) fsleep(interval);
6134         }
6135
6136         if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
6137                 (void) printf(gettext("no pools available\n"));
6138                 ret = 0;
6139         }
6140
6141         pool_list_free(list);
6142         zprop_free_list(cb.cb_proplist);
6143         return (ret);
6144 }
6145
6146 static int
6147 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
6148 {
6149         boolean_t force = B_FALSE;
6150         boolean_t rebuild = B_FALSE;
6151         boolean_t wait = B_FALSE;
6152         int c;
6153         nvlist_t *nvroot;
6154         char *poolname, *old_disk, *new_disk;
6155         zpool_handle_t *zhp;
6156         nvlist_t *props = NULL;
6157         char *propval;
6158         int ret;
6159
6160         /* check options */
6161         while ((c = getopt(argc, argv, "fo:sw")) != -1) {
6162                 switch (c) {
6163                 case 'f':
6164                         force = B_TRUE;
6165                         break;
6166                 case 'o':
6167                         if ((propval = strchr(optarg, '=')) == NULL) {
6168                                 (void) fprintf(stderr, gettext("missing "
6169                                     "'=' for -o option\n"));
6170                                 usage(B_FALSE);
6171                         }
6172                         *propval = '\0';
6173                         propval++;
6174
6175                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
6176                             (add_prop_list(optarg, propval, &props, B_TRUE)))
6177                                 usage(B_FALSE);
6178                         break;
6179                 case 's':
6180                         rebuild = B_TRUE;
6181                         break;
6182                 case 'w':
6183                         wait = B_TRUE;
6184                         break;
6185                 case '?':
6186                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6187                             optopt);
6188                         usage(B_FALSE);
6189                 }
6190         }
6191
6192         argc -= optind;
6193         argv += optind;
6194
6195         /* get pool name and check number of arguments */
6196         if (argc < 1) {
6197                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6198                 usage(B_FALSE);
6199         }
6200
6201         poolname = argv[0];
6202
6203         if (argc < 2) {
6204                 (void) fprintf(stderr,
6205                     gettext("missing <device> specification\n"));
6206                 usage(B_FALSE);
6207         }
6208
6209         old_disk = argv[1];
6210
6211         if (argc < 3) {
6212                 if (!replacing) {
6213                         (void) fprintf(stderr,
6214                             gettext("missing <new_device> specification\n"));
6215                         usage(B_FALSE);
6216                 }
6217                 new_disk = old_disk;
6218                 argc -= 1;
6219                 argv += 1;
6220         } else {
6221                 new_disk = argv[2];
6222                 argc -= 2;
6223                 argv += 2;
6224         }
6225
6226         if (argc > 1) {
6227                 (void) fprintf(stderr, gettext("too many arguments\n"));
6228                 usage(B_FALSE);
6229         }
6230
6231         if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
6232                 nvlist_free(props);
6233                 return (1);
6234         }
6235
6236         if (zpool_get_config(zhp, NULL) == NULL) {
6237                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
6238                     poolname);
6239                 zpool_close(zhp);
6240                 nvlist_free(props);
6241                 return (1);
6242         }
6243
6244         /* unless manually specified use "ashift" pool property (if set) */
6245         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
6246                 int intval;
6247                 zprop_source_t src;
6248                 char strval[ZPOOL_MAXPROPLEN];
6249
6250                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
6251                 if (src != ZPROP_SRC_DEFAULT) {
6252                         (void) sprintf(strval, "%" PRId32, intval);
6253                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
6254                             &props, B_TRUE) == 0);
6255                 }
6256         }
6257
6258         nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
6259             argc, argv);
6260         if (nvroot == NULL) {
6261                 zpool_close(zhp);
6262                 nvlist_free(props);
6263                 return (1);
6264         }
6265
6266         ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
6267             rebuild);
6268
6269         if (ret == 0 && wait)
6270                 ret = zpool_wait(zhp,
6271                     replacing ? ZPOOL_WAIT_REPLACE : ZPOOL_WAIT_RESILVER);
6272
6273         nvlist_free(props);
6274         nvlist_free(nvroot);
6275         zpool_close(zhp);
6276
6277         return (ret);
6278 }
6279
6280 /*
6281  * zpool replace [-fsw] [-o property=value] <pool> <device> <new_device>
6282  *
6283  *      -f      Force attach, even if <new_device> appears to be in use.
6284  *      -s      Use sequential instead of healing reconstruction for resilver.
6285  *      -o      Set property=value.
6286  *      -w      Wait for replacing to complete before returning
6287  *
6288  * Replace <device> with <new_device>.
6289  */
6290 /* ARGSUSED */
6291 int
6292 zpool_do_replace(int argc, char **argv)
6293 {
6294         return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
6295 }
6296
6297 /*
6298  * zpool attach [-fsw] [-o property=value] <pool> <device> <new_device>
6299  *
6300  *      -f      Force attach, even if <new_device> appears to be in use.
6301  *      -s      Use sequential instead of healing reconstruction for resilver.
6302  *      -o      Set property=value.
6303  *      -w      Wait for resilvering to complete before returning
6304  *
6305  * Attach <new_device> to the mirror containing <device>.  If <device> is not
6306  * part of a mirror, then <device> will be transformed into a mirror of
6307  * <device> and <new_device>.  In either case, <new_device> will begin life
6308  * with a DTL of [0, now], and will immediately begin to resilver itself.
6309  */
6310 int
6311 zpool_do_attach(int argc, char **argv)
6312 {
6313         return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
6314 }
6315
6316 /*
6317  * zpool detach [-f] <pool> <device>
6318  *
6319  *      -f      Force detach of <device>, even if DTLs argue against it
6320  *              (not supported yet)
6321  *
6322  * Detach a device from a mirror.  The operation will be refused if <device>
6323  * is the last device in the mirror, or if the DTLs indicate that this device
6324  * has the only valid copy of some data.
6325  */
6326 /* ARGSUSED */
6327 int
6328 zpool_do_detach(int argc, char **argv)
6329 {
6330         int c;
6331         char *poolname, *path;
6332         zpool_handle_t *zhp;
6333         int ret;
6334
6335         /* check options */
6336         while ((c = getopt(argc, argv, "")) != -1) {
6337                 switch (c) {
6338                 case '?':
6339                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6340                             optopt);
6341                         usage(B_FALSE);
6342                 }
6343         }
6344
6345         argc -= optind;
6346         argv += optind;
6347
6348         /* get pool name and check number of arguments */
6349         if (argc < 1) {
6350                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6351                 usage(B_FALSE);
6352         }
6353
6354         if (argc < 2) {
6355                 (void) fprintf(stderr,
6356                     gettext("missing <device> specification\n"));
6357                 usage(B_FALSE);
6358         }
6359
6360         poolname = argv[0];
6361         path = argv[1];
6362
6363         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6364                 return (1);
6365
6366         ret = zpool_vdev_detach(zhp, path);
6367
6368         zpool_close(zhp);
6369
6370         return (ret);
6371 }
6372
6373 /*
6374  * zpool split [-gLnP] [-o prop=val] ...
6375  *              [-o mntopt] ...
6376  *              [-R altroot] <pool> <newpool> [<device> ...]
6377  *
6378  *      -g      Display guid for individual vdev name.
6379  *      -L      Follow links when resolving vdev path name.
6380  *      -n      Do not split the pool, but display the resulting layout if
6381  *              it were to be split.
6382  *      -o      Set property=value, or set mount options.
6383  *      -P      Display full path for vdev name.
6384  *      -R      Mount the split-off pool under an alternate root.
6385  *      -l      Load encryption keys while importing.
6386  *
6387  * Splits the named pool and gives it the new pool name.  Devices to be split
6388  * off may be listed, provided that no more than one device is specified
6389  * per top-level vdev mirror.  The newly split pool is left in an exported
6390  * state unless -R is specified.
6391  *
6392  * Restrictions: the top-level of the pool pool must only be made up of
6393  * mirrors; all devices in the pool must be healthy; no device may be
6394  * undergoing a resilvering operation.
6395  */
6396 int
6397 zpool_do_split(int argc, char **argv)
6398 {
6399         char *srcpool, *newpool, *propval;
6400         char *mntopts = NULL;
6401         splitflags_t flags;
6402         int c, ret = 0;
6403         boolean_t loadkeys = B_FALSE;
6404         zpool_handle_t *zhp;
6405         nvlist_t *config, *props = NULL;
6406
6407         flags.dryrun = B_FALSE;
6408         flags.import = B_FALSE;
6409         flags.name_flags = 0;
6410
6411         /* check options */
6412         while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
6413                 switch (c) {
6414                 case 'g':
6415                         flags.name_flags |= VDEV_NAME_GUID;
6416                         break;
6417                 case 'L':
6418                         flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
6419                         break;
6420                 case 'R':
6421                         flags.import = B_TRUE;
6422                         if (add_prop_list(
6423                             zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
6424                             &props, B_TRUE) != 0) {
6425                                 nvlist_free(props);
6426                                 usage(B_FALSE);
6427                         }
6428                         break;
6429                 case 'l':
6430                         loadkeys = B_TRUE;
6431                         break;
6432                 case 'n':
6433                         flags.dryrun = B_TRUE;
6434                         break;
6435                 case 'o':
6436                         if ((propval = strchr(optarg, '=')) != NULL) {
6437                                 *propval = '\0';
6438                                 propval++;
6439                                 if (add_prop_list(optarg, propval,
6440                                     &props, B_TRUE) != 0) {
6441                                         nvlist_free(props);
6442                                         usage(B_FALSE);
6443                                 }
6444                         } else {
6445                                 mntopts = optarg;
6446                         }
6447                         break;
6448                 case 'P':
6449                         flags.name_flags |= VDEV_NAME_PATH;
6450                         break;
6451                 case ':':
6452                         (void) fprintf(stderr, gettext("missing argument for "
6453                             "'%c' option\n"), optopt);
6454                         usage(B_FALSE);
6455                         break;
6456                 case '?':
6457                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6458                             optopt);
6459                         usage(B_FALSE);
6460                         break;
6461                 }
6462         }
6463
6464         if (!flags.import && mntopts != NULL) {
6465                 (void) fprintf(stderr, gettext("setting mntopts is only "
6466                     "valid when importing the pool\n"));
6467                 usage(B_FALSE);
6468         }
6469
6470         if (!flags.import && loadkeys) {
6471                 (void) fprintf(stderr, gettext("loading keys is only "
6472                     "valid when importing the pool\n"));
6473                 usage(B_FALSE);
6474         }
6475
6476         argc -= optind;
6477         argv += optind;
6478
6479         if (argc < 1) {
6480                 (void) fprintf(stderr, gettext("Missing pool name\n"));
6481                 usage(B_FALSE);
6482         }
6483         if (argc < 2) {
6484                 (void) fprintf(stderr, gettext("Missing new pool name\n"));
6485                 usage(B_FALSE);
6486         }
6487
6488         srcpool = argv[0];
6489         newpool = argv[1];
6490
6491         argc -= 2;
6492         argv += 2;
6493
6494         if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
6495                 nvlist_free(props);
6496                 return (1);
6497         }
6498
6499         config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
6500         if (config == NULL) {
6501                 ret = 1;
6502         } else {
6503                 if (flags.dryrun) {
6504                         (void) printf(gettext("would create '%s' with the "
6505                             "following layout:\n\n"), newpool);
6506                         print_vdev_tree(NULL, newpool, config, 0, "",
6507                             flags.name_flags);
6508                 }
6509         }
6510
6511         zpool_close(zhp);
6512
6513         if (ret != 0 || flags.dryrun || !flags.import) {
6514                 nvlist_free(config);
6515                 nvlist_free(props);
6516                 return (ret);
6517         }
6518
6519         /*
6520          * The split was successful. Now we need to open the new
6521          * pool and import it.
6522          */
6523         if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
6524                 nvlist_free(config);
6525                 nvlist_free(props);
6526                 return (1);
6527         }
6528
6529         if (loadkeys) {
6530                 ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
6531                 if (ret != 0)
6532                         ret = 1;
6533         }
6534
6535         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
6536             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
6537                 ret = 1;
6538                 (void) fprintf(stderr, gettext("Split was successful, but "
6539                     "the datasets could not all be mounted\n"));
6540                 (void) fprintf(stderr, gettext("Try doing '%s' with a "
6541                     "different altroot\n"), "zpool import");
6542         }
6543         zpool_close(zhp);
6544         nvlist_free(config);
6545         nvlist_free(props);
6546
6547         return (ret);
6548 }
6549
6550
6551
6552 /*
6553  * zpool online <pool> <device> ...
6554  */
6555 int
6556 zpool_do_online(int argc, char **argv)
6557 {
6558         int c, i;
6559         char *poolname;
6560         zpool_handle_t *zhp;
6561         int ret = 0;
6562         vdev_state_t newstate;
6563         int flags = 0;
6564
6565         /* check options */
6566         while ((c = getopt(argc, argv, "e")) != -1) {
6567                 switch (c) {
6568                 case 'e':
6569                         flags |= ZFS_ONLINE_EXPAND;
6570                         break;
6571                 case '?':
6572                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6573                             optopt);
6574                         usage(B_FALSE);
6575                 }
6576         }
6577
6578         argc -= optind;
6579         argv += optind;
6580
6581         /* get pool name and check number of arguments */
6582         if (argc < 1) {
6583                 (void) fprintf(stderr, gettext("missing pool name\n"));
6584                 usage(B_FALSE);
6585         }
6586         if (argc < 2) {
6587                 (void) fprintf(stderr, gettext("missing device name\n"));
6588                 usage(B_FALSE);
6589         }
6590
6591         poolname = argv[0];
6592
6593         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6594                 return (1);
6595
6596         for (i = 1; i < argc; i++) {
6597                 if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
6598                         if (newstate != VDEV_STATE_HEALTHY) {
6599                                 (void) printf(gettext("warning: device '%s' "
6600                                     "onlined, but remains in faulted state\n"),
6601                                     argv[i]);
6602                                 if (newstate == VDEV_STATE_FAULTED)
6603                                         (void) printf(gettext("use 'zpool "
6604                                             "clear' to restore a faulted "
6605                                             "device\n"));
6606                                 else
6607                                         (void) printf(gettext("use 'zpool "
6608                                             "replace' to replace devices "
6609                                             "that are no longer present\n"));
6610                         }
6611                 } else {
6612                         ret = 1;
6613                 }
6614         }
6615
6616         zpool_close(zhp);
6617
6618         return (ret);
6619 }
6620
6621 /*
6622  * zpool offline [-ft] <pool> <device> ...
6623  *
6624  *      -f      Force the device into a faulted state.
6625  *
6626  *      -t      Only take the device off-line temporarily.  The offline/faulted
6627  *              state will not be persistent across reboots.
6628  */
6629 /* ARGSUSED */
6630 int
6631 zpool_do_offline(int argc, char **argv)
6632 {
6633         int c, i;
6634         char *poolname;
6635         zpool_handle_t *zhp;
6636         int ret = 0;
6637         boolean_t istmp = B_FALSE;
6638         boolean_t fault = B_FALSE;
6639
6640         /* check options */
6641         while ((c = getopt(argc, argv, "ft")) != -1) {
6642                 switch (c) {
6643                 case 'f':
6644                         fault = B_TRUE;
6645                         break;
6646                 case 't':
6647                         istmp = B_TRUE;
6648                         break;
6649                 case '?':
6650                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6651                             optopt);
6652                         usage(B_FALSE);
6653                 }
6654         }
6655
6656         argc -= optind;
6657         argv += optind;
6658
6659         /* get pool name and check number of arguments */
6660         if (argc < 1) {
6661                 (void) fprintf(stderr, gettext("missing pool name\n"));
6662                 usage(B_FALSE);
6663         }
6664         if (argc < 2) {
6665                 (void) fprintf(stderr, gettext("missing device name\n"));
6666                 usage(B_FALSE);
6667         }
6668
6669         poolname = argv[0];
6670
6671         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6672                 return (1);
6673
6674         for (i = 1; i < argc; i++) {
6675                 if (fault) {
6676                         uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
6677                         vdev_aux_t aux;
6678                         if (istmp == B_FALSE) {
6679                                 /* Force the fault to persist across imports */
6680                                 aux = VDEV_AUX_EXTERNAL_PERSIST;
6681                         } else {
6682                                 aux = VDEV_AUX_EXTERNAL;
6683                         }
6684
6685                         if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
6686                                 ret = 1;
6687                 } else {
6688                         if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
6689                                 ret = 1;
6690                 }
6691         }
6692
6693         zpool_close(zhp);
6694
6695         return (ret);
6696 }
6697
6698 /*
6699  * zpool clear <pool> [device]
6700  *
6701  * Clear all errors associated with a pool or a particular device.
6702  */
6703 int
6704 zpool_do_clear(int argc, char **argv)
6705 {
6706         int c;
6707         int ret = 0;
6708         boolean_t dryrun = B_FALSE;
6709         boolean_t do_rewind = B_FALSE;
6710         boolean_t xtreme_rewind = B_FALSE;
6711         uint32_t rewind_policy = ZPOOL_NO_REWIND;
6712         nvlist_t *policy = NULL;
6713         zpool_handle_t *zhp;
6714         char *pool, *device;
6715
6716         /* check options */
6717         while ((c = getopt(argc, argv, "FnX")) != -1) {
6718                 switch (c) {
6719                 case 'F':
6720                         do_rewind = B_TRUE;
6721                         break;
6722                 case 'n':
6723                         dryrun = B_TRUE;
6724                         break;
6725                 case 'X':
6726                         xtreme_rewind = B_TRUE;
6727                         break;
6728                 case '?':
6729                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6730                             optopt);
6731                         usage(B_FALSE);
6732                 }
6733         }
6734
6735         argc -= optind;
6736         argv += optind;
6737
6738         if (argc < 1) {
6739                 (void) fprintf(stderr, gettext("missing pool name\n"));
6740                 usage(B_FALSE);
6741         }
6742
6743         if (argc > 2) {
6744                 (void) fprintf(stderr, gettext("too many arguments\n"));
6745                 usage(B_FALSE);
6746         }
6747
6748         if ((dryrun || xtreme_rewind) && !do_rewind) {
6749                 (void) fprintf(stderr,
6750                     gettext("-n or -X only meaningful with -F\n"));
6751                 usage(B_FALSE);
6752         }
6753         if (dryrun)
6754                 rewind_policy = ZPOOL_TRY_REWIND;
6755         else if (do_rewind)
6756                 rewind_policy = ZPOOL_DO_REWIND;
6757         if (xtreme_rewind)
6758                 rewind_policy |= ZPOOL_EXTREME_REWIND;
6759
6760         /* In future, further rewind policy choices can be passed along here */
6761         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
6762             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
6763             rewind_policy) != 0) {
6764                 return (1);
6765         }
6766
6767         pool = argv[0];
6768         device = argc == 2 ? argv[1] : NULL;
6769
6770         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
6771                 nvlist_free(policy);
6772                 return (1);
6773         }
6774
6775         if (zpool_clear(zhp, device, policy) != 0)
6776                 ret = 1;
6777
6778         zpool_close(zhp);
6779
6780         nvlist_free(policy);
6781
6782         return (ret);
6783 }
6784
6785 /*
6786  * zpool reguid <pool>
6787  */
6788 int
6789 zpool_do_reguid(int argc, char **argv)
6790 {
6791         int c;
6792         char *poolname;
6793         zpool_handle_t *zhp;
6794         int ret = 0;
6795
6796         /* check options */
6797         while ((c = getopt(argc, argv, "")) != -1) {
6798                 switch (c) {
6799                 case '?':
6800                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6801                             optopt);
6802                         usage(B_FALSE);
6803                 }
6804         }
6805
6806         argc -= optind;
6807         argv += optind;
6808
6809         /* get pool name and check number of arguments */
6810         if (argc < 1) {
6811                 (void) fprintf(stderr, gettext("missing pool name\n"));
6812                 usage(B_FALSE);
6813         }
6814
6815         if (argc > 1) {
6816                 (void) fprintf(stderr, gettext("too many arguments\n"));
6817                 usage(B_FALSE);
6818         }
6819
6820         poolname = argv[0];
6821         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6822                 return (1);
6823
6824         ret = zpool_reguid(zhp);
6825
6826         zpool_close(zhp);
6827         return (ret);
6828 }
6829
6830
6831 /*
6832  * zpool reopen <pool>
6833  *
6834  * Reopen the pool so that the kernel can update the sizes of all vdevs.
6835  */
6836 int
6837 zpool_do_reopen(int argc, char **argv)
6838 {
6839         int c;
6840         int ret = 0;
6841         boolean_t scrub_restart = B_TRUE;
6842
6843         /* check options */
6844         while ((c = getopt(argc, argv, "n")) != -1) {
6845                 switch (c) {
6846                 case 'n':
6847                         scrub_restart = B_FALSE;
6848                         break;
6849                 case '?':
6850                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6851                             optopt);
6852                         usage(B_FALSE);
6853                 }
6854         }
6855
6856         argc -= optind;
6857         argv += optind;
6858
6859         /* if argc == 0 we will execute zpool_reopen_one on all pools */
6860         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_reopen_one,
6861             &scrub_restart);
6862
6863         return (ret);
6864 }
6865
6866 typedef struct scrub_cbdata {
6867         int     cb_type;
6868         pool_scrub_cmd_t cb_scrub_cmd;
6869 } scrub_cbdata_t;
6870
6871 static boolean_t
6872 zpool_has_checkpoint(zpool_handle_t *zhp)
6873 {
6874         nvlist_t *config, *nvroot;
6875
6876         config = zpool_get_config(zhp, NULL);
6877
6878         if (config != NULL) {
6879                 pool_checkpoint_stat_t *pcs = NULL;
6880                 uint_t c;
6881
6882                 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6883                 (void) nvlist_lookup_uint64_array(nvroot,
6884                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
6885
6886                 if (pcs == NULL || pcs->pcs_state == CS_NONE)
6887                         return (B_FALSE);
6888
6889                 assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
6890                     pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
6891                 return (B_TRUE);
6892         }
6893
6894         return (B_FALSE);
6895 }
6896
6897 static int
6898 scrub_callback(zpool_handle_t *zhp, void *data)
6899 {
6900         scrub_cbdata_t *cb = data;
6901         int err;
6902
6903         /*
6904          * Ignore faulted pools.
6905          */
6906         if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
6907                 (void) fprintf(stderr, gettext("cannot scan '%s': pool is "
6908                     "currently unavailable\n"), zpool_get_name(zhp));
6909                 return (1);
6910         }
6911
6912         err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
6913
6914         if (err == 0 && zpool_has_checkpoint(zhp) &&
6915             cb->cb_type == POOL_SCAN_SCRUB) {
6916                 (void) printf(gettext("warning: will not scrub state that "
6917                     "belongs to the checkpoint of pool '%s'\n"),
6918                     zpool_get_name(zhp));
6919         }
6920
6921         return (err != 0);
6922 }
6923
6924 static int
6925 wait_callback(zpool_handle_t *zhp, void *data)
6926 {
6927         zpool_wait_activity_t *act = data;
6928         return (zpool_wait(zhp, *act));
6929 }
6930
6931 /*
6932  * zpool scrub [-s | -p] [-w] <pool> ...
6933  *
6934  *      -s      Stop.  Stops any in-progress scrub.
6935  *      -p      Pause. Pause in-progress scrub.
6936  *      -w      Wait.  Blocks until scrub has completed.
6937  */
6938 int
6939 zpool_do_scrub(int argc, char **argv)
6940 {
6941         int c;
6942         scrub_cbdata_t cb;
6943         boolean_t wait = B_FALSE;
6944         int error;
6945
6946         cb.cb_type = POOL_SCAN_SCRUB;
6947         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
6948
6949         /* check options */
6950         while ((c = getopt(argc, argv, "spw")) != -1) {
6951                 switch (c) {
6952                 case 's':
6953                         cb.cb_type = POOL_SCAN_NONE;
6954                         break;
6955                 case 'p':
6956                         cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
6957                         break;
6958                 case 'w':
6959                         wait = B_TRUE;
6960                         break;
6961                 case '?':
6962                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6963                             optopt);
6964                         usage(B_FALSE);
6965                 }
6966         }
6967
6968         if (cb.cb_type == POOL_SCAN_NONE &&
6969             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
6970                 (void) fprintf(stderr, gettext("invalid option combination: "
6971                     "-s and -p are mutually exclusive\n"));
6972                 usage(B_FALSE);
6973         }
6974
6975         if (wait && (cb.cb_type == POOL_SCAN_NONE ||
6976             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
6977                 (void) fprintf(stderr, gettext("invalid option combination: "
6978                     "-w cannot be used with -p or -s\n"));
6979                 usage(B_FALSE);
6980         }
6981
6982         argc -= optind;
6983         argv += optind;
6984
6985         if (argc < 1) {
6986                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6987                 usage(B_FALSE);
6988         }
6989
6990         error = for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb);
6991
6992         if (wait && !error) {
6993                 zpool_wait_activity_t act = ZPOOL_WAIT_SCRUB;
6994                 error = for_each_pool(argc, argv, B_TRUE, NULL, wait_callback,
6995                     &act);
6996         }
6997
6998         return (error);
6999 }
7000
7001 /*
7002  * zpool resilver <pool> ...
7003  *
7004  *      Restarts any in-progress resilver
7005  */
7006 int
7007 zpool_do_resilver(int argc, char **argv)
7008 {
7009         int c;
7010         scrub_cbdata_t cb;
7011
7012         cb.cb_type = POOL_SCAN_RESILVER;
7013         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7014
7015         /* check options */
7016         while ((c = getopt(argc, argv, "")) != -1) {
7017                 switch (c) {
7018                 case '?':
7019                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7020                             optopt);
7021                         usage(B_FALSE);
7022                 }
7023         }
7024
7025         argc -= optind;
7026         argv += optind;
7027
7028         if (argc < 1) {
7029                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7030                 usage(B_FALSE);
7031         }
7032
7033         return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
7034 }
7035
7036 /*
7037  * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
7038  *
7039  *      -c              Cancel. Ends any in-progress trim.
7040  *      -d              Secure trim.  Requires kernel and device support.
7041  *      -r <rate>       Sets the TRIM rate in bytes (per second). Supports
7042  *                      adding a multiplier suffix such as 'k' or 'm'.
7043  *      -s              Suspend. TRIM can then be restarted with no flags.
7044  *      -w              Wait. Blocks until trimming has completed.
7045  */
7046 int
7047 zpool_do_trim(int argc, char **argv)
7048 {
7049         struct option long_options[] = {
7050                 {"cancel",      no_argument,            NULL,   'c'},
7051                 {"secure",      no_argument,            NULL,   'd'},
7052                 {"rate",        required_argument,      NULL,   'r'},
7053                 {"suspend",     no_argument,            NULL,   's'},
7054                 {"wait",        no_argument,            NULL,   'w'},
7055                 {0, 0, 0, 0}
7056         };
7057
7058         pool_trim_func_t cmd_type = POOL_TRIM_START;
7059         uint64_t rate = 0;
7060         boolean_t secure = B_FALSE;
7061         boolean_t wait = B_FALSE;
7062
7063         int c;
7064         while ((c = getopt_long(argc, argv, "cdr:sw", long_options, NULL))
7065             != -1) {
7066                 switch (c) {
7067                 case 'c':
7068                         if (cmd_type != POOL_TRIM_START &&
7069                             cmd_type != POOL_TRIM_CANCEL) {
7070                                 (void) fprintf(stderr, gettext("-c cannot be "
7071                                     "combined with other options\n"));
7072                                 usage(B_FALSE);
7073                         }
7074                         cmd_type = POOL_TRIM_CANCEL;
7075                         break;
7076                 case 'd':
7077                         if (cmd_type != POOL_TRIM_START) {
7078                                 (void) fprintf(stderr, gettext("-d cannot be "
7079                                     "combined with the -c or -s options\n"));
7080                                 usage(B_FALSE);
7081                         }
7082                         secure = B_TRUE;
7083                         break;
7084                 case 'r':
7085                         if (cmd_type != POOL_TRIM_START) {
7086                                 (void) fprintf(stderr, gettext("-r cannot be "
7087                                     "combined with the -c or -s options\n"));
7088                                 usage(B_FALSE);
7089                         }
7090                         if (zfs_nicestrtonum(NULL, optarg, &rate) == -1) {
7091                                 (void) fprintf(stderr,
7092                                     gettext("invalid value for rate\n"));
7093                                 usage(B_FALSE);
7094                         }
7095                         break;
7096                 case 's':
7097                         if (cmd_type != POOL_TRIM_START &&
7098                             cmd_type != POOL_TRIM_SUSPEND) {
7099                                 (void) fprintf(stderr, gettext("-s cannot be "
7100                                     "combined with other options\n"));
7101                                 usage(B_FALSE);
7102                         }
7103                         cmd_type = POOL_TRIM_SUSPEND;
7104                         break;
7105                 case 'w':
7106                         wait = B_TRUE;
7107                         break;
7108                 case '?':
7109                         if (optopt != 0) {
7110                                 (void) fprintf(stderr,
7111                                     gettext("invalid option '%c'\n"), optopt);
7112                         } else {
7113                                 (void) fprintf(stderr,
7114                                     gettext("invalid option '%s'\n"),
7115                                     argv[optind - 1]);
7116                         }
7117                         usage(B_FALSE);
7118                 }
7119         }
7120
7121         argc -= optind;
7122         argv += optind;
7123
7124         if (argc < 1) {
7125                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7126                 usage(B_FALSE);
7127                 return (-1);
7128         }
7129
7130         if (wait && (cmd_type != POOL_TRIM_START)) {
7131                 (void) fprintf(stderr, gettext("-w cannot be used with -c or "
7132                     "-s\n"));
7133                 usage(B_FALSE);
7134         }
7135
7136         char *poolname = argv[0];
7137         zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
7138         if (zhp == NULL)
7139                 return (-1);
7140
7141         trimflags_t trim_flags = {
7142                 .secure = secure,
7143                 .rate = rate,
7144                 .wait = wait,
7145         };
7146
7147         nvlist_t *vdevs = fnvlist_alloc();
7148         if (argc == 1) {
7149                 /* no individual leaf vdevs specified, so add them all */
7150                 nvlist_t *config = zpool_get_config(zhp, NULL);
7151                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
7152                     ZPOOL_CONFIG_VDEV_TREE);
7153                 zpool_collect_leaves(zhp, nvroot, vdevs);
7154                 trim_flags.fullpool = B_TRUE;
7155         } else {
7156                 trim_flags.fullpool = B_FALSE;
7157                 for (int i = 1; i < argc; i++) {
7158                         fnvlist_add_boolean(vdevs, argv[i]);
7159                 }
7160         }
7161
7162         int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
7163
7164         fnvlist_free(vdevs);
7165         zpool_close(zhp);
7166
7167         return (error);
7168 }
7169
7170 /*
7171  * Converts a total number of seconds to a human readable string broken
7172  * down in to days/hours/minutes/seconds.
7173  */
7174 static void
7175 secs_to_dhms(uint64_t total, char *buf)
7176 {
7177         uint64_t days = total / 60 / 60 / 24;
7178         uint64_t hours = (total / 60 / 60) % 24;
7179         uint64_t mins = (total / 60) % 60;
7180         uint64_t secs = (total % 60);
7181
7182         if (days > 0) {
7183                 (void) sprintf(buf, "%llu days %02llu:%02llu:%02llu",
7184                     (u_longlong_t)days, (u_longlong_t)hours,
7185                     (u_longlong_t)mins, (u_longlong_t)secs);
7186         } else {
7187                 (void) sprintf(buf, "%02llu:%02llu:%02llu",
7188                     (u_longlong_t)hours, (u_longlong_t)mins,
7189                     (u_longlong_t)secs);
7190         }
7191 }
7192
7193 /*
7194  * Print out detailed scrub status.
7195  */
7196 static void
7197 print_scan_scrub_resilver_status(pool_scan_stat_t *ps)
7198 {
7199         time_t start, end, pause;
7200         uint64_t pass_scanned, scanned, pass_issued, issued, total;
7201         uint64_t elapsed, scan_rate, issue_rate;
7202         double fraction_done;
7203         char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
7204         char srate_buf[7], irate_buf[7], time_buf[32];
7205
7206         printf("  ");
7207         printf_color(ANSI_BOLD, gettext("scan:"));
7208         printf(" ");
7209
7210         /* If there's never been a scan, there's not much to say. */
7211         if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
7212             ps->pss_func >= POOL_SCAN_FUNCS) {
7213                 (void) printf(gettext("none requested\n"));
7214                 return;
7215         }
7216
7217         start = ps->pss_start_time;
7218         end = ps->pss_end_time;
7219         pause = ps->pss_pass_scrub_pause;
7220
7221         zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
7222
7223         assert(ps->pss_func == POOL_SCAN_SCRUB ||
7224             ps->pss_func == POOL_SCAN_RESILVER);
7225
7226         /* Scan is finished or canceled. */
7227         if (ps->pss_state == DSS_FINISHED) {
7228                 secs_to_dhms(end - start, time_buf);
7229
7230                 if (ps->pss_func == POOL_SCAN_SCRUB) {
7231                         (void) printf(gettext("scrub repaired %s "
7232                             "in %s with %llu errors on %s"), processed_buf,
7233                             time_buf, (u_longlong_t)ps->pss_errors,
7234                             ctime(&end));
7235                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
7236                         (void) printf(gettext("resilvered %s "
7237                             "in %s with %llu errors on %s"), processed_buf,
7238                             time_buf, (u_longlong_t)ps->pss_errors,
7239                             ctime(&end));
7240                 }
7241                 return;
7242         } else if (ps->pss_state == DSS_CANCELED) {
7243                 if (ps->pss_func == POOL_SCAN_SCRUB) {
7244                         (void) printf(gettext("scrub canceled on %s"),
7245                             ctime(&end));
7246                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
7247                         (void) printf(gettext("resilver canceled on %s"),
7248                             ctime(&end));
7249                 }
7250                 return;
7251         }
7252
7253         assert(ps->pss_state == DSS_SCANNING);
7254
7255         /* Scan is in progress. Resilvers can't be paused. */
7256         if (ps->pss_func == POOL_SCAN_SCRUB) {
7257                 if (pause == 0) {
7258                         (void) printf(gettext("scrub in progress since %s"),
7259                             ctime(&start));
7260                 } else {
7261                         (void) printf(gettext("scrub paused since %s"),
7262                             ctime(&pause));
7263                         (void) printf(gettext("\tscrub started on %s"),
7264                             ctime(&start));
7265                 }
7266         } else if (ps->pss_func == POOL_SCAN_RESILVER) {
7267                 (void) printf(gettext("resilver in progress since %s"),
7268                     ctime(&start));
7269         }
7270
7271         scanned = ps->pss_examined;
7272         pass_scanned = ps->pss_pass_exam;
7273         issued = ps->pss_issued;
7274         pass_issued = ps->pss_pass_issued;
7275         total = ps->pss_to_examine;
7276
7277         /* we are only done with a block once we have issued the IO for it */
7278         fraction_done = (double)issued / total;
7279
7280         /* elapsed time for this pass, rounding up to 1 if it's 0 */
7281         elapsed = time(NULL) - ps->pss_pass_start;
7282         elapsed -= ps->pss_pass_scrub_spent_paused;
7283         elapsed = (elapsed != 0) ? elapsed : 1;
7284
7285         scan_rate = pass_scanned / elapsed;
7286         issue_rate = pass_issued / elapsed;
7287         uint64_t total_secs_left = (issue_rate != 0 && total >= issued) ?
7288             ((total - issued) / issue_rate) : UINT64_MAX;
7289         secs_to_dhms(total_secs_left, time_buf);
7290
7291         /* format all of the numbers we will be reporting */
7292         zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
7293         zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
7294         zfs_nicebytes(total, total_buf, sizeof (total_buf));
7295         zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
7296         zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
7297
7298         /* do not print estimated time if we have a paused scrub */
7299         if (pause == 0) {
7300                 (void) printf(gettext("\t%s scanned at %s/s, "
7301                     "%s issued at %s/s, %s total\n"),
7302                     scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
7303         } else {
7304                 (void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
7305                     scanned_buf, issued_buf, total_buf);
7306         }
7307
7308         if (ps->pss_func == POOL_SCAN_RESILVER) {
7309                 (void) printf(gettext("\t%s resilvered, %.2f%% done"),
7310                     processed_buf, 100 * fraction_done);
7311         } else if (ps->pss_func == POOL_SCAN_SCRUB) {
7312                 (void) printf(gettext("\t%s repaired, %.2f%% done"),
7313                     processed_buf, 100 * fraction_done);
7314         }
7315
7316         if (pause == 0) {
7317                 if (total_secs_left != UINT64_MAX &&
7318                     issue_rate >= 10 * 1024 * 1024) {
7319                         (void) printf(gettext(", %s to go\n"), time_buf);
7320                 } else {
7321                         (void) printf(gettext(", no estimated "
7322                             "completion time\n"));
7323                 }
7324         } else {
7325                 (void) printf(gettext("\n"));
7326         }
7327 }
7328
7329 static void
7330 print_rebuild_status_impl(vdev_rebuild_stat_t *vrs, char *vdev_name)
7331 {
7332         if (vrs == NULL || vrs->vrs_state == VDEV_REBUILD_NONE)
7333                 return;
7334
7335         printf("  ");
7336         printf_color(ANSI_BOLD, gettext("scan:"));
7337         printf(" ");
7338
7339         uint64_t bytes_scanned = vrs->vrs_bytes_scanned;
7340         uint64_t bytes_issued = vrs->vrs_bytes_issued;
7341         uint64_t bytes_rebuilt = vrs->vrs_bytes_rebuilt;
7342         uint64_t bytes_est = vrs->vrs_bytes_est;
7343         uint64_t scan_rate = (vrs->vrs_pass_bytes_scanned /
7344             (vrs->vrs_pass_time_ms + 1)) * 1000;
7345         uint64_t issue_rate = (vrs->vrs_pass_bytes_issued /
7346             (vrs->vrs_pass_time_ms + 1)) * 1000;
7347         double scan_pct = MIN((double)bytes_scanned * 100 /
7348             (bytes_est + 1), 100);
7349
7350         /* Format all of the numbers we will be reporting */
7351         char bytes_scanned_buf[7], bytes_issued_buf[7];
7352         char bytes_rebuilt_buf[7], bytes_est_buf[7];
7353         char scan_rate_buf[7], issue_rate_buf[7], time_buf[32];
7354         zfs_nicebytes(bytes_scanned, bytes_scanned_buf,
7355             sizeof (bytes_scanned_buf));
7356         zfs_nicebytes(bytes_issued, bytes_issued_buf,
7357             sizeof (bytes_issued_buf));
7358         zfs_nicebytes(bytes_rebuilt, bytes_rebuilt_buf,
7359             sizeof (bytes_rebuilt_buf));
7360         zfs_nicebytes(bytes_est, bytes_est_buf, sizeof (bytes_est_buf));
7361         zfs_nicebytes(scan_rate, scan_rate_buf, sizeof (scan_rate_buf));
7362         zfs_nicebytes(issue_rate, issue_rate_buf, sizeof (issue_rate_buf));
7363
7364         time_t start = vrs->vrs_start_time;
7365         time_t end = vrs->vrs_end_time;
7366
7367         /* Rebuild is finished or canceled. */
7368         if (vrs->vrs_state == VDEV_REBUILD_COMPLETE) {
7369                 secs_to_dhms(vrs->vrs_scan_time_ms / 1000, time_buf);
7370                 (void) printf(gettext("resilvered (%s) %s in %s "
7371                     "with %llu errors on %s"), vdev_name, bytes_rebuilt_buf,
7372                     time_buf, (u_longlong_t)vrs->vrs_errors, ctime(&end));
7373                 return;
7374         } else if (vrs->vrs_state == VDEV_REBUILD_CANCELED) {
7375                 (void) printf(gettext("resilver (%s) canceled on %s"),
7376                     vdev_name, ctime(&end));
7377                 return;
7378         } else if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7379                 (void) printf(gettext("resilver (%s) in progress since %s"),
7380                     vdev_name, ctime(&start));
7381         }
7382
7383         assert(vrs->vrs_state == VDEV_REBUILD_ACTIVE);
7384
7385         secs_to_dhms(MAX((int64_t)bytes_est - (int64_t)bytes_scanned, 0) /
7386             MAX(scan_rate, 1), time_buf);
7387
7388         (void) printf(gettext("\t%s scanned at %s/s, %s issued %s/s, "
7389             "%s total\n"), bytes_scanned_buf, scan_rate_buf,
7390             bytes_issued_buf, issue_rate_buf, bytes_est_buf);
7391         (void) printf(gettext("\t%s resilvered, %.2f%% done"),
7392             bytes_rebuilt_buf, scan_pct);
7393
7394         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7395                 if (scan_rate >= 10 * 1024 * 1024) {
7396                         (void) printf(gettext(", %s to go\n"), time_buf);
7397                 } else {
7398                         (void) printf(gettext(", no estimated "
7399                             "completion time\n"));
7400                 }
7401         } else {
7402                 (void) printf(gettext("\n"));
7403         }
7404 }
7405
7406 /*
7407  * Print rebuild status for top-level vdevs.
7408  */
7409 static void
7410 print_rebuild_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7411 {
7412         nvlist_t **child;
7413         uint_t children;
7414
7415         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7416             &child, &children) != 0)
7417                 children = 0;
7418
7419         for (uint_t c = 0; c < children; c++) {
7420                 vdev_rebuild_stat_t *vrs;
7421                 uint_t i;
7422
7423                 if (nvlist_lookup_uint64_array(child[c],
7424                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7425                         char *name = zpool_vdev_name(g_zfs, zhp,
7426                             child[c], VDEV_NAME_TYPE_ID);
7427                         print_rebuild_status_impl(vrs, name);
7428                         free(name);
7429                 }
7430         }
7431 }
7432
7433 /*
7434  * As we don't scrub checkpointed blocks, we want to warn the user that we
7435  * skipped scanning some blocks if a checkpoint exists or existed at any
7436  * time during the scan.  If a sequential instead of healing reconstruction
7437  * was performed then the blocks were reconstructed.  However, their checksums
7438  * have not been verified so we still print the warning.
7439  */
7440 static void
7441 print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
7442 {
7443         if (ps == NULL || pcs == NULL)
7444                 return;
7445
7446         if (pcs->pcs_state == CS_NONE ||
7447             pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
7448                 return;
7449
7450         assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
7451
7452         if (ps->pss_state == DSS_NONE)
7453                 return;
7454
7455         if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
7456             ps->pss_end_time < pcs->pcs_start_time)
7457                 return;
7458
7459         if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
7460                 (void) printf(gettext("    scan warning: skipped blocks "
7461                     "that are only referenced by the checkpoint.\n"));
7462         } else {
7463                 assert(ps->pss_state == DSS_SCANNING);
7464                 (void) printf(gettext("    scan warning: skipping blocks "
7465                     "that are only referenced by the checkpoint.\n"));
7466         }
7467 }
7468
7469 /*
7470  * Returns B_TRUE if there is an active rebuild in progress.  Otherwise,
7471  * B_FALSE is returned and 'rebuild_end_time' is set to the end time for
7472  * the last completed (or cancelled) rebuild.
7473  */
7474 static boolean_t
7475 check_rebuilding(nvlist_t *nvroot, uint64_t *rebuild_end_time)
7476 {
7477         nvlist_t **child;
7478         uint_t children;
7479         boolean_t rebuilding = B_FALSE;
7480         uint64_t end_time = 0;
7481
7482         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7483             &child, &children) != 0)
7484                 children = 0;
7485
7486         for (uint_t c = 0; c < children; c++) {
7487                 vdev_rebuild_stat_t *vrs;
7488                 uint_t i;
7489
7490                 if (nvlist_lookup_uint64_array(child[c],
7491                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7492
7493                         if (vrs->vrs_end_time > end_time)
7494                                 end_time = vrs->vrs_end_time;
7495
7496                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7497                                 rebuilding = B_TRUE;
7498                                 end_time = 0;
7499                                 break;
7500                         }
7501                 }
7502         }
7503
7504         if (rebuild_end_time != NULL)
7505                 *rebuild_end_time = end_time;
7506
7507         return (rebuilding);
7508 }
7509
7510 /*
7511  * Print the scan status.
7512  */
7513 static void
7514 print_scan_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7515 {
7516         uint64_t rebuild_end_time = 0, resilver_end_time = 0;
7517         boolean_t have_resilver = B_FALSE, have_scrub = B_FALSE;
7518         boolean_t active_resilver = B_FALSE;
7519         pool_checkpoint_stat_t *pcs = NULL;
7520         pool_scan_stat_t *ps = NULL;
7521         uint_t c;
7522
7523         if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
7524             (uint64_t **)&ps, &c) == 0) {
7525                 if (ps->pss_func == POOL_SCAN_RESILVER) {
7526                         resilver_end_time = ps->pss_end_time;
7527                         active_resilver = (ps->pss_state == DSS_SCANNING);
7528                 }
7529
7530                 have_resilver = (ps->pss_func == POOL_SCAN_RESILVER);
7531                 have_scrub = (ps->pss_func == POOL_SCAN_SCRUB);
7532         }
7533
7534         boolean_t active_rebuild = check_rebuilding(nvroot, &rebuild_end_time);
7535         boolean_t have_rebuild = (active_rebuild || (rebuild_end_time > 0));
7536
7537         /* Always print the scrub status when available. */
7538         if (have_scrub)
7539                 print_scan_scrub_resilver_status(ps);
7540
7541         /*
7542          * When there is an active resilver or rebuild print its status.
7543          * Otherwise print the status of the last resilver or rebuild.
7544          */
7545         if (active_resilver || (!active_rebuild && have_resilver &&
7546             resilver_end_time && resilver_end_time > rebuild_end_time)) {
7547                 print_scan_scrub_resilver_status(ps);
7548         } else if (active_rebuild || (!active_resilver && have_rebuild &&
7549             rebuild_end_time && rebuild_end_time > resilver_end_time)) {
7550                 print_rebuild_status(zhp, nvroot);
7551         }
7552
7553         (void) nvlist_lookup_uint64_array(nvroot,
7554             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7555         print_checkpoint_scan_warning(ps, pcs);
7556 }
7557
7558 /*
7559  * Print out detailed removal status.
7560  */
7561 static void
7562 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
7563 {
7564         char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
7565         time_t start, end;
7566         nvlist_t *config, *nvroot;
7567         nvlist_t **child;
7568         uint_t children;
7569         char *vdev_name;
7570
7571         if (prs == NULL || prs->prs_state == DSS_NONE)
7572                 return;
7573
7574         /*
7575          * Determine name of vdev.
7576          */
7577         config = zpool_get_config(zhp, NULL);
7578         nvroot = fnvlist_lookup_nvlist(config,
7579             ZPOOL_CONFIG_VDEV_TREE);
7580         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7581             &child, &children) == 0);
7582         assert(prs->prs_removing_vdev < children);
7583         vdev_name = zpool_vdev_name(g_zfs, zhp,
7584             child[prs->prs_removing_vdev], B_TRUE);
7585
7586         (void) printf(gettext("remove: "));
7587
7588         start = prs->prs_start_time;
7589         end = prs->prs_end_time;
7590         zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
7591
7592         /*
7593          * Removal is finished or canceled.
7594          */
7595         if (prs->prs_state == DSS_FINISHED) {
7596                 uint64_t minutes_taken = (end - start) / 60;
7597
7598                 (void) printf(gettext("Removal of vdev %llu copied %s "
7599                     "in %lluh%um, completed on %s"),
7600                     (longlong_t)prs->prs_removing_vdev,
7601                     copied_buf,
7602                     (u_longlong_t)(minutes_taken / 60),
7603                     (uint_t)(minutes_taken % 60),
7604                     ctime((time_t *)&end));
7605         } else if (prs->prs_state == DSS_CANCELED) {
7606                 (void) printf(gettext("Removal of %s canceled on %s"),
7607                     vdev_name, ctime(&end));
7608         } else {
7609                 uint64_t copied, total, elapsed, mins_left, hours_left;
7610                 double fraction_done;
7611                 uint_t rate;
7612
7613                 assert(prs->prs_state == DSS_SCANNING);
7614
7615                 /*
7616                  * Removal is in progress.
7617                  */
7618                 (void) printf(gettext(
7619                     "Evacuation of %s in progress since %s"),
7620                     vdev_name, ctime(&start));
7621
7622                 copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
7623                 total = prs->prs_to_copy;
7624                 fraction_done = (double)copied / total;
7625
7626                 /* elapsed time for this pass */
7627                 elapsed = time(NULL) - prs->prs_start_time;
7628                 elapsed = elapsed > 0 ? elapsed : 1;
7629                 rate = copied / elapsed;
7630                 rate = rate > 0 ? rate : 1;
7631                 mins_left = ((total - copied) / rate) / 60;
7632                 hours_left = mins_left / 60;
7633
7634                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
7635                 zfs_nicenum(total, total_buf, sizeof (total_buf));
7636                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
7637
7638                 /*
7639                  * do not print estimated time if hours_left is more than
7640                  * 30 days
7641                  */
7642                 (void) printf(gettext("    %s copied out of %s at %s/s, "
7643                     "%.2f%% done"),
7644                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
7645                 if (hours_left < (30 * 24)) {
7646                         (void) printf(gettext(", %lluh%um to go\n"),
7647                             (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
7648                 } else {
7649                         (void) printf(gettext(
7650                             ", (copy is slow, no estimated time)\n"));
7651                 }
7652         }
7653         free(vdev_name);
7654
7655         if (prs->prs_mapping_memory > 0) {
7656                 char mem_buf[7];
7657                 zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
7658                 (void) printf(gettext("    %s memory used for "
7659                     "removed device mappings\n"),
7660                     mem_buf);
7661         }
7662 }
7663
7664 static void
7665 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
7666 {
7667         time_t start;
7668         char space_buf[7];
7669
7670         if (pcs == NULL || pcs->pcs_state == CS_NONE)
7671                 return;
7672
7673         (void) printf(gettext("checkpoint: "));
7674
7675         start = pcs->pcs_start_time;
7676         zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
7677
7678         if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
7679                 char *date = ctime(&start);
7680
7681                 /*
7682                  * ctime() adds a newline at the end of the generated
7683                  * string, thus the weird format specifier and the
7684                  * strlen() call used to chop it off from the output.
7685                  */
7686                 (void) printf(gettext("created %.*s, consumes %s\n"),
7687                     (int)(strlen(date) - 1), date, space_buf);
7688                 return;
7689         }
7690
7691         assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
7692
7693         (void) printf(gettext("discarding, %s remaining.\n"),
7694             space_buf);
7695 }
7696
7697 static void
7698 print_error_log(zpool_handle_t *zhp)
7699 {
7700         nvlist_t *nverrlist = NULL;
7701         nvpair_t *elem;
7702         char *pathname;
7703         size_t len = MAXPATHLEN * 2;
7704
7705         if (zpool_get_errlog(zhp, &nverrlist) != 0)
7706                 return;
7707
7708         (void) printf("errors: Permanent errors have been "
7709             "detected in the following files:\n\n");
7710
7711         pathname = safe_malloc(len);
7712         elem = NULL;
7713         while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
7714                 nvlist_t *nv;
7715                 uint64_t dsobj, obj;
7716
7717                 verify(nvpair_value_nvlist(elem, &nv) == 0);
7718                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
7719                     &dsobj) == 0);
7720                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
7721                     &obj) == 0);
7722                 zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
7723                 (void) printf("%7s %s\n", "", pathname);
7724         }
7725         free(pathname);
7726         nvlist_free(nverrlist);
7727 }
7728
7729 static void
7730 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
7731     uint_t nspares)
7732 {
7733         uint_t i;
7734         char *name;
7735
7736         if (nspares == 0)
7737                 return;
7738
7739         (void) printf(gettext("\tspares\n"));
7740
7741         for (i = 0; i < nspares; i++) {
7742                 name = zpool_vdev_name(g_zfs, zhp, spares[i],
7743                     cb->cb_name_flags);
7744                 print_status_config(zhp, cb, name, spares[i], 2, B_TRUE, NULL);
7745                 free(name);
7746         }
7747 }
7748
7749 static void
7750 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
7751     uint_t nl2cache)
7752 {
7753         uint_t i;
7754         char *name;
7755
7756         if (nl2cache == 0)
7757                 return;
7758
7759         (void) printf(gettext("\tcache\n"));
7760
7761         for (i = 0; i < nl2cache; i++) {
7762                 name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
7763                     cb->cb_name_flags);
7764                 print_status_config(zhp, cb, name, l2cache[i], 2,
7765                     B_FALSE, NULL);
7766                 free(name);
7767         }
7768 }
7769
7770 static void
7771 print_dedup_stats(nvlist_t *config)
7772 {
7773         ddt_histogram_t *ddh;
7774         ddt_stat_t *dds;
7775         ddt_object_t *ddo;
7776         uint_t c;
7777         char dspace[6], mspace[6];
7778
7779         /*
7780          * If the pool was faulted then we may not have been able to
7781          * obtain the config. Otherwise, if we have anything in the dedup
7782          * table continue processing the stats.
7783          */
7784         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
7785             (uint64_t **)&ddo, &c) != 0)
7786                 return;
7787
7788         (void) printf("\n");
7789         (void) printf(gettext(" dedup: "));
7790         if (ddo->ddo_count == 0) {
7791                 (void) printf(gettext("no DDT entries\n"));
7792                 return;
7793         }
7794
7795         zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
7796         zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
7797         (void) printf("DDT entries %llu, size %s on disk, %s in core\n",
7798             (u_longlong_t)ddo->ddo_count,
7799             dspace,
7800             mspace);
7801
7802         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
7803             (uint64_t **)&dds, &c) == 0);
7804         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
7805             (uint64_t **)&ddh, &c) == 0);
7806         zpool_dump_ddt(dds, ddh);
7807 }
7808
7809 /*
7810  * Display a summary of pool status.  Displays a summary such as:
7811  *
7812  *        pool: tank
7813  *      status: DEGRADED
7814  *      reason: One or more devices ...
7815  *         see: https://openzfs.github.io/openzfs-docs/msg/ZFS-xxxx-01
7816  *      config:
7817  *              mirror          DEGRADED
7818  *                c1t0d0        OK
7819  *                c2t0d0        UNAVAIL
7820  *
7821  * When given the '-v' option, we print out the complete config.  If the '-e'
7822  * option is specified, then we print out error rate information as well.
7823  */
7824 static int
7825 status_callback(zpool_handle_t *zhp, void *data)
7826 {
7827         status_cbdata_t *cbp = data;
7828         nvlist_t *config, *nvroot;
7829         char *msgid;
7830         zpool_status_t reason;
7831         zpool_errata_t errata;
7832         const char *health;
7833         uint_t c;
7834         vdev_stat_t *vs;
7835
7836         config = zpool_get_config(zhp, NULL);
7837         reason = zpool_get_status(zhp, &msgid, &errata);
7838
7839         cbp->cb_count++;
7840
7841         /*
7842          * If we were given 'zpool status -x', only report those pools with
7843          * problems.
7844          */
7845         if (cbp->cb_explain &&
7846             (reason == ZPOOL_STATUS_OK ||
7847             reason == ZPOOL_STATUS_VERSION_OLDER ||
7848             reason == ZPOOL_STATUS_FEAT_DISABLED)) {
7849                 if (!cbp->cb_allpools) {
7850                         (void) printf(gettext("pool '%s' is healthy\n"),
7851                             zpool_get_name(zhp));
7852                         if (cbp->cb_first)
7853                                 cbp->cb_first = B_FALSE;
7854                 }
7855                 return (0);
7856         }
7857
7858         if (cbp->cb_first)
7859                 cbp->cb_first = B_FALSE;
7860         else
7861                 (void) printf("\n");
7862
7863         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
7864         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
7865             (uint64_t **)&vs, &c) == 0);
7866
7867         health = zpool_get_state_str(zhp);
7868
7869         printf("  ");
7870         printf_color(ANSI_BOLD, gettext("pool:"));
7871         printf(" %s\n", zpool_get_name(zhp));
7872         printf(" ");
7873         printf_color(ANSI_BOLD, gettext("state: "));
7874
7875         printf_color(health_str_to_color(health), "%s", health);
7876
7877         printf("\n");
7878
7879         switch (reason) {
7880         case ZPOOL_STATUS_MISSING_DEV_R:
7881                 printf_color(ANSI_BOLD, gettext("status: "));
7882                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7883                     "not be opened.  Sufficient replicas exist for\n\tthe pool "
7884                     "to continue functioning in a degraded state.\n"));
7885                 printf_color(ANSI_BOLD, gettext("action: "));
7886                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
7887                     "and online it using 'zpool online'.\n"));
7888                 break;
7889
7890         case ZPOOL_STATUS_MISSING_DEV_NR:
7891                 printf_color(ANSI_BOLD, gettext("status: "));
7892                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7893                     "not be opened.  There are insufficient\n\treplicas for the"
7894                     " pool to continue functioning.\n"));
7895                 printf_color(ANSI_BOLD, gettext("action: "));
7896                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
7897                     "and online it using 'zpool online'.\n"));
7898                 break;
7899
7900         case ZPOOL_STATUS_CORRUPT_LABEL_R:
7901                 printf_color(ANSI_BOLD, gettext("status: "));
7902                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7903                     "not be used because the label is missing or\n\tinvalid.  "
7904                     "Sufficient replicas exist for the pool to continue\n\t"
7905                     "functioning in a degraded state.\n"));
7906                 printf_color(ANSI_BOLD, gettext("action: "));
7907                 printf_color(ANSI_YELLOW, gettext("Replace the device using "
7908                     "'zpool replace'.\n"));
7909                 break;
7910
7911         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
7912                 printf_color(ANSI_BOLD, gettext("status: "));
7913                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7914                     "not be used because the label is missing \n\tor invalid.  "
7915                     "There are insufficient replicas for the pool to "
7916                     "continue\n\tfunctioning.\n"));
7917                 zpool_explain_recover(zpool_get_handle(zhp),
7918                     zpool_get_name(zhp), reason, config);
7919                 break;
7920
7921         case ZPOOL_STATUS_FAILING_DEV:
7922                 printf_color(ANSI_BOLD, gettext("status: "));
7923                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7924                     "experienced an unrecoverable error.  An\n\tattempt was "
7925                     "made to correct the error.  Applications are "
7926                     "unaffected.\n"));
7927                 printf_color(ANSI_BOLD, gettext("action: "));
7928                         printf_color(ANSI_YELLOW, gettext("Determine if the "
7929                     "device needs to be replaced, and clear the errors\n\tusing"
7930                     " 'zpool clear' or replace the device with 'zpool "
7931                     "replace'.\n"));
7932                 break;
7933
7934         case ZPOOL_STATUS_OFFLINE_DEV:
7935                 printf_color(ANSI_BOLD, gettext("status: "));
7936                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7937                     "been taken offline by the administrator.\n\tSufficient "
7938                     "replicas exist for the pool to continue functioning in "
7939                     "a\n\tdegraded state.\n"));
7940                 printf_color(ANSI_BOLD, gettext("action: "));
7941                 printf_color(ANSI_YELLOW, gettext("Online the device "
7942                     "using 'zpool online' or replace the device with\n\t'zpool "
7943                     "replace'.\n"));
7944                 break;
7945
7946         case ZPOOL_STATUS_REMOVED_DEV:
7947                 printf_color(ANSI_BOLD, gettext("status: "));
7948                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7949                     "been removed by the administrator.\n\tSufficient "
7950                     "replicas exist for the pool to continue functioning in "
7951                     "a\n\tdegraded state.\n"));
7952                 printf_color(ANSI_BOLD, gettext("action: "));
7953                 printf_color(ANSI_YELLOW, gettext("Online the device "
7954                     "using zpool online' or replace the device with\n\t'zpool "
7955                     "replace'.\n"));
7956                 break;
7957
7958         case ZPOOL_STATUS_RESILVERING:
7959         case ZPOOL_STATUS_REBUILDING:
7960                 printf_color(ANSI_BOLD, gettext("status: "));
7961                 printf_color(ANSI_YELLOW, gettext("One or more devices is "
7962                     "currently being resilvered.  The pool will\n\tcontinue "
7963                     "to function, possibly in a degraded state.\n"));
7964                 printf_color(ANSI_BOLD, gettext("action: "));
7965                 printf_color(ANSI_YELLOW, gettext("Wait for the resilver to "
7966                     "complete.\n"));
7967                 break;
7968
7969         case ZPOOL_STATUS_REBUILD_SCRUB:
7970                 printf_color(ANSI_BOLD, gettext("status: "));
7971                 printf_color(ANSI_YELLOW, gettext("One or more devices have "
7972                     "been sequentially resilvered, scrubbing\n\tthe pool "
7973                     "is recommended.\n"));
7974                 printf_color(ANSI_BOLD, gettext("action: "));
7975                 printf_color(ANSI_YELLOW, gettext("Use 'zpool scrub' to "
7976                     "verify all data checksums.\n"));
7977                 break;
7978
7979         case ZPOOL_STATUS_CORRUPT_DATA:
7980                 printf_color(ANSI_BOLD, gettext("status: "));
7981                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7982                     "experienced an error resulting in data\n\tcorruption.  "
7983                     "Applications may be affected.\n"));
7984                 printf_color(ANSI_BOLD, gettext("action: "));
7985                 printf_color(ANSI_YELLOW, gettext("Restore the file in question"
7986                     " if possible.  Otherwise restore the\n\tentire pool from "
7987                     "backup.\n"));
7988                 break;
7989
7990         case ZPOOL_STATUS_CORRUPT_POOL:
7991                 printf_color(ANSI_BOLD, gettext("status: "));
7992                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
7993                     "corrupted and the pool cannot be opened.\n"));
7994                 zpool_explain_recover(zpool_get_handle(zhp),
7995                     zpool_get_name(zhp), reason, config);
7996                 break;
7997
7998         case ZPOOL_STATUS_VERSION_OLDER:
7999                 printf_color(ANSI_BOLD, gettext("status: "));
8000                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
8001                     "a legacy on-disk format.  The pool can\n\tstill be used, "
8002                     "but some features are unavailable.\n"));
8003                 printf_color(ANSI_BOLD, gettext("action: "));
8004                 printf_color(ANSI_YELLOW, gettext("Upgrade the pool using "
8005                     "'zpool upgrade'.  Once this is done, the\n\tpool will no "
8006                     "longer be accessible on software that does not support\n\t"
8007                     "feature flags.\n"));
8008                 break;
8009
8010         case ZPOOL_STATUS_VERSION_NEWER:
8011                 printf_color(ANSI_BOLD, gettext("status: "));
8012                 printf_color(ANSI_YELLOW, gettext("The pool has been upgraded "
8013                     "to a newer, incompatible on-disk version.\n\tThe pool "
8014                     "cannot be accessed on this system.\n"));
8015                 printf_color(ANSI_BOLD, gettext("action: "));
8016                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8017                     "system running more recent software, or\n\trestore the "
8018                     "pool from backup.\n"));
8019                 break;
8020
8021         case ZPOOL_STATUS_FEAT_DISABLED:
8022                 printf_color(ANSI_BOLD, gettext("status: "));
8023                 printf_color(ANSI_YELLOW, gettext("Some supported features are "
8024                     "not enabled on the pool. The pool can\n\tstill be used, "
8025                     "but some features are unavailable.\n"));
8026                 printf_color(ANSI_BOLD, gettext("action: "));
8027                 printf_color(ANSI_YELLOW, gettext("Enable all features using "
8028                     "'zpool upgrade'. Once this is done,\n\tthe pool may no "
8029                     "longer be accessible by software that does not support\n\t"
8030                     "the features. See zpool-features(5) for details.\n"));
8031                 break;
8032
8033         case ZPOOL_STATUS_UNSUP_FEAT_READ:
8034                 printf_color(ANSI_BOLD, gettext("status: "));
8035                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8036                     "on this system because it uses the\n\tfollowing feature(s)"
8037                     " not supported on this system:\n"));
8038                 zpool_print_unsup_feat(config);
8039                 (void) printf("\n");
8040                 printf_color(ANSI_BOLD, gettext("action: "));
8041                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8042                     "system that supports the required feature(s),\n\tor "
8043                     "restore the pool from backup.\n"));
8044                 break;
8045
8046         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
8047                 printf_color(ANSI_BOLD, gettext("status: "));
8048                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
8049                     "accessed in read-only mode on this system. It\n\tcannot be"
8050                     " accessed in read-write mode because it uses the "
8051                     "following\n\tfeature(s) not supported on this system:\n"));
8052                 zpool_print_unsup_feat(config);
8053                 (void) printf("\n");
8054                 printf_color(ANSI_BOLD, gettext("action: "));
8055                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8056                     "in read-write mode. Import the pool with\n"
8057                     "\t\"-o readonly=on\", access the pool from a system that "
8058                     "supports the\n\trequired feature(s), or restore the "
8059                     "pool from backup.\n"));
8060                 break;
8061
8062         case ZPOOL_STATUS_FAULTED_DEV_R:
8063                 printf_color(ANSI_BOLD, gettext("status: "));
8064                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8065                     "faulted in response to persistent errors.\n\tSufficient "
8066                     "replicas exist for the pool to continue functioning "
8067                     "in a\n\tdegraded state.\n"));
8068                 printf_color(ANSI_BOLD, gettext("action: "));
8069                 printf_color(ANSI_YELLOW, gettext("Replace the faulted device, "
8070                     "or use 'zpool clear' to mark the device\n\trepaired.\n"));
8071                 break;
8072
8073         case ZPOOL_STATUS_FAULTED_DEV_NR:
8074                 printf_color(ANSI_BOLD, gettext("status: "));
8075                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8076                     "faulted in response to persistent errors.  There are "
8077                     "insufficient replicas for the pool to\n\tcontinue "
8078                     "functioning.\n"));
8079                 printf_color(ANSI_BOLD, gettext("action: "));
8080                 printf_color(ANSI_YELLOW, gettext("Destroy and re-create the "
8081                     "pool from a backup source.  Manually marking the device\n"
8082                     "\trepaired using 'zpool clear' may allow some data "
8083                     "to be recovered.\n"));
8084                 break;
8085
8086         case ZPOOL_STATUS_IO_FAILURE_MMP:
8087                 printf_color(ANSI_BOLD, gettext("status: "));
8088                 printf_color(ANSI_YELLOW, gettext("The pool is suspended "
8089                     "because multihost writes failed or were delayed;\n\t"
8090                     "another system could import the pool undetected.\n"));
8091                 printf_color(ANSI_BOLD, gettext("action: "));
8092                 printf_color(ANSI_YELLOW, gettext("Make sure the pool's devices"
8093                     " are connected, then reboot your system and\n\timport the "
8094                     "pool.\n"));
8095                 break;
8096
8097         case ZPOOL_STATUS_IO_FAILURE_WAIT:
8098         case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
8099                 printf_color(ANSI_BOLD, gettext("status: "));
8100                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8101                     "faulted in response to IO failures.\n"));
8102                 printf_color(ANSI_BOLD, gettext("action: "));
8103                 printf_color(ANSI_YELLOW, gettext("Make sure the affected "
8104                     "devices are connected, then run 'zpool clear'.\n"));
8105                 break;
8106
8107         case ZPOOL_STATUS_BAD_LOG:
8108                 printf_color(ANSI_BOLD, gettext("status: "));
8109                 printf_color(ANSI_YELLOW, gettext("An intent log record "
8110                     "could not be read.\n"
8111                     "\tWaiting for administrator intervention to fix the "
8112                     "faulted pool.\n"));
8113                 printf_color(ANSI_BOLD, gettext("action: "));
8114                 printf_color(ANSI_YELLOW, gettext("Either restore the affected "
8115                     "device(s) and run 'zpool online',\n"
8116                     "\tor ignore the intent log records by running "
8117                     "'zpool clear'.\n"));
8118                 break;
8119
8120         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
8121                 (void) printf(gettext("status: One or more devices are "
8122                     "configured to use a non-native block size.\n"
8123                     "\tExpect reduced performance.\n"));
8124                 (void) printf(gettext("action: Replace affected devices with "
8125                     "devices that support the\n\tconfigured block size, or "
8126                     "migrate data to a properly configured\n\tpool.\n"));
8127                 break;
8128
8129         case ZPOOL_STATUS_HOSTID_MISMATCH:
8130                 printf_color(ANSI_BOLD, gettext("status: "));
8131                 printf_color(ANSI_YELLOW, gettext("Mismatch between pool hostid"
8132                     " and system hostid on imported pool.\n\tThis pool was "
8133                     "previously imported into a system with a different "
8134                     "hostid,\n\tand then was verbatim imported into this "
8135                     "system.\n"));
8136                 printf_color(ANSI_BOLD, gettext("action: "));
8137                 printf_color(ANSI_YELLOW, gettext("Export this pool on all "
8138                     "systems on which it is imported.\n"
8139                     "\tThen import it to correct the mismatch.\n"));
8140                 break;
8141
8142         case ZPOOL_STATUS_ERRATA:
8143                 printf_color(ANSI_BOLD, gettext("status: "));
8144                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
8145                     errata);
8146
8147                 switch (errata) {
8148                 case ZPOOL_ERRATA_NONE:
8149                         break;
8150
8151                 case ZPOOL_ERRATA_ZOL_2094_SCRUB:
8152                         printf_color(ANSI_BOLD, gettext("action: "));
8153                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
8154                             " run 'zpool scrub'.\n"));
8155                         break;
8156
8157                 case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
8158                         (void) printf(gettext("\tExisting encrypted datasets "
8159                             "contain an on-disk incompatibility\n\twhich "
8160                             "needs to be corrected.\n"));
8161                         printf_color(ANSI_BOLD, gettext("action: "));
8162                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
8163                             " backup existing encrypted datasets to new\n\t"
8164                             "encrypted datasets and destroy the old ones. "
8165                             "'zfs mount -o ro' can\n\tbe used to temporarily "
8166                             "mount existing encrypted datasets readonly.\n"));
8167                         break;
8168
8169                 case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
8170                         (void) printf(gettext("\tExisting encrypted snapshots "
8171                             "and bookmarks contain an on-disk\n\tincompat"
8172                             "ibility. This may cause on-disk corruption if "
8173                             "they are used\n\twith 'zfs recv'.\n"));
8174                         printf_color(ANSI_BOLD, gettext("action: "));
8175                         printf_color(ANSI_YELLOW, gettext("To correct the"
8176                             "issue, enable the bookmark_v2 feature. No "
8177                             "additional\n\taction is needed if there are no "
8178                             "encrypted snapshots or bookmarks.\n\tIf preserving"
8179                             "the encrypted snapshots and bookmarks is required,"
8180                             " use\n\ta non-raw send to backup and restore them."
8181                             " Alternately, they may be\n\tremoved to resolve "
8182                             "the incompatibility.\n"));
8183                         break;
8184
8185                 default:
8186                         /*
8187                          * All errata which allow the pool to be imported
8188                          * must contain an action message.
8189                          */
8190                         assert(0);
8191                 }
8192                 break;
8193
8194         default:
8195                 /*
8196                  * The remaining errors can't actually be generated, yet.
8197                  */
8198                 assert(reason == ZPOOL_STATUS_OK);
8199         }
8200
8201         if (msgid != NULL) {
8202                 printf("   ");
8203                 printf_color(ANSI_BOLD, gettext("see:"));
8204                 printf(gettext(
8205                     " https://openzfs.github.io/openzfs-docs/msg/%s\n"),
8206                     msgid);
8207         }
8208
8209         if (config != NULL) {
8210                 uint64_t nerr;
8211                 nvlist_t **spares, **l2cache;
8212                 uint_t nspares, nl2cache;
8213                 pool_checkpoint_stat_t *pcs = NULL;
8214                 pool_removal_stat_t *prs = NULL;
8215
8216                 print_scan_status(zhp, nvroot);
8217
8218                 (void) nvlist_lookup_uint64_array(nvroot,
8219                     ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
8220                 print_removal_status(zhp, prs);
8221
8222                 (void) nvlist_lookup_uint64_array(nvroot,
8223                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
8224                 print_checkpoint_status(pcs);
8225
8226                 cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
8227                     cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
8228                 if (cbp->cb_namewidth < 10)
8229                         cbp->cb_namewidth = 10;
8230
8231                 color_start(ANSI_BOLD);
8232                 (void) printf(gettext("config:\n\n"));
8233                 (void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
8234                     cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
8235                     "CKSUM");
8236                 color_end();
8237
8238                 if (cbp->cb_print_slow_ios) {
8239                         printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
8240                 }
8241
8242                 if (cbp->vcdl != NULL)
8243                         print_cmd_columns(cbp->vcdl, 0);
8244
8245                 printf("\n");
8246
8247                 print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
8248                     B_FALSE, NULL);
8249
8250                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
8251                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
8252                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
8253
8254                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
8255                     &l2cache, &nl2cache) == 0)
8256                         print_l2cache(zhp, cbp, l2cache, nl2cache);
8257
8258                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
8259                     &spares, &nspares) == 0)
8260                         print_spares(zhp, cbp, spares, nspares);
8261
8262                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
8263                     &nerr) == 0) {
8264                         nvlist_t *nverrlist = NULL;
8265
8266                         /*
8267                          * If the approximate error count is small, get a
8268                          * precise count by fetching the entire log and
8269                          * uniquifying the results.
8270                          */
8271                         if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
8272                             zpool_get_errlog(zhp, &nverrlist) == 0) {
8273                                 nvpair_t *elem;
8274
8275                                 elem = NULL;
8276                                 nerr = 0;
8277                                 while ((elem = nvlist_next_nvpair(nverrlist,
8278                                     elem)) != NULL) {
8279                                         nerr++;
8280                                 }
8281                         }
8282                         nvlist_free(nverrlist);
8283
8284                         (void) printf("\n");
8285
8286                         if (nerr == 0)
8287                                 (void) printf(gettext("errors: No known data "
8288                                     "errors\n"));
8289                         else if (!cbp->cb_verbose)
8290                                 (void) printf(gettext("errors: %llu data "
8291                                     "errors, use '-v' for a list\n"),
8292                                     (u_longlong_t)nerr);
8293                         else
8294                                 print_error_log(zhp);
8295                 }
8296
8297                 if (cbp->cb_dedup_stats)
8298                         print_dedup_stats(config);
8299         } else {
8300                 (void) printf(gettext("config: The configuration cannot be "
8301                     "determined.\n"));
8302         }
8303
8304         return (0);
8305 }
8306
8307 /*
8308  * zpool status [-c [script1,script2,...]] [-igLpPstvx] [-T d|u] [pool] ...
8309  *              [interval [count]]
8310  *
8311  *      -c CMD  For each vdev, run command CMD
8312  *      -i      Display vdev initialization status.
8313  *      -g      Display guid for individual vdev name.
8314  *      -L      Follow links when resolving vdev path name.
8315  *      -p      Display values in parsable (exact) format.
8316  *      -P      Display full path for vdev name.
8317  *      -s      Display slow IOs column.
8318  *      -v      Display complete error logs
8319  *      -x      Display only pools with potential problems
8320  *      -D      Display dedup status (undocumented)
8321  *      -t      Display vdev TRIM status.
8322  *      -T      Display a timestamp in date(1) or Unix format
8323  *
8324  * Describes the health status of all pools or some subset.
8325  */
8326 int
8327 zpool_do_status(int argc, char **argv)
8328 {
8329         int c;
8330         int ret;
8331         float interval = 0;
8332         unsigned long count = 0;
8333         status_cbdata_t cb = { 0 };
8334         char *cmd = NULL;
8335
8336         /* check options */
8337         while ((c = getopt(argc, argv, "c:igLpPsvxDtT:")) != -1) {
8338                 switch (c) {
8339                 case 'c':
8340                         if (cmd != NULL) {
8341                                 fprintf(stderr,
8342                                     gettext("Can't set -c flag twice\n"));
8343                                 exit(1);
8344                         }
8345
8346                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
8347                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
8348                                 fprintf(stderr, gettext(
8349                                     "Can't run -c, disabled by "
8350                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
8351                                 exit(1);
8352                         }
8353
8354                         if ((getuid() <= 0 || geteuid() <= 0) &&
8355                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
8356                                 fprintf(stderr, gettext(
8357                                     "Can't run -c with root privileges "
8358                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
8359                                 exit(1);
8360                         }
8361                         cmd = optarg;
8362                         break;
8363                 case 'i':
8364                         cb.cb_print_vdev_init = B_TRUE;
8365                         break;
8366                 case 'g':
8367                         cb.cb_name_flags |= VDEV_NAME_GUID;
8368                         break;
8369                 case 'L':
8370                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
8371                         break;
8372                 case 'p':
8373                         cb.cb_literal = B_TRUE;
8374                         break;
8375                 case 'P':
8376                         cb.cb_name_flags |= VDEV_NAME_PATH;
8377                         break;
8378                 case 's':
8379                         cb.cb_print_slow_ios = B_TRUE;
8380                         break;
8381                 case 'v':
8382                         cb.cb_verbose = B_TRUE;
8383                         break;
8384                 case 'x':
8385                         cb.cb_explain = B_TRUE;
8386                         break;
8387                 case 'D':
8388                         cb.cb_dedup_stats = B_TRUE;
8389                         break;
8390                 case 't':
8391                         cb.cb_print_vdev_trim = B_TRUE;
8392                         break;
8393                 case 'T':
8394                         get_timestamp_arg(*optarg);
8395                         break;
8396                 case '?':
8397                         if (optopt == 'c') {
8398                                 print_zpool_script_list("status");
8399                                 exit(0);
8400                         } else {
8401                                 fprintf(stderr,
8402                                     gettext("invalid option '%c'\n"), optopt);
8403                         }
8404                         usage(B_FALSE);
8405                 }
8406         }
8407
8408         argc -= optind;
8409         argv += optind;
8410
8411         get_interval_count(&argc, argv, &interval, &count);
8412
8413         if (argc == 0)
8414                 cb.cb_allpools = B_TRUE;
8415
8416         cb.cb_first = B_TRUE;
8417         cb.cb_print_status = B_TRUE;
8418
8419         for (;;) {
8420                 if (timestamp_fmt != NODATE)
8421                         print_timestamp(timestamp_fmt);
8422
8423                 if (cmd != NULL)
8424                         cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
8425                             NULL, NULL, 0, 0);
8426
8427                 ret = for_each_pool(argc, argv, B_TRUE, NULL,
8428                     status_callback, &cb);
8429
8430                 if (cb.vcdl != NULL)
8431                         free_vdev_cmd_data_list(cb.vcdl);
8432
8433                 if (argc == 0 && cb.cb_count == 0)
8434                         (void) fprintf(stderr, gettext("no pools available\n"));
8435                 else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
8436                         (void) printf(gettext("all pools are healthy\n"));
8437
8438                 if (ret != 0)
8439                         return (ret);
8440
8441                 if (interval == 0)
8442                         break;
8443
8444                 if (count != 0 && --count == 0)
8445                         break;
8446
8447                 (void) fsleep(interval);
8448         }
8449
8450         return (0);
8451 }
8452
8453 typedef struct upgrade_cbdata {
8454         int     cb_first;
8455         int     cb_argc;
8456         uint64_t cb_version;
8457         char    **cb_argv;
8458 } upgrade_cbdata_t;
8459
8460 static int
8461 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
8462 {
8463         int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
8464         int *count = (int *)unsupp_fs;
8465
8466         if (zfs_version > ZPL_VERSION) {
8467                 (void) printf(gettext("%s (v%d) is not supported by this "
8468                     "implementation of ZFS.\n"),
8469                     zfs_get_name(zhp), zfs_version);
8470                 (*count)++;
8471         }
8472
8473         zfs_iter_filesystems(zhp, check_unsupp_fs, unsupp_fs);
8474
8475         zfs_close(zhp);
8476
8477         return (0);
8478 }
8479
8480 static int
8481 upgrade_version(zpool_handle_t *zhp, uint64_t version)
8482 {
8483         int ret;
8484         nvlist_t *config;
8485         uint64_t oldversion;
8486         int unsupp_fs = 0;
8487
8488         config = zpool_get_config(zhp, NULL);
8489         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8490             &oldversion) == 0);
8491
8492         assert(SPA_VERSION_IS_SUPPORTED(oldversion));
8493         assert(oldversion < version);
8494
8495         ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
8496         if (ret != 0)
8497                 return (ret);
8498
8499         if (unsupp_fs) {
8500                 (void) fprintf(stderr, gettext("Upgrade not performed due "
8501                     "to %d unsupported filesystems (max v%d).\n"),
8502                     unsupp_fs, (int)ZPL_VERSION);
8503                 return (1);
8504         }
8505
8506         ret = zpool_upgrade(zhp, version);
8507         if (ret != 0)
8508                 return (ret);
8509
8510         if (version >= SPA_VERSION_FEATURES) {
8511                 (void) printf(gettext("Successfully upgraded "
8512                     "'%s' from version %llu to feature flags.\n"),
8513                     zpool_get_name(zhp), (u_longlong_t)oldversion);
8514         } else {
8515                 (void) printf(gettext("Successfully upgraded "
8516                     "'%s' from version %llu to version %llu.\n"),
8517                     zpool_get_name(zhp), (u_longlong_t)oldversion,
8518                     (u_longlong_t)version);
8519         }
8520
8521         return (0);
8522 }
8523
8524 static int
8525 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
8526 {
8527         int i, ret, count;
8528         boolean_t firstff = B_TRUE;
8529         nvlist_t *enabled = zpool_get_features(zhp);
8530
8531         count = 0;
8532         for (i = 0; i < SPA_FEATURES; i++) {
8533                 const char *fname = spa_feature_table[i].fi_uname;
8534                 const char *fguid = spa_feature_table[i].fi_guid;
8535                 if (!nvlist_exists(enabled, fguid)) {
8536                         char *propname;
8537                         verify(-1 != asprintf(&propname, "feature@%s", fname));
8538                         ret = zpool_set_prop(zhp, propname,
8539                             ZFS_FEATURE_ENABLED);
8540                         if (ret != 0) {
8541                                 free(propname);
8542                                 return (ret);
8543                         }
8544                         count++;
8545
8546                         if (firstff) {
8547                                 (void) printf(gettext("Enabled the "
8548                                     "following features on '%s':\n"),
8549                                     zpool_get_name(zhp));
8550                                 firstff = B_FALSE;
8551                         }
8552                         (void) printf(gettext("  %s\n"), fname);
8553                         free(propname);
8554                 }
8555         }
8556
8557         if (countp != NULL)
8558                 *countp = count;
8559         return (0);
8560 }
8561
8562 static int
8563 upgrade_cb(zpool_handle_t *zhp, void *arg)
8564 {
8565         upgrade_cbdata_t *cbp = arg;
8566         nvlist_t *config;
8567         uint64_t version;
8568         boolean_t printnl = B_FALSE;
8569         int ret;
8570
8571         config = zpool_get_config(zhp, NULL);
8572         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8573             &version) == 0);
8574
8575         assert(SPA_VERSION_IS_SUPPORTED(version));
8576
8577         if (version < cbp->cb_version) {
8578                 cbp->cb_first = B_FALSE;
8579                 ret = upgrade_version(zhp, cbp->cb_version);
8580                 if (ret != 0)
8581                         return (ret);
8582                 printnl = B_TRUE;
8583
8584                 /*
8585                  * If they did "zpool upgrade -a", then we could
8586                  * be doing ioctls to different pools.  We need
8587                  * to log this history once to each pool, and bypass
8588                  * the normal history logging that happens in main().
8589                  */
8590                 (void) zpool_log_history(g_zfs, history_str);
8591                 log_history = B_FALSE;
8592         }
8593
8594         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
8595                 int count;
8596                 ret = upgrade_enable_all(zhp, &count);
8597                 if (ret != 0)
8598                         return (ret);
8599
8600                 if (count > 0) {
8601                         cbp->cb_first = B_FALSE;
8602                         printnl = B_TRUE;
8603                 }
8604         }
8605
8606         if (printnl) {
8607                 (void) printf(gettext("\n"));
8608         }
8609
8610         return (0);
8611 }
8612
8613 static int
8614 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
8615 {
8616         upgrade_cbdata_t *cbp = arg;
8617         nvlist_t *config;
8618         uint64_t version;
8619
8620         config = zpool_get_config(zhp, NULL);
8621         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8622             &version) == 0);
8623
8624         assert(SPA_VERSION_IS_SUPPORTED(version));
8625
8626         if (version < SPA_VERSION_FEATURES) {
8627                 if (cbp->cb_first) {
8628                         (void) printf(gettext("The following pools are "
8629                             "formatted with legacy version numbers and can\n"
8630                             "be upgraded to use feature flags.  After "
8631                             "being upgraded, these pools\nwill no "
8632                             "longer be accessible by software that does not "
8633                             "support feature\nflags.\n\n"));
8634                         (void) printf(gettext("VER  POOL\n"));
8635                         (void) printf(gettext("---  ------------\n"));
8636                         cbp->cb_first = B_FALSE;
8637                 }
8638
8639                 (void) printf("%2llu   %s\n", (u_longlong_t)version,
8640                     zpool_get_name(zhp));
8641         }
8642
8643         return (0);
8644 }
8645
8646 static int
8647 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
8648 {
8649         upgrade_cbdata_t *cbp = arg;
8650         nvlist_t *config;
8651         uint64_t version;
8652
8653         config = zpool_get_config(zhp, NULL);
8654         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8655             &version) == 0);
8656
8657         if (version >= SPA_VERSION_FEATURES) {
8658                 int i;
8659                 boolean_t poolfirst = B_TRUE;
8660                 nvlist_t *enabled = zpool_get_features(zhp);
8661
8662                 for (i = 0; i < SPA_FEATURES; i++) {
8663                         const char *fguid = spa_feature_table[i].fi_guid;
8664                         const char *fname = spa_feature_table[i].fi_uname;
8665                         if (!nvlist_exists(enabled, fguid)) {
8666                                 if (cbp->cb_first) {
8667                                         (void) printf(gettext("\nSome "
8668                                             "supported features are not "
8669                                             "enabled on the following pools. "
8670                                             "Once a\nfeature is enabled the "
8671                                             "pool may become incompatible with "
8672                                             "software\nthat does not support "
8673                                             "the feature. See "
8674                                             "zpool-features(5) for "
8675                                             "details.\n\n"));
8676                                         (void) printf(gettext("POOL  "
8677                                             "FEATURE\n"));
8678                                         (void) printf(gettext("------"
8679                                             "---------\n"));
8680                                         cbp->cb_first = B_FALSE;
8681                                 }
8682
8683                                 if (poolfirst) {
8684                                         (void) printf(gettext("%s\n"),
8685                                             zpool_get_name(zhp));
8686                                         poolfirst = B_FALSE;
8687                                 }
8688
8689                                 (void) printf(gettext("      %s\n"), fname);
8690                         }
8691                         /*
8692                          * If they did "zpool upgrade -a", then we could
8693                          * be doing ioctls to different pools.  We need
8694                          * to log this history once to each pool, and bypass
8695                          * the normal history logging that happens in main().
8696                          */
8697                         (void) zpool_log_history(g_zfs, history_str);
8698                         log_history = B_FALSE;
8699                 }
8700         }
8701
8702         return (0);
8703 }
8704
8705 /* ARGSUSED */
8706 static int
8707 upgrade_one(zpool_handle_t *zhp, void *data)
8708 {
8709         boolean_t printnl = B_FALSE;
8710         upgrade_cbdata_t *cbp = data;
8711         uint64_t cur_version;
8712         int ret;
8713
8714         if (strcmp("log", zpool_get_name(zhp)) == 0) {
8715                 (void) fprintf(stderr, gettext("'log' is now a reserved word\n"
8716                     "Pool 'log' must be renamed using export and import"
8717                     " to upgrade.\n"));
8718                 return (1);
8719         }
8720
8721         cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
8722         if (cur_version > cbp->cb_version) {
8723                 (void) printf(gettext("Pool '%s' is already formatted "
8724                     "using more current version '%llu'.\n\n"),
8725                     zpool_get_name(zhp), (u_longlong_t)cur_version);
8726                 return (0);
8727         }
8728
8729         if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
8730                 (void) printf(gettext("Pool '%s' is already formatted "
8731                     "using version %llu.\n\n"), zpool_get_name(zhp),
8732                     (u_longlong_t)cbp->cb_version);
8733                 return (0);
8734         }
8735
8736         if (cur_version != cbp->cb_version) {
8737                 printnl = B_TRUE;
8738                 ret = upgrade_version(zhp, cbp->cb_version);
8739                 if (ret != 0)
8740                         return (ret);
8741         }
8742
8743         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
8744                 int count = 0;
8745                 ret = upgrade_enable_all(zhp, &count);
8746                 if (ret != 0)
8747                         return (ret);
8748
8749                 if (count != 0) {
8750                         printnl = B_TRUE;
8751                 } else if (cur_version == SPA_VERSION) {
8752                         (void) printf(gettext("Pool '%s' already has all "
8753                             "supported features enabled.\n"),
8754                             zpool_get_name(zhp));
8755                 }
8756         }
8757
8758         if (printnl) {
8759                 (void) printf(gettext("\n"));
8760         }
8761
8762         return (0);
8763 }
8764
8765 /*
8766  * zpool upgrade
8767  * zpool upgrade -v
8768  * zpool upgrade [-V version] <-a | pool ...>
8769  *
8770  * With no arguments, display downrev'd ZFS pool available for upgrade.
8771  * Individual pools can be upgraded by specifying the pool, and '-a' will
8772  * upgrade all pools.
8773  */
8774 int
8775 zpool_do_upgrade(int argc, char **argv)
8776 {
8777         int c;
8778         upgrade_cbdata_t cb = { 0 };
8779         int ret = 0;
8780         boolean_t showversions = B_FALSE;
8781         boolean_t upgradeall = B_FALSE;
8782         char *end;
8783
8784
8785         /* check options */
8786         while ((c = getopt(argc, argv, ":avV:")) != -1) {
8787                 switch (c) {
8788                 case 'a':
8789                         upgradeall = B_TRUE;
8790                         break;
8791                 case 'v':
8792                         showversions = B_TRUE;
8793                         break;
8794                 case 'V':
8795                         cb.cb_version = strtoll(optarg, &end, 10);
8796                         if (*end != '\0' ||
8797                             !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
8798                                 (void) fprintf(stderr,
8799                                     gettext("invalid version '%s'\n"), optarg);
8800                                 usage(B_FALSE);
8801                         }
8802                         break;
8803                 case ':':
8804                         (void) fprintf(stderr, gettext("missing argument for "
8805                             "'%c' option\n"), optopt);
8806                         usage(B_FALSE);
8807                         break;
8808                 case '?':
8809                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
8810                             optopt);
8811                         usage(B_FALSE);
8812                 }
8813         }
8814
8815         cb.cb_argc = argc;
8816         cb.cb_argv = argv;
8817         argc -= optind;
8818         argv += optind;
8819
8820         if (cb.cb_version == 0) {
8821                 cb.cb_version = SPA_VERSION;
8822         } else if (!upgradeall && argc == 0) {
8823                 (void) fprintf(stderr, gettext("-V option is "
8824                     "incompatible with other arguments\n"));
8825                 usage(B_FALSE);
8826         }
8827
8828         if (showversions) {
8829                 if (upgradeall || argc != 0) {
8830                         (void) fprintf(stderr, gettext("-v option is "
8831                             "incompatible with other arguments\n"));
8832                         usage(B_FALSE);
8833                 }
8834         } else if (upgradeall) {
8835                 if (argc != 0) {
8836                         (void) fprintf(stderr, gettext("-a option should not "
8837                             "be used along with a pool name\n"));
8838                         usage(B_FALSE);
8839                 }
8840         }
8841
8842         (void) printf(gettext("This system supports ZFS pool feature "
8843             "flags.\n\n"));
8844         if (showversions) {
8845                 int i;
8846
8847                 (void) printf(gettext("The following features are "
8848                     "supported:\n\n"));
8849                 (void) printf(gettext("FEAT DESCRIPTION\n"));
8850                 (void) printf("----------------------------------------------"
8851                     "---------------\n");
8852                 for (i = 0; i < SPA_FEATURES; i++) {
8853                         zfeature_info_t *fi = &spa_feature_table[i];
8854                         const char *ro =
8855                             (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
8856                             " (read-only compatible)" : "";
8857
8858                         (void) printf("%-37s%s\n", fi->fi_uname, ro);
8859                         (void) printf("     %s\n", fi->fi_desc);
8860                 }
8861                 (void) printf("\n");
8862
8863                 (void) printf(gettext("The following legacy versions are also "
8864                     "supported:\n\n"));
8865                 (void) printf(gettext("VER  DESCRIPTION\n"));
8866                 (void) printf("---  -----------------------------------------"
8867                     "---------------\n");
8868                 (void) printf(gettext(" 1   Initial ZFS version\n"));
8869                 (void) printf(gettext(" 2   Ditto blocks "
8870                     "(replicated metadata)\n"));
8871                 (void) printf(gettext(" 3   Hot spares and double parity "
8872                     "RAID-Z\n"));
8873                 (void) printf(gettext(" 4   zpool history\n"));
8874                 (void) printf(gettext(" 5   Compression using the gzip "
8875                     "algorithm\n"));
8876                 (void) printf(gettext(" 6   bootfs pool property\n"));
8877                 (void) printf(gettext(" 7   Separate intent log devices\n"));
8878                 (void) printf(gettext(" 8   Delegated administration\n"));
8879                 (void) printf(gettext(" 9   refquota and refreservation "
8880                     "properties\n"));
8881                 (void) printf(gettext(" 10  Cache devices\n"));
8882                 (void) printf(gettext(" 11  Improved scrub performance\n"));
8883                 (void) printf(gettext(" 12  Snapshot properties\n"));
8884                 (void) printf(gettext(" 13  snapused property\n"));
8885                 (void) printf(gettext(" 14  passthrough-x aclinherit\n"));
8886                 (void) printf(gettext(" 15  user/group space accounting\n"));
8887                 (void) printf(gettext(" 16  stmf property support\n"));
8888                 (void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
8889                 (void) printf(gettext(" 18  Snapshot user holds\n"));
8890                 (void) printf(gettext(" 19  Log device removal\n"));
8891                 (void) printf(gettext(" 20  Compression using zle "
8892                     "(zero-length encoding)\n"));
8893                 (void) printf(gettext(" 21  Deduplication\n"));
8894                 (void) printf(gettext(" 22  Received properties\n"));
8895                 (void) printf(gettext(" 23  Slim ZIL\n"));
8896                 (void) printf(gettext(" 24  System attributes\n"));
8897                 (void) printf(gettext(" 25  Improved scrub stats\n"));
8898                 (void) printf(gettext(" 26  Improved snapshot deletion "
8899                     "performance\n"));
8900                 (void) printf(gettext(" 27  Improved snapshot creation "
8901                     "performance\n"));
8902                 (void) printf(gettext(" 28  Multiple vdev replacements\n"));
8903                 (void) printf(gettext("\nFor more information on a particular "
8904                     "version, including supported releases,\n"));
8905                 (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
8906         } else if (argc == 0 && upgradeall) {
8907                 cb.cb_first = B_TRUE;
8908                 ret = zpool_iter(g_zfs, upgrade_cb, &cb);
8909                 if (ret == 0 && cb.cb_first) {
8910                         if (cb.cb_version == SPA_VERSION) {
8911                                 (void) printf(gettext("All pools are already "
8912                                     "formatted using feature flags.\n\n"));
8913                                 (void) printf(gettext("Every feature flags "
8914                                     "pool already has all supported features "
8915                                     "enabled.\n"));
8916                         } else {
8917                                 (void) printf(gettext("All pools are already "
8918                                     "formatted with version %llu or higher.\n"),
8919                                     (u_longlong_t)cb.cb_version);
8920                         }
8921                 }
8922         } else if (argc == 0) {
8923                 cb.cb_first = B_TRUE;
8924                 ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
8925                 assert(ret == 0);
8926
8927                 if (cb.cb_first) {
8928                         (void) printf(gettext("All pools are formatted "
8929                             "using feature flags.\n\n"));
8930                 } else {
8931                         (void) printf(gettext("\nUse 'zpool upgrade -v' "
8932                             "for a list of available legacy versions.\n"));
8933                 }
8934
8935                 cb.cb_first = B_TRUE;
8936                 ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
8937                 assert(ret == 0);
8938
8939                 if (cb.cb_first) {
8940                         (void) printf(gettext("Every feature flags pool has "
8941                             "all supported features enabled.\n"));
8942                 } else {
8943                         (void) printf(gettext("\n"));
8944                 }
8945         } else {
8946                 ret = for_each_pool(argc, argv, B_FALSE, NULL,
8947                     upgrade_one, &cb);
8948         }
8949
8950         return (ret);
8951 }
8952
8953 typedef struct hist_cbdata {
8954         boolean_t first;
8955         boolean_t longfmt;
8956         boolean_t internal;
8957 } hist_cbdata_t;
8958
8959 static void
8960 print_history_records(nvlist_t *nvhis, hist_cbdata_t *cb)
8961 {
8962         nvlist_t **records;
8963         uint_t numrecords;
8964         int i;
8965
8966         verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
8967             &records, &numrecords) == 0);
8968         for (i = 0; i < numrecords; i++) {
8969                 nvlist_t *rec = records[i];
8970                 char tbuf[30] = "";
8971
8972                 if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
8973                         time_t tsec;
8974                         struct tm t;
8975
8976                         tsec = fnvlist_lookup_uint64(records[i],
8977                             ZPOOL_HIST_TIME);
8978                         (void) localtime_r(&tsec, &t);
8979                         (void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
8980                 }
8981
8982                 if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
8983                         (void) printf("%s %s", tbuf,
8984                             fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
8985                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
8986                         int ievent =
8987                             fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
8988                         if (!cb->internal)
8989                                 continue;
8990                         if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
8991                                 (void) printf("%s unrecognized record:\n",
8992                                     tbuf);
8993                                 dump_nvlist(rec, 4);
8994                                 continue;
8995                         }
8996                         (void) printf("%s [internal %s txg:%lld] %s", tbuf,
8997                             zfs_history_event_names[ievent],
8998                             (longlong_t)fnvlist_lookup_uint64(
8999                             rec, ZPOOL_HIST_TXG),
9000                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
9001                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
9002                         if (!cb->internal)
9003                                 continue;
9004                         (void) printf("%s [txg:%lld] %s", tbuf,
9005                             (longlong_t)fnvlist_lookup_uint64(
9006                             rec, ZPOOL_HIST_TXG),
9007                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
9008                         if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
9009                                 (void) printf(" %s (%llu)",
9010                                     fnvlist_lookup_string(rec,
9011                                     ZPOOL_HIST_DSNAME),
9012                                     (u_longlong_t)fnvlist_lookup_uint64(rec,
9013                                     ZPOOL_HIST_DSID));
9014                         }
9015                         (void) printf(" %s", fnvlist_lookup_string(rec,
9016                             ZPOOL_HIST_INT_STR));
9017                 } else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
9018                         if (!cb->internal)
9019                                 continue;
9020                         (void) printf("%s ioctl %s\n", tbuf,
9021                             fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
9022                         if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
9023                                 (void) printf("    input:\n");
9024                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9025                                     ZPOOL_HIST_INPUT_NVL), 8);
9026                         }
9027                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
9028                                 (void) printf("    output:\n");
9029                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9030                                     ZPOOL_HIST_OUTPUT_NVL), 8);
9031                         }
9032                         if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
9033                                 (void) printf("    errno: %lld\n",
9034                                     (longlong_t)fnvlist_lookup_int64(rec,
9035                                     ZPOOL_HIST_ERRNO));
9036                         }
9037                 } else {
9038                         if (!cb->internal)
9039                                 continue;
9040                         (void) printf("%s unrecognized record:\n", tbuf);
9041                         dump_nvlist(rec, 4);
9042                 }
9043
9044                 if (!cb->longfmt) {
9045                         (void) printf("\n");
9046                         continue;
9047                 }
9048                 (void) printf(" [");
9049                 if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
9050                         uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
9051                         struct passwd *pwd = getpwuid(who);
9052                         (void) printf("user %d ", (int)who);
9053                         if (pwd != NULL)
9054                                 (void) printf("(%s) ", pwd->pw_name);
9055                 }
9056                 if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
9057                         (void) printf("on %s",
9058                             fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
9059                 }
9060                 if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
9061                         (void) printf(":%s",
9062                             fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
9063                 }
9064
9065                 (void) printf("]");
9066                 (void) printf("\n");
9067         }
9068 }
9069
9070 /*
9071  * Print out the command history for a specific pool.
9072  */
9073 static int
9074 get_history_one(zpool_handle_t *zhp, void *data)
9075 {
9076         nvlist_t *nvhis;
9077         int ret;
9078         hist_cbdata_t *cb = (hist_cbdata_t *)data;
9079         uint64_t off = 0;
9080         boolean_t eof = B_FALSE;
9081
9082         cb->first = B_FALSE;
9083
9084         (void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
9085
9086         while (!eof) {
9087                 if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
9088                         return (ret);
9089
9090                 print_history_records(nvhis, cb);
9091                 nvlist_free(nvhis);
9092         }
9093         (void) printf("\n");
9094
9095         return (ret);
9096 }
9097
9098 /*
9099  * zpool history <pool>
9100  *
9101  * Displays the history of commands that modified pools.
9102  */
9103 int
9104 zpool_do_history(int argc, char **argv)
9105 {
9106         hist_cbdata_t cbdata = { 0 };
9107         int ret;
9108         int c;
9109
9110         cbdata.first = B_TRUE;
9111         /* check options */
9112         while ((c = getopt(argc, argv, "li")) != -1) {
9113                 switch (c) {
9114                 case 'l':
9115                         cbdata.longfmt = B_TRUE;
9116                         break;
9117                 case 'i':
9118                         cbdata.internal = B_TRUE;
9119                         break;
9120                 case '?':
9121                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9122                             optopt);
9123                         usage(B_FALSE);
9124                 }
9125         }
9126         argc -= optind;
9127         argv += optind;
9128
9129         ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
9130             &cbdata);
9131
9132         if (argc == 0 && cbdata.first == B_TRUE) {
9133                 (void) fprintf(stderr, gettext("no pools available\n"));
9134                 return (0);
9135         }
9136
9137         return (ret);
9138 }
9139
9140 typedef struct ev_opts {
9141         int verbose;
9142         int scripted;
9143         int follow;
9144         int clear;
9145         char poolname[ZFS_MAX_DATASET_NAME_LEN];
9146 } ev_opts_t;
9147
9148 static void
9149 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
9150 {
9151         char ctime_str[26], str[32], *ptr;
9152         int64_t *tv;
9153         uint_t n;
9154
9155         verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
9156         memset(str, ' ', 32);
9157         (void) ctime_r((const time_t *)&tv[0], ctime_str);
9158         (void) memcpy(str, ctime_str+4,  6);            /* 'Jun 30' */
9159         (void) memcpy(str+7, ctime_str+20, 4);          /* '1993' */
9160         (void) memcpy(str+12, ctime_str+11, 8);         /* '21:49:08' */
9161         (void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
9162         if (opts->scripted)
9163                 (void) printf(gettext("%s\t"), str);
9164         else
9165                 (void) printf(gettext("%s "), str);
9166
9167         verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
9168         (void) printf(gettext("%s\n"), ptr);
9169 }
9170
9171 static void
9172 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
9173 {
9174         nvpair_t *nvp;
9175
9176         for (nvp = nvlist_next_nvpair(nvl, NULL);
9177             nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
9178
9179                 data_type_t type = nvpair_type(nvp);
9180                 const char *name = nvpair_name(nvp);
9181
9182                 boolean_t b;
9183                 uint8_t i8;
9184                 uint16_t i16;
9185                 uint32_t i32;
9186                 uint64_t i64;
9187                 char *str;
9188                 nvlist_t *cnv;
9189
9190                 printf(gettext("%*s%s = "), depth, "", name);
9191
9192                 switch (type) {
9193                 case DATA_TYPE_BOOLEAN:
9194                         printf(gettext("%s"), "1");
9195                         break;
9196
9197                 case DATA_TYPE_BOOLEAN_VALUE:
9198                         (void) nvpair_value_boolean_value(nvp, &b);
9199                         printf(gettext("%s"), b ? "1" : "0");
9200                         break;
9201
9202                 case DATA_TYPE_BYTE:
9203                         (void) nvpair_value_byte(nvp, &i8);
9204                         printf(gettext("0x%x"), i8);
9205                         break;
9206
9207                 case DATA_TYPE_INT8:
9208                         (void) nvpair_value_int8(nvp, (void *)&i8);
9209                         printf(gettext("0x%x"), i8);
9210                         break;
9211
9212                 case DATA_TYPE_UINT8:
9213                         (void) nvpair_value_uint8(nvp, &i8);
9214                         printf(gettext("0x%x"), i8);
9215                         break;
9216
9217                 case DATA_TYPE_INT16:
9218                         (void) nvpair_value_int16(nvp, (void *)&i16);
9219                         printf(gettext("0x%x"), i16);
9220                         break;
9221
9222                 case DATA_TYPE_UINT16:
9223                         (void) nvpair_value_uint16(nvp, &i16);
9224                         printf(gettext("0x%x"), i16);
9225                         break;
9226
9227                 case DATA_TYPE_INT32:
9228                         (void) nvpair_value_int32(nvp, (void *)&i32);
9229                         printf(gettext("0x%x"), i32);
9230                         break;
9231
9232                 case DATA_TYPE_UINT32:
9233                         (void) nvpair_value_uint32(nvp, &i32);
9234                         printf(gettext("0x%x"), i32);
9235                         break;
9236
9237                 case DATA_TYPE_INT64:
9238                         (void) nvpair_value_int64(nvp, (void *)&i64);
9239                         printf(gettext("0x%llx"), (u_longlong_t)i64);
9240                         break;
9241
9242                 case DATA_TYPE_UINT64:
9243                         (void) nvpair_value_uint64(nvp, &i64);
9244                         /*
9245                          * translate vdev state values to readable
9246                          * strings to aide zpool events consumers
9247                          */
9248                         if (strcmp(name,
9249                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
9250                             strcmp(name,
9251                             FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
9252                                 printf(gettext("\"%s\" (0x%llx)"),
9253                                     zpool_state_to_name(i64, VDEV_AUX_NONE),
9254                                     (u_longlong_t)i64);
9255                         } else {
9256                                 printf(gettext("0x%llx"), (u_longlong_t)i64);
9257                         }
9258                         break;
9259
9260                 case DATA_TYPE_HRTIME:
9261                         (void) nvpair_value_hrtime(nvp, (void *)&i64);
9262                         printf(gettext("0x%llx"), (u_longlong_t)i64);
9263                         break;
9264
9265                 case DATA_TYPE_STRING:
9266                         (void) nvpair_value_string(nvp, &str);
9267                         printf(gettext("\"%s\""), str ? str : "<NULL>");
9268                         break;
9269
9270                 case DATA_TYPE_NVLIST:
9271                         printf(gettext("(embedded nvlist)\n"));
9272                         (void) nvpair_value_nvlist(nvp, &cnv);
9273                         zpool_do_events_nvprint(cnv, depth + 8);
9274                         printf(gettext("%*s(end %s)"), depth, "", name);
9275                         break;
9276
9277                 case DATA_TYPE_NVLIST_ARRAY: {
9278                         nvlist_t **val;
9279                         uint_t i, nelem;
9280
9281                         (void) nvpair_value_nvlist_array(nvp, &val, &nelem);
9282                         printf(gettext("(%d embedded nvlists)\n"), nelem);
9283                         for (i = 0; i < nelem; i++) {
9284                                 printf(gettext("%*s%s[%d] = %s\n"),
9285                                     depth, "", name, i, "(embedded nvlist)");
9286                                 zpool_do_events_nvprint(val[i], depth + 8);
9287                                 printf(gettext("%*s(end %s[%i])\n"),
9288                                     depth, "", name, i);
9289                         }
9290                         printf(gettext("%*s(end %s)\n"), depth, "", name);
9291                         }
9292                         break;
9293
9294                 case DATA_TYPE_INT8_ARRAY: {
9295                         int8_t *val;
9296                         uint_t i, nelem;
9297
9298                         (void) nvpair_value_int8_array(nvp, &val, &nelem);
9299                         for (i = 0; i < nelem; i++)
9300                                 printf(gettext("0x%x "), val[i]);
9301
9302                         break;
9303                         }
9304
9305                 case DATA_TYPE_UINT8_ARRAY: {
9306                         uint8_t *val;
9307                         uint_t i, nelem;
9308
9309                         (void) nvpair_value_uint8_array(nvp, &val, &nelem);
9310                         for (i = 0; i < nelem; i++)
9311                                 printf(gettext("0x%x "), val[i]);
9312
9313                         break;
9314                         }
9315
9316                 case DATA_TYPE_INT16_ARRAY: {
9317                         int16_t *val;
9318                         uint_t i, nelem;
9319
9320                         (void) nvpair_value_int16_array(nvp, &val, &nelem);
9321                         for (i = 0; i < nelem; i++)
9322                                 printf(gettext("0x%x "), val[i]);
9323
9324                         break;
9325                         }
9326
9327                 case DATA_TYPE_UINT16_ARRAY: {
9328                         uint16_t *val;
9329                         uint_t i, nelem;
9330
9331                         (void) nvpair_value_uint16_array(nvp, &val, &nelem);
9332                         for (i = 0; i < nelem; i++)
9333                                 printf(gettext("0x%x "), val[i]);
9334
9335                         break;
9336                         }
9337
9338                 case DATA_TYPE_INT32_ARRAY: {
9339                         int32_t *val;
9340                         uint_t i, nelem;
9341
9342                         (void) nvpair_value_int32_array(nvp, &val, &nelem);
9343                         for (i = 0; i < nelem; i++)
9344                                 printf(gettext("0x%x "), val[i]);
9345
9346                         break;
9347                         }
9348
9349                 case DATA_TYPE_UINT32_ARRAY: {
9350                         uint32_t *val;
9351                         uint_t i, nelem;
9352
9353                         (void) nvpair_value_uint32_array(nvp, &val, &nelem);
9354                         for (i = 0; i < nelem; i++)
9355                                 printf(gettext("0x%x "), val[i]);
9356
9357                         break;
9358                         }
9359
9360                 case DATA_TYPE_INT64_ARRAY: {
9361                         int64_t *val;
9362                         uint_t i, nelem;
9363
9364                         (void) nvpair_value_int64_array(nvp, &val, &nelem);
9365                         for (i = 0; i < nelem; i++)
9366                                 printf(gettext("0x%llx "),
9367                                     (u_longlong_t)val[i]);
9368
9369                         break;
9370                         }
9371
9372                 case DATA_TYPE_UINT64_ARRAY: {
9373                         uint64_t *val;
9374                         uint_t i, nelem;
9375
9376                         (void) nvpair_value_uint64_array(nvp, &val, &nelem);
9377                         for (i = 0; i < nelem; i++)
9378                                 printf(gettext("0x%llx "),
9379                                     (u_longlong_t)val[i]);
9380
9381                         break;
9382                         }
9383
9384                 case DATA_TYPE_STRING_ARRAY: {
9385                         char **str;
9386                         uint_t i, nelem;
9387
9388                         (void) nvpair_value_string_array(nvp, &str, &nelem);
9389                         for (i = 0; i < nelem; i++)
9390                                 printf(gettext("\"%s\" "),
9391                                     str[i] ? str[i] : "<NULL>");
9392
9393                         break;
9394                         }
9395
9396                 case DATA_TYPE_BOOLEAN_ARRAY:
9397                 case DATA_TYPE_BYTE_ARRAY:
9398                 case DATA_TYPE_DOUBLE:
9399                 case DATA_TYPE_DONTCARE:
9400                 case DATA_TYPE_UNKNOWN:
9401                         printf(gettext("<unknown>"));
9402                         break;
9403                 }
9404
9405                 printf(gettext("\n"));
9406         }
9407 }
9408
9409 static int
9410 zpool_do_events_next(ev_opts_t *opts)
9411 {
9412         nvlist_t *nvl;
9413         int zevent_fd, ret, dropped;
9414         char *pool;
9415
9416         zevent_fd = open(ZFS_DEV, O_RDWR);
9417         VERIFY(zevent_fd >= 0);
9418
9419         if (!opts->scripted)
9420                 (void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
9421
9422         while (1) {
9423                 ret = zpool_events_next(g_zfs, &nvl, &dropped,
9424                     (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
9425                 if (ret || nvl == NULL)
9426                         break;
9427
9428                 if (dropped > 0)
9429                         (void) printf(gettext("dropped %d events\n"), dropped);
9430
9431                 if (strlen(opts->poolname) > 0 &&
9432                     nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
9433                     strcmp(opts->poolname, pool) != 0)
9434                         continue;
9435
9436                 zpool_do_events_short(nvl, opts);
9437
9438                 if (opts->verbose) {
9439                         zpool_do_events_nvprint(nvl, 8);
9440                         printf(gettext("\n"));
9441                 }
9442                 (void) fflush(stdout);
9443
9444                 nvlist_free(nvl);
9445         }
9446
9447         VERIFY(0 == close(zevent_fd));
9448
9449         return (ret);
9450 }
9451
9452 static int
9453 zpool_do_events_clear(ev_opts_t *opts)
9454 {
9455         int count, ret;
9456
9457         ret = zpool_events_clear(g_zfs, &count);
9458         if (!ret)
9459                 (void) printf(gettext("cleared %d events\n"), count);
9460
9461         return (ret);
9462 }
9463
9464 /*
9465  * zpool events [-vHf [pool] | -c]
9466  *
9467  * Displays events logs by ZFS.
9468  */
9469 int
9470 zpool_do_events(int argc, char **argv)
9471 {
9472         ev_opts_t opts = { 0 };
9473         int ret;
9474         int c;
9475
9476         /* check options */
9477         while ((c = getopt(argc, argv, "vHfc")) != -1) {
9478                 switch (c) {
9479                 case 'v':
9480                         opts.verbose = 1;
9481                         break;
9482                 case 'H':
9483                         opts.scripted = 1;
9484                         break;
9485                 case 'f':
9486                         opts.follow = 1;
9487                         break;
9488                 case 'c':
9489                         opts.clear = 1;
9490                         break;
9491                 case '?':
9492                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9493                             optopt);
9494                         usage(B_FALSE);
9495                 }
9496         }
9497         argc -= optind;
9498         argv += optind;
9499
9500         if (argc > 1) {
9501                 (void) fprintf(stderr, gettext("too many arguments\n"));
9502                 usage(B_FALSE);
9503         } else if (argc == 1) {
9504                 (void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
9505                 if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
9506                         (void) fprintf(stderr,
9507                             gettext("invalid pool name '%s'\n"), opts.poolname);
9508                         usage(B_FALSE);
9509                 }
9510         }
9511
9512         if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
9513             opts.clear) {
9514                 (void) fprintf(stderr,
9515                     gettext("invalid options combined with -c\n"));
9516                 usage(B_FALSE);
9517         }
9518
9519         if (opts.clear)
9520                 ret = zpool_do_events_clear(&opts);
9521         else
9522                 ret = zpool_do_events_next(&opts);
9523
9524         return (ret);
9525 }
9526
9527 static int
9528 get_callback(zpool_handle_t *zhp, void *data)
9529 {
9530         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
9531         char value[MAXNAMELEN];
9532         zprop_source_t srctype;
9533         zprop_list_t *pl;
9534
9535         for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
9536
9537                 /*
9538                  * Skip the special fake placeholder. This will also skip
9539                  * over the name property when 'all' is specified.
9540                  */
9541                 if (pl->pl_prop == ZPOOL_PROP_NAME &&
9542                     pl == cbp->cb_proplist)
9543                         continue;
9544
9545                 if (pl->pl_prop == ZPROP_INVAL &&
9546                     (zpool_prop_feature(pl->pl_user_prop) ||
9547                     zpool_prop_unsupported(pl->pl_user_prop))) {
9548                         srctype = ZPROP_SRC_LOCAL;
9549
9550                         if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
9551                             value, sizeof (value)) == 0) {
9552                                 zprop_print_one_property(zpool_get_name(zhp),
9553                                     cbp, pl->pl_user_prop, value, srctype,
9554                                     NULL, NULL);
9555                         }
9556                 } else {
9557                         if (zpool_get_prop(zhp, pl->pl_prop, value,
9558                             sizeof (value), &srctype, cbp->cb_literal) != 0)
9559                                 continue;
9560
9561                         zprop_print_one_property(zpool_get_name(zhp), cbp,
9562                             zpool_prop_to_name(pl->pl_prop), value, srctype,
9563                             NULL, NULL);
9564                 }
9565         }
9566         return (0);
9567 }
9568
9569 /*
9570  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
9571  *
9572  *      -H      Scripted mode.  Don't display headers, and separate properties
9573  *              by a single tab.
9574  *      -o      List of columns to display.  Defaults to
9575  *              "name,property,value,source".
9576  *      -p      Display values in parsable (exact) format.
9577  *
9578  * Get properties of pools in the system. Output space statistics
9579  * for each one as well as other attributes.
9580  */
9581 int
9582 zpool_do_get(int argc, char **argv)
9583 {
9584         zprop_get_cbdata_t cb = { 0 };
9585         zprop_list_t fake_name = { 0 };
9586         int ret;
9587         int c, i;
9588         char *value;
9589
9590         cb.cb_first = B_TRUE;
9591
9592         /*
9593          * Set up default columns and sources.
9594          */
9595         cb.cb_sources = ZPROP_SRC_ALL;
9596         cb.cb_columns[0] = GET_COL_NAME;
9597         cb.cb_columns[1] = GET_COL_PROPERTY;
9598         cb.cb_columns[2] = GET_COL_VALUE;
9599         cb.cb_columns[3] = GET_COL_SOURCE;
9600         cb.cb_type = ZFS_TYPE_POOL;
9601
9602         /* check options */
9603         while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
9604                 switch (c) {
9605                 case 'p':
9606                         cb.cb_literal = B_TRUE;
9607                         break;
9608                 case 'H':
9609                         cb.cb_scripted = B_TRUE;
9610                         break;
9611                 case 'o':
9612                         bzero(&cb.cb_columns, sizeof (cb.cb_columns));
9613                         i = 0;
9614                         while (*optarg != '\0') {
9615                                 static char *col_subopts[] =
9616                                 { "name", "property", "value", "source",
9617                                 "all", NULL };
9618
9619                                 if (i == ZFS_GET_NCOLS) {
9620                                         (void) fprintf(stderr, gettext("too "
9621                                         "many fields given to -o "
9622                                         "option\n"));
9623                                         usage(B_FALSE);
9624                                 }
9625
9626                                 switch (getsubopt(&optarg, col_subopts,
9627                                     &value)) {
9628                                 case 0:
9629                                         cb.cb_columns[i++] = GET_COL_NAME;
9630                                         break;
9631                                 case 1:
9632                                         cb.cb_columns[i++] = GET_COL_PROPERTY;
9633                                         break;
9634                                 case 2:
9635                                         cb.cb_columns[i++] = GET_COL_VALUE;
9636                                         break;
9637                                 case 3:
9638                                         cb.cb_columns[i++] = GET_COL_SOURCE;
9639                                         break;
9640                                 case 4:
9641                                         if (i > 0) {
9642                                                 (void) fprintf(stderr,
9643                                                     gettext("\"all\" conflicts "
9644                                                     "with specific fields "
9645                                                     "given to -o option\n"));
9646                                                 usage(B_FALSE);
9647                                         }
9648                                         cb.cb_columns[0] = GET_COL_NAME;
9649                                         cb.cb_columns[1] = GET_COL_PROPERTY;
9650                                         cb.cb_columns[2] = GET_COL_VALUE;
9651                                         cb.cb_columns[3] = GET_COL_SOURCE;
9652                                         i = ZFS_GET_NCOLS;
9653                                         break;
9654                                 default:
9655                                         (void) fprintf(stderr,
9656                                             gettext("invalid column name "
9657                                             "'%s'\n"), value);
9658                                         usage(B_FALSE);
9659                                 }
9660                         }
9661                         break;
9662                 case '?':
9663                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9664                             optopt);
9665                         usage(B_FALSE);
9666                 }
9667         }
9668
9669         argc -= optind;
9670         argv += optind;
9671
9672         if (argc < 1) {
9673                 (void) fprintf(stderr, gettext("missing property "
9674                     "argument\n"));
9675                 usage(B_FALSE);
9676         }
9677
9678         if (zprop_get_list(g_zfs, argv[0], &cb.cb_proplist,
9679             ZFS_TYPE_POOL) != 0)
9680                 usage(B_FALSE);
9681
9682         argc--;
9683         argv++;
9684
9685         if (cb.cb_proplist != NULL) {
9686                 fake_name.pl_prop = ZPOOL_PROP_NAME;
9687                 fake_name.pl_width = strlen(gettext("NAME"));
9688                 fake_name.pl_next = cb.cb_proplist;
9689                 cb.cb_proplist = &fake_name;
9690         }
9691
9692         ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
9693             get_callback, &cb);
9694
9695         if (cb.cb_proplist == &fake_name)
9696                 zprop_free_list(fake_name.pl_next);
9697         else
9698                 zprop_free_list(cb.cb_proplist);
9699
9700         return (ret);
9701 }
9702
9703 typedef struct set_cbdata {
9704         char *cb_propname;
9705         char *cb_value;
9706         boolean_t cb_any_successful;
9707 } set_cbdata_t;
9708
9709 static int
9710 set_callback(zpool_handle_t *zhp, void *data)
9711 {
9712         int error;
9713         set_cbdata_t *cb = (set_cbdata_t *)data;
9714
9715         error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
9716
9717         if (!error)
9718                 cb->cb_any_successful = B_TRUE;
9719
9720         return (error);
9721 }
9722
9723 int
9724 zpool_do_set(int argc, char **argv)
9725 {
9726         set_cbdata_t cb = { 0 };
9727         int error;
9728
9729         if (argc > 1 && argv[1][0] == '-') {
9730                 (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9731                     argv[1][1]);
9732                 usage(B_FALSE);
9733         }
9734
9735         if (argc < 2) {
9736                 (void) fprintf(stderr, gettext("missing property=value "
9737                     "argument\n"));
9738                 usage(B_FALSE);
9739         }
9740
9741         if (argc < 3) {
9742                 (void) fprintf(stderr, gettext("missing pool name\n"));
9743                 usage(B_FALSE);
9744         }
9745
9746         if (argc > 3) {
9747                 (void) fprintf(stderr, gettext("too many pool names\n"));
9748                 usage(B_FALSE);
9749         }
9750
9751         cb.cb_propname = argv[1];
9752         cb.cb_value = strchr(cb.cb_propname, '=');
9753         if (cb.cb_value == NULL) {
9754                 (void) fprintf(stderr, gettext("missing value in "
9755                     "property=value argument\n"));
9756                 usage(B_FALSE);
9757         }
9758
9759         *(cb.cb_value) = '\0';
9760         cb.cb_value++;
9761
9762         error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
9763             set_callback, &cb);
9764
9765         return (error);
9766 }
9767
9768 /* Add up the total number of bytes left to initialize/trim across all vdevs */
9769 static uint64_t
9770 vdev_activity_remaining(nvlist_t *nv, zpool_wait_activity_t activity)
9771 {
9772         uint64_t bytes_remaining;
9773         nvlist_t **child;
9774         uint_t c, children;
9775         vdev_stat_t *vs;
9776
9777         assert(activity == ZPOOL_WAIT_INITIALIZE ||
9778             activity == ZPOOL_WAIT_TRIM);
9779
9780         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
9781             (uint64_t **)&vs, &c) == 0);
9782
9783         if (activity == ZPOOL_WAIT_INITIALIZE &&
9784             vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
9785                 bytes_remaining = vs->vs_initialize_bytes_est -
9786                     vs->vs_initialize_bytes_done;
9787         else if (activity == ZPOOL_WAIT_TRIM &&
9788             vs->vs_trim_state == VDEV_TRIM_ACTIVE)
9789                 bytes_remaining = vs->vs_trim_bytes_est -
9790                     vs->vs_trim_bytes_done;
9791         else
9792                 bytes_remaining = 0;
9793
9794         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
9795             &child, &children) != 0)
9796                 children = 0;
9797
9798         for (c = 0; c < children; c++)
9799                 bytes_remaining += vdev_activity_remaining(child[c], activity);
9800
9801         return (bytes_remaining);
9802 }
9803
9804 /* Add up the total number of bytes left to rebuild across top-level vdevs */
9805 static uint64_t
9806 vdev_activity_top_remaining(nvlist_t *nv)
9807 {
9808         uint64_t bytes_remaining = 0;
9809         nvlist_t **child;
9810         uint_t children;
9811         int error;
9812
9813         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
9814             &child, &children) != 0)
9815                 children = 0;
9816
9817         for (uint_t c = 0; c < children; c++) {
9818                 vdev_rebuild_stat_t *vrs;
9819                 uint_t i;
9820
9821                 error = nvlist_lookup_uint64_array(child[c],
9822                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i);
9823                 if (error == 0) {
9824                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
9825                                 bytes_remaining += (vrs->vrs_bytes_est -
9826                                     vrs->vrs_bytes_rebuilt);
9827                         }
9828                 }
9829         }
9830
9831         return (bytes_remaining);
9832 }
9833
9834 /* Whether any vdevs are 'spare' or 'replacing' vdevs */
9835 static boolean_t
9836 vdev_any_spare_replacing(nvlist_t *nv)
9837 {
9838         nvlist_t **child;
9839         uint_t c, children;
9840         char *vdev_type;
9841
9842         (void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &vdev_type);
9843
9844         if (strcmp(vdev_type, VDEV_TYPE_REPLACING) == 0 ||
9845             strcmp(vdev_type, VDEV_TYPE_SPARE) == 0) {
9846                 return (B_TRUE);
9847         }
9848
9849         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
9850             &child, &children) != 0)
9851                 children = 0;
9852
9853         for (c = 0; c < children; c++) {
9854                 if (vdev_any_spare_replacing(child[c]))
9855                         return (B_TRUE);
9856         }
9857
9858         return (B_FALSE);
9859 }
9860
9861 typedef struct wait_data {
9862         char *wd_poolname;
9863         boolean_t wd_scripted;
9864         boolean_t wd_exact;
9865         boolean_t wd_headers_once;
9866         boolean_t wd_should_exit;
9867         /* Which activities to wait for */
9868         boolean_t wd_enabled[ZPOOL_WAIT_NUM_ACTIVITIES];
9869         float wd_interval;
9870         pthread_cond_t wd_cv;
9871         pthread_mutex_t wd_mutex;
9872 } wait_data_t;
9873
9874 /*
9875  * Print to stdout a single line, containing one column for each activity that
9876  * we are waiting for specifying how many bytes of work are left for that
9877  * activity.
9878  */
9879 static void
9880 print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
9881 {
9882         nvlist_t *config, *nvroot;
9883         uint_t c;
9884         int i;
9885         pool_checkpoint_stat_t *pcs = NULL;
9886         pool_scan_stat_t *pss = NULL;
9887         pool_removal_stat_t *prs = NULL;
9888         char *headers[] = {"DISCARD", "FREE", "INITIALIZE", "REPLACE",
9889             "REMOVE", "RESILVER", "SCRUB", "TRIM"};
9890         int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];
9891
9892         /* Calculate the width of each column */
9893         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
9894                 /*
9895                  * Make sure we have enough space in the col for pretty-printed
9896                  * numbers and for the column header, and then leave a couple
9897                  * spaces between cols for readability.
9898                  */
9899                 col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
9900         }
9901
9902         /* Print header if appropriate */
9903         int term_height = terminal_height();
9904         boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
9905             row % (term_height-1) == 0);
9906         if (!wd->wd_scripted && (row == 0 || reprint_header)) {
9907                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
9908                         if (wd->wd_enabled[i])
9909                                 (void) printf("%*s", col_widths[i], headers[i]);
9910                 }
9911                 (void) printf("\n");
9912         }
9913
9914         /* Bytes of work remaining in each activity */
9915         int64_t bytes_rem[ZPOOL_WAIT_NUM_ACTIVITIES] = {0};
9916
9917         bytes_rem[ZPOOL_WAIT_FREE] =
9918             zpool_get_prop_int(zhp, ZPOOL_PROP_FREEING, NULL);
9919
9920         config = zpool_get_config(zhp, NULL);
9921         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
9922
9923         (void) nvlist_lookup_uint64_array(nvroot,
9924             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
9925         if (pcs != NULL && pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
9926                 bytes_rem[ZPOOL_WAIT_CKPT_DISCARD] = pcs->pcs_space;
9927
9928         (void) nvlist_lookup_uint64_array(nvroot,
9929             ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
9930         if (prs != NULL && prs->prs_state == DSS_SCANNING)
9931                 bytes_rem[ZPOOL_WAIT_REMOVE] = prs->prs_to_copy -
9932                     prs->prs_copied;
9933
9934         (void) nvlist_lookup_uint64_array(nvroot,
9935             ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&pss, &c);
9936         if (pss != NULL && pss->pss_state == DSS_SCANNING &&
9937             pss->pss_pass_scrub_pause == 0) {
9938                 int64_t rem = pss->pss_to_examine - pss->pss_issued;
9939                 if (pss->pss_func == POOL_SCAN_SCRUB)
9940                         bytes_rem[ZPOOL_WAIT_SCRUB] = rem;
9941                 else
9942                         bytes_rem[ZPOOL_WAIT_RESILVER] = rem;
9943         } else if (check_rebuilding(nvroot, NULL)) {
9944                 bytes_rem[ZPOOL_WAIT_RESILVER] =
9945                     vdev_activity_top_remaining(nvroot);
9946         }
9947
9948         bytes_rem[ZPOOL_WAIT_INITIALIZE] =
9949             vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
9950         bytes_rem[ZPOOL_WAIT_TRIM] =
9951             vdev_activity_remaining(nvroot, ZPOOL_WAIT_TRIM);
9952
9953         /*
9954          * A replace finishes after resilvering finishes, so the amount of work
9955          * left for a replace is the same as for resilvering.
9956          *
9957          * It isn't quite correct to say that if we have any 'spare' or
9958          * 'replacing' vdevs and a resilver is happening, then a replace is in
9959          * progress, like we do here. When a hot spare is used, the faulted vdev
9960          * is not removed after the hot spare is resilvered, so parent 'spare'
9961          * vdev is not removed either. So we could have a 'spare' vdev, but be
9962          * resilvering for a different reason. However, we use it as a heuristic
9963          * because we don't have access to the DTLs, which could tell us whether
9964          * or not we have really finished resilvering a hot spare.
9965          */
9966         if (vdev_any_spare_replacing(nvroot))
9967                 bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
9968
9969         if (timestamp_fmt != NODATE)
9970                 print_timestamp(timestamp_fmt);
9971
9972         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
9973                 char buf[64];
9974                 if (!wd->wd_enabled[i])
9975                         continue;
9976
9977                 if (wd->wd_exact)
9978                         (void) snprintf(buf, sizeof (buf), "%" PRIi64,
9979                             bytes_rem[i]);
9980                 else
9981                         zfs_nicenum(bytes_rem[i], buf, sizeof (buf));
9982
9983                 if (wd->wd_scripted)
9984                         (void) printf(i == 0 ? "%s" : "\t%s", buf);
9985                 else
9986                         (void) printf(" %*s", col_widths[i] - 1, buf);
9987         }
9988         (void) printf("\n");
9989         (void) fflush(stdout);
9990 }
9991
9992 static void *
9993 wait_status_thread(void *arg)
9994 {
9995         wait_data_t *wd = (wait_data_t *)arg;
9996         zpool_handle_t *zhp;
9997
9998         if ((zhp = zpool_open(g_zfs, wd->wd_poolname)) == NULL)
9999                 return (void *)(1);
10000
10001         for (int row = 0; ; row++) {
10002                 boolean_t missing;
10003                 struct timespec timeout;
10004                 int ret = 0;
10005                 (void) clock_gettime(CLOCK_REALTIME, &timeout);
10006
10007                 if (zpool_refresh_stats(zhp, &missing) != 0 || missing ||
10008                     zpool_props_refresh(zhp) != 0) {
10009                         zpool_close(zhp);
10010                         return (void *)(uintptr_t)(missing ? 0 : 1);
10011                 }
10012
10013                 print_wait_status_row(wd, zhp, row);
10014
10015                 timeout.tv_sec += floor(wd->wd_interval);
10016                 long nanos = timeout.tv_nsec +
10017                     (wd->wd_interval - floor(wd->wd_interval)) * NANOSEC;
10018                 if (nanos >= NANOSEC) {
10019                         timeout.tv_sec++;
10020                         timeout.tv_nsec = nanos - NANOSEC;
10021                 } else {
10022                         timeout.tv_nsec = nanos;
10023                 }
10024                 pthread_mutex_lock(&wd->wd_mutex);
10025                 if (!wd->wd_should_exit)
10026                         ret = pthread_cond_timedwait(&wd->wd_cv, &wd->wd_mutex,
10027                             &timeout);
10028                 pthread_mutex_unlock(&wd->wd_mutex);
10029                 if (ret == 0) {
10030                         break; /* signaled by main thread */
10031                 } else if (ret != ETIMEDOUT) {
10032                         (void) fprintf(stderr, gettext("pthread_cond_timedwait "
10033                             "failed: %s\n"), strerror(ret));
10034                         zpool_close(zhp);
10035                         return (void *)(uintptr_t)(1);
10036                 }
10037         }
10038
10039         zpool_close(zhp);
10040         return (void *)(0);
10041 }
10042
10043 int
10044 zpool_do_wait(int argc, char **argv)
10045 {
10046         boolean_t verbose = B_FALSE;
10047         char c;
10048         char *value;
10049         int i;
10050         unsigned long count;
10051         pthread_t status_thr;
10052         int error = 0;
10053         zpool_handle_t *zhp;
10054
10055         wait_data_t wd;
10056         wd.wd_scripted = B_FALSE;
10057         wd.wd_exact = B_FALSE;
10058         wd.wd_headers_once = B_FALSE;
10059         wd.wd_should_exit = B_FALSE;
10060
10061         pthread_mutex_init(&wd.wd_mutex, NULL);
10062         pthread_cond_init(&wd.wd_cv, NULL);
10063
10064         /* By default, wait for all types of activity. */
10065         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++)
10066                 wd.wd_enabled[i] = B_TRUE;
10067
10068         while ((c = getopt(argc, argv, "HpT:t:")) != -1) {
10069                 switch (c) {
10070                 case 'H':
10071                         wd.wd_scripted = B_TRUE;
10072                         break;
10073                 case 'n':
10074                         wd.wd_headers_once = B_TRUE;
10075                         break;
10076                 case 'p':
10077                         wd.wd_exact = B_TRUE;
10078                         break;
10079                 case 'T':
10080                         get_timestamp_arg(*optarg);
10081                         break;
10082                 case 't':
10083                 {
10084                         static char *col_subopts[] = { "discard", "free",
10085                             "initialize", "replace", "remove", "resilver",
10086                             "scrub", "trim", NULL };
10087
10088                         /* Reset activities array */
10089                         bzero(&wd.wd_enabled, sizeof (wd.wd_enabled));
10090                         while (*optarg != '\0') {
10091                                 int activity = getsubopt(&optarg, col_subopts,
10092                                     &value);
10093
10094                                 if (activity < 0) {
10095                                         (void) fprintf(stderr,
10096                                             gettext("invalid activity '%s'\n"),
10097                                             value);
10098                                         usage(B_FALSE);
10099                                 }
10100
10101                                 wd.wd_enabled[activity] = B_TRUE;
10102                         }
10103                         break;
10104                 }
10105                 case '?':
10106                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10107                             optopt);
10108                         usage(B_FALSE);
10109                 }
10110         }
10111
10112         argc -= optind;
10113         argv += optind;
10114
10115         get_interval_count(&argc, argv, &wd.wd_interval, &count);
10116         if (count != 0) {
10117                 /* This subcmd only accepts an interval, not a count */
10118                 (void) fprintf(stderr, gettext("too many arguments\n"));
10119                 usage(B_FALSE);
10120         }
10121
10122         if (wd.wd_interval != 0)
10123                 verbose = B_TRUE;
10124
10125         if (argc < 1) {
10126                 (void) fprintf(stderr, gettext("missing 'pool' argument\n"));
10127                 usage(B_FALSE);
10128         }
10129         if (argc > 1) {
10130                 (void) fprintf(stderr, gettext("too many arguments\n"));
10131                 usage(B_FALSE);
10132         }
10133
10134         wd.wd_poolname = argv[0];
10135
10136         if ((zhp = zpool_open(g_zfs, wd.wd_poolname)) == NULL)
10137                 return (1);
10138
10139         if (verbose) {
10140                 /*
10141                  * We use a separate thread for printing status updates because
10142                  * the main thread will call lzc_wait(), which blocks as long
10143                  * as an activity is in progress, which can be a long time.
10144                  */
10145                 if (pthread_create(&status_thr, NULL, wait_status_thread, &wd)
10146                     != 0) {
10147                         (void) fprintf(stderr, gettext("failed to create status"
10148                             "thread: %s\n"), strerror(errno));
10149                         zpool_close(zhp);
10150                         return (1);
10151                 }
10152         }
10153
10154         /*
10155          * Loop over all activities that we are supposed to wait for until none
10156          * of them are in progress. Note that this means we can end up waiting
10157          * for more activities to complete than just those that were in progress
10158          * when we began waiting; if an activity we are interested in begins
10159          * while we are waiting for another activity, we will wait for both to
10160          * complete before exiting.
10161          */
10162         for (;;) {
10163                 boolean_t missing = B_FALSE;
10164                 boolean_t any_waited = B_FALSE;
10165
10166                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10167                         boolean_t waited;
10168
10169                         if (!wd.wd_enabled[i])
10170                                 continue;
10171
10172                         error = zpool_wait_status(zhp, i, &missing, &waited);
10173                         if (error != 0 || missing)
10174                                 break;
10175
10176                         any_waited = (any_waited || waited);
10177                 }
10178
10179                 if (error != 0 || missing || !any_waited)
10180                         break;
10181         }
10182
10183         zpool_close(zhp);
10184
10185         if (verbose) {
10186                 uintptr_t status;
10187                 pthread_mutex_lock(&wd.wd_mutex);
10188                 wd.wd_should_exit = B_TRUE;
10189                 pthread_cond_signal(&wd.wd_cv);
10190                 pthread_mutex_unlock(&wd.wd_mutex);
10191                 (void) pthread_join(status_thr, (void *)&status);
10192                 if (status != 0)
10193                         error = status;
10194         }
10195
10196         pthread_mutex_destroy(&wd.wd_mutex);
10197         pthread_cond_destroy(&wd.wd_cv);
10198         return (error);
10199 }
10200
10201 static int
10202 find_command_idx(char *command, int *idx)
10203 {
10204         int i;
10205
10206         for (i = 0; i < NCOMMAND; i++) {
10207                 if (command_table[i].name == NULL)
10208                         continue;
10209
10210                 if (strcmp(command, command_table[i].name) == 0) {
10211                         *idx = i;
10212                         return (0);
10213                 }
10214         }
10215         return (1);
10216 }
10217
10218 /*
10219  * Display version message
10220  */
10221 static int
10222 zpool_do_version(int argc, char **argv)
10223 {
10224         if (zfs_version_print() == -1)
10225                 return (1);
10226
10227         return (0);
10228 }
10229
10230 int
10231 main(int argc, char **argv)
10232 {
10233         int ret = 0;
10234         int i = 0;
10235         char *cmdname;
10236         char **newargv;
10237
10238         (void) setlocale(LC_ALL, "");
10239         (void) setlocale(LC_NUMERIC, "C");
10240         (void) textdomain(TEXT_DOMAIN);
10241         srand(time(NULL));
10242
10243         opterr = 0;
10244
10245         /*
10246          * Make sure the user has specified some command.
10247          */
10248         if (argc < 2) {
10249                 (void) fprintf(stderr, gettext("missing command\n"));
10250                 usage(B_FALSE);
10251         }
10252
10253         cmdname = argv[1];
10254
10255         /*
10256          * Special case '-?'
10257          */
10258         if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
10259                 usage(B_TRUE);
10260
10261         /*
10262          * Special case '-V|--version'
10263          */
10264         if ((strcmp(cmdname, "-V") == 0) || (strcmp(cmdname, "--version") == 0))
10265                 return (zpool_do_version(argc, argv));
10266
10267         if ((g_zfs = libzfs_init()) == NULL) {
10268                 (void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
10269                 return (1);
10270         }
10271
10272         libzfs_print_on_error(g_zfs, B_TRUE);
10273
10274         zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
10275
10276         /*
10277          * Many commands modify input strings for string parsing reasons.
10278          * We create a copy to protect the original argv.
10279          */
10280         newargv = malloc((argc + 1) * sizeof (newargv[0]));
10281         for (i = 0; i < argc; i++)
10282                 newargv[i] = strdup(argv[i]);
10283         newargv[argc] = NULL;
10284
10285         /*
10286          * Run the appropriate command.
10287          */
10288         if (find_command_idx(cmdname, &i) == 0) {
10289                 current_command = &command_table[i];
10290                 ret = command_table[i].func(argc - 1, newargv + 1);
10291         } else if (strchr(cmdname, '=')) {
10292                 verify(find_command_idx("set", &i) == 0);
10293                 current_command = &command_table[i];
10294                 ret = command_table[i].func(argc, newargv);
10295         } else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
10296                 /*
10297                  * 'freeze' is a vile debugging abomination, so we treat
10298                  * it as such.
10299                  */
10300                 zfs_cmd_t zc = {"\0"};
10301
10302                 (void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
10303                 ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
10304                 if (ret != 0) {
10305                         (void) fprintf(stderr,
10306                         gettext("failed to freeze pool: %d\n"), errno);
10307                         ret = 1;
10308                 }
10309
10310                 log_history = 0;
10311         } else {
10312                 (void) fprintf(stderr, gettext("unrecognized "
10313                     "command '%s'\n"), cmdname);
10314                 usage(B_FALSE);
10315                 ret = 1;
10316         }
10317
10318         for (i = 0; i < argc; i++)
10319                 free(newargv[i]);
10320         free(newargv);
10321
10322         if (ret == 0 && log_history)
10323                 (void) zpool_log_history(g_zfs, history_str);
10324
10325         libzfs_fini(g_zfs);
10326
10327         /*
10328          * The 'ZFS_ABORT' environment variable causes us to dump core on exit
10329          * for the purposes of running ::findleaks.
10330          */
10331         if (getenv("ZFS_ABORT") != NULL) {
10332                 (void) printf("dumping core by request\n");
10333                 abort();
10334         }
10335
10336         return (ret);
10337 }