sys/contrib/openzfs/cmd/zpool/zpool_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  30  * Copyright (c) 2017 Datto Inc.
  31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  32  * Copyright (c) 2017, Intel Corporation.
  33  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  34  */
  35
  36 #include <assert.h>
  37 #include <ctype.h>
  38 #include <dirent.h>
  39 #include <errno.h>
  40 #include <fcntl.h>
  41 #include <getopt.h>
  42 #include <libgen.h>
  43 #include <libintl.h>
  44 #include <libuutil.h>
  45 #include <locale.h>
  46 #include <pthread.h>
  47 #include <stdio.h>
  48 #include <stdlib.h>
  49 #include <string.h>
  50 #include <strings.h>
  51 #include <time.h>
  52 #include <unistd.h>
  53 #include <pwd.h>
  54 #include <zone.h>
  55 #include <sys/wait.h>
  56 #include <zfs_prop.h>
  57 #include <sys/fs/zfs.h>
  58 #include <sys/stat.h>
  59 #include <sys/systeminfo.h>
  60 #include <sys/fm/fs/zfs.h>
  61 #include <sys/fm/util.h>
  62 #include <sys/fm/protocol.h>
  63 #include <sys/zfs_ioctl.h>
  64 #include <sys/mount.h>
  65 #include <sys/sysmacros.h>
  66
  67 #include <math.h>
  68
  69 #include <libzfs.h>
  70 #include <libzutil.h>
  71
  72 #include "zpool_util.h"
  73 #include "zfs_comutil.h"
  74 #include "zfeature_common.h"
  75
  76 #include "statcommon.h"
  77
  78 libzfs_handle_t *g_zfs;
  79
  80 static int zpool_do_create(int, char **);
  81 static int zpool_do_destroy(int, char **);
  82
  83 static int zpool_do_add(int, char **);
  84 static int zpool_do_remove(int, char **);
  85 static int zpool_do_labelclear(int, char **);
  86
  87 static int zpool_do_checkpoint(int, char **);
  88
  89 static int zpool_do_list(int, char **);
  90 static int zpool_do_iostat(int, char **);
  91 static int zpool_do_status(int, char **);
  92
  93 static int zpool_do_online(int, char **);
  94 static int zpool_do_offline(int, char **);
  95 static int zpool_do_clear(int, char **);
  96 static int zpool_do_reopen(int, char **);
  97
  98 static int zpool_do_reguid(int, char **);
  99
 100 static int zpool_do_attach(int, char **);
 101 static int zpool_do_detach(int, char **);
 102 static int zpool_do_replace(int, char **);
 103 static int zpool_do_split(int, char **);
 104
 105 static int zpool_do_initialize(int, char **);
 106 static int zpool_do_scrub(int, char **);
 107 static int zpool_do_resilver(int, char **);
 108 static int zpool_do_trim(int, char **);
 109
 110 static int zpool_do_import(int, char **);
 111 static int zpool_do_export(int, char **);
 112
 113 static int zpool_do_upgrade(int, char **);
 114
 115 static int zpool_do_history(int, char **);
 116 static int zpool_do_events(int, char **);
 117
 118 static int zpool_do_get(int, char **);
 119 static int zpool_do_set(int, char **);
 120
 121 static int zpool_do_sync(int, char **);
 122
 123 static int zpool_do_version(int, char **);
 124
 125 static int zpool_do_wait(int, char **);
 126
 127 /*
 128  * These libumem hooks provide a reasonable set of defaults for the allocator's
 129  * debugging facilities.
 130  */
 131
 132 #ifdef DEBUG
 133 const char *
 134 _umem_debug_init(void)
 135 {
 136         return ("default,verbose"); /* $UMEM_DEBUG setting */
 137 }
 138
 139 const char *
 140 _umem_logging_init(void)
 141 {
 142         return ("fail,contents"); /* $UMEM_LOGGING setting */
 143 }
 144 #endif
 145
 146 typedef enum {
 147         HELP_ADD,
 148         HELP_ATTACH,
 149         HELP_CLEAR,
 150         HELP_CREATE,
 151         HELP_CHECKPOINT,
 152         HELP_DESTROY,
 153         HELP_DETACH,
 154         HELP_EXPORT,
 155         HELP_HISTORY,
 156         HELP_IMPORT,
 157         HELP_IOSTAT,
 158         HELP_LABELCLEAR,
 159         HELP_LIST,
 160         HELP_OFFLINE,
 161         HELP_ONLINE,
 162         HELP_REPLACE,
 163         HELP_REMOVE,
 164         HELP_INITIALIZE,
 165         HELP_SCRUB,
 166         HELP_RESILVER,
 167         HELP_TRIM,
 168         HELP_STATUS,
 169         HELP_UPGRADE,
 170         HELP_EVENTS,
 171         HELP_GET,
 172         HELP_SET,
 173         HELP_SPLIT,
 174         HELP_SYNC,
 175         HELP_REGUID,
 176         HELP_REOPEN,
 177         HELP_VERSION,
 178         HELP_WAIT
 179 } zpool_help_t;
 180
 181
 182 /*
 183  * Flags for stats to display with "zpool iostats"
 184  */
 185 enum iostat_type {
 186         IOS_DEFAULT = 0,
 187         IOS_LATENCY = 1,
 188         IOS_QUEUES = 2,
 189         IOS_L_HISTO = 3,
 190         IOS_RQ_HISTO = 4,
 191         IOS_COUNT,      /* always last element */
 192 };
 193
 194 /* iostat_type entries as bitmasks */
 195 #define IOS_DEFAULT_M   (1ULL << IOS_DEFAULT)
 196 #define IOS_LATENCY_M   (1ULL << IOS_LATENCY)
 197 #define IOS_QUEUES_M    (1ULL << IOS_QUEUES)
 198 #define IOS_L_HISTO_M   (1ULL << IOS_L_HISTO)
 199 #define IOS_RQ_HISTO_M  (1ULL << IOS_RQ_HISTO)
 200
 201 /* Mask of all the histo bits */
 202 #define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
 203
 204 /*
 205  * Lookup table for iostat flags to nvlist names.  Basically a list
 206  * of all the nvlists a flag requires.  Also specifies the order in
 207  * which data gets printed in zpool iostat.
 208  */
 209 static const char *vsx_type_to_nvlist[IOS_COUNT][13] = {
 210         [IOS_L_HISTO] = {
 211             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 212             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 213             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 214             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 215             ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
 216             ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
 217             ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
 218             ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
 219             ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
 220             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 221             NULL},
 222         [IOS_LATENCY] = {
 223             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 224             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 225             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 226             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 227             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 228             NULL},
 229         [IOS_QUEUES] = {
 230             ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
 231             ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
 232             ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
 233             ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
 234             ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
 235             ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
 236             NULL},
 237         [IOS_RQ_HISTO] = {
 238             ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
 239             ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
 240             ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
 241             ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
 242             ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
 243             ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
 244             ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
 245             ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
 246             ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
 247             ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
 248             ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
 249             ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
 250             NULL},
 251 };
 252
 253
 254 /*
 255  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
 256  * Right now, only one histo bit is ever set at one time, so we can
 257  * just do a highbit64(a)
 258  */
 259 #define IOS_HISTO_IDX(a)        (highbit64(a & IOS_ANYHISTO_M) - 1)
 260
 261 typedef struct zpool_command {
 262         const char      *name;
 263         int             (*func)(int, char **);
 264         zpool_help_t    usage;
 265 } zpool_command_t;
 266
 267 /*
 268  * Master command table.  Each ZFS command has a name, associated function, and
 269  * usage message.  The usage messages need to be internationalized, so we have
 270  * to have a function to return the usage message based on a command index.
 271  *
 272  * These commands are organized according to how they are displayed in the usage
 273  * message.  An empty command (one with a NULL name) indicates an empty line in
 274  * the generic usage message.
 275  */
 276 static zpool_command_t command_table[] = {
 277         { "version",    zpool_do_version,       HELP_VERSION            },
 278         { NULL },
 279         { "create",     zpool_do_create,        HELP_CREATE             },
 280         { "destroy",    zpool_do_destroy,       HELP_DESTROY            },
 281         { NULL },
 282         { "add",        zpool_do_add,           HELP_ADD                },
 283         { "remove",     zpool_do_remove,        HELP_REMOVE             },
 284         { NULL },
 285         { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
 286         { NULL },
 287         { "checkpoint", zpool_do_checkpoint,    HELP_CHECKPOINT         },
 288         { NULL },
 289         { "list",       zpool_do_list,          HELP_LIST               },
 290         { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
 291         { "status",     zpool_do_status,        HELP_STATUS             },
 292         { NULL },
 293         { "online",     zpool_do_online,        HELP_ONLINE             },
 294         { "offline",    zpool_do_offline,       HELP_OFFLINE            },
 295         { "clear",      zpool_do_clear,         HELP_CLEAR              },
 296         { "reopen",     zpool_do_reopen,        HELP_REOPEN             },
 297         { NULL },
 298         { "attach",     zpool_do_attach,        HELP_ATTACH             },
 299         { "detach",     zpool_do_detach,        HELP_DETACH             },
 300         { "replace",    zpool_do_replace,       HELP_REPLACE            },
 301         { "split",      zpool_do_split,         HELP_SPLIT              },
 302         { NULL },
 303         { "initialize", zpool_do_initialize,    HELP_INITIALIZE         },
 304         { "resilver",   zpool_do_resilver,      HELP_RESILVER           },
 305         { "scrub",      zpool_do_scrub,         HELP_SCRUB              },
 306         { "trim",       zpool_do_trim,          HELP_TRIM               },
 307         { NULL },
 308         { "import",     zpool_do_import,        HELP_IMPORT             },
 309         { "export",     zpool_do_export,        HELP_EXPORT             },
 310         { "upgrade",    zpool_do_upgrade,       HELP_UPGRADE            },
 311         { "reguid",     zpool_do_reguid,        HELP_REGUID             },
 312         { NULL },
 313         { "history",    zpool_do_history,       HELP_HISTORY            },
 314         { "events",     zpool_do_events,        HELP_EVENTS             },
 315         { NULL },
 316         { "get",        zpool_do_get,           HELP_GET                },
 317         { "set",        zpool_do_set,           HELP_SET                },
 318         { "sync",       zpool_do_sync,          HELP_SYNC               },
 319         { NULL },
 320         { "wait",       zpool_do_wait,          HELP_WAIT               },
 321 };
 322
 323 #define NCOMMAND        (ARRAY_SIZE(command_table))
 324
 325 #define VDEV_ALLOC_CLASS_LOGS   "logs"
 326
 327 static zpool_command_t *current_command;
 328 static char history_str[HIS_MAX_RECORD_LEN];
 329 static boolean_t log_history = B_TRUE;
 330 static uint_t timestamp_fmt = NODATE;
 331
 332 static const char *
 333 get_usage(zpool_help_t idx)
 334 {
 335         switch (idx) {
 336         case HELP_ADD:
 337                 return (gettext("\tadd [-fgLnP] [-o property=value] "
 338                     "<pool> <vdev> ...\n"));
 339         case HELP_ATTACH:
 340                 return (gettext("\tattach [-fsw] [-o property=value] "
 341                     "<pool> <device> <new-device>\n"));
 342         case HELP_CLEAR:
 343                 return (gettext("\tclear [-nF] <pool> [device]\n"));
 344         case HELP_CREATE:
 345                 return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 346                     "\t    [-O file-system-property=value] ... \n"
 347                     "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 348         case HELP_CHECKPOINT:
 349                 return (gettext("\tcheckpoint [-d [-w]] <pool> ...\n"));
 350         case HELP_DESTROY:
 351                 return (gettext("\tdestroy [-f] <pool>\n"));
 352         case HELP_DETACH:
 353                 return (gettext("\tdetach <pool> <device>\n"));
 354         case HELP_EXPORT:
 355                 return (gettext("\texport [-af] <pool> ...\n"));
 356         case HELP_HISTORY:
 357                 return (gettext("\thistory [-il] [<pool>] ...\n"));
 358         case HELP_IMPORT:
 359                 return (gettext("\timport [-d dir] [-D]\n"
 360                     "\timport [-o mntopts] [-o property=value] ... \n"
 361                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 362                     "[-R root] [-F [-n]] -a\n"
 363                     "\timport [-o mntopts] [-o property=value] ... \n"
 364                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 365                     "[-R root] [-F [-n]]\n"
 366                     "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
 367         case HELP_IOSTAT:
 368                 return (gettext("\tiostat [[[-c [script1,script2,...]"
 369                     "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
 370                     "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
 371                     " [[-n] interval [count]]\n"));
 372         case HELP_LABELCLEAR:
 373                 return (gettext("\tlabelclear [-f] <vdev>\n"));
 374         case HELP_LIST:
 375                 return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 376                     "[-T d|u] [pool] ... \n"
 377                     "\t    [interval [count]]\n"));
 378         case HELP_OFFLINE:
 379                 return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
 380         case HELP_ONLINE:
 381                 return (gettext("\tonline [-e] <pool> <device> ...\n"));
 382         case HELP_REPLACE:
 383                 return (gettext("\treplace [-fsw] [-o property=value] "
 384                     "<pool> <device> [new-device]\n"));
 385         case HELP_REMOVE:
 386                 return (gettext("\tremove [-npsw] <pool> <device> ...\n"));
 387         case HELP_REOPEN:
 388                 return (gettext("\treopen [-n] <pool>\n"));
 389         case HELP_INITIALIZE:
 390                 return (gettext("\tinitialize [-c | -s] [-w] <pool> "
 391                     "[<device> ...]\n"));
 392         case HELP_SCRUB:
 393                 return (gettext("\tscrub [-s | -p] [-w] <pool> ...\n"));
 394         case HELP_RESILVER:
 395                 return (gettext("\tresilver <pool> ...\n"));
 396         case HELP_TRIM:
 397                 return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
 398                     "[<device> ...]\n"));
 399         case HELP_STATUS:
 400                 return (gettext("\tstatus [-c [script1,script2,...]] "
 401                     "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
 402                     "\t    [interval [count]]\n"));
 403         case HELP_UPGRADE:
 404                 return (gettext("\tupgrade\n"
 405                     "\tupgrade -v\n"
 406                     "\tupgrade [-V version] <-a | pool ...>\n"));
 407         case HELP_EVENTS:
 408                 return (gettext("\tevents [-vHf [pool] | -c]\n"));
 409         case HELP_GET:
 410                 return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
 411                     "<\"all\" | property[,...]> <pool> ...\n"));
 412         case HELP_SET:
 413                 return (gettext("\tset <property=value> <pool> \n"));
 414         case HELP_SPLIT:
 415                 return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
 416                     "\t    [-o property=value] <pool> <newpool> "
 417                     "[<device> ...]\n"));
 418         case HELP_REGUID:
 419                 return (gettext("\treguid <pool>\n"));
 420         case HELP_SYNC:
 421                 return (gettext("\tsync [pool] ...\n"));
 422         case HELP_VERSION:
 423                 return (gettext("\tversion\n"));
 424         case HELP_WAIT:
 425                 return (gettext("\twait [-Hp] [-T d|u] [-t <activity>[,...]] "
 426                     "<pool> [interval]\n"));
 427         }
 428
 429         abort();
 430         /* NOTREACHED */
 431 }
 432
 433 static void
 434 zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
 435 {
 436         uint_t children = 0;
 437         nvlist_t **child;
 438         uint_t i;
 439
 440         (void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
 441             &child, &children);
 442
 443         if (children == 0) {
 444                 char *path = zpool_vdev_name(g_zfs, zhp, nvroot,
 445                     VDEV_NAME_PATH);
 446
 447                 if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 &&
 448                     strcmp(path, VDEV_TYPE_HOLE) != 0)
 449                         fnvlist_add_boolean(res, path);
 450
 451                 free(path);
 452                 return;
 453         }
 454
 455         for (i = 0; i < children; i++) {
 456                 zpool_collect_leaves(zhp, child[i], res);
 457         }
 458 }
 459
 460 /*
 461  * Callback routine that will print out a pool property value.
 462  */
 463 static int
 464 print_prop_cb(int prop, void *cb)
 465 {
 466         FILE *fp = cb;
 467
 468         (void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
 469
 470         if (zpool_prop_readonly(prop))
 471                 (void) fprintf(fp, "  NO   ");
 472         else
 473                 (void) fprintf(fp, " YES   ");
 474
 475         if (zpool_prop_values(prop) == NULL)
 476                 (void) fprintf(fp, "-\n");
 477         else
 478                 (void) fprintf(fp, "%s\n", zpool_prop_values(prop));
 479
 480         return (ZPROP_CONT);
 481 }
 482
 483 /*
 484  * Display usage message.  If we're inside a command, display only the usage for
 485  * that command.  Otherwise, iterate over the entire command table and display
 486  * a complete usage message.
 487  */
 488 static void
 489 usage(boolean_t requested)
 490 {
 491         FILE *fp = requested ? stdout : stderr;
 492
 493         if (current_command == NULL) {
 494                 int i;
 495
 496                 (void) fprintf(fp, gettext("usage: zpool command args ...\n"));
 497                 (void) fprintf(fp,
 498                     gettext("where 'command' is one of the following:\n\n"));
 499
 500                 for (i = 0; i < NCOMMAND; i++) {
 501                         if (command_table[i].name == NULL)
 502                                 (void) fprintf(fp, "\n");
 503                         else
 504                                 (void) fprintf(fp, "%s",
 505                                     get_usage(command_table[i].usage));
 506                 }
 507         } else {
 508                 (void) fprintf(fp, gettext("usage:\n"));
 509                 (void) fprintf(fp, "%s", get_usage(current_command->usage));
 510         }
 511
 512         if (current_command != NULL &&
 513             ((strcmp(current_command->name, "set") == 0) ||
 514             (strcmp(current_command->name, "get") == 0) ||
 515             (strcmp(current_command->name, "list") == 0))) {
 516
 517                 (void) fprintf(fp,
 518                     gettext("\nthe following properties are supported:\n"));
 519
 520                 (void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
 521                     "PROPERTY", "EDIT", "VALUES");
 522
 523                 /* Iterate over all properties */
 524                 (void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
 525                     ZFS_TYPE_POOL);
 526
 527                 (void) fprintf(fp, "\t%-19s   ", "feature@...");
 528                 (void) fprintf(fp, "YES   disabled | enabled | active\n");
 529
 530                 (void) fprintf(fp, gettext("\nThe feature@ properties must be "
 531                     "appended with a feature name.\nSee zpool-features(5).\n"));
 532         }
 533
 534         /*
 535          * See comments at end of main().
 536          */
 537         if (getenv("ZFS_ABORT") != NULL) {
 538                 (void) printf("dumping core by request\n");
 539                 abort();
 540         }
 541
 542         exit(requested ? 0 : 2);
 543 }
 544
 545 /*
 546  * zpool initialize [-c | -s] [-w] <pool> [<vdev> ...]
 547  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
 548  * if none specified.
 549  *
 550  *      -c      Cancel. Ends active initializing.
 551  *      -s      Suspend. Initializing can then be restarted with no flags.
 552  *      -w      Wait. Blocks until initializing has completed.
 553  */
 554 int
 555 zpool_do_initialize(int argc, char **argv)
 556 {
 557         int c;
 558         char *poolname;
 559         zpool_handle_t *zhp;
 560         nvlist_t *vdevs;
 561         int err = 0;
 562         boolean_t wait = B_FALSE;
 563
 564         struct option long_options[] = {
 565                 {"cancel",      no_argument,            NULL, 'c'},
 566                 {"suspend",     no_argument,            NULL, 's'},
 567                 {"wait",        no_argument,            NULL, 'w'},
 568                 {0, 0, 0, 0}
 569         };
 570
 571         pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
 572         while ((c = getopt_long(argc, argv, "csw", long_options, NULL)) != -1) {
 573                 switch (c) {
 574                 case 'c':
 575                         if (cmd_type != POOL_INITIALIZE_START &&
 576                             cmd_type != POOL_INITIALIZE_CANCEL) {
 577                                 (void) fprintf(stderr, gettext("-c cannot be "
 578                                     "combined with other options\n"));
 579                                 usage(B_FALSE);
 580                         }
 581                         cmd_type = POOL_INITIALIZE_CANCEL;
 582                         break;
 583                 case 's':
 584                         if (cmd_type != POOL_INITIALIZE_START &&
 585                             cmd_type != POOL_INITIALIZE_SUSPEND) {
 586                                 (void) fprintf(stderr, gettext("-s cannot be "
 587                                     "combined with other options\n"));
 588                                 usage(B_FALSE);
 589                         }
 590                         cmd_type = POOL_INITIALIZE_SUSPEND;
 591                         break;
 592                 case 'w':
 593                         wait = B_TRUE;
 594                         break;
 595                 case '?':
 596                         if (optopt != 0) {
 597                                 (void) fprintf(stderr,
 598                                     gettext("invalid option '%c'\n"), optopt);
 599                         } else {
 600                                 (void) fprintf(stderr,
 601                                     gettext("invalid option '%s'\n"),
 602                                     argv[optind - 1]);
 603                         }
 604                         usage(B_FALSE);
 605                 }
 606         }
 607
 608         argc -= optind;
 609         argv += optind;
 610
 611         if (argc < 1) {
 612                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 613                 usage(B_FALSE);
 614                 return (-1);
 615         }
 616
 617         if (wait && (cmd_type != POOL_INITIALIZE_START)) {
 618                 (void) fprintf(stderr, gettext("-w cannot be used with -c or "
 619                     "-s\n"));
 620                 usage(B_FALSE);
 621         }
 622
 623         poolname = argv[0];
 624         zhp = zpool_open(g_zfs, poolname);
 625         if (zhp == NULL)
 626                 return (-1);
 627
 628         vdevs = fnvlist_alloc();
 629         if (argc == 1) {
 630                 /* no individual leaf vdevs specified, so add them all */
 631                 nvlist_t *config = zpool_get_config(zhp, NULL);
 632                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
 633                     ZPOOL_CONFIG_VDEV_TREE);
 634                 zpool_collect_leaves(zhp, nvroot, vdevs);
 635         } else {
 636                 for (int i = 1; i < argc; i++) {
 637                         fnvlist_add_boolean(vdevs, argv[i]);
 638                 }
 639         }
 640
 641         if (wait)
 642                 err = zpool_initialize_wait(zhp, cmd_type, vdevs);
 643         else
 644                 err = zpool_initialize(zhp, cmd_type, vdevs);
 645
 646         fnvlist_free(vdevs);
 647         zpool_close(zhp);
 648
 649         return (err);
 650 }
 651
 652 /*
 653  * print a pool vdev config for dry runs
 654  */
 655 static void
 656 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 657     const char *match, int name_flags)
 658 {
 659         nvlist_t **child;
 660         uint_t c, children;
 661         char *vname;
 662         boolean_t printed = B_FALSE;
 663
 664         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 665             &child, &children) != 0) {
 666                 if (name != NULL)
 667                         (void) printf("\t%*s%s\n", indent, "", name);
 668                 return;
 669         }
 670
 671         for (c = 0; c < children; c++) {
 672                 uint64_t is_log = B_FALSE;
 673                 char *class = "";
 674
 675                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 676                     &is_log);
 677                 if (is_log)
 678                         class = VDEV_ALLOC_BIAS_LOG;
 679                 (void) nvlist_lookup_string(child[c],
 680                     ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
 681                 if (strcmp(match, class) != 0)
 682                         continue;
 683
 684                 if (!printed && name != NULL) {
 685                         (void) printf("\t%*s%s\n", indent, "", name);
 686                         printed = B_TRUE;
 687                 }
 688                 vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
 689                 print_vdev_tree(zhp, vname, child[c], indent + 2, "",
 690                     name_flags);
 691                 free(vname);
 692         }
 693 }
 694
 695 static boolean_t
 696 prop_list_contains_feature(nvlist_t *proplist)
 697 {
 698         nvpair_t *nvp;
 699         for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
 700             nvp = nvlist_next_nvpair(proplist, nvp)) {
 701                 if (zpool_prop_feature(nvpair_name(nvp)))
 702                         return (B_TRUE);
 703         }
 704         return (B_FALSE);
 705 }
 706
 707 /*
 708  * Add a property pair (name, string-value) into a property nvlist.
 709  */
 710 static int
 711 add_prop_list(const char *propname, char *propval, nvlist_t **props,
 712     boolean_t poolprop)
 713 {
 714         zpool_prop_t prop = ZPOOL_PROP_INVAL;
 715         nvlist_t *proplist;
 716         const char *normnm;
 717         char *strval;
 718
 719         if (*props == NULL &&
 720             nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
 721                 (void) fprintf(stderr,
 722                     gettext("internal error: out of memory\n"));
 723                 return (1);
 724         }
 725
 726         proplist = *props;
 727
 728         if (poolprop) {
 729                 const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 730
 731                 if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 732                     !zpool_prop_feature(propname)) {
 733                         (void) fprintf(stderr, gettext("property '%s' is "
 734                             "not a valid pool property\n"), propname);
 735                         return (2);
 736                 }
 737
 738                 /*
 739                  * feature@ properties and version should not be specified
 740                  * at the same time.
 741                  */
 742                 if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
 743                     nvlist_exists(proplist, vname)) ||
 744                     (prop == ZPOOL_PROP_VERSION &&
 745                     prop_list_contains_feature(proplist))) {
 746                         (void) fprintf(stderr, gettext("'feature@' and "
 747                             "'version' properties cannot be specified "
 748                             "together\n"));
 749                         return (2);
 750                 }
 751
 752
 753                 if (zpool_prop_feature(propname))
 754                         normnm = propname;
 755                 else
 756                         normnm = zpool_prop_to_name(prop);
 757         } else {
 758                 zfs_prop_t fsprop = zfs_name_to_prop(propname);
 759
 760                 if (zfs_prop_valid_for_type(fsprop, ZFS_TYPE_FILESYSTEM,
 761                     B_FALSE)) {
 762                         normnm = zfs_prop_to_name(fsprop);
 763                 } else if (zfs_prop_user(propname) ||
 764                     zfs_prop_userquota(propname)) {
 765                         normnm = propname;
 766                 } else {
 767                         (void) fprintf(stderr, gettext("property '%s' is "
 768                             "not a valid filesystem property\n"), propname);
 769                         return (2);
 770                 }
 771         }
 772
 773         if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
 774             prop != ZPOOL_PROP_CACHEFILE) {
 775                 (void) fprintf(stderr, gettext("property '%s' "
 776                     "specified multiple times\n"), propname);
 777                 return (2);
 778         }
 779
 780         if (nvlist_add_string(proplist, normnm, propval) != 0) {
 781                 (void) fprintf(stderr, gettext("internal "
 782                     "error: out of memory\n"));
 783                 return (1);
 784         }
 785
 786         return (0);
 787 }
 788
 789 /*
 790  * Set a default property pair (name, string-value) in a property nvlist
 791  */
 792 static int
 793 add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
 794     boolean_t poolprop)
 795 {
 796         char *pval;
 797
 798         if (nvlist_lookup_string(*props, propname, &pval) == 0)
 799                 return (0);
 800
 801         return (add_prop_list(propname, propval, props, B_TRUE));
 802 }
 803
 804 /*
 805  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
 806  *
 807  *      -f      Force addition of devices, even if they appear in use
 808  *      -g      Display guid for individual vdev name.
 809  *      -L      Follow links when resolving vdev path name.
 810  *      -n      Do not add the devices, but display the resulting layout if
 811  *              they were to be added.
 812  *      -o      Set property=value.
 813  *      -P      Display full path for vdev name.
 814  *
 815  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
 816  * handled by make_root_vdev(), which constructs the nvlist needed to pass to
 817  * libzfs.
 818  */
 819 int
 820 zpool_do_add(int argc, char **argv)
 821 {
 822         boolean_t force = B_FALSE;
 823         boolean_t dryrun = B_FALSE;
 824         int name_flags = 0;
 825         int c;
 826         nvlist_t *nvroot;
 827         char *poolname;
 828         int ret;
 829         zpool_handle_t *zhp;
 830         nvlist_t *config;
 831         nvlist_t *props = NULL;
 832         char *propval;
 833
 834         /* check options */
 835         while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
 836                 switch (c) {
 837                 case 'f':
 838                         force = B_TRUE;
 839                         break;
 840                 case 'g':
 841                         name_flags |= VDEV_NAME_GUID;
 842                         break;
 843                 case 'L':
 844                         name_flags |= VDEV_NAME_FOLLOW_LINKS;
 845                         break;
 846                 case 'n':
 847                         dryrun = B_TRUE;
 848                         break;
 849                 case 'o':
 850                         if ((propval = strchr(optarg, '=')) == NULL) {
 851                                 (void) fprintf(stderr, gettext("missing "
 852                                     "'=' for -o option\n"));
 853                                 usage(B_FALSE);
 854                         }
 855                         *propval = '\0';
 856                         propval++;
 857
 858                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
 859                             (add_prop_list(optarg, propval, &props, B_TRUE)))
 860                                 usage(B_FALSE);
 861                         break;
 862                 case 'P':
 863                         name_flags |= VDEV_NAME_PATH;
 864                         break;
 865                 case '?':
 866                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 867                             optopt);
 868                         usage(B_FALSE);
 869                 }
 870         }
 871
 872         argc -= optind;
 873         argv += optind;
 874
 875         /* get pool name and check number of arguments */
 876         if (argc < 1) {
 877                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 878                 usage(B_FALSE);
 879         }
 880         if (argc < 2) {
 881                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
 882                 usage(B_FALSE);
 883         }
 884
 885         poolname = argv[0];
 886
 887         argc--;
 888         argv++;
 889
 890         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 891                 return (1);
 892
 893         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
 894                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
 895                     poolname);
 896                 zpool_close(zhp);
 897                 return (1);
 898         }
 899
 900         /* unless manually specified use "ashift" pool property (if set) */
 901         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
 902                 int intval;
 903                 zprop_source_t src;
 904                 char strval[ZPOOL_MAXPROPLEN];
 905
 906                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
 907                 if (src != ZPROP_SRC_DEFAULT) {
 908                         (void) sprintf(strval, "%" PRId32, intval);
 909                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
 910                             &props, B_TRUE) == 0);
 911                 }
 912         }
 913
 914         /* pass off to make_root_vdev for processing */
 915         nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
 916             argc, argv);
 917         if (nvroot == NULL) {
 918                 zpool_close(zhp);
 919                 return (1);
 920         }
 921
 922         if (dryrun) {
 923                 nvlist_t *poolnvroot;
 924                 nvlist_t **l2child;
 925                 uint_t l2children, c;
 926                 char *vname;
 927                 boolean_t hadcache = B_FALSE;
 928
 929                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
 930                     &poolnvroot) == 0);
 931
 932                 (void) printf(gettext("would update '%s' to the following "
 933                     "configuration:\n"), zpool_get_name(zhp));
 934
 935                 /* print original main pool and new tree */
 936                 print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
 937                     name_flags | VDEV_NAME_TYPE_ID);
 938                 print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
 939
 940                 /* print other classes: 'dedup', 'special', and 'log' */
 941                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_DEDUP)) {
 942                         print_vdev_tree(zhp, "dedup", poolnvroot, 0,
 943                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
 944                         print_vdev_tree(zhp, NULL, nvroot, 0,
 945                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
 946                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_DEDUP)) {
 947                         print_vdev_tree(zhp, "dedup", nvroot, 0,
 948                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
 949                 }
 950
 951                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
 952                         print_vdev_tree(zhp, "special", poolnvroot, 0,
 953                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
 954                         print_vdev_tree(zhp, NULL, nvroot, 0,
 955                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
 956                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
 957                         print_vdev_tree(zhp, "special", nvroot, 0,
 958                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
 959                 }
 960
 961                 if (num_logs(poolnvroot) > 0) {
 962                         print_vdev_tree(zhp, "logs", poolnvroot, 0,
 963                             VDEV_ALLOC_BIAS_LOG, name_flags);
 964                         print_vdev_tree(zhp, NULL, nvroot, 0,
 965                             VDEV_ALLOC_BIAS_LOG, name_flags);
 966                 } else if (num_logs(nvroot) > 0) {
 967                         print_vdev_tree(zhp, "logs", nvroot, 0,
 968                             VDEV_ALLOC_BIAS_LOG, name_flags);
 969                 }
 970
 971                 /* Do the same for the caches */
 972                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
 973                     &l2child, &l2children) == 0 && l2children) {
 974                         hadcache = B_TRUE;
 975                         (void) printf(gettext("\tcache\n"));
 976                         for (c = 0; c < l2children; c++) {
 977                                 vname = zpool_vdev_name(g_zfs, NULL,
 978                                     l2child[c], name_flags);
 979                                 (void) printf("\t  %s\n", vname);
 980                                 free(vname);
 981                         }
 982                 }
 983                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
 984                     &l2child, &l2children) == 0 && l2children) {
 985                         if (!hadcache)
 986                                 (void) printf(gettext("\tcache\n"));
 987                         for (c = 0; c < l2children; c++) {
 988                                 vname = zpool_vdev_name(g_zfs, NULL,
 989                                     l2child[c], name_flags);
 990                                 (void) printf("\t  %s\n", vname);
 991                                 free(vname);
 992                         }
 993                 }
 994
 995                 ret = 0;
 996         } else {
 997                 ret = (zpool_add(zhp, nvroot) != 0);
 998         }
 999
1000         nvlist_free(props);
1001         nvlist_free(nvroot);
1002         zpool_close(zhp);
1003
1004         return (ret);
1005 }
1006
1007 /*
1008  * zpool remove [-npsw] <pool> <vdev> ...
1009  *
1010  * Removes the given vdev from the pool.
1011  */
1012 int
1013 zpool_do_remove(int argc, char **argv)
1014 {
1015         char *poolname;
1016         int i, ret = 0;
1017         zpool_handle_t *zhp = NULL;
1018         boolean_t stop = B_FALSE;
1019         int c;
1020         boolean_t noop = B_FALSE;
1021         boolean_t parsable = B_FALSE;
1022         boolean_t wait = B_FALSE;
1023
1024         /* check options */
1025         while ((c = getopt(argc, argv, "npsw")) != -1) {
1026                 switch (c) {
1027                 case 'n':
1028                         noop = B_TRUE;
1029                         break;
1030                 case 'p':
1031                         parsable = B_TRUE;
1032                         break;
1033                 case 's':
1034                         stop = B_TRUE;
1035                         break;
1036                 case 'w':
1037                         wait = B_TRUE;
1038                         break;
1039                 case '?':
1040                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1041                             optopt);
1042                         usage(B_FALSE);
1043                 }
1044         }
1045
1046         argc -= optind;
1047         argv += optind;
1048
1049         /* get pool name and check number of arguments */
1050         if (argc < 1) {
1051                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1052                 usage(B_FALSE);
1053         }
1054
1055         poolname = argv[0];
1056
1057         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1058                 return (1);
1059
1060         if (stop && noop) {
1061                 (void) fprintf(stderr, gettext("stop request ignored\n"));
1062                 return (0);
1063         }
1064
1065         if (stop) {
1066                 if (argc > 1) {
1067                         (void) fprintf(stderr, gettext("too many arguments\n"));
1068                         usage(B_FALSE);
1069                 }
1070                 if (zpool_vdev_remove_cancel(zhp) != 0)
1071                         ret = 1;
1072                 if (wait) {
1073                         (void) fprintf(stderr, gettext("invalid option "
1074                             "combination: -w cannot be used with -s\n"));
1075                         usage(B_FALSE);
1076                 }
1077         } else {
1078                 if (argc < 2) {
1079                         (void) fprintf(stderr, gettext("missing device\n"));
1080                         usage(B_FALSE);
1081                 }
1082
1083                 for (i = 1; i < argc; i++) {
1084                         if (noop) {
1085                                 uint64_t size;
1086
1087                                 if (zpool_vdev_indirect_size(zhp, argv[i],
1088                                     &size) != 0) {
1089                                         ret = 1;
1090                                         break;
1091                                 }
1092                                 if (parsable) {
1093                                         (void) printf("%s %llu\n",
1094                                             argv[i], (unsigned long long)size);
1095                                 } else {
1096                                         char valstr[32];
1097                                         zfs_nicenum(size, valstr,
1098                                             sizeof (valstr));
1099                                         (void) printf("Memory that will be "
1100                                             "used after removing %s: %s\n",
1101                                             argv[i], valstr);
1102                                 }
1103                         } else {
1104                                 if (zpool_vdev_remove(zhp, argv[i]) != 0)
1105                                         ret = 1;
1106                         }
1107                 }
1108
1109                 if (ret == 0 && wait)
1110                         ret = zpool_wait(zhp, ZPOOL_WAIT_REMOVE);
1111         }
1112         zpool_close(zhp);
1113
1114         return (ret);
1115 }
1116
1117 /*
1118  * zpool labelclear [-f] <vdev>
1119  *
1120  *      -f      Force clearing the label for the vdevs which are members of
1121  *              the exported or foreign pools.
1122  *
1123  * Verifies that the vdev is not active and zeros out the label information
1124  * on the device.
1125  */
1126 int
1127 zpool_do_labelclear(int argc, char **argv)
1128 {
1129         char vdev[MAXPATHLEN];
1130         char *name = NULL;
1131         struct stat st;
1132         int c, fd = -1, ret = 0;
1133         nvlist_t *config;
1134         pool_state_t state;
1135         boolean_t inuse = B_FALSE;
1136         boolean_t force = B_FALSE;
1137
1138         /* check options */
1139         while ((c = getopt(argc, argv, "f")) != -1) {
1140                 switch (c) {
1141                 case 'f':
1142                         force = B_TRUE;
1143                         break;
1144                 default:
1145                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1146                             optopt);
1147                         usage(B_FALSE);
1148                 }
1149         }
1150
1151         argc -= optind;
1152         argv += optind;
1153
1154         /* get vdev name */
1155         if (argc < 1) {
1156                 (void) fprintf(stderr, gettext("missing vdev name\n"));
1157                 usage(B_FALSE);
1158         }
1159         if (argc > 1) {
1160                 (void) fprintf(stderr, gettext("too many arguments\n"));
1161                 usage(B_FALSE);
1162         }
1163
1164         /*
1165          * Check if we were given absolute path and use it as is.
1166          * Otherwise if the provided vdev name doesn't point to a file,
1167          * try prepending expected disk paths and partition numbers.
1168          */
1169         (void) strlcpy(vdev, argv[0], sizeof (vdev));
1170         if (vdev[0] != '/' && stat(vdev, &st) != 0) {
1171                 int error;
1172
1173                 error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
1174                 if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
1175                         if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
1176                                 error = ENOENT;
1177                 }
1178
1179                 if (error || (stat(vdev, &st) != 0)) {
1180                         (void) fprintf(stderr, gettext(
1181                             "failed to find device %s, try specifying absolute "
1182                             "path instead\n"), argv[0]);
1183                         return (1);
1184                 }
1185         }
1186
1187         if ((fd = open(vdev, O_RDWR)) < 0) {
1188                 (void) fprintf(stderr, gettext("failed to open %s: %s\n"),
1189                     vdev, strerror(errno));
1190                 return (1);
1191         }
1192
1193         /*
1194          * Flush all dirty pages for the block device.  This should not be
1195          * fatal when the device does not support BLKFLSBUF as would be the
1196          * case for a file vdev.
1197          */
1198         if ((zfs_dev_flush(fd) != 0) && (errno != ENOTTY))
1199                 (void) fprintf(stderr, gettext("failed to invalidate "
1200                     "cache for %s: %s\n"), vdev, strerror(errno));
1201
1202         if (zpool_read_label(fd, &config, NULL) != 0) {
1203                 (void) fprintf(stderr,
1204                     gettext("failed to read label from %s\n"), vdev);
1205                 ret = 1;
1206                 goto errout;
1207         }
1208         nvlist_free(config);
1209
1210         ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
1211         if (ret != 0) {
1212                 (void) fprintf(stderr,
1213                     gettext("failed to check state for %s\n"), vdev);
1214                 ret = 1;
1215                 goto errout;
1216         }
1217
1218         if (!inuse)
1219                 goto wipe_label;
1220
1221         switch (state) {
1222         default:
1223         case POOL_STATE_ACTIVE:
1224         case POOL_STATE_SPARE:
1225         case POOL_STATE_L2CACHE:
1226                 (void) fprintf(stderr, gettext(
1227                     "%s is a member (%s) of pool \"%s\"\n"),
1228                     vdev, zpool_pool_state_to_name(state), name);
1229                 ret = 1;
1230                 goto errout;
1231
1232         case POOL_STATE_EXPORTED:
1233                 if (force)
1234                         break;
1235                 (void) fprintf(stderr, gettext(
1236                     "use '-f' to override the following error:\n"
1237                     "%s is a member of exported pool \"%s\"\n"),
1238                     vdev, name);
1239                 ret = 1;
1240                 goto errout;
1241
1242         case POOL_STATE_POTENTIALLY_ACTIVE:
1243                 if (force)
1244                         break;
1245                 (void) fprintf(stderr, gettext(
1246                     "use '-f' to override the following error:\n"
1247                     "%s is a member of potentially active pool \"%s\"\n"),
1248                     vdev, name);
1249                 ret = 1;
1250                 goto errout;
1251
1252         case POOL_STATE_DESTROYED:
1253                 /* inuse should never be set for a destroyed pool */
1254                 assert(0);
1255                 break;
1256         }
1257
1258 wipe_label:
1259         ret = zpool_clear_label(fd);
1260         if (ret != 0) {
1261                 (void) fprintf(stderr,
1262                     gettext("failed to clear label for %s\n"), vdev);
1263         }
1264
1265 errout:
1266         free(name);
1267         (void) close(fd);
1268
1269         return (ret);
1270 }
1271
1272 /*
1273  * zpool create [-fnd] [-o property=value] ...
1274  *              [-O file-system-property=value] ...
1275  *              [-R root] [-m mountpoint] <pool> <dev> ...
1276  *
1277  *      -f      Force creation, even if devices appear in use
1278  *      -n      Do not create the pool, but display the resulting layout if it
1279  *              were to be created.
1280  *      -R      Create a pool under an alternate root
1281  *      -m      Set default mountpoint for the root dataset.  By default it's
1282  *              '/<pool>'
1283  *      -o      Set property=value.
1284  *      -o      Set feature@feature=enabled|disabled.
1285  *      -d      Don't automatically enable all supported pool features
1286  *              (individual features can be enabled with -o).
1287  *      -O      Set fsproperty=value in the pool's root file system
1288  *
1289  * Creates the named pool according to the given vdev specification.  The
1290  * bulk of the vdev processing is done in make_root_vdev() in zpool_vdev.c.
1291  * Once we get the nvlist back from make_root_vdev(), we either print out the
1292  * contents (if '-n' was specified), or pass it to libzfs to do the creation.
1293  */
1294 int
1295 zpool_do_create(int argc, char **argv)
1296 {
1297         boolean_t force = B_FALSE;
1298         boolean_t dryrun = B_FALSE;
1299         boolean_t enable_all_pool_feat = B_TRUE;
1300         int c;
1301         nvlist_t *nvroot = NULL;
1302         char *poolname;
1303         char *tname = NULL;
1304         int ret = 1;
1305         char *altroot = NULL;
1306         char *mountpoint = NULL;
1307         nvlist_t *fsprops = NULL;
1308         nvlist_t *props = NULL;
1309         char *propval;
1310
1311         /* check options */
1312         while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1313                 switch (c) {
1314                 case 'f':
1315                         force = B_TRUE;
1316                         break;
1317                 case 'n':
1318                         dryrun = B_TRUE;
1319                         break;
1320                 case 'd':
1321                         enable_all_pool_feat = B_FALSE;
1322                         break;
1323                 case 'R':
1324                         altroot = optarg;
1325                         if (add_prop_list(zpool_prop_to_name(
1326                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1327                                 goto errout;
1328                         if (add_prop_list_default(zpool_prop_to_name(
1329                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1330                                 goto errout;
1331                         break;
1332                 case 'm':
1333                         /* Equivalent to -O mountpoint=optarg */
1334                         mountpoint = optarg;
1335                         break;
1336                 case 'o':
1337                         if ((propval = strchr(optarg, '=')) == NULL) {
1338                                 (void) fprintf(stderr, gettext("missing "
1339                                     "'=' for -o option\n"));
1340                                 goto errout;
1341                         }
1342                         *propval = '\0';
1343                         propval++;
1344
1345                         if (add_prop_list(optarg, propval, &props, B_TRUE))
1346                                 goto errout;
1347
1348                         /*
1349                          * If the user is creating a pool that doesn't support
1350                          * feature flags, don't enable any features.
1351                          */
1352                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1353                                 char *end;
1354                                 u_longlong_t ver;
1355
1356                                 ver = strtoull(propval, &end, 10);
1357                                 if (*end == '\0' &&
1358                                     ver < SPA_VERSION_FEATURES) {
1359                                         enable_all_pool_feat = B_FALSE;
1360                                 }
1361                         }
1362                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1363                                 altroot = propval;
1364                         break;
1365                 case 'O':
1366                         if ((propval = strchr(optarg, '=')) == NULL) {
1367                                 (void) fprintf(stderr, gettext("missing "
1368                                     "'=' for -O option\n"));
1369                                 goto errout;
1370                         }
1371                         *propval = '\0';
1372                         propval++;
1373
1374                         /*
1375                          * Mountpoints are checked and then added later.
1376                          * Uniquely among properties, they can be specified
1377                          * more than once, to avoid conflict with -m.
1378                          */
1379                         if (0 == strcmp(optarg,
1380                             zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1381                                 mountpoint = propval;
1382                         } else if (add_prop_list(optarg, propval, &fsprops,
1383                             B_FALSE)) {
1384                                 goto errout;
1385                         }
1386                         break;
1387                 case 't':
1388                         /*
1389                          * Sanity check temporary pool name.
1390                          */
1391                         if (strchr(optarg, '/') != NULL) {
1392                                 (void) fprintf(stderr, gettext("cannot create "
1393                                     "'%s': invalid character '/' in temporary "
1394                                     "name\n"), optarg);
1395                                 (void) fprintf(stderr, gettext("use 'zfs "
1396                                     "create' to create a dataset\n"));
1397                                 goto errout;
1398                         }
1399
1400                         if (add_prop_list(zpool_prop_to_name(
1401                             ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1402                                 goto errout;
1403                         if (add_prop_list_default(zpool_prop_to_name(
1404                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1405                                 goto errout;
1406                         tname = optarg;
1407                         break;
1408                 case ':':
1409                         (void) fprintf(stderr, gettext("missing argument for "
1410                             "'%c' option\n"), optopt);
1411                         goto badusage;
1412                 case '?':
1413                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1414                             optopt);
1415                         goto badusage;
1416                 }
1417         }
1418
1419         argc -= optind;
1420         argv += optind;
1421
1422         /* get pool name and check number of arguments */
1423         if (argc < 1) {
1424                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1425                 goto badusage;
1426         }
1427         if (argc < 2) {
1428                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1429                 goto badusage;
1430         }
1431
1432         poolname = argv[0];
1433
1434         /*
1435          * As a special case, check for use of '/' in the name, and direct the
1436          * user to use 'zfs create' instead.
1437          */
1438         if (strchr(poolname, '/') != NULL) {
1439                 (void) fprintf(stderr, gettext("cannot create '%s': invalid "
1440                     "character '/' in pool name\n"), poolname);
1441                 (void) fprintf(stderr, gettext("use 'zfs create' to "
1442                     "create a dataset\n"));
1443                 goto errout;
1444         }
1445
1446         /* pass off to make_root_vdev for bulk processing */
1447         nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1448             argc - 1, argv + 1);
1449         if (nvroot == NULL)
1450                 goto errout;
1451
1452         /* make_root_vdev() allows 0 toplevel children if there are spares */
1453         if (!zfs_allocatable_devs(nvroot)) {
1454                 (void) fprintf(stderr, gettext("invalid vdev "
1455                     "specification: at least one toplevel vdev must be "
1456                     "specified\n"));
1457                 goto errout;
1458         }
1459
1460         if (altroot != NULL && altroot[0] != '/') {
1461                 (void) fprintf(stderr, gettext("invalid alternate root '%s': "
1462                     "must be an absolute path\n"), altroot);
1463                 goto errout;
1464         }
1465
1466         /*
1467          * Check the validity of the mountpoint and direct the user to use the
1468          * '-m' mountpoint option if it looks like its in use.
1469          */
1470         if (mountpoint == NULL ||
1471             (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1472             strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1473                 char buf[MAXPATHLEN];
1474                 DIR *dirp;
1475
1476                 if (mountpoint && mountpoint[0] != '/') {
1477                         (void) fprintf(stderr, gettext("invalid mountpoint "
1478                             "'%s': must be an absolute path, 'legacy', or "
1479                             "'none'\n"), mountpoint);
1480                         goto errout;
1481                 }
1482
1483                 if (mountpoint == NULL) {
1484                         if (altroot != NULL)
1485                                 (void) snprintf(buf, sizeof (buf), "%s/%s",
1486                                     altroot, poolname);
1487                         else
1488                                 (void) snprintf(buf, sizeof (buf), "/%s",
1489                                     poolname);
1490                 } else {
1491                         if (altroot != NULL)
1492                                 (void) snprintf(buf, sizeof (buf), "%s%s",
1493                                     altroot, mountpoint);
1494                         else
1495                                 (void) snprintf(buf, sizeof (buf), "%s",
1496                                     mountpoint);
1497                 }
1498
1499                 if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1500                         (void) fprintf(stderr, gettext("mountpoint '%s' : "
1501                             "%s\n"), buf, strerror(errno));
1502                         (void) fprintf(stderr, gettext("use '-m' "
1503                             "option to provide a different default\n"));
1504                         goto errout;
1505                 } else if (dirp) {
1506                         int count = 0;
1507
1508                         while (count < 3 && readdir(dirp) != NULL)
1509                                 count++;
1510                         (void) closedir(dirp);
1511
1512                         if (count > 2) {
1513                                 (void) fprintf(stderr, gettext("mountpoint "
1514                                     "'%s' exists and is not empty\n"), buf);
1515                                 (void) fprintf(stderr, gettext("use '-m' "
1516                                     "option to provide a "
1517                                     "different default\n"));
1518                                 goto errout;
1519                         }
1520                 }
1521         }
1522
1523         /*
1524          * Now that the mountpoint's validity has been checked, ensure that
1525          * the property is set appropriately prior to creating the pool.
1526          */
1527         if (mountpoint != NULL) {
1528                 ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1529                     mountpoint, &fsprops, B_FALSE);
1530                 if (ret != 0)
1531                         goto errout;
1532         }
1533
1534         ret = 1;
1535         if (dryrun) {
1536                 /*
1537                  * For a dry run invocation, print out a basic message and run
1538                  * through all the vdevs in the list and print out in an
1539                  * appropriate hierarchy.
1540                  */
1541                 (void) printf(gettext("would create '%s' with the "
1542                     "following layout:\n\n"), poolname);
1543
1544                 print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1545                 print_vdev_tree(NULL, "dedup", nvroot, 0,
1546                     VDEV_ALLOC_BIAS_DEDUP, 0);
1547                 print_vdev_tree(NULL, "special", nvroot, 0,
1548                     VDEV_ALLOC_BIAS_SPECIAL, 0);
1549                 print_vdev_tree(NULL, "logs", nvroot, 0,
1550                     VDEV_ALLOC_BIAS_LOG, 0);
1551
1552                 ret = 0;
1553         } else {
1554                 /*
1555                  * Hand off to libzfs.
1556                  */
1557                 spa_feature_t i;
1558                 for (i = 0; i < SPA_FEATURES; i++) {
1559                         char propname[MAXPATHLEN];
1560                         char *propval;
1561                         zfeature_info_t *feat = &spa_feature_table[i];
1562
1563                         (void) snprintf(propname, sizeof (propname),
1564                             "feature@%s", feat->fi_uname);
1565
1566                         /*
1567                          * Only features contained in props will be enabled:
1568                          * remove from the nvlist every ZFS_FEATURE_DISABLED
1569                          * value and add every missing ZFS_FEATURE_ENABLED if
1570                          * enable_all_pool_feat is set.
1571                          */
1572                         if (!nvlist_lookup_string(props, propname, &propval)) {
1573                                 if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
1574                                         (void) nvlist_remove_all(props,
1575                                             propname);
1576                         } else if (enable_all_pool_feat) {
1577                                 ret = add_prop_list(propname,
1578                                     ZFS_FEATURE_ENABLED, &props, B_TRUE);
1579                                 if (ret != 0)
1580                                         goto errout;
1581                         }
1582                 }
1583
1584                 ret = 1;
1585                 if (zpool_create(g_zfs, poolname,
1586                     nvroot, props, fsprops) == 0) {
1587                         zfs_handle_t *pool = zfs_open(g_zfs,
1588                             tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1589                         if (pool != NULL) {
1590                                 if (zfs_mount(pool, NULL, 0) == 0) {
1591                                         ret = zfs_shareall(pool);
1592                                         zfs_commit_all_shares();
1593                                 }
1594                                 zfs_close(pool);
1595                         }
1596                 } else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1597                         (void) fprintf(stderr, gettext("pool name may have "
1598                             "been omitted\n"));
1599                 }
1600         }
1601
1602 errout:
1603         nvlist_free(nvroot);
1604         nvlist_free(fsprops);
1605         nvlist_free(props);
1606         return (ret);
1607 badusage:
1608         nvlist_free(fsprops);
1609         nvlist_free(props);
1610         usage(B_FALSE);
1611         return (2);
1612 }
1613
1614 /*
1615  * zpool destroy <pool>
1616  *
1617  *      -f      Forcefully unmount any datasets
1618  *
1619  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1620  */
1621 int
1622 zpool_do_destroy(int argc, char **argv)
1623 {
1624         boolean_t force = B_FALSE;
1625         int c;
1626         char *pool;
1627         zpool_handle_t *zhp;
1628         int ret;
1629
1630         /* check options */
1631         while ((c = getopt(argc, argv, "f")) != -1) {
1632                 switch (c) {
1633                 case 'f':
1634                         force = B_TRUE;
1635                         break;
1636                 case '?':
1637                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1638                             optopt);
1639                         usage(B_FALSE);
1640                 }
1641         }
1642
1643         argc -= optind;
1644         argv += optind;
1645
1646         /* check arguments */
1647         if (argc < 1) {
1648                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1649                 usage(B_FALSE);
1650         }
1651         if (argc > 1) {
1652                 (void) fprintf(stderr, gettext("too many arguments\n"));
1653                 usage(B_FALSE);
1654         }
1655
1656         pool = argv[0];
1657
1658         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1659                 /*
1660                  * As a special case, check for use of '/' in the name, and
1661                  * direct the user to use 'zfs destroy' instead.
1662                  */
1663                 if (strchr(pool, '/') != NULL)
1664                         (void) fprintf(stderr, gettext("use 'zfs destroy' to "
1665                             "destroy a dataset\n"));
1666                 return (1);
1667         }
1668
1669         if (zpool_disable_datasets(zhp, force) != 0) {
1670                 (void) fprintf(stderr, gettext("could not destroy '%s': "
1671                     "could not unmount datasets\n"), zpool_get_name(zhp));
1672                 zpool_close(zhp);
1673                 return (1);
1674         }
1675
1676         /* The history must be logged as part of the export */
1677         log_history = B_FALSE;
1678
1679         ret = (zpool_destroy(zhp, history_str) != 0);
1680
1681         zpool_close(zhp);
1682
1683         return (ret);
1684 }
1685
1686 typedef struct export_cbdata {
1687         boolean_t force;
1688         boolean_t hardforce;
1689 } export_cbdata_t;
1690
1691 /*
1692  * Export one pool
1693  */
1694 static int
1695 zpool_export_one(zpool_handle_t *zhp, void *data)
1696 {
1697         export_cbdata_t *cb = data;
1698
1699         if (zpool_disable_datasets(zhp, cb->force) != 0)
1700                 return (1);
1701
1702         /* The history must be logged as part of the export */
1703         log_history = B_FALSE;
1704
1705         if (cb->hardforce) {
1706                 if (zpool_export_force(zhp, history_str) != 0)
1707                         return (1);
1708         } else if (zpool_export(zhp, cb->force, history_str) != 0) {
1709                 return (1);
1710         }
1711
1712         return (0);
1713 }
1714
1715 /*
1716  * zpool export [-f] <pool> ...
1717  *
1718  *      -a      Export all pools
1719  *      -f      Forcefully unmount datasets
1720  *
1721  * Export the given pools.  By default, the command will attempt to cleanly
1722  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1723  * then the datasets will be forcefully unmounted.
1724  */
1725 int
1726 zpool_do_export(int argc, char **argv)
1727 {
1728         export_cbdata_t cb;
1729         boolean_t do_all = B_FALSE;
1730         boolean_t force = B_FALSE;
1731         boolean_t hardforce = B_FALSE;
1732         int c, ret;
1733
1734         /* check options */
1735         while ((c = getopt(argc, argv, "afF")) != -1) {
1736                 switch (c) {
1737                 case 'a':
1738                         do_all = B_TRUE;
1739                         break;
1740                 case 'f':
1741                         force = B_TRUE;
1742                         break;
1743                 case 'F':
1744                         hardforce = B_TRUE;
1745                         break;
1746                 case '?':
1747                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1748                             optopt);
1749                         usage(B_FALSE);
1750                 }
1751         }
1752
1753         cb.force = force;
1754         cb.hardforce = hardforce;
1755         argc -= optind;
1756         argv += optind;
1757
1758         if (do_all) {
1759                 if (argc != 0) {
1760                         (void) fprintf(stderr, gettext("too many arguments\n"));
1761                         usage(B_FALSE);
1762                 }
1763
1764                 return (for_each_pool(argc, argv, B_TRUE, NULL,
1765                     zpool_export_one, &cb));
1766         }
1767
1768         /* check arguments */
1769         if (argc < 1) {
1770                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1771                 usage(B_FALSE);
1772         }
1773
1774         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_export_one, &cb);
1775
1776         return (ret);
1777 }
1778
1779 /*
1780  * Given a vdev configuration, determine the maximum width needed for the device
1781  * name column.
1782  */
1783 static int
1784 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1785     int name_flags)
1786 {
1787         char *name;
1788         nvlist_t **child;
1789         uint_t c, children;
1790         int ret;
1791
1792         name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
1793         if (strlen(name) + depth > max)
1794                 max = strlen(name) + depth;
1795
1796         free(name);
1797
1798         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1799             &child, &children) == 0) {
1800                 for (c = 0; c < children; c++)
1801                         if ((ret = max_width(zhp, child[c], depth + 2,
1802                             max, name_flags)) > max)
1803                                 max = ret;
1804         }
1805
1806         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1807             &child, &children) == 0) {
1808                 for (c = 0; c < children; c++)
1809                         if ((ret = max_width(zhp, child[c], depth + 2,
1810                             max, name_flags)) > max)
1811                                 max = ret;
1812         }
1813
1814         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1815             &child, &children) == 0) {
1816                 for (c = 0; c < children; c++)
1817                         if ((ret = max_width(zhp, child[c], depth + 2,
1818                             max, name_flags)) > max)
1819                                 max = ret;
1820         }
1821
1822         return (max);
1823 }
1824
1825 typedef struct spare_cbdata {
1826         uint64_t        cb_guid;
1827         zpool_handle_t  *cb_zhp;
1828 } spare_cbdata_t;
1829
1830 static boolean_t
1831 find_vdev(nvlist_t *nv, uint64_t search)
1832 {
1833         uint64_t guid;
1834         nvlist_t **child;
1835         uint_t c, children;
1836
1837         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1838             search == guid)
1839                 return (B_TRUE);
1840
1841         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1842             &child, &children) == 0) {
1843                 for (c = 0; c < children; c++)
1844                         if (find_vdev(child[c], search))
1845                                 return (B_TRUE);
1846         }
1847
1848         return (B_FALSE);
1849 }
1850
1851 static int
1852 find_spare(zpool_handle_t *zhp, void *data)
1853 {
1854         spare_cbdata_t *cbp = data;
1855         nvlist_t *config, *nvroot;
1856
1857         config = zpool_get_config(zhp, NULL);
1858         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1859             &nvroot) == 0);
1860
1861         if (find_vdev(nvroot, cbp->cb_guid)) {
1862                 cbp->cb_zhp = zhp;
1863                 return (1);
1864         }
1865
1866         zpool_close(zhp);
1867         return (0);
1868 }
1869
1870 typedef struct status_cbdata {
1871         int             cb_count;
1872         int             cb_name_flags;
1873         int             cb_namewidth;
1874         boolean_t       cb_allpools;
1875         boolean_t       cb_verbose;
1876         boolean_t       cb_literal;
1877         boolean_t       cb_explain;
1878         boolean_t       cb_first;
1879         boolean_t       cb_dedup_stats;
1880         boolean_t       cb_print_status;
1881         boolean_t       cb_print_slow_ios;
1882         boolean_t       cb_print_vdev_init;
1883         boolean_t       cb_print_vdev_trim;
1884         vdev_cmd_data_list_t    *vcdl;
1885 } status_cbdata_t;
1886
1887 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
1888 static int
1889 is_blank_str(char *str)
1890 {
1891         while (str != NULL && *str != '\0') {
1892                 if (!isblank(*str))
1893                         return (0);
1894                 str++;
1895         }
1896         return (1);
1897 }
1898
1899 /* Print command output lines for specific vdev in a specific pool */
1900 static void
1901 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
1902 {
1903         vdev_cmd_data_t *data;
1904         int i, j;
1905         char *val;
1906
1907         for (i = 0; i < vcdl->count; i++) {
1908                 if ((strcmp(vcdl->data[i].path, path) != 0) ||
1909                     (strcmp(vcdl->data[i].pool, pool) != 0)) {
1910                         /* Not the vdev we're looking for */
1911                         continue;
1912                 }
1913
1914                 data = &vcdl->data[i];
1915                 /* Print out all the output values for this vdev */
1916                 for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
1917                         val = NULL;
1918                         /* Does this vdev have values for this column? */
1919                         for (int k = 0; k < data->cols_cnt; k++) {
1920                                 if (strcmp(data->cols[k],
1921                                     vcdl->uniq_cols[j]) == 0) {
1922                                         /* yes it does, record the value */
1923                                         val = data->lines[k];
1924                                         break;
1925                                 }
1926                         }
1927                         /*
1928                          * Mark empty values with dashes to make output
1929                          * awk-able.
1930                          */
1931                         if (is_blank_str(val))
1932                                 val = "-";
1933
1934                         printf("%*s", vcdl->uniq_cols_width[j], val);
1935                         if (j < vcdl->uniq_cols_cnt - 1)
1936                                 printf("  ");
1937                 }
1938
1939                 /* Print out any values that aren't in a column at the end */
1940                 for (j = data->cols_cnt; j < data->lines_cnt; j++) {
1941                         /* Did we have any columns?  If so print a spacer. */
1942                         if (vcdl->uniq_cols_cnt > 0)
1943                                 printf("  ");
1944
1945                         val = data->lines[j];
1946                         printf("%s", val ? val : "");
1947                 }
1948                 break;
1949         }
1950 }
1951
1952 /*
1953  * Print vdev initialization status for leaves
1954  */
1955 static void
1956 print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
1957 {
1958         if (verbose) {
1959                 if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
1960                     vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
1961                     vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
1962                     !vs->vs_scan_removing) {
1963                         char zbuf[1024];
1964                         char tbuf[256];
1965                         struct tm zaction_ts;
1966
1967                         time_t t = vs->vs_initialize_action_time;
1968                         int initialize_pct = 100;
1969                         if (vs->vs_initialize_state !=
1970                             VDEV_INITIALIZE_COMPLETE) {
1971                                 initialize_pct = (vs->vs_initialize_bytes_done *
1972                                     100 / (vs->vs_initialize_bytes_est + 1));
1973                         }
1974
1975                         (void) localtime_r(&t, &zaction_ts);
1976                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
1977
1978                         switch (vs->vs_initialize_state) {
1979                         case VDEV_INITIALIZE_SUSPENDED:
1980                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1981                                     gettext("suspended, started at"), tbuf);
1982                                 break;
1983                         case VDEV_INITIALIZE_ACTIVE:
1984                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1985                                     gettext("started at"), tbuf);
1986                                 break;
1987                         case VDEV_INITIALIZE_COMPLETE:
1988                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
1989                                     gettext("completed at"), tbuf);
1990                                 break;
1991                         }
1992
1993                         (void) printf(gettext("  (%d%% initialized%s)"),
1994                             initialize_pct, zbuf);
1995                 } else {
1996                         (void) printf(gettext("  (uninitialized)"));
1997                 }
1998         } else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
1999                 (void) printf(gettext("  (initializing)"));
2000         }
2001 }
2002
2003 /*
2004  * Print vdev TRIM status for leaves
2005  */
2006 static void
2007 print_status_trim(vdev_stat_t *vs, boolean_t verbose)
2008 {
2009         if (verbose) {
2010                 if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
2011                     vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
2012                     vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
2013                     !vs->vs_scan_removing) {
2014                         char zbuf[1024];
2015                         char tbuf[256];
2016                         struct tm zaction_ts;
2017
2018                         time_t t = vs->vs_trim_action_time;
2019                         int trim_pct = 100;
2020                         if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
2021                                 trim_pct = (vs->vs_trim_bytes_done *
2022                                     100 / (vs->vs_trim_bytes_est + 1));
2023                         }
2024
2025                         (void) localtime_r(&t, &zaction_ts);
2026                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2027
2028                         switch (vs->vs_trim_state) {
2029                         case VDEV_TRIM_SUSPENDED:
2030                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2031                                     gettext("suspended, started at"), tbuf);
2032                                 break;
2033                         case VDEV_TRIM_ACTIVE:
2034                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2035                                     gettext("started at"), tbuf);
2036                                 break;
2037                         case VDEV_TRIM_COMPLETE:
2038                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2039                                     gettext("completed at"), tbuf);
2040                                 break;
2041                         }
2042
2043                         (void) printf(gettext("  (%d%% trimmed%s)"),
2044                             trim_pct, zbuf);
2045                 } else if (vs->vs_trim_notsup) {
2046                         (void) printf(gettext("  (trim unsupported)"));
2047                 } else {
2048                         (void) printf(gettext("  (untrimmed)"));
2049                 }
2050         } else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
2051                 (void) printf(gettext("  (trimming)"));
2052         }
2053 }
2054
2055 /*
2056  * Return the color associated with a health string.  This includes returning
2057  * NULL for no color change.
2058  */
2059 static char *
2060 health_str_to_color(const char *health)
2061 {
2062         if (strcmp(health, gettext("FAULTED")) == 0 ||
2063             strcmp(health, gettext("SUSPENDED")) == 0 ||
2064             strcmp(health, gettext("UNAVAIL")) == 0) {
2065                 return (ANSI_RED);
2066         }
2067
2068         if (strcmp(health, gettext("OFFLINE")) == 0 ||
2069             strcmp(health, gettext("DEGRADED")) == 0 ||
2070             strcmp(health, gettext("REMOVED")) == 0) {
2071                 return (ANSI_YELLOW);
2072         }
2073
2074         return (NULL);
2075 }
2076
2077 /*
2078  * Print out configuration state as requested by status_callback.
2079  */
2080 static void
2081 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
2082     nvlist_t *nv, int depth, boolean_t isspare, vdev_rebuild_stat_t *vrs)
2083 {
2084         nvlist_t **child, *root;
2085         uint_t c, i, vsc, children;
2086         pool_scan_stat_t *ps = NULL;
2087         vdev_stat_t *vs;
2088         char rbuf[6], wbuf[6], cbuf[6];
2089         char *vname;
2090         uint64_t notpresent;
2091         spare_cbdata_t spare_cb;
2092         const char *state;
2093         char *type;
2094         char *path = NULL;
2095         char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL;
2096
2097         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2098             &child, &children) != 0)
2099                 children = 0;
2100
2101         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2102             (uint64_t **)&vs, &vsc) == 0);
2103
2104         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2105
2106         if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2107                 return;
2108
2109         state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2110
2111         if (isspare) {
2112                 /*
2113                  * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
2114                  * online drives.
2115                  */
2116                 if (vs->vs_aux == VDEV_AUX_SPARED)
2117                         state = gettext("INUSE");
2118                 else if (vs->vs_state == VDEV_STATE_HEALTHY)
2119                         state = gettext("AVAIL");
2120         }
2121
2122         printf_color(health_str_to_color(state),
2123             "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
2124             name, state);
2125
2126         if (!isspare) {
2127                 if (vs->vs_read_errors)
2128                         rcolor = ANSI_RED;
2129
2130                 if (vs->vs_write_errors)
2131                         wcolor = ANSI_RED;
2132
2133                 if (vs->vs_checksum_errors)
2134                         ccolor = ANSI_RED;
2135
2136                 if (cb->cb_literal) {
2137                         printf(" ");
2138                         printf_color(rcolor, "%5llu",
2139                             (u_longlong_t)vs->vs_read_errors);
2140                         printf(" ");
2141                         printf_color(wcolor, "%5llu",
2142                             (u_longlong_t)vs->vs_write_errors);
2143                         printf(" ");
2144                         printf_color(ccolor, "%5llu",
2145                             (u_longlong_t)vs->vs_checksum_errors);
2146                 } else {
2147                         zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
2148                         zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
2149                         zfs_nicenum(vs->vs_checksum_errors, cbuf,
2150                             sizeof (cbuf));
2151                         printf(" ");
2152                         printf_color(rcolor, "%5s", rbuf);
2153                         printf(" ");
2154                         printf_color(wcolor, "%5s", wbuf);
2155                         printf(" ");
2156                         printf_color(ccolor, "%5s", cbuf);
2157                 }
2158                 if (cb->cb_print_slow_ios) {
2159                         if (children == 0)  {
2160                                 /* Only leafs vdevs have slow IOs */
2161                                 zfs_nicenum(vs->vs_slow_ios, rbuf,
2162                                     sizeof (rbuf));
2163                         } else {
2164                                 snprintf(rbuf, sizeof (rbuf), "-");
2165                         }
2166
2167                         if (cb->cb_literal)
2168                                 printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
2169                         else
2170                                 printf(" %5s", rbuf);
2171                 }
2172         }
2173
2174         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
2175             &notpresent) == 0) {
2176                 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
2177                 (void) printf("  %s %s", gettext("was"), path);
2178         } else if (vs->vs_aux != 0) {
2179                 (void) printf("  ");
2180                 color_start(ANSI_RED);
2181                 switch (vs->vs_aux) {
2182                 case VDEV_AUX_OPEN_FAILED:
2183                         (void) printf(gettext("cannot open"));
2184                         break;
2185
2186                 case VDEV_AUX_BAD_GUID_SUM:
2187                         (void) printf(gettext("missing device"));
2188                         break;
2189
2190                 case VDEV_AUX_NO_REPLICAS:
2191                         (void) printf(gettext("insufficient replicas"));
2192                         break;
2193
2194                 case VDEV_AUX_VERSION_NEWER:
2195                         (void) printf(gettext("newer version"));
2196                         break;
2197
2198                 case VDEV_AUX_UNSUP_FEAT:
2199                         (void) printf(gettext("unsupported feature(s)"));
2200                         break;
2201
2202                 case VDEV_AUX_ASHIFT_TOO_BIG:
2203                         (void) printf(gettext("unsupported minimum blocksize"));
2204                         break;
2205
2206                 case VDEV_AUX_SPARED:
2207                         verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
2208                             &spare_cb.cb_guid) == 0);
2209                         if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
2210                                 if (strcmp(zpool_get_name(spare_cb.cb_zhp),
2211                                     zpool_get_name(zhp)) == 0)
2212                                         (void) printf(gettext("currently in "
2213                                             "use"));
2214                                 else
2215                                         (void) printf(gettext("in use by "
2216                                             "pool '%s'"),
2217                                             zpool_get_name(spare_cb.cb_zhp));
2218                                 zpool_close(spare_cb.cb_zhp);
2219                         } else {
2220                                 (void) printf(gettext("currently in use"));
2221                         }
2222                         break;
2223
2224                 case VDEV_AUX_ERR_EXCEEDED:
2225                         (void) printf(gettext("too many errors"));
2226                         break;
2227
2228                 case VDEV_AUX_IO_FAILURE:
2229                         (void) printf(gettext("experienced I/O failures"));
2230                         break;
2231
2232                 case VDEV_AUX_BAD_LOG:
2233                         (void) printf(gettext("bad intent log"));
2234                         break;
2235
2236                 case VDEV_AUX_EXTERNAL:
2237                         (void) printf(gettext("external device fault"));
2238                         break;
2239
2240                 case VDEV_AUX_SPLIT_POOL:
2241                         (void) printf(gettext("split into new pool"));
2242                         break;
2243
2244                 case VDEV_AUX_ACTIVE:
2245                         (void) printf(gettext("currently in use"));
2246                         break;
2247
2248                 case VDEV_AUX_CHILDREN_OFFLINE:
2249                         (void) printf(gettext("all children offline"));
2250                         break;
2251
2252                 default:
2253                         (void) printf(gettext("corrupted data"));
2254                         break;
2255                 }
2256                 color_end();
2257         }
2258
2259         /* The root vdev has the scrub/resilver stats */
2260         root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
2261             ZPOOL_CONFIG_VDEV_TREE);
2262         (void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
2263             (uint64_t **)&ps, &c);
2264
2265         if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0) {
2266                 if (vs->vs_scan_processed != 0) {
2267                         (void) printf(gettext("  (%s)"),
2268                             (ps->pss_func == POOL_SCAN_RESILVER) ?
2269                             "resilvering" : "repairing");
2270                 } else if (vs->vs_resilver_deferred) {
2271                         (void) printf(gettext("  (awaiting resilver)"));
2272                 }
2273         }
2274
2275         /* The top-level vdevs have the rebuild stats */
2276         if (vrs != NULL && vrs->vrs_state == VDEV_REBUILD_ACTIVE &&
2277             children == 0) {
2278                 if (vs->vs_rebuild_processed != 0) {
2279                         (void) printf(gettext("  (resilvering)"));
2280                 }
2281         }
2282
2283         if (cb->vcdl != NULL) {
2284                 if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
2285                         printf("  ");
2286                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
2287                 }
2288         }
2289
2290         /* Display vdev initialization and trim status for leaves */
2291         if (children == 0) {
2292                 print_status_initialize(vs, cb->cb_print_vdev_init);
2293                 print_status_trim(vs, cb->cb_print_vdev_trim);
2294         }
2295
2296         (void) printf("\n");
2297
2298         for (c = 0; c < children; c++) {
2299                 uint64_t islog = B_FALSE, ishole = B_FALSE;
2300
2301                 /* Don't print logs or holes here */
2302                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2303                     &islog);
2304                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
2305                     &ishole);
2306                 if (islog || ishole)
2307                         continue;
2308                 /* Only print normal classes here */
2309                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2310                         continue;
2311
2312                 /* Provide vdev_rebuild_stats to children if available */
2313                 if (vrs == NULL) {
2314                         (void) nvlist_lookup_uint64_array(nv,
2315                             ZPOOL_CONFIG_REBUILD_STATS,
2316                             (uint64_t **)&vrs, &i);
2317                 }
2318
2319                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
2320                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2321                 print_status_config(zhp, cb, vname, child[c], depth + 2,
2322                     isspare, vrs);
2323                 free(vname);
2324         }
2325 }
2326
2327 /*
2328  * Print the configuration of an exported pool.  Iterate over all vdevs in the
2329  * pool, printing out the name and status for each one.
2330  */
2331 static void
2332 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
2333     int depth)
2334 {
2335         nvlist_t **child;
2336         uint_t c, children;
2337         vdev_stat_t *vs;
2338         char *type, *vname;
2339
2340         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2341         if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
2342             strcmp(type, VDEV_TYPE_HOLE) == 0)
2343                 return;
2344
2345         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2346             (uint64_t **)&vs, &c) == 0);
2347
2348         (void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
2349         (void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
2350
2351         if (vs->vs_aux != 0) {
2352                 (void) printf("  ");
2353
2354                 switch (vs->vs_aux) {
2355                 case VDEV_AUX_OPEN_FAILED:
2356                         (void) printf(gettext("cannot open"));
2357                         break;
2358
2359                 case VDEV_AUX_BAD_GUID_SUM:
2360                         (void) printf(gettext("missing device"));
2361                         break;
2362
2363                 case VDEV_AUX_NO_REPLICAS:
2364                         (void) printf(gettext("insufficient replicas"));
2365                         break;
2366
2367                 case VDEV_AUX_VERSION_NEWER:
2368                         (void) printf(gettext("newer version"));
2369                         break;
2370
2371                 case VDEV_AUX_UNSUP_FEAT:
2372                         (void) printf(gettext("unsupported feature(s)"));
2373                         break;
2374
2375                 case VDEV_AUX_ERR_EXCEEDED:
2376                         (void) printf(gettext("too many errors"));
2377                         break;
2378
2379                 case VDEV_AUX_ACTIVE:
2380                         (void) printf(gettext("currently in use"));
2381                         break;
2382
2383                 case VDEV_AUX_CHILDREN_OFFLINE:
2384                         (void) printf(gettext("all children offline"));
2385                         break;
2386
2387                 default:
2388                         (void) printf(gettext("corrupted data"));
2389                         break;
2390                 }
2391         }
2392         (void) printf("\n");
2393
2394         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2395             &child, &children) != 0)
2396                 return;
2397
2398         for (c = 0; c < children; c++) {
2399                 uint64_t is_log = B_FALSE;
2400
2401                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2402                     &is_log);
2403                 if (is_log)
2404                         continue;
2405                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2406                         continue;
2407
2408                 vname = zpool_vdev_name(g_zfs, NULL, child[c],
2409                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2410                 print_import_config(cb, vname, child[c], depth + 2);
2411                 free(vname);
2412         }
2413
2414         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2415             &child, &children) == 0) {
2416                 (void) printf(gettext("\tcache\n"));
2417                 for (c = 0; c < children; c++) {
2418                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2419                             cb->cb_name_flags);
2420                         (void) printf("\t  %s\n", vname);
2421                         free(vname);
2422                 }
2423         }
2424
2425         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2426             &child, &children) == 0) {
2427                 (void) printf(gettext("\tspares\n"));
2428                 for (c = 0; c < children; c++) {
2429                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2430                             cb->cb_name_flags);
2431                         (void) printf("\t  %s\n", vname);
2432                         free(vname);
2433                 }
2434         }
2435 }
2436
2437 /*
2438  * Print specialized class vdevs.
2439  *
2440  * These are recorded as top level vdevs in the main pool child array
2441  * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2442  * print_status_config() or print_import_config() to print the top level
2443  * class vdevs then any of their children (eg mirrored slogs) are printed
2444  * recursively - which works because only the top level vdev is marked.
2445  */
2446 static void
2447 print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2448     const char *class)
2449 {
2450         uint_t c, children;
2451         nvlist_t **child;
2452         boolean_t printed = B_FALSE;
2453
2454         assert(zhp != NULL || !cb->cb_verbose);
2455
2456         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2457             &children) != 0)
2458                 return;
2459
2460         for (c = 0; c < children; c++) {
2461                 uint64_t is_log = B_FALSE;
2462                 char *bias = NULL;
2463                 char *type = NULL;
2464
2465                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2466                     &is_log);
2467
2468                 if (is_log) {
2469                         bias = VDEV_ALLOC_CLASS_LOGS;
2470                 } else {
2471                         (void) nvlist_lookup_string(child[c],
2472                             ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2473                         (void) nvlist_lookup_string(child[c],
2474                             ZPOOL_CONFIG_TYPE, &type);
2475                 }
2476
2477                 if (bias == NULL || strcmp(bias, class) != 0)
2478                         continue;
2479                 if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2480                         continue;
2481
2482                 if (!printed) {
2483                         (void) printf("\t%s\t\n", gettext(class));
2484                         printed = B_TRUE;
2485                 }
2486
2487                 char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2488                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2489                 if (cb->cb_print_status)
2490                         print_status_config(zhp, cb, name, child[c], 2,
2491                             B_FALSE, NULL);
2492                 else
2493                         print_import_config(cb, name, child[c], 2);
2494                 free(name);
2495         }
2496 }
2497
2498 /*
2499  * Display the status for the given pool.
2500  */
2501 static void
2502 show_import(nvlist_t *config)
2503 {
2504         uint64_t pool_state;
2505         vdev_stat_t *vs;
2506         char *name;
2507         uint64_t guid;
2508         uint64_t hostid = 0;
2509         char *msgid;
2510         char *hostname = "unknown";
2511         nvlist_t *nvroot, *nvinfo;
2512         zpool_status_t reason;
2513         zpool_errata_t errata;
2514         const char *health;
2515         uint_t vsc;
2516         char *comment;
2517         status_cbdata_t cb = { 0 };
2518
2519         verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2520             &name) == 0);
2521         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2522             &guid) == 0);
2523         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2524             &pool_state) == 0);
2525         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2526             &nvroot) == 0);
2527
2528         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2529             (uint64_t **)&vs, &vsc) == 0);
2530         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2531
2532         reason = zpool_import_status(config, &msgid, &errata);
2533
2534         (void) printf(gettext("   pool: %s\n"), name);
2535         (void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2536         (void) printf(gettext("  state: %s"), health);
2537         if (pool_state == POOL_STATE_DESTROYED)
2538                 (void) printf(gettext(" (DESTROYED)"));
2539         (void) printf("\n");
2540
2541         switch (reason) {
2542         case ZPOOL_STATUS_MISSING_DEV_R:
2543         case ZPOOL_STATUS_MISSING_DEV_NR:
2544         case ZPOOL_STATUS_BAD_GUID_SUM:
2545                 printf_color(ANSI_BOLD, gettext("status: "));
2546                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2547                     "missing from the system.\n"));
2548                 break;
2549
2550         case ZPOOL_STATUS_CORRUPT_LABEL_R:
2551         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2552                 printf_color(ANSI_BOLD, gettext("status: "));
2553                 printf_color(ANSI_YELLOW, gettext("One or more devices contains"
2554                     " corrupted data.\n"));
2555                 break;
2556
2557         case ZPOOL_STATUS_CORRUPT_DATA:
2558                 (void) printf(
2559                     gettext(" status: The pool data is corrupted.\n"));
2560                 break;
2561
2562         case ZPOOL_STATUS_OFFLINE_DEV:
2563                 printf_color(ANSI_BOLD, gettext("status: "));
2564                 printf_color(ANSI_YELLOW, gettext("One or more devices "
2565                     "are offlined.\n"));
2566                 break;
2567
2568         case ZPOOL_STATUS_CORRUPT_POOL:
2569                 printf_color(ANSI_BOLD, gettext("status: "));
2570                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
2571                     "corrupted.\n"));
2572                 break;
2573
2574         case ZPOOL_STATUS_VERSION_OLDER:
2575                 printf_color(ANSI_BOLD, gettext("status: "));
2576                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2577                     "a legacy on-disk version.\n"));
2578                 break;
2579
2580         case ZPOOL_STATUS_VERSION_NEWER:
2581                 printf_color(ANSI_BOLD, gettext("status: "));
2582                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2583                     "an incompatible version.\n"));
2584                 break;
2585
2586         case ZPOOL_STATUS_FEAT_DISABLED:
2587                 printf_color(ANSI_BOLD, gettext("status: "));
2588                 printf_color(ANSI_YELLOW, gettext("Some supported features are "
2589                     "not enabled on the pool.\n"));
2590                 break;
2591
2592         case ZPOOL_STATUS_UNSUP_FEAT_READ:
2593                 printf_color(ANSI_BOLD, gettext("status: "));
2594                 printf_color(ANSI_YELLOW, gettext("The pool uses the following "
2595                     "feature(s) not supported on this system:\n"));
2596                 color_start(ANSI_YELLOW);
2597                 zpool_print_unsup_feat(config);
2598                 color_end();
2599                 break;
2600
2601         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2602                 printf_color(ANSI_BOLD, gettext("status: "));
2603                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
2604                     "accessed in read-only mode on this system. It\n\tcannot be"
2605                     " accessed in read-write mode because it uses the "
2606                     "following\n\tfeature(s) not supported on this system:\n"));
2607                 color_start(ANSI_YELLOW);
2608                 zpool_print_unsup_feat(config);
2609                 color_end();
2610                 break;
2611
2612         case ZPOOL_STATUS_HOSTID_ACTIVE:
2613                 printf_color(ANSI_BOLD, gettext("status: "));
2614                 printf_color(ANSI_YELLOW, gettext("The pool is currently "
2615                     "imported by another system.\n"));
2616                 break;
2617
2618         case ZPOOL_STATUS_HOSTID_REQUIRED:
2619                 printf_color(ANSI_BOLD, gettext("status: "));
2620                 printf_color(ANSI_YELLOW, gettext("The pool has the "
2621                     "multihost property on.  It cannot\n\tbe safely imported "
2622                     "when the system hostid is not set.\n"));
2623                 break;
2624
2625         case ZPOOL_STATUS_HOSTID_MISMATCH:
2626                 printf_color(ANSI_BOLD, gettext("status: "));
2627                 printf_color(ANSI_YELLOW, gettext("The pool was last accessed "
2628                     "by another system.\n"));
2629                 break;
2630
2631         case ZPOOL_STATUS_FAULTED_DEV_R:
2632         case ZPOOL_STATUS_FAULTED_DEV_NR:
2633                 printf_color(ANSI_BOLD, gettext("status: "));
2634                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2635                     "faulted.\n"));
2636                 break;
2637
2638         case ZPOOL_STATUS_BAD_LOG:
2639                 printf_color(ANSI_BOLD, gettext("status: "));
2640                 printf_color(ANSI_YELLOW, gettext("An intent log record cannot "
2641                     "be read.\n"));
2642                 break;
2643
2644         case ZPOOL_STATUS_RESILVERING:
2645         case ZPOOL_STATUS_REBUILDING:
2646                 printf_color(ANSI_BOLD, gettext("status: "));
2647                 printf_color(ANSI_YELLOW, gettext("One or more devices were "
2648                     "being resilvered.\n"));
2649                 break;
2650
2651         case ZPOOL_STATUS_ERRATA:
2652                 printf_color(ANSI_BOLD, gettext("status: "));
2653                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
2654                     errata);
2655                 break;
2656
2657         default:
2658                 /*
2659                  * No other status can be seen when importing pools.
2660                  */
2661                 assert(reason == ZPOOL_STATUS_OK);
2662         }
2663
2664         /*
2665          * Print out an action according to the overall state of the pool.
2666          */
2667         if (vs->vs_state == VDEV_STATE_HEALTHY) {
2668                 if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2669                     reason == ZPOOL_STATUS_FEAT_DISABLED) {
2670                         (void) printf(gettext(" action: The pool can be "
2671                             "imported using its name or numeric identifier, "
2672                             "though\n\tsome features will not be available "
2673                             "without an explicit 'zpool upgrade'.\n"));
2674                 } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2675                         (void) printf(gettext(" action: The pool can be "
2676                             "imported using its name or numeric "
2677                             "identifier and\n\tthe '-f' flag.\n"));
2678                 } else if (reason == ZPOOL_STATUS_ERRATA) {
2679                         switch (errata) {
2680                         case ZPOOL_ERRATA_NONE:
2681                                 break;
2682
2683                         case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2684                                 (void) printf(gettext(" action: The pool can "
2685                                     "be imported using its name or numeric "
2686                                     "identifier,\n\thowever there is a compat"
2687                                     "ibility issue which should be corrected"
2688                                     "\n\tby running 'zpool scrub'\n"));
2689                                 break;
2690
2691                         case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2692                                 (void) printf(gettext(" action: The pool can"
2693                                     "not be imported with this version of ZFS "
2694                                     "due to\n\tan active asynchronous destroy. "
2695                                     "Revert to an earlier version\n\tand "
2696                                     "allow the destroy to complete before "
2697                                     "updating.\n"));
2698                                 break;
2699
2700                         case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2701                                 (void) printf(gettext(" action: Existing "
2702                                     "encrypted datasets contain an on-disk "
2703                                     "incompatibility, which\n\tneeds to be "
2704                                     "corrected. Backup these datasets to new "
2705                                     "encrypted datasets\n\tand destroy the "
2706                                     "old ones.\n"));
2707                                 break;
2708
2709                         case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
2710                                 (void) printf(gettext(" action: Existing "
2711                                     "encrypted snapshots and bookmarks contain "
2712                                     "an on-disk\n\tincompatibility. This may "
2713                                     "cause on-disk corruption if they are used"
2714                                     "\n\twith 'zfs recv'. To correct the "
2715                                     "issue, enable the bookmark_v2 feature.\n\t"
2716                                     "No additional action is needed if there "
2717                                     "are no encrypted snapshots or\n\t"
2718                                     "bookmarks. If preserving the encrypted "
2719                                     "snapshots and bookmarks is\n\trequired, "
2720                                     "use a non-raw send to backup and restore "
2721                                     "them. Alternately,\n\tthey may be removed"
2722                                     " to resolve the incompatibility.\n"));
2723                                 break;
2724                         default:
2725                                 /*
2726                                  * All errata must contain an action message.
2727                                  */
2728                                 assert(0);
2729                         }
2730                 } else {
2731                         (void) printf(gettext(" action: The pool can be "
2732                             "imported using its name or numeric "
2733                             "identifier.\n"));
2734                 }
2735         } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2736                 (void) printf(gettext(" action: The pool can be imported "
2737                     "despite missing or damaged devices.  The\n\tfault "
2738                     "tolerance of the pool may be compromised if imported.\n"));
2739         } else {
2740                 switch (reason) {
2741                 case ZPOOL_STATUS_VERSION_NEWER:
2742                         (void) printf(gettext(" action: The pool cannot be "
2743                             "imported.  Access the pool on a system running "
2744                             "newer\n\tsoftware, or recreate the pool from "
2745                             "backup.\n"));
2746                         break;
2747                 case ZPOOL_STATUS_UNSUP_FEAT_READ:
2748                         printf_color(ANSI_BOLD, gettext("action: "));
2749                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
2750                             "imported. Access the pool on a system that "
2751                             "supports\n\tthe required feature(s), or recreate "
2752                             "the pool from backup.\n"));
2753                         break;
2754                 case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2755                         printf_color(ANSI_BOLD, gettext("action: "));
2756                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
2757                             "imported in read-write mode. Import the pool "
2758                             "with\n"
2759                             "\t\"-o readonly=on\", access the pool on a system "
2760                             "that supports the\n\trequired feature(s), or "
2761                             "recreate the pool from backup.\n"));
2762                         break;
2763                 case ZPOOL_STATUS_MISSING_DEV_R:
2764                 case ZPOOL_STATUS_MISSING_DEV_NR:
2765                 case ZPOOL_STATUS_BAD_GUID_SUM:
2766                         (void) printf(gettext(" action: The pool cannot be "
2767                             "imported. Attach the missing\n\tdevices and try "
2768                             "again.\n"));
2769                         break;
2770                 case ZPOOL_STATUS_HOSTID_ACTIVE:
2771                         VERIFY0(nvlist_lookup_nvlist(config,
2772                             ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
2773
2774                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2775                                 hostname = fnvlist_lookup_string(nvinfo,
2776                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2777
2778                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2779                                 hostid = fnvlist_lookup_uint64(nvinfo,
2780                                     ZPOOL_CONFIG_MMP_HOSTID);
2781
2782                         (void) printf(gettext(" action: The pool must be "
2783                             "exported from %s (hostid=%lx)\n\tbefore it "
2784                             "can be safely imported.\n"), hostname,
2785                             (unsigned long) hostid);
2786                         break;
2787                 case ZPOOL_STATUS_HOSTID_REQUIRED:
2788                         (void) printf(gettext(" action: Set a unique system "
2789                             "hostid with the zgenhostid(8) command.\n"));
2790                         break;
2791                 default:
2792                         (void) printf(gettext(" action: The pool cannot be "
2793                             "imported due to damaged devices or data.\n"));
2794                 }
2795         }
2796
2797         /* Print the comment attached to the pool. */
2798         if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2799                 (void) printf(gettext("comment: %s\n"), comment);
2800
2801         /*
2802          * If the state is "closed" or "can't open", and the aux state
2803          * is "corrupt data":
2804          */
2805         if (((vs->vs_state == VDEV_STATE_CLOSED) ||
2806             (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
2807             (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
2808                 if (pool_state == POOL_STATE_DESTROYED)
2809                         (void) printf(gettext("\tThe pool was destroyed, "
2810                             "but can be imported using the '-Df' flags.\n"));
2811                 else if (pool_state != POOL_STATE_EXPORTED)
2812                         (void) printf(gettext("\tThe pool may be active on "
2813                             "another system, but can be imported using\n\t"
2814                             "the '-f' flag.\n"));
2815         }
2816
2817         if (msgid != NULL) {
2818                 (void) printf(gettext(
2819                     "   see: https://openzfs.github.io/openzfs-docs/msg/%s\n"),
2820                     msgid);
2821         }
2822
2823         (void) printf(gettext(" config:\n\n"));
2824
2825         cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
2826             VDEV_NAME_TYPE_ID);
2827         if (cb.cb_namewidth < 10)
2828                 cb.cb_namewidth = 10;
2829
2830         print_import_config(&cb, name, nvroot, 0);
2831
2832         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
2833         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
2834         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
2835
2836         if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
2837                 (void) printf(gettext("\n\tAdditional devices are known to "
2838                     "be part of this pool, though their\n\texact "
2839                     "configuration cannot be determined.\n"));
2840         }
2841 }
2842
2843 static boolean_t
2844 zfs_force_import_required(nvlist_t *config)
2845 {
2846         uint64_t state;
2847         uint64_t hostid = 0;
2848         nvlist_t *nvinfo;
2849
2850         state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
2851         (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
2852
2853         if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
2854                 return (B_TRUE);
2855
2856         nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2857         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
2858                 mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
2859                     ZPOOL_CONFIG_MMP_STATE);
2860
2861                 if (mmp_state != MMP_STATE_INACTIVE)
2862                         return (B_TRUE);
2863         }
2864
2865         return (B_FALSE);
2866 }
2867
2868 /*
2869  * Perform the import for the given configuration.  This passes the heavy
2870  * lifting off to zpool_import_props(), and then mounts the datasets contained
2871  * within the pool.
2872  */
2873 static int
2874 do_import(nvlist_t *config, const char *newname, const char *mntopts,
2875     nvlist_t *props, int flags)
2876 {
2877         int ret = 0;
2878         zpool_handle_t *zhp;
2879         char *name;
2880         uint64_t version;
2881
2882         name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
2883         version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
2884
2885         if (!SPA_VERSION_IS_SUPPORTED(version)) {
2886                 (void) fprintf(stderr, gettext("cannot import '%s': pool "
2887                     "is formatted using an unsupported ZFS version\n"), name);
2888                 return (1);
2889         } else if (zfs_force_import_required(config) &&
2890             !(flags & ZFS_IMPORT_ANY_HOST)) {
2891                 mmp_state_t mmp_state = MMP_STATE_INACTIVE;
2892                 nvlist_t *nvinfo;
2893
2894                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2895                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
2896                         mmp_state = fnvlist_lookup_uint64(nvinfo,
2897                             ZPOOL_CONFIG_MMP_STATE);
2898
2899                 if (mmp_state == MMP_STATE_ACTIVE) {
2900                         char *hostname = "<unknown>";
2901                         uint64_t hostid = 0;
2902
2903                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2904                                 hostname = fnvlist_lookup_string(nvinfo,
2905                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2906
2907                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2908                                 hostid = fnvlist_lookup_uint64(nvinfo,
2909                                     ZPOOL_CONFIG_MMP_HOSTID);
2910
2911                         (void) fprintf(stderr, gettext("cannot import '%s': "
2912                             "pool is imported on %s (hostid: "
2913                             "0x%lx)\nExport the pool on the other system, "
2914                             "then run 'zpool import'.\n"),
2915                             name, hostname, (unsigned long) hostid);
2916                 } else if (mmp_state == MMP_STATE_NO_HOSTID) {
2917                         (void) fprintf(stderr, gettext("Cannot import '%s': "
2918                             "pool has the multihost property on and the\n"
2919                             "system's hostid is not set. Set a unique hostid "
2920                             "with the zgenhostid(8) command.\n"), name);
2921                 } else {
2922                         char *hostname = "<unknown>";
2923                         uint64_t timestamp = 0;
2924                         uint64_t hostid = 0;
2925
2926                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
2927                                 hostname = fnvlist_lookup_string(config,
2928                                     ZPOOL_CONFIG_HOSTNAME);
2929
2930                         if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
2931                                 timestamp = fnvlist_lookup_uint64(config,
2932                                     ZPOOL_CONFIG_TIMESTAMP);
2933
2934                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
2935                                 hostid = fnvlist_lookup_uint64(config,
2936                                     ZPOOL_CONFIG_HOSTID);
2937
2938                         (void) fprintf(stderr, gettext("cannot import '%s': "
2939                             "pool was previously in use from another system.\n"
2940                             "Last accessed by %s (hostid=%lx) at %s"
2941                             "The pool can be imported, use 'zpool import -f' "
2942                             "to import the pool.\n"), name, hostname,
2943                             (unsigned long)hostid, ctime((time_t *)&timestamp));
2944                 }
2945
2946                 return (1);
2947         }
2948
2949         if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2950                 return (1);
2951
2952         if (newname != NULL)
2953                 name = (char *)newname;
2954
2955         if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2956                 return (1);
2957
2958         /*
2959          * Loading keys is best effort. We don't want to return immediately
2960          * if it fails but we do want to give the error to the caller.
2961          */
2962         if (flags & ZFS_IMPORT_LOAD_KEYS) {
2963                 ret = zfs_crypto_attempt_load_keys(g_zfs, name);
2964                 if (ret != 0)
2965                         ret = 1;
2966         }
2967
2968         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2969             !(flags & ZFS_IMPORT_ONLY) &&
2970             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2971                 zpool_close(zhp);
2972                 return (1);
2973         }
2974
2975         zpool_close(zhp);
2976         return (ret);
2977 }
2978
2979 typedef struct target_exists_args {
2980         const char      *poolname;
2981         uint64_t        poolguid;
2982 } target_exists_args_t;
2983
2984 static int
2985 name_or_guid_exists(zpool_handle_t *zhp, void *data)
2986 {
2987         target_exists_args_t *args = data;
2988         nvlist_t *config = zpool_get_config(zhp, NULL);
2989         int found = 0;
2990
2991         if (config == NULL)
2992                 return (0);
2993
2994         if (args->poolname != NULL) {
2995                 char *pool_name;
2996
2997                 verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2998                     &pool_name) == 0);
2999                 if (strcmp(pool_name, args->poolname) == 0)
3000                         found = 1;
3001         } else {
3002                 uint64_t pool_guid;
3003
3004                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3005                     &pool_guid) == 0);
3006                 if (pool_guid == args->poolguid)
3007                         found = 1;
3008         }
3009         zpool_close(zhp);
3010
3011         return (found);
3012 }
3013 /*
3014  * zpool checkpoint <pool>
3015  *       checkpoint --discard <pool>
3016  *
3017  *       -d         Discard the checkpoint from a checkpointed
3018  *       --discard  pool.
3019  *
3020  *       -w         Wait for discarding a checkpoint to complete.
3021  *       --wait
3022  *
3023  * Checkpoints the specified pool, by taking a "snapshot" of its
3024  * current state. A pool can only have one checkpoint at a time.
3025  */
3026 int
3027 zpool_do_checkpoint(int argc, char **argv)
3028 {
3029         boolean_t discard, wait;
3030         char *pool;
3031         zpool_handle_t *zhp;
3032         int c, err;
3033
3034         struct option long_options[] = {
3035                 {"discard", no_argument, NULL, 'd'},
3036                 {"wait", no_argument, NULL, 'w'},
3037                 {0, 0, 0, 0}
3038         };
3039
3040         discard = B_FALSE;
3041         wait = B_FALSE;
3042         while ((c = getopt_long(argc, argv, ":dw", long_options, NULL)) != -1) {
3043                 switch (c) {
3044                 case 'd':
3045                         discard = B_TRUE;
3046                         break;
3047                 case 'w':
3048                         wait = B_TRUE;
3049                         break;
3050                 case '?':
3051                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3052                             optopt);
3053                         usage(B_FALSE);
3054                 }
3055         }
3056
3057         if (wait && !discard) {
3058                 (void) fprintf(stderr, gettext("--wait only valid when "
3059                     "--discard also specified\n"));
3060                 usage(B_FALSE);
3061         }
3062
3063         argc -= optind;
3064         argv += optind;
3065
3066         if (argc < 1) {
3067                 (void) fprintf(stderr, gettext("missing pool argument\n"));
3068                 usage(B_FALSE);
3069         }
3070
3071         if (argc > 1) {
3072                 (void) fprintf(stderr, gettext("too many arguments\n"));
3073                 usage(B_FALSE);
3074         }
3075
3076         pool = argv[0];
3077
3078         if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
3079                 /* As a special case, check for use of '/' in the name */
3080                 if (strchr(pool, '/') != NULL)
3081                         (void) fprintf(stderr, gettext("'zpool checkpoint' "
3082                             "doesn't work on datasets. To save the state "
3083                             "of a dataset from a specific point in time "
3084                             "please use 'zfs snapshot'\n"));
3085                 return (1);
3086         }
3087
3088         if (discard) {
3089                 err = (zpool_discard_checkpoint(zhp) != 0);
3090                 if (err == 0 && wait)
3091                         err = zpool_wait(zhp, ZPOOL_WAIT_CKPT_DISCARD);
3092         } else {
3093                 err = (zpool_checkpoint(zhp) != 0);
3094         }
3095
3096         zpool_close(zhp);
3097
3098         return (err);
3099 }
3100
3101 #define CHECKPOINT_OPT  1024
3102
3103 /*
3104  * zpool import [-d dir] [-D]
3105  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3106  *              [-d dir | -c cachefile] [-f] -a
3107  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3108  *              [-d dir | -c cachefile] [-f] [-n] [-F] <pool | id> [newpool]
3109  *
3110  *       -c     Read pool information from a cachefile instead of searching
3111  *              devices.
3112  *
3113  *       -d     Scan in a specific directory, other than /dev/.  More than
3114  *              one directory can be specified using multiple '-d' options.
3115  *
3116  *       -D     Scan for previously destroyed pools or import all or only
3117  *              specified destroyed pools.
3118  *
3119  *       -R     Temporarily import the pool, with all mountpoints relative to
3120  *              the given root.  The pool will remain exported when the machine
3121  *              is rebooted.
3122  *
3123  *       -V     Import even in the presence of faulted vdevs.  This is an
3124  *              intentionally undocumented option for testing purposes, and
3125  *              treats the pool configuration as complete, leaving any bad
3126  *              vdevs in the FAULTED state. In other words, it does verbatim
3127  *              import.
3128  *
3129  *       -f     Force import, even if it appears that the pool is active.
3130  *
3131  *       -F     Attempt rewind if necessary.
3132  *
3133  *       -n     See if rewind would work, but don't actually rewind.
3134  *
3135  *       -N     Import the pool but don't mount datasets.
3136  *
3137  *       -T     Specify a starting txg to use for import. This option is
3138  *              intentionally undocumented option for testing purposes.
3139  *
3140  *       -a     Import all pools found.
3141  *
3142  *       -l     Load encryption keys while importing.
3143  *
3144  *       -o     Set property=value and/or temporary mount options (without '=').
3145  *
3146  *       -s     Scan using the default search path, the libblkid cache will
3147  *              not be consulted.
3148  *
3149  *       --rewind-to-checkpoint
3150  *              Import the pool and revert back to the checkpoint.
3151  *
3152  * The import command scans for pools to import, and import pools based on pool
3153  * name and GUID.  The pool can also be renamed as part of the import process.
3154  */
3155 int
3156 zpool_do_import(int argc, char **argv)
3157 {
3158         char **searchdirs = NULL;
3159         char *env, *envdup = NULL;
3160         int nsearch = 0;
3161         int c;
3162         int err = 0;
3163         nvlist_t *pools = NULL;
3164         boolean_t do_all = B_FALSE;
3165         boolean_t do_destroyed = B_FALSE;
3166         char *mntopts = NULL;
3167         nvpair_t *elem;
3168         nvlist_t *config;
3169         uint64_t searchguid = 0;
3170         char *searchname = NULL;
3171         char *propval;
3172         nvlist_t *found_config;
3173         nvlist_t *policy = NULL;
3174         nvlist_t *props = NULL;
3175         boolean_t first;
3176         int flags = ZFS_IMPORT_NORMAL;
3177         uint32_t rewind_policy = ZPOOL_NO_REWIND;
3178         boolean_t dryrun = B_FALSE;
3179         boolean_t do_rewind = B_FALSE;
3180         boolean_t xtreme_rewind = B_FALSE;
3181         boolean_t do_scan = B_FALSE;
3182         boolean_t pool_exists = B_FALSE;
3183         uint64_t pool_state, txg = -1ULL;
3184         char *cachefile = NULL;
3185         importargs_t idata = { 0 };
3186         char *endptr;
3187
3188         struct option long_options[] = {
3189                 {"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
3190                 {0, 0, 0, 0}
3191         };
3192
3193         /* check options */
3194         while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
3195             long_options, NULL)) != -1) {
3196                 switch (c) {
3197                 case 'a':
3198                         do_all = B_TRUE;
3199                         break;
3200                 case 'c':
3201                         cachefile = optarg;
3202                         break;
3203                 case 'd':
3204                         if (searchdirs == NULL) {
3205                                 searchdirs = safe_malloc(sizeof (char *));
3206                         } else {
3207                                 char **tmp = safe_malloc((nsearch + 1) *
3208                                     sizeof (char *));
3209                                 bcopy(searchdirs, tmp, nsearch *
3210                                     sizeof (char *));
3211                                 free(searchdirs);
3212                                 searchdirs = tmp;
3213                         }
3214                         searchdirs[nsearch++] = optarg;
3215                         break;
3216                 case 'D':
3217                         do_destroyed = B_TRUE;
3218                         break;
3219                 case 'f':
3220                         flags |= ZFS_IMPORT_ANY_HOST;
3221                         break;
3222                 case 'F':
3223                         do_rewind = B_TRUE;
3224                         break;
3225                 case 'l':
3226                         flags |= ZFS_IMPORT_LOAD_KEYS;
3227                         break;
3228                 case 'm':
3229                         flags |= ZFS_IMPORT_MISSING_LOG;
3230                         break;
3231                 case 'n':
3232                         dryrun = B_TRUE;
3233                         break;
3234                 case 'N':
3235                         flags |= ZFS_IMPORT_ONLY;
3236                         break;
3237                 case 'o':
3238                         if ((propval = strchr(optarg, '=')) != NULL) {
3239                                 *propval = '\0';
3240                                 propval++;
3241                                 if (add_prop_list(optarg, propval,
3242                                     &props, B_TRUE))
3243                                         goto error;
3244                         } else {
3245                                 mntopts = optarg;
3246                         }
3247                         break;
3248                 case 'R':
3249                         if (add_prop_list(zpool_prop_to_name(
3250                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
3251                                 goto error;
3252                         if (add_prop_list_default(zpool_prop_to_name(
3253                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
3254                                 goto error;
3255                         break;
3256                 case 's':
3257                         do_scan = B_TRUE;
3258                         break;
3259                 case 't':
3260                         flags |= ZFS_IMPORT_TEMP_NAME;
3261                         if (add_prop_list_default(zpool_prop_to_name(
3262                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
3263                                 goto error;
3264                         break;
3265
3266                 case 'T':
3267                         errno = 0;
3268                         txg = strtoull(optarg, &endptr, 0);
3269                         if (errno != 0 || *endptr != '\0') {
3270                                 (void) fprintf(stderr,
3271                                     gettext("invalid txg value\n"));
3272                                 usage(B_FALSE);
3273                         }
3274                         rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
3275                         break;
3276                 case 'V':
3277                         flags |= ZFS_IMPORT_VERBATIM;
3278                         break;
3279                 case 'X':
3280                         xtreme_rewind = B_TRUE;
3281                         break;
3282                 case CHECKPOINT_OPT:
3283                         flags |= ZFS_IMPORT_CHECKPOINT;
3284                         break;
3285                 case ':':
3286                         (void) fprintf(stderr, gettext("missing argument for "
3287                             "'%c' option\n"), optopt);
3288                         usage(B_FALSE);
3289                         break;
3290                 case '?':
3291                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3292                             optopt);
3293                         usage(B_FALSE);
3294                 }
3295         }
3296
3297         argc -= optind;
3298         argv += optind;
3299
3300         if (cachefile && nsearch != 0) {
3301                 (void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
3302                 usage(B_FALSE);
3303         }
3304
3305         if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
3306                 (void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
3307                 usage(B_FALSE);
3308         }
3309
3310         if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
3311                 (void) fprintf(stderr, gettext("-l is only meaningful during "
3312                     "an import\n"));
3313                 usage(B_FALSE);
3314         }
3315
3316         if ((dryrun || xtreme_rewind) && !do_rewind) {
3317                 (void) fprintf(stderr,
3318                     gettext("-n or -X only meaningful with -F\n"));
3319                 usage(B_FALSE);
3320         }
3321         if (dryrun)
3322                 rewind_policy = ZPOOL_TRY_REWIND;
3323         else if (do_rewind)
3324                 rewind_policy = ZPOOL_DO_REWIND;
3325         if (xtreme_rewind)
3326                 rewind_policy |= ZPOOL_EXTREME_REWIND;
3327
3328         /* In the future, we can capture further policy and include it here */
3329         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
3330             nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
3331             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
3332             rewind_policy) != 0)
3333                 goto error;
3334
3335         /* check argument count */
3336         if (do_all) {
3337                 if (argc != 0) {
3338                         (void) fprintf(stderr, gettext("too many arguments\n"));
3339                         usage(B_FALSE);
3340                 }
3341         } else {
3342                 if (argc > 2) {
3343                         (void) fprintf(stderr, gettext("too many arguments\n"));
3344                         usage(B_FALSE);
3345                 }
3346         }
3347
3348         /*
3349          * Check for the effective uid.  We do this explicitly here because
3350          * otherwise any attempt to discover pools will silently fail.
3351          */
3352         if (argc == 0 && geteuid() != 0) {
3353                 (void) fprintf(stderr, gettext("cannot "
3354                     "discover pools: permission denied\n"));
3355                 if (searchdirs != NULL)
3356                         free(searchdirs);
3357
3358                 nvlist_free(props);
3359                 nvlist_free(policy);
3360                 return (1);
3361         }
3362
3363         /*
3364          * Depending on the arguments given, we do one of the following:
3365          *
3366          *      <none>  Iterate through all pools and display information about
3367          *              each one.
3368          *
3369          *      -a      Iterate through all pools and try to import each one.
3370          *
3371          *      <id>    Find the pool that corresponds to the given GUID/pool
3372          *              name and import that one.
3373          *
3374          *      -D      Above options applies only to destroyed pools.
3375          */
3376         if (argc != 0) {
3377                 char *endptr;
3378
3379                 errno = 0;
3380                 searchguid = strtoull(argv[0], &endptr, 10);
3381                 if (errno != 0 || *endptr != '\0') {
3382                         searchname = argv[0];
3383                         searchguid = 0;
3384                 }
3385                 found_config = NULL;
3386
3387                 /*
3388                  * User specified a name or guid.  Ensure it's unique.
3389                  */
3390                 target_exists_args_t search = {searchname, searchguid};
3391                 pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
3392         }
3393
3394         /*
3395          * Check the environment for the preferred search path.
3396          */
3397         if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
3398                 char *dir;
3399
3400                 envdup = strdup(env);
3401
3402                 dir = strtok(envdup, ":");
3403                 while (dir != NULL) {
3404                         if (searchdirs == NULL) {
3405                                 searchdirs = safe_malloc(sizeof (char *));
3406                         } else {
3407                                 char **tmp = safe_malloc((nsearch + 1) *
3408                                     sizeof (char *));
3409                                 bcopy(searchdirs, tmp, nsearch *
3410                                     sizeof (char *));
3411                                 free(searchdirs);
3412                                 searchdirs = tmp;
3413                         }
3414                         searchdirs[nsearch++] = dir;
3415                         dir = strtok(NULL, ":");
3416                 }
3417         }
3418
3419         idata.path = searchdirs;
3420         idata.paths = nsearch;
3421         idata.poolname = searchname;
3422         idata.guid = searchguid;
3423         idata.cachefile = cachefile;
3424         idata.scan = do_scan;
3425         idata.policy = policy;
3426
3427         pools = zpool_search_import(g_zfs, &idata, &libzfs_config_ops);
3428
3429         if (pools != NULL && pool_exists &&
3430             (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
3431                 (void) fprintf(stderr, gettext("cannot import '%s': "
3432                     "a pool with that name already exists\n"),
3433                     argv[0]);
3434                 (void) fprintf(stderr, gettext("use the form '%s "
3435                     "<pool | id> <newpool>' to give it a new name\n"),
3436                     "zpool import");
3437                 err = 1;
3438         } else if (pools == NULL && pool_exists) {
3439                 (void) fprintf(stderr, gettext("cannot import '%s': "
3440                     "a pool with that name is already created/imported,\n"),
3441                     argv[0]);
3442                 (void) fprintf(stderr, gettext("and no additional pools "
3443                     "with that name were found\n"));
3444                 err = 1;
3445         } else if (pools == NULL) {
3446                 if (argc != 0) {
3447                         (void) fprintf(stderr, gettext("cannot import '%s': "
3448                             "no such pool available\n"), argv[0]);
3449                 }
3450                 err = 1;
3451         }
3452
3453         if (err == 1) {
3454                 if (searchdirs != NULL)
3455                         free(searchdirs);
3456                 if (envdup != NULL)
3457                         free(envdup);
3458                 nvlist_free(policy);
3459                 nvlist_free(pools);
3460                 nvlist_free(props);
3461                 return (1);
3462         }
3463
3464         /*
3465          * At this point we have a list of import candidate configs. Even if
3466          * we were searching by pool name or guid, we still need to
3467          * post-process the list to deal with pool state and possible
3468          * duplicate names.
3469          */
3470         err = 0;
3471         elem = NULL;
3472         first = B_TRUE;
3473         while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
3474
3475                 verify(nvpair_value_nvlist(elem, &config) == 0);
3476
3477                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
3478                     &pool_state) == 0);
3479                 if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
3480                         continue;
3481                 if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
3482                         continue;
3483
3484                 verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
3485                     policy) == 0);
3486
3487                 if (argc == 0) {
3488                         if (first)
3489                                 first = B_FALSE;
3490                         else if (!do_all)
3491                                 (void) printf("\n");
3492
3493                         if (do_all) {
3494                                 err |= do_import(config, NULL, mntopts,
3495                                     props, flags);
3496                         } else {
3497                                 show_import(config);
3498                         }
3499                 } else if (searchname != NULL) {
3500                         char *name;
3501
3502                         /*
3503                          * We are searching for a pool based on name.
3504                          */
3505                         verify(nvlist_lookup_string(config,
3506                             ZPOOL_CONFIG_POOL_NAME, &name) == 0);
3507
3508                         if (strcmp(name, searchname) == 0) {
3509                                 if (found_config != NULL) {
3510                                         (void) fprintf(stderr, gettext(
3511                                             "cannot import '%s': more than "
3512                                             "one matching pool\n"), searchname);
3513                                         (void) fprintf(stderr, gettext(
3514                                             "import by numeric ID instead\n"));
3515                                         err = B_TRUE;
3516                                 }
3517                                 found_config = config;
3518                         }
3519                 } else {
3520                         uint64_t guid;
3521
3522                         /*
3523                          * Search for a pool by guid.
3524                          */
3525                         verify(nvlist_lookup_uint64(config,
3526                             ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3527
3528                         if (guid == searchguid)
3529                                 found_config = config;
3530                 }
3531         }
3532
3533         /*
3534          * If we were searching for a specific pool, verify that we found a
3535          * pool, and then do the import.
3536          */
3537         if (argc != 0 && err == 0) {
3538                 if (found_config == NULL) {
3539                         (void) fprintf(stderr, gettext("cannot import '%s': "
3540                             "no such pool available\n"), argv[0]);
3541                         err = B_TRUE;
3542                 } else {
3543                         err |= do_import(found_config, argc == 1 ? NULL :
3544                             argv[1], mntopts, props, flags);
3545                 }
3546         }
3547
3548         /*
3549          * If we were just looking for pools, report an error if none were
3550          * found.
3551          */
3552         if (argc == 0 && first)
3553                 (void) fprintf(stderr,
3554                     gettext("no pools available to import\n"));
3555
3556 error:
3557         nvlist_free(props);
3558         nvlist_free(pools);
3559         nvlist_free(policy);
3560         if (searchdirs != NULL)
3561                 free(searchdirs);
3562         if (envdup != NULL)
3563                 free(envdup);
3564
3565         return (err ? 1 : 0);
3566 }
3567
3568 /*
3569  * zpool sync [-f] [pool] ...
3570  *
3571  * -f (undocumented) force uberblock (and config including zpool cache file)
3572  *    update.
3573  *
3574  * Sync the specified pool(s).
3575  * Without arguments "zpool sync" will sync all pools.
3576  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
3577  *
3578  */
3579 static int
3580 zpool_do_sync(int argc, char **argv)
3581 {
3582         int ret;
3583         boolean_t force = B_FALSE;
3584
3585         /* check options */
3586         while ((ret  = getopt(argc, argv, "f")) != -1) {
3587                 switch (ret) {
3588                 case 'f':
3589                         force = B_TRUE;
3590                         break;
3591                 case '?':
3592                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3593                             optopt);
3594                         usage(B_FALSE);
3595                 }
3596         }
3597
3598         argc -= optind;
3599         argv += optind;
3600
3601         /* if argc == 0 we will execute zpool_sync_one on all pools */
3602         ret = for_each_pool(argc, argv, B_FALSE, NULL, zpool_sync_one, &force);
3603
3604         return (ret);
3605 }
3606
3607 typedef struct iostat_cbdata {
3608         uint64_t cb_flags;
3609         int cb_name_flags;
3610         int cb_namewidth;
3611         int cb_iteration;
3612         char **cb_vdev_names; /* Only show these vdevs */
3613         unsigned int cb_vdev_names_count;
3614         boolean_t cb_verbose;
3615         boolean_t cb_literal;
3616         boolean_t cb_scripted;
3617         zpool_list_t *cb_list;
3618         vdev_cmd_data_list_t *vcdl;
3619 } iostat_cbdata_t;
3620
3621 /*  iostat labels */
3622 typedef struct name_and_columns {
3623         const char *name;       /* Column name */
3624         unsigned int columns;   /* Center name to this number of columns */
3625 } name_and_columns_t;
3626
3627 #define IOSTAT_MAX_LABELS       13      /* Max number of labels on one line */
3628
3629 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
3630 {
3631         [IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
3632             {NULL}},
3633         [IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3634             {"asyncq_wait", 2}, {"scrub", 1}, {"trim", 1}, {NULL}},
3635         [IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
3636             {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
3637             {"trimq_write", 2}, {NULL}},
3638         [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3639             {"asyncq_wait", 2}, {NULL}},
3640         [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
3641             {"async_read", 2}, {"async_write", 2}, {"scrub", 2},
3642             {"trim", 2}, {NULL}},
3643 };
3644
3645 /* Shorthand - if "columns" field not set, default to 1 column */
3646 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
3647 {
3648         [IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
3649             {"write"}, {NULL}},
3650         [IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3651             {"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {NULL}},
3652         [IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
3653             {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
3654             {"pend"}, {"activ"}, {NULL}},
3655         [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3656             {"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {NULL}},
3657         [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3658             {"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"}, {NULL}},
3659 };
3660
3661 static const char *histo_to_title[] = {
3662         [IOS_L_HISTO] = "latency",
3663         [IOS_RQ_HISTO] = "req_size",
3664 };
3665
3666 /*
3667  * Return the number of labels in a null-terminated name_and_columns_t
3668  * array.
3669  *
3670  */
3671 static unsigned int
3672 label_array_len(const name_and_columns_t *labels)
3673 {
3674         int i = 0;
3675
3676         while (labels[i].name)
3677                 i++;
3678
3679         return (i);
3680 }
3681
3682 /*
3683  * Return the number of strings in a null-terminated string array.
3684  * For example:
3685  *
3686  *     const char foo[] = {"bar", "baz", NULL}
3687  *
3688  * returns 2
3689  */
3690 static uint64_t
3691 str_array_len(const char *array[])
3692 {
3693         uint64_t i = 0;
3694         while (array[i])
3695                 i++;
3696
3697         return (i);
3698 }
3699
3700
3701 /*
3702  * Return a default column width for default/latency/queue columns. This does
3703  * not include histograms, which have their columns autosized.
3704  */
3705 static unsigned int
3706 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
3707 {
3708         unsigned long column_width = 5; /* Normal niceprint */
3709         static unsigned long widths[] = {
3710                 /*
3711                  * Choose some sane default column sizes for printing the
3712                  * raw numbers.
3713                  */
3714                 [IOS_DEFAULT] = 15, /* 1PB capacity */
3715                 [IOS_LATENCY] = 10, /* 1B ns = 10sec */
3716                 [IOS_QUEUES] = 6,   /* 1M queue entries */
3717                 [IOS_L_HISTO] = 10, /* 1B ns = 10sec */
3718                 [IOS_RQ_HISTO] = 6, /* 1M queue entries */
3719         };
3720
3721         if (cb->cb_literal)
3722                 column_width = widths[type];
3723
3724         return (column_width);
3725 }
3726
3727 /*
3728  * Print the column labels, i.e:
3729  *
3730  *   capacity     operations     bandwidth
3731  * alloc   free   read  write   read  write  ...
3732  *
3733  * If force_column_width is set, use it for the column width.  If not set, use
3734  * the default column width.
3735  */
3736 static void
3737 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
3738     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
3739 {
3740         int i, idx, s;
3741         int text_start, rw_column_width, spaces_to_end;
3742         uint64_t flags = cb->cb_flags;
3743         uint64_t f;
3744         unsigned int column_width = force_column_width;
3745
3746         /* For each bit set in flags */
3747         for (f = flags; f; f &= ~(1ULL << idx)) {
3748                 idx = lowbit64(f) - 1;
3749                 if (!force_column_width)
3750                         column_width = default_column_width(cb, idx);
3751                 /* Print our top labels centered over "read  write" label. */
3752                 for (i = 0; i < label_array_len(labels[idx]); i++) {
3753                         const char *name = labels[idx][i].name;
3754                         /*
3755                          * We treat labels[][].columns == 0 as shorthand
3756                          * for one column.  It makes writing out the label
3757                          * tables more concise.
3758                          */
3759                         unsigned int columns = MAX(1, labels[idx][i].columns);
3760                         unsigned int slen = strlen(name);
3761
3762                         rw_column_width = (column_width * columns) +
3763                             (2 * (columns - 1));
3764
3765                         text_start = (int)((rw_column_width) / columns -
3766                             slen / columns);
3767                         if (text_start < 0)
3768                                 text_start = 0;
3769
3770                         printf("  ");   /* Two spaces between columns */
3771
3772                         /* Space from beginning of column to label */
3773                         for (s = 0; s < text_start; s++)
3774                                 printf(" ");
3775
3776                         printf("%s", name);
3777
3778                         /* Print space after label to end of column */
3779                         spaces_to_end = rw_column_width - text_start - slen;
3780                         if (spaces_to_end < 0)
3781                                 spaces_to_end = 0;
3782
3783                         for (s = 0; s < spaces_to_end; s++)
3784                                 printf(" ");
3785                 }
3786         }
3787 }
3788
3789
3790 /*
3791  * print_cmd_columns - Print custom column titles from -c
3792  *
3793  * If the user specified the "zpool status|iostat -c" then print their custom
3794  * column titles in the header.  For example, print_cmd_columns() would print
3795  * the "  col1  col2" part of this:
3796  *
3797  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
3798  * ...
3799  *            capacity     operations     bandwidth
3800  * pool        alloc   free   read  write   read  write  col1  col2
3801  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3802  * mypool       269K  1008M      0      0    107    946
3803  *   mirror     269K  1008M      0      0    107    946
3804  *     sdb         -      -      0      0    102    473  val1  val2
3805  *     sdc         -      -      0      0      5    473  val1  val2
3806  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3807  */
3808 static void
3809 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
3810 {
3811         int i, j;
3812         vdev_cmd_data_t *data = &vcdl->data[0];
3813
3814         if (vcdl->count == 0 || data == NULL)
3815                 return;
3816
3817         /*
3818          * Each vdev cmd should have the same column names unless the user did
3819          * something weird with their cmd.  Just take the column names from the
3820          * first vdev and assume it works for all of them.
3821          */
3822         for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
3823                 printf("  ");
3824                 if (use_dashes) {
3825                         for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
3826                                 printf("-");
3827                 } else {
3828                         printf_color(ANSI_BOLD, "%*s", vcdl->uniq_cols_width[i],
3829                             vcdl->uniq_cols[i]);
3830                 }
3831         }
3832 }
3833
3834
3835 /*
3836  * Utility function to print out a line of dashes like:
3837  *
3838  *      --------------------------------  -----  -----  -----  -----  -----
3839  *
3840  * ...or a dashed named-row line like:
3841  *
3842  *      logs                                  -      -      -      -      -
3843  *
3844  * @cb:                         iostat data
3845  *
3846  * @force_column_width          If non-zero, use the value as the column width.
3847  *                              Otherwise use the default column widths.
3848  *
3849  * @name:                       Print a dashed named-row line starting
3850  *                              with @name.  Otherwise, print a regular
3851  *                              dashed line.
3852  */
3853 static void
3854 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
3855     const char *name)
3856 {
3857         int i;
3858         unsigned int namewidth;
3859         uint64_t flags = cb->cb_flags;
3860         uint64_t f;
3861         int idx;
3862         const name_and_columns_t *labels;
3863         const char *title;
3864
3865
3866         if (cb->cb_flags & IOS_ANYHISTO_M) {
3867                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3868         } else if (cb->cb_vdev_names_count) {
3869                 title = "vdev";
3870         } else  {
3871                 title = "pool";
3872         }
3873
3874         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3875             name ? strlen(name) : 0);
3876
3877
3878         if (name) {
3879                 printf("%-*s", namewidth, name);
3880         } else {
3881                 for (i = 0; i < namewidth; i++)
3882                         (void) printf("-");
3883         }
3884
3885         /* For each bit in flags */
3886         for (f = flags; f; f &= ~(1ULL << idx)) {
3887                 unsigned int column_width;
3888                 idx = lowbit64(f) - 1;
3889                 if (force_column_width)
3890                         column_width = force_column_width;
3891                 else
3892                         column_width = default_column_width(cb, idx);
3893
3894                 labels = iostat_bottom_labels[idx];
3895                 for (i = 0; i < label_array_len(labels); i++) {
3896                         if (name)
3897                                 printf("  %*s-", column_width - 1, " ");
3898                         else
3899                                 printf("  %.*s", column_width,
3900                                     "--------------------");
3901                 }
3902         }
3903 }
3904
3905
3906 static void
3907 print_iostat_separator_impl(iostat_cbdata_t *cb,
3908     unsigned int force_column_width)
3909 {
3910         print_iostat_dashes(cb, force_column_width, NULL);
3911 }
3912
3913 static void
3914 print_iostat_separator(iostat_cbdata_t *cb)
3915 {
3916         print_iostat_separator_impl(cb, 0);
3917 }
3918
3919 static void
3920 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
3921     const char *histo_vdev_name)
3922 {
3923         unsigned int namewidth;
3924         const char *title;
3925
3926         if (cb->cb_flags & IOS_ANYHISTO_M) {
3927                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3928         } else if (cb->cb_vdev_names_count) {
3929                 title = "vdev";
3930         } else  {
3931                 title = "pool";
3932         }
3933
3934         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3935             histo_vdev_name ? strlen(histo_vdev_name) : 0);
3936
3937         if (histo_vdev_name)
3938                 printf("%-*s", namewidth, histo_vdev_name);
3939         else
3940                 printf("%*s", namewidth, "");
3941
3942
3943         print_iostat_labels(cb, force_column_width, iostat_top_labels);
3944         printf("\n");
3945
3946         printf("%-*s", namewidth, title);
3947
3948         print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
3949         if (cb->vcdl != NULL)
3950                 print_cmd_columns(cb->vcdl, 0);
3951
3952         printf("\n");
3953
3954         print_iostat_separator_impl(cb, force_column_width);
3955
3956         if (cb->vcdl != NULL)
3957                 print_cmd_columns(cb->vcdl, 1);
3958
3959         printf("\n");
3960 }
3961
3962 static void
3963 print_iostat_header(iostat_cbdata_t *cb)
3964 {
3965         print_iostat_header_impl(cb, 0, NULL);
3966 }
3967
3968
3969 /*
3970  * Display a single statistic.
3971  */
3972 static void
3973 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
3974     unsigned int column_size, boolean_t scripted)
3975 {
3976         char buf[64];
3977
3978         zfs_nicenum_format(value, buf, sizeof (buf), format);
3979
3980         if (scripted)
3981                 printf("\t%s", buf);
3982         else
3983                 printf("  %*s", column_size, buf);
3984 }
3985
3986 /*
3987  * Calculate the default vdev stats
3988  *
3989  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
3990  * stats into calcvs.
3991  */
3992 static void
3993 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
3994     vdev_stat_t *calcvs)
3995 {
3996         int i;
3997
3998         memcpy(calcvs, newvs, sizeof (*calcvs));
3999         for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
4000                 calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
4001
4002         for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
4003                 calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
4004 }
4005
4006 /*
4007  * Internal representation of the extended iostats data.
4008  *
4009  * The extended iostat stats are exported in nvlists as either uint64_t arrays
4010  * or single uint64_t's.  We make both look like arrays to make them easier
4011  * to process.  In order to make single uint64_t's look like arrays, we set
4012  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
4013  * we can just use *data and count.
4014  */
4015 struct stat_array {
4016         uint64_t *data;
4017         uint_t count;   /* Number of entries in data[] */
4018         uint64_t __data; /* Only used when data is a single uint64_t */
4019 };
4020
4021 static uint64_t
4022 stat_histo_max(struct stat_array *nva, unsigned int len)
4023 {
4024         uint64_t max = 0;
4025         int i;
4026         for (i = 0; i < len; i++)
4027                 max = MAX(max, array64_max(nva[i].data, nva[i].count));
4028
4029         return (max);
4030 }
4031
4032 /*
4033  * Helper function to lookup a uint64_t array or uint64_t value and store its
4034  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
4035  * it look like a one element array to make it easier to process.
4036  */
4037 static int
4038 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
4039     struct stat_array *nva)
4040 {
4041         nvpair_t *tmp;
4042         int ret;
4043
4044         verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
4045         switch (nvpair_type(tmp)) {
4046         case DATA_TYPE_UINT64_ARRAY:
4047                 ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
4048                 break;
4049         case DATA_TYPE_UINT64:
4050                 ret = nvpair_value_uint64(tmp, &nva->__data);
4051                 nva->data = &nva->__data;
4052                 nva->count = 1;
4053                 break;
4054         default:
4055                 /* Not a uint64_t */
4056                 ret = EINVAL;
4057                 break;
4058         }
4059
4060         return (ret);
4061 }
4062
4063 /*
4064  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
4065  * subtract them, and return the results in a newly allocated stat_array.
4066  * You must free the returned array after you are done with it with
4067  * free_calc_stats().
4068  *
4069  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
4070  * values.
4071  */
4072 static struct stat_array *
4073 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
4074     nvlist_t *newnv)
4075 {
4076         nvlist_t *oldnvx = NULL, *newnvx;
4077         struct stat_array *oldnva, *newnva, *calcnva;
4078         int i, j;
4079         unsigned int alloc_size = (sizeof (struct stat_array)) * len;
4080
4081         /* Extract our extended stats nvlist from the main list */
4082         verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4083             &newnvx) == 0);
4084         if (oldnv) {
4085                 verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4086                     &oldnvx) == 0);
4087         }
4088
4089         newnva = safe_malloc(alloc_size);
4090         oldnva = safe_malloc(alloc_size);
4091         calcnva = safe_malloc(alloc_size);
4092
4093         for (j = 0; j < len; j++) {
4094                 verify(nvpair64_to_stat_array(newnvx, names[j],
4095                     &newnva[j]) == 0);
4096                 calcnva[j].count = newnva[j].count;
4097                 alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
4098                 calcnva[j].data = safe_malloc(alloc_size);
4099                 memcpy(calcnva[j].data, newnva[j].data, alloc_size);
4100
4101                 if (oldnvx) {
4102                         verify(nvpair64_to_stat_array(oldnvx, names[j],
4103                             &oldnva[j]) == 0);
4104                         for (i = 0; i < oldnva[j].count; i++)
4105                                 calcnva[j].data[i] -= oldnva[j].data[i];
4106                 }
4107         }
4108         free(newnva);
4109         free(oldnva);
4110         return (calcnva);
4111 }
4112
4113 static void
4114 free_calc_stats(struct stat_array *nva, unsigned int len)
4115 {
4116         int i;
4117         for (i = 0; i < len; i++)
4118                 free(nva[i].data);
4119
4120         free(nva);
4121 }
4122
4123 static void
4124 print_iostat_histo(struct stat_array *nva, unsigned int len,
4125     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
4126     double scale)
4127 {
4128         int i, j;
4129         char buf[6];
4130         uint64_t val;
4131         enum zfs_nicenum_format format;
4132         unsigned int buckets;
4133         unsigned int start_bucket;
4134
4135         if (cb->cb_literal)
4136                 format = ZFS_NICENUM_RAW;
4137         else
4138                 format = ZFS_NICENUM_1024;
4139
4140         /* All these histos are the same size, so just use nva[0].count */
4141         buckets = nva[0].count;
4142
4143         if (cb->cb_flags & IOS_RQ_HISTO_M) {
4144                 /* Start at 512 - req size should never be lower than this */
4145                 start_bucket = 9;
4146         } else {
4147                 start_bucket = 0;
4148         }
4149
4150         for (j = start_bucket; j < buckets; j++) {
4151                 /* Print histogram bucket label */
4152                 if (cb->cb_flags & IOS_L_HISTO_M) {
4153                         /* Ending range of this bucket */
4154                         val = (1UL << (j + 1)) - 1;
4155                         zfs_nicetime(val, buf, sizeof (buf));
4156                 } else {
4157                         /* Request size (starting range of bucket) */
4158                         val = (1UL << j);
4159                         zfs_nicenum(val, buf, sizeof (buf));
4160                 }
4161
4162                 if (cb->cb_scripted)
4163                         printf("%llu", (u_longlong_t)val);
4164                 else
4165                         printf("%-*s", namewidth, buf);
4166
4167                 /* Print the values on the line */
4168                 for (i = 0; i < len; i++) {
4169                         print_one_stat(nva[i].data[j] * scale, format,
4170                             column_width, cb->cb_scripted);
4171                 }
4172                 printf("\n");
4173         }
4174 }
4175
4176 static void
4177 print_solid_separator(unsigned int length)
4178 {
4179         while (length--)
4180                 printf("-");
4181         printf("\n");
4182 }
4183
4184 static void
4185 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
4186     nvlist_t *newnv, double scale, const char *name)
4187 {
4188         unsigned int column_width;
4189         unsigned int namewidth;
4190         unsigned int entire_width;
4191         enum iostat_type type;
4192         struct stat_array *nva;
4193         const char **names;
4194         unsigned int names_len;
4195
4196         /* What type of histo are we? */
4197         type = IOS_HISTO_IDX(cb->cb_flags);
4198
4199         /* Get NULL-terminated array of nvlist names for our histo */
4200         names = vsx_type_to_nvlist[type];
4201         names_len = str_array_len(names); /* num of names */
4202
4203         nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
4204
4205         if (cb->cb_literal) {
4206                 column_width = MAX(5,
4207                     (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
4208         } else {
4209                 column_width = 5;
4210         }
4211
4212         namewidth = MAX(cb->cb_namewidth,
4213             strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
4214
4215         /*
4216          * Calculate the entire line width of what we're printing.  The
4217          * +2 is for the two spaces between columns:
4218          */
4219         /*       read  write                            */
4220         /*      -----  -----                            */
4221         /*      |___|  <---------- column_width         */
4222         /*                                              */
4223         /*      |__________|  <--- entire_width         */
4224         /*                                              */
4225         entire_width = namewidth + (column_width + 2) *
4226             label_array_len(iostat_bottom_labels[type]);
4227
4228         if (cb->cb_scripted)
4229                 printf("%s\n", name);
4230         else
4231                 print_iostat_header_impl(cb, column_width, name);
4232
4233         print_iostat_histo(nva, names_len, cb, column_width,
4234             namewidth, scale);
4235
4236         free_calc_stats(nva, names_len);
4237         if (!cb->cb_scripted)
4238                 print_solid_separator(entire_width);
4239 }
4240
4241 /*
4242  * Calculate the average latency of a power-of-two latency histogram
4243  */
4244 static uint64_t
4245 single_histo_average(uint64_t *histo, unsigned int buckets)
4246 {
4247         int i;
4248         uint64_t count = 0, total = 0;
4249
4250         for (i = 0; i < buckets; i++) {
4251                 /*
4252                  * Our buckets are power-of-two latency ranges.  Use the
4253                  * midpoint latency of each bucket to calculate the average.
4254                  * For example:
4255                  *
4256                  * Bucket          Midpoint
4257                  * 8ns-15ns:       12ns
4258                  * 16ns-31ns:      24ns
4259                  * ...
4260                  */
4261                 if (histo[i] != 0) {
4262                         total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
4263                         count += histo[i];
4264                 }
4265         }
4266
4267         /* Prevent divide by zero */
4268         return (count == 0 ? 0 : total / count);
4269 }
4270
4271 static void
4272 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *oldnv,
4273     nvlist_t *newnv)
4274 {
4275         int i;
4276         uint64_t val;
4277         const char *names[] = {
4278                 ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
4279                 ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
4280                 ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
4281                 ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
4282                 ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
4283                 ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
4284                 ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
4285                 ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
4286                 ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
4287                 ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
4288                 ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
4289                 ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
4290         };
4291
4292         struct stat_array *nva;
4293
4294         unsigned int column_width = default_column_width(cb, IOS_QUEUES);
4295         enum zfs_nicenum_format format;
4296
4297         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
4298
4299         if (cb->cb_literal)
4300                 format = ZFS_NICENUM_RAW;
4301         else
4302                 format = ZFS_NICENUM_1024;
4303
4304         for (i = 0; i < ARRAY_SIZE(names); i++) {
4305                 val = nva[i].data[0];
4306                 print_one_stat(val, format, column_width, cb->cb_scripted);
4307         }
4308
4309         free_calc_stats(nva, ARRAY_SIZE(names));
4310 }
4311
4312 static void
4313 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
4314     nvlist_t *newnv)
4315 {
4316         int i;
4317         uint64_t val;
4318         const char *names[] = {
4319                 ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
4320                 ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
4321                 ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
4322                 ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
4323                 ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
4324                 ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
4325                 ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
4326                 ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
4327                 ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
4328                 ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
4329         };
4330         struct stat_array *nva;
4331
4332         unsigned int column_width = default_column_width(cb, IOS_LATENCY);
4333         enum zfs_nicenum_format format;
4334
4335         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
4336
4337         if (cb->cb_literal)
4338                 format = ZFS_NICENUM_RAWTIME;
4339         else
4340                 format = ZFS_NICENUM_TIME;
4341
4342         /* Print our avg latencies on the line */
4343         for (i = 0; i < ARRAY_SIZE(names); i++) {
4344                 /* Compute average latency for a latency histo */
4345                 val = single_histo_average(nva[i].data, nva[i].count);
4346                 print_one_stat(val, format, column_width, cb->cb_scripted);
4347         }
4348         free_calc_stats(nva, ARRAY_SIZE(names));
4349 }
4350
4351 /*
4352  * Print default statistics (capacity/operations/bandwidth)
4353  */
4354 static void
4355 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
4356 {
4357         unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
4358         enum zfs_nicenum_format format;
4359         char na;        /* char to print for "not applicable" values */
4360
4361         if (cb->cb_literal) {
4362                 format = ZFS_NICENUM_RAW;
4363                 na = '0';
4364         } else {
4365                 format = ZFS_NICENUM_1024;
4366                 na = '-';
4367         }
4368
4369         /* only toplevel vdevs have capacity stats */
4370         if (vs->vs_space == 0) {
4371                 if (cb->cb_scripted)
4372                         printf("\t%c\t%c", na, na);
4373                 else
4374                         printf("  %*c  %*c", column_width, na, column_width,
4375                             na);
4376         } else {
4377                 print_one_stat(vs->vs_alloc, format, column_width,
4378                     cb->cb_scripted);
4379                 print_one_stat(vs->vs_space - vs->vs_alloc, format,
4380                     column_width, cb->cb_scripted);
4381         }
4382
4383         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
4384             format, column_width, cb->cb_scripted);
4385         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
4386             format, column_width, cb->cb_scripted);
4387         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
4388             format, column_width, cb->cb_scripted);
4389         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
4390             format, column_width, cb->cb_scripted);
4391 }
4392
4393 static const char *class_name[] = {
4394         VDEV_ALLOC_BIAS_DEDUP,
4395         VDEV_ALLOC_BIAS_SPECIAL,
4396         VDEV_ALLOC_CLASS_LOGS
4397 };
4398
4399 /*
4400  * Print out all the statistics for the given vdev.  This can either be the
4401  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
4402  * is a verbose output, and we don't want to display the toplevel pool stats.
4403  *
4404  * Returns the number of stat lines printed.
4405  */
4406 static unsigned int
4407 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
4408     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
4409 {
4410         nvlist_t **oldchild, **newchild;
4411         uint_t c, children, oldchildren;
4412         vdev_stat_t *oldvs, *newvs, *calcvs;
4413         vdev_stat_t zerovs = { 0 };
4414         char *vname;
4415         int i;
4416         int ret = 0;
4417         uint64_t tdelta;
4418         double scale;
4419
4420         if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
4421                 return (ret);
4422
4423         calcvs = safe_malloc(sizeof (*calcvs));
4424
4425         if (oldnv != NULL) {
4426                 verify(nvlist_lookup_uint64_array(oldnv,
4427                     ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
4428         } else {
4429                 oldvs = &zerovs;
4430         }
4431
4432         /* Do we only want to see a specific vdev? */
4433         for (i = 0; i < cb->cb_vdev_names_count; i++) {
4434                 /* Yes we do.  Is this the vdev? */
4435                 if (strcmp(name, cb->cb_vdev_names[i]) == 0) {
4436                         /*
4437                          * This is our vdev.  Since it is the only vdev we
4438                          * will be displaying, make depth = 0 so that it
4439                          * doesn't get indented.
4440                          */
4441                         depth = 0;
4442                         break;
4443                 }
4444         }
4445
4446         if (cb->cb_vdev_names_count && (i == cb->cb_vdev_names_count)) {
4447                 /* Couldn't match the name */
4448                 goto children;
4449         }
4450
4451
4452         verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
4453             (uint64_t **)&newvs, &c) == 0);
4454
4455         /*
4456          * Print the vdev name unless it's is a histogram.  Histograms
4457          * display the vdev name in the header itself.
4458          */
4459         if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
4460                 if (cb->cb_scripted) {
4461                         printf("%s", name);
4462                 } else {
4463                         if (strlen(name) + depth > cb->cb_namewidth)
4464                                 (void) printf("%*s%s", depth, "", name);
4465                         else
4466                                 (void) printf("%*s%s%*s", depth, "", name,
4467                                     (int)(cb->cb_namewidth - strlen(name) -
4468                                     depth), "");
4469                 }
4470         }
4471
4472         /* Calculate our scaling factor */
4473         tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
4474         if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
4475                 /*
4476                  * If we specify printing histograms with no time interval, then
4477                  * print the histogram numbers over the entire lifetime of the
4478                  * vdev.
4479                  */
4480                 scale = 1;
4481         } else {
4482                 if (tdelta == 0)
4483                         scale = 1.0;
4484                 else
4485                         scale = (double)NANOSEC / tdelta;
4486         }
4487
4488         if (cb->cb_flags & IOS_DEFAULT_M) {
4489                 calc_default_iostats(oldvs, newvs, calcvs);
4490                 print_iostat_default(calcvs, cb, scale);
4491         }
4492         if (cb->cb_flags & IOS_LATENCY_M)
4493                 print_iostat_latency(cb, oldnv, newnv);
4494         if (cb->cb_flags & IOS_QUEUES_M)
4495                 print_iostat_queues(cb, oldnv, newnv);
4496         if (cb->cb_flags & IOS_ANYHISTO_M) {
4497                 printf("\n");
4498                 print_iostat_histos(cb, oldnv, newnv, scale, name);
4499         }
4500
4501         if (cb->vcdl != NULL) {
4502                 char *path;
4503                 if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
4504                     &path) == 0) {
4505                         printf("  ");
4506                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
4507                 }
4508         }
4509
4510         if (!(cb->cb_flags & IOS_ANYHISTO_M))
4511                 printf("\n");
4512
4513         ret++;
4514
4515 children:
4516
4517         free(calcvs);
4518
4519         if (!cb->cb_verbose)
4520                 return (ret);
4521
4522         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
4523             &newchild, &children) != 0)
4524                 return (ret);
4525
4526         if (oldnv) {
4527                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
4528                     &oldchild, &oldchildren) != 0)
4529                         return (ret);
4530
4531                 children = MIN(oldchildren, children);
4532         }
4533
4534         /*
4535          * print normal top-level devices
4536          */
4537         for (c = 0; c < children; c++) {
4538                 uint64_t ishole = B_FALSE, islog = B_FALSE;
4539
4540                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
4541                     &ishole);
4542
4543                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
4544                     &islog);
4545
4546                 if (ishole || islog)
4547                         continue;
4548
4549                 if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
4550                         continue;
4551
4552                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4553                     cb->cb_name_flags);
4554                 ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
4555                     newchild[c], cb, depth + 2);
4556                 free(vname);
4557         }
4558
4559         /*
4560          * print all other top-level devices
4561          */
4562         for (uint_t n = 0; n < 3; n++) {
4563                 boolean_t printed = B_FALSE;
4564
4565                 for (c = 0; c < children; c++) {
4566                         uint64_t islog = B_FALSE;
4567                         char *bias = NULL;
4568                         char *type = NULL;
4569
4570                         (void) nvlist_lookup_uint64(newchild[c],
4571                             ZPOOL_CONFIG_IS_LOG, &islog);
4572                         if (islog) {
4573                                 bias = VDEV_ALLOC_CLASS_LOGS;
4574                         } else {
4575                                 (void) nvlist_lookup_string(newchild[c],
4576                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
4577                                 (void) nvlist_lookup_string(newchild[c],
4578                                     ZPOOL_CONFIG_TYPE, &type);
4579                         }
4580                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
4581                                 continue;
4582                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
4583                                 continue;
4584
4585                         if (!printed) {
4586                                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
4587                                     !cb->cb_scripted && !cb->cb_vdev_names) {
4588                                         print_iostat_dashes(cb, 0,
4589                                             class_name[n]);
4590                                 }
4591                                 printf("\n");
4592                                 printed = B_TRUE;
4593                         }
4594
4595                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4596                             cb->cb_name_flags);
4597                         ret += print_vdev_stats(zhp, vname, oldnv ?
4598                             oldchild[c] : NULL, newchild[c], cb, depth + 2);
4599                         free(vname);
4600                 }
4601         }
4602
4603         /*
4604          * Include level 2 ARC devices in iostat output
4605          */
4606         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
4607             &newchild, &children) != 0)
4608                 return (ret);
4609
4610         if (oldnv) {
4611                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
4612                     &oldchild, &oldchildren) != 0)
4613                         return (ret);
4614
4615                 children = MIN(oldchildren, children);
4616         }
4617
4618         if (children > 0) {
4619                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
4620                     !cb->cb_vdev_names) {
4621                         print_iostat_dashes(cb, 0, "cache");
4622                 }
4623                 printf("\n");
4624
4625                 for (c = 0; c < children; c++) {
4626                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4627                             cb->cb_name_flags);
4628                         ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
4629                             : NULL, newchild[c], cb, depth + 2);
4630                         free(vname);
4631                 }
4632         }
4633
4634         return (ret);
4635 }
4636
4637 static int
4638 refresh_iostat(zpool_handle_t *zhp, void *data)
4639 {
4640         iostat_cbdata_t *cb = data;
4641         boolean_t missing;
4642
4643         /*
4644          * If the pool has disappeared, remove it from the list and continue.
4645          */
4646         if (zpool_refresh_stats(zhp, &missing) != 0)
4647                 return (-1);
4648
4649         if (missing)
4650                 pool_list_remove(cb->cb_list, zhp);
4651
4652         return (0);
4653 }
4654
4655 /*
4656  * Callback to print out the iostats for the given pool.
4657  */
4658 static int
4659 print_iostat(zpool_handle_t *zhp, void *data)
4660 {
4661         iostat_cbdata_t *cb = data;
4662         nvlist_t *oldconfig, *newconfig;
4663         nvlist_t *oldnvroot, *newnvroot;
4664         int ret;
4665
4666         newconfig = zpool_get_config(zhp, &oldconfig);
4667
4668         if (cb->cb_iteration == 1)
4669                 oldconfig = NULL;
4670
4671         verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
4672             &newnvroot) == 0);
4673
4674         if (oldconfig == NULL)
4675                 oldnvroot = NULL;
4676         else
4677                 verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
4678                     &oldnvroot) == 0);
4679
4680         ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
4681             cb, 0);
4682         if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
4683             !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
4684                 print_iostat_separator(cb);
4685                 if (cb->vcdl != NULL) {
4686                         print_cmd_columns(cb->vcdl, 1);
4687                 }
4688                 printf("\n");
4689         }
4690
4691         return (ret);
4692 }
4693
4694 static int
4695 get_columns(void)
4696 {
4697         struct winsize ws;
4698         int columns = 80;
4699         int error;
4700
4701         if (isatty(STDOUT_FILENO)) {
4702                 error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
4703                 if (error == 0)
4704                         columns = ws.ws_col;
4705         } else {
4706                 columns = 999;
4707         }
4708
4709         return (columns);
4710 }
4711
4712 /*
4713  * Return the required length of the pool/vdev name column.  The minimum
4714  * allowed width and output formatting flags must be provided.
4715  */
4716 static int
4717 get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
4718 {
4719         nvlist_t *config, *nvroot;
4720         int width = min_width;
4721
4722         if ((config = zpool_get_config(zhp, NULL)) != NULL) {
4723                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4724                     &nvroot) == 0);
4725                 unsigned int poolname_len = strlen(zpool_get_name(zhp));
4726                 if (verbose == B_FALSE) {
4727                         width = MAX(poolname_len, min_width);
4728                 } else {
4729                         width = MAX(poolname_len,
4730                             max_width(zhp, nvroot, 0, min_width, flags));
4731                 }
4732         }
4733
4734         return (width);
4735 }
4736
4737 /*
4738  * Parse the input string, get the 'interval' and 'count' value if there is one.
4739  */
4740 static void
4741 get_interval_count(int *argcp, char **argv, float *iv,
4742     unsigned long *cnt)
4743 {
4744         float interval = 0;
4745         unsigned long count = 0;
4746         int argc = *argcp;
4747
4748         /*
4749          * Determine if the last argument is an integer or a pool name
4750          */
4751         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
4752                 char *end;
4753
4754                 errno = 0;
4755                 interval = strtof(argv[argc - 1], &end);
4756
4757                 if (*end == '\0' && errno == 0) {
4758                         if (interval == 0) {
4759                                 (void) fprintf(stderr, gettext("interval "
4760                                     "cannot be zero\n"));
4761                                 usage(B_FALSE);
4762                         }
4763                         /*
4764                          * Ignore the last parameter
4765                          */
4766                         argc--;
4767                 } else {
4768                         /*
4769                          * If this is not a valid number, just plow on.  The
4770                          * user will get a more informative error message later
4771                          * on.
4772                          */
4773                         interval = 0;
4774                 }
4775         }
4776
4777         /*
4778          * If the last argument is also an integer, then we have both a count
4779          * and an interval.
4780          */
4781         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
4782                 char *end;
4783
4784                 errno = 0;
4785                 count = interval;
4786                 interval = strtof(argv[argc - 1], &end);
4787
4788                 if (*end == '\0' && errno == 0) {
4789                         if (interval == 0) {
4790                                 (void) fprintf(stderr, gettext("interval "
4791                                     "cannot be zero\n"));
4792                                 usage(B_FALSE);
4793                         }
4794
4795                         /*
4796                          * Ignore the last parameter
4797                          */
4798                         argc--;
4799                 } else {
4800                         interval = 0;
4801                 }
4802         }
4803
4804         *iv = interval;
4805         *cnt = count;
4806         *argcp = argc;
4807 }
4808
4809 static void
4810 get_timestamp_arg(char c)
4811 {
4812         if (c == 'u')
4813                 timestamp_fmt = UDATE;
4814         else if (c == 'd')
4815                 timestamp_fmt = DDATE;
4816         else
4817                 usage(B_FALSE);
4818 }
4819
4820 /*
4821  * Return stat flags that are supported by all pools by both the module and
4822  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
4823  * It will get ANDed down until only the flags that are supported on all pools
4824  * remain.
4825  */
4826 static int
4827 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
4828 {
4829         uint64_t *mask = data;
4830         nvlist_t *config, *nvroot, *nvx;
4831         uint64_t flags = 0;
4832         int i, j;
4833
4834         config = zpool_get_config(zhp, NULL);
4835         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4836             &nvroot) == 0);
4837
4838         /* Default stats are always supported, but for completeness.. */
4839         if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
4840                 flags |= IOS_DEFAULT_M;
4841
4842         /* Get our extended stats nvlist from the main list */
4843         if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
4844             &nvx) != 0) {
4845                 /*
4846                  * No extended stats; they're probably running an older
4847                  * module.  No big deal, we support that too.
4848                  */
4849                 goto end;
4850         }
4851
4852         /* For each extended stat, make sure all its nvpairs are supported */
4853         for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
4854                 if (!vsx_type_to_nvlist[j][0])
4855                         continue;
4856
4857                 /* Start off by assuming the flag is supported, then check */
4858                 flags |= (1ULL << j);
4859                 for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
4860                         if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
4861                                 /* flag isn't supported */
4862                                 flags = flags & ~(1ULL  << j);
4863                                 break;
4864                         }
4865                 }
4866         }
4867 end:
4868         *mask = *mask & flags;
4869         return (0);
4870 }
4871
4872 /*
4873  * Return a bitmask of stats that are supported on all pools by both the module
4874  * and zpool iostat.
4875  */
4876 static uint64_t
4877 get_stat_flags(zpool_list_t *list)
4878 {
4879         uint64_t mask = -1;
4880
4881         /*
4882          * get_stat_flags_cb() will lop off bits from "mask" until only the
4883          * flags that are supported on all pools remain.
4884          */
4885         pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
4886         return (mask);
4887 }
4888
4889 /*
4890  * Return 1 if cb_data->cb_vdev_names[0] is this vdev's name, 0 otherwise.
4891  */
4892 static int
4893 is_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_data)
4894 {
4895         iostat_cbdata_t *cb = cb_data;
4896         char *name = NULL;
4897         int ret = 0;
4898
4899         name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
4900
4901         if (strcmp(name, cb->cb_vdev_names[0]) == 0)
4902                 ret = 1; /* match */
4903         free(name);
4904
4905         return (ret);
4906 }
4907
4908 /*
4909  * Returns 1 if cb_data->cb_vdev_names[0] is a vdev name, 0 otherwise.
4910  */
4911 static int
4912 is_vdev(zpool_handle_t *zhp, void *cb_data)
4913 {
4914         return (for_each_vdev(zhp, is_vdev_cb, cb_data));
4915 }
4916
4917 /*
4918  * Check if vdevs are in a pool
4919  *
4920  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
4921  * return 0.  If pool_name is NULL, then search all pools.
4922  */
4923 static int
4924 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
4925     iostat_cbdata_t *cb)
4926 {
4927         char **tmp_name;
4928         int ret = 0;
4929         int i;
4930         int pool_count = 0;
4931
4932         if ((argc == 0) || !*argv)
4933                 return (0);
4934
4935         if (pool_name)
4936                 pool_count = 1;
4937
4938         /* Temporarily hijack cb_vdev_names for a second... */
4939         tmp_name = cb->cb_vdev_names;
4940
4941         /* Go though our list of prospective vdev names */
4942         for (i = 0; i < argc; i++) {
4943                 cb->cb_vdev_names = argv + i;
4944
4945                 /* Is this name a vdev in our pools? */
4946                 ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
4947                     is_vdev, cb);
4948                 if (!ret) {
4949                         /* No match */
4950                         break;
4951                 }
4952         }
4953
4954         cb->cb_vdev_names = tmp_name;
4955
4956         return (ret);
4957 }
4958
4959 static int
4960 is_pool_cb(zpool_handle_t *zhp, void *data)
4961 {
4962         char *name = data;
4963         if (strcmp(name, zpool_get_name(zhp)) == 0)
4964                 return (1);
4965
4966         return (0);
4967 }
4968
4969 /*
4970  * Do we have a pool named *name?  If so, return 1, otherwise 0.
4971  */
4972 static int
4973 is_pool(char *name)
4974 {
4975         return (for_each_pool(0, NULL, B_TRUE, NULL,  is_pool_cb, name));
4976 }
4977
4978 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
4979 static int
4980 are_all_pools(int argc, char **argv)
4981 {
4982         if ((argc == 0) || !*argv)
4983                 return (0);
4984
4985         while (--argc >= 0)
4986                 if (!is_pool(argv[argc]))
4987                         return (0);
4988
4989         return (1);
4990 }
4991
4992 /*
4993  * Helper function to print out vdev/pool names we can't resolve.  Used for an
4994  * error message.
4995  */
4996 static void
4997 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
4998     iostat_cbdata_t *cb)
4999 {
5000         int i;
5001         char *name;
5002         char *str;
5003         for (i = 0; i < argc; i++) {
5004                 name = argv[i];
5005
5006                 if (is_pool(name))
5007                         str = gettext("pool");
5008                 else if (are_vdevs_in_pool(1, &name, pool_name, cb))
5009                         str = gettext("vdev in this pool");
5010                 else if (are_vdevs_in_pool(1, &name, NULL, cb))
5011                         str = gettext("vdev in another pool");
5012                 else
5013                         str = gettext("unknown");
5014
5015                 fprintf(stderr, "\t%s (%s)\n", name, str);
5016         }
5017 }
5018
5019 /*
5020  * Same as get_interval_count(), but with additional checks to not misinterpret
5021  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
5022  * cb.cb_name_flags.
5023  */
5024 static void
5025 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
5026     unsigned long *count, iostat_cbdata_t *cb)
5027 {
5028         char **tmpargv = argv;
5029         int argc_for_interval = 0;
5030
5031         /* Is the last arg an interval value?  Or a guid? */
5032         if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL, cb)) {
5033                 /*
5034                  * The last arg is not a guid, so it's probably an
5035                  * interval value.
5036                  */
5037                 argc_for_interval++;
5038
5039                 if (*argc >= 2 &&
5040                     !are_vdevs_in_pool(1, &argv[*argc - 2], NULL, cb)) {
5041                         /*
5042                          * The 2nd to last arg is not a guid, so it's probably
5043                          * an interval value.
5044                          */
5045                         argc_for_interval++;
5046                 }
5047         }
5048
5049         /* Point to our list of possible intervals */
5050         tmpargv = &argv[*argc - argc_for_interval];
5051
5052         *argc = *argc - argc_for_interval;
5053         get_interval_count(&argc_for_interval, tmpargv,
5054             interval, count);
5055 }
5056
5057 /*
5058  * Floating point sleep().  Allows you to pass in a floating point value for
5059  * seconds.
5060  */
5061 static void
5062 fsleep(float sec)
5063 {
5064         struct timespec req;
5065         req.tv_sec = floor(sec);
5066         req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
5067         nanosleep(&req, NULL);
5068 }
5069
5070 /*
5071  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
5072  * if we were unable to determine its size.
5073  */
5074 static int
5075 terminal_height(void)
5076 {
5077         struct winsize win;
5078
5079         if (isatty(STDOUT_FILENO) == 0)
5080                 return (-1);
5081
5082         if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) != -1 && win.ws_row > 0)
5083                 return (win.ws_row);
5084
5085         return (-1);
5086 }
5087
5088 /*
5089  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
5090  * print the result.
5091  *
5092  * name:        Short name of the script ('iostat').
5093  * path:        Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
5094  */
5095 static void
5096 print_zpool_script_help(char *name, char *path)
5097 {
5098         char *argv[] = {path, "-h", NULL};
5099         char **lines = NULL;
5100         int lines_cnt = 0;
5101         int rc;
5102
5103         rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
5104             &lines_cnt);
5105         if (rc != 0 || lines == NULL || lines_cnt <= 0) {
5106                 if (lines != NULL)
5107                         libzfs_free_str_array(lines, lines_cnt);
5108                 return;
5109         }
5110
5111         for (int i = 0; i < lines_cnt; i++)
5112                 if (!is_blank_str(lines[i]))
5113                         printf("  %-14s  %s\n", name, lines[i]);
5114
5115         libzfs_free_str_array(lines, lines_cnt);
5116 }
5117
5118 /*
5119  * Go though the zpool status/iostat -c scripts in the user's path, run their
5120  * help option (-h), and print out the results.
5121  */
5122 static void
5123 print_zpool_dir_scripts(char *dirpath)
5124 {
5125         DIR *dir;
5126         struct dirent *ent;
5127         char fullpath[MAXPATHLEN];
5128         struct stat dir_stat;
5129
5130         if ((dir = opendir(dirpath)) != NULL) {
5131                 /* print all the files and directories within directory */
5132                 while ((ent = readdir(dir)) != NULL) {
5133                         sprintf(fullpath, "%s/%s", dirpath, ent->d_name);
5134
5135                         /* Print the scripts */
5136                         if (stat(fullpath, &dir_stat) == 0)
5137                                 if (dir_stat.st_mode & S_IXUSR &&
5138                                     S_ISREG(dir_stat.st_mode))
5139                                         print_zpool_script_help(ent->d_name,
5140                                             fullpath);
5141                 }
5142                 closedir(dir);
5143         }
5144 }
5145
5146 /*
5147  * Print out help text for all zpool status/iostat -c scripts.
5148  */
5149 static void
5150 print_zpool_script_list(char *subcommand)
5151 {
5152         char *dir, *sp;
5153
5154         printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
5155
5156         sp = zpool_get_cmd_search_path();
5157         if (sp == NULL)
5158                 return;
5159
5160         dir = strtok(sp, ":");
5161         while (dir != NULL) {
5162                 print_zpool_dir_scripts(dir);
5163                 dir = strtok(NULL, ":");
5164         }
5165
5166         free(sp);
5167 }
5168
5169 /*
5170  * Set the minimum pool/vdev name column width.  The width must be at least 10,
5171  * but may be as large as the column width - 42 so it still fits on one line.
5172  * NOTE: 42 is the width of the default capacity/operations/bandwidth output
5173  */
5174 static int
5175 get_namewidth_iostat(zpool_handle_t *zhp, void *data)
5176 {
5177         iostat_cbdata_t *cb = data;
5178         int width, available_width;
5179
5180         /*
5181          * get_namewidth() returns the maximum width of any name in that column
5182          * for any pool/vdev/device line that will be output.
5183          */
5184         width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
5185             cb->cb_verbose);
5186
5187         /*
5188          * The width we are calculating is the width of the header and also the
5189          * padding width for names that are less than maximum width.  The stats
5190          * take up 42 characters, so the width available for names is:
5191          */
5192         available_width = get_columns() - 42;
5193
5194         /*
5195          * If the maximum width fits on a screen, then great!  Make everything
5196          * line up by justifying all lines to the same width.  If that max
5197          * width is larger than what's available, the name plus stats won't fit
5198          * on one line, and justifying to that width would cause every line to
5199          * wrap on the screen.  We only want lines with long names to wrap.
5200          * Limit the padding to what won't wrap.
5201          */
5202         if (width > available_width)
5203                 width = available_width;
5204
5205         /*
5206          * And regardless of whatever the screen width is (get_columns can
5207          * return 0 if the width is not known or less than 42 for a narrow
5208          * terminal) have the width be a minimum of 10.
5209          */
5210         if (width < 10)
5211                 width = 10;
5212
5213         /* Save the calculated width */
5214         cb->cb_namewidth = width;
5215
5216         return (0);
5217 }
5218
5219 /*
5220  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
5221  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
5222  *              [interval [count]]
5223  *
5224  *      -c CMD  For each vdev, run command CMD
5225  *      -g      Display guid for individual vdev name.
5226  *      -L      Follow links when resolving vdev path name.
5227  *      -P      Display full path for vdev name.
5228  *      -v      Display statistics for individual vdevs
5229  *      -h      Display help
5230  *      -p      Display values in parsable (exact) format.
5231  *      -H      Scripted mode.  Don't display headers, and separate properties
5232  *              by a single tab.
5233  *      -l      Display average latency
5234  *      -q      Display queue depths
5235  *      -w      Display latency histograms
5236  *      -r      Display request size histogram
5237  *      -T      Display a timestamp in date(1) or Unix format
5238  *      -n      Only print headers once
5239  *
5240  * This command can be tricky because we want to be able to deal with pool
5241  * creation/destruction as well as vdev configuration changes.  The bulk of this
5242  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
5243  * on pool_list_update() to detect the addition of new pools.  Configuration
5244  * changes are all handled within libzfs.
5245  */
5246 int
5247 zpool_do_iostat(int argc, char **argv)
5248 {
5249         int c;
5250         int ret;
5251         int npools;
5252         float interval = 0;
5253         unsigned long count = 0;
5254         int winheight = 24;
5255         zpool_list_t *list;
5256         boolean_t verbose = B_FALSE;
5257         boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
5258         boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
5259         boolean_t omit_since_boot = B_FALSE;
5260         boolean_t guid = B_FALSE;
5261         boolean_t follow_links = B_FALSE;
5262         boolean_t full_name = B_FALSE;
5263         boolean_t headers_once = B_FALSE;
5264         iostat_cbdata_t cb = { 0 };
5265         char *cmd = NULL;
5266
5267         /* Used for printing error message */
5268         const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
5269             [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
5270
5271         uint64_t unsupported_flags;
5272
5273         /* check options */
5274         while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwnH")) != -1) {
5275                 switch (c) {
5276                 case 'c':
5277                         if (cmd != NULL) {
5278                                 fprintf(stderr,
5279                                     gettext("Can't set -c flag twice\n"));
5280                                 exit(1);
5281                         }
5282
5283                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
5284                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
5285                                 fprintf(stderr, gettext(
5286                                     "Can't run -c, disabled by "
5287                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
5288                                 exit(1);
5289                         }
5290
5291                         if ((getuid() <= 0 || geteuid() <= 0) &&
5292                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
5293                                 fprintf(stderr, gettext(
5294                                     "Can't run -c with root privileges "
5295                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
5296                                 exit(1);
5297                         }
5298                         cmd = optarg;
5299                         verbose = B_TRUE;
5300                         break;
5301                 case 'g':
5302                         guid = B_TRUE;
5303                         break;
5304                 case 'L':
5305                         follow_links = B_TRUE;
5306                         break;
5307                 case 'P':
5308                         full_name = B_TRUE;
5309                         break;
5310                 case 'T':
5311                         get_timestamp_arg(*optarg);
5312                         break;
5313                 case 'v':
5314                         verbose = B_TRUE;
5315                         break;
5316                 case 'p':
5317                         parsable = B_TRUE;
5318                         break;
5319                 case 'l':
5320                         latency = B_TRUE;
5321                         break;
5322                 case 'q':
5323                         queues = B_TRUE;
5324                         break;
5325                 case 'H':
5326                         scripted = B_TRUE;
5327                         break;
5328                 case 'w':
5329                         l_histo = B_TRUE;
5330                         break;
5331                 case 'r':
5332                         rq_histo = B_TRUE;
5333                         break;
5334                 case 'y':
5335                         omit_since_boot = B_TRUE;
5336                         break;
5337                 case 'n':
5338                         headers_once = B_TRUE;
5339                         break;
5340                 case 'h':
5341                         usage(B_FALSE);
5342                         break;
5343                 case '?':
5344                         if (optopt == 'c') {
5345                                 print_zpool_script_list("iostat");
5346                                 exit(0);
5347                         } else {
5348                                 fprintf(stderr,
5349                                     gettext("invalid option '%c'\n"), optopt);
5350                         }
5351                         usage(B_FALSE);
5352                 }
5353         }
5354
5355         argc -= optind;
5356         argv += optind;
5357
5358         cb.cb_literal = parsable;
5359         cb.cb_scripted = scripted;
5360
5361         if (guid)
5362                 cb.cb_name_flags |= VDEV_NAME_GUID;
5363         if (follow_links)
5364                 cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5365         if (full_name)
5366                 cb.cb_name_flags |= VDEV_NAME_PATH;
5367         cb.cb_iteration = 0;
5368         cb.cb_namewidth = 0;
5369         cb.cb_verbose = verbose;
5370
5371         /* Get our interval and count values (if any) */
5372         if (guid) {
5373                 get_interval_count_filter_guids(&argc, argv, &interval,
5374                     &count, &cb);
5375         } else {
5376                 get_interval_count(&argc, argv, &interval, &count);
5377         }
5378
5379         if (argc == 0) {
5380                 /* No args, so just print the defaults. */
5381         } else if (are_all_pools(argc, argv)) {
5382                 /* All the args are pool names */
5383         } else if (are_vdevs_in_pool(argc, argv, NULL, &cb)) {
5384                 /* All the args are vdevs */
5385                 cb.cb_vdev_names = argv;
5386                 cb.cb_vdev_names_count = argc;
5387                 argc = 0; /* No pools to process */
5388         } else if (are_all_pools(1, argv)) {
5389                 /* The first arg is a pool name */
5390                 if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0], &cb)) {
5391                         /* ...and the rest are vdev names */
5392                         cb.cb_vdev_names = argv + 1;
5393                         cb.cb_vdev_names_count = argc - 1;
5394                         argc = 1; /* One pool to process */
5395                 } else {
5396                         fprintf(stderr, gettext("Expected either a list of "));
5397                         fprintf(stderr, gettext("pools, or list of vdevs in"));
5398                         fprintf(stderr, " \"%s\", ", argv[0]);
5399                         fprintf(stderr, gettext("but got:\n"));
5400                         error_list_unresolved_vdevs(argc - 1, argv + 1,
5401                             argv[0], &cb);
5402                         fprintf(stderr, "\n");
5403                         usage(B_FALSE);
5404                         return (1);
5405                 }
5406         } else {
5407                 /*
5408                  * The args don't make sense. The first arg isn't a pool name,
5409                  * nor are all the args vdevs.
5410                  */
5411                 fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
5412                 fprintf(stderr, "\n");
5413                 return (1);
5414         }
5415
5416         if (cb.cb_vdev_names_count != 0) {
5417                 /*
5418                  * If user specified vdevs, it implies verbose.
5419                  */
5420                 cb.cb_verbose = B_TRUE;
5421         }
5422
5423         /*
5424          * Construct the list of all interesting pools.
5425          */
5426         ret = 0;
5427         if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
5428                 return (1);
5429
5430         if (pool_list_count(list) == 0 && argc != 0) {
5431                 pool_list_free(list);
5432                 return (1);
5433         }
5434
5435         if (pool_list_count(list) == 0 && interval == 0) {
5436                 pool_list_free(list);
5437                 (void) fprintf(stderr, gettext("no pools available\n"));
5438                 return (1);
5439         }
5440
5441         if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
5442                 pool_list_free(list);
5443                 (void) fprintf(stderr,
5444                     gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
5445                 usage(B_FALSE);
5446                 return (1);
5447         }
5448
5449         if (l_histo && rq_histo) {
5450                 pool_list_free(list);
5451                 (void) fprintf(stderr,
5452                     gettext("Only one of [-r|-w] can be passed at a time\n"));
5453                 usage(B_FALSE);
5454                 return (1);
5455         }
5456
5457         /*
5458          * Enter the main iostat loop.
5459          */
5460         cb.cb_list = list;
5461
5462         if (l_histo) {
5463                 /*
5464                  * Histograms tables look out of place when you try to display
5465                  * them with the other stats, so make a rule that you can only
5466                  * print histograms by themselves.
5467                  */
5468                 cb.cb_flags = IOS_L_HISTO_M;
5469         } else if (rq_histo) {
5470                 cb.cb_flags = IOS_RQ_HISTO_M;
5471         } else {
5472                 cb.cb_flags = IOS_DEFAULT_M;
5473                 if (latency)
5474                         cb.cb_flags |= IOS_LATENCY_M;
5475                 if (queues)
5476                         cb.cb_flags |= IOS_QUEUES_M;
5477         }
5478
5479         /*
5480          * See if the module supports all the stats we want to display.
5481          */
5482         unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
5483         if (unsupported_flags) {
5484                 uint64_t f;
5485                 int idx;
5486                 fprintf(stderr,
5487                     gettext("The loaded zfs module doesn't support:"));
5488
5489                 /* for each bit set in unsupported_flags */
5490                 for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
5491                         idx = lowbit64(f) - 1;
5492                         fprintf(stderr, " -%c", flag_to_arg[idx]);
5493                 }
5494
5495                 fprintf(stderr, ".  Try running a newer module.\n");
5496                 pool_list_free(list);
5497
5498                 return (1);
5499         }
5500
5501         for (;;) {
5502                 if ((npools = pool_list_count(list)) == 0)
5503                         (void) fprintf(stderr, gettext("no pools available\n"));
5504                 else {
5505                         /*
5506                          * If this is the first iteration and -y was supplied
5507                          * we skip any printing.
5508                          */
5509                         boolean_t skip = (omit_since_boot &&
5510                             cb.cb_iteration == 0);
5511
5512                         /*
5513                          * Refresh all statistics.  This is done as an
5514                          * explicit step before calculating the maximum name
5515                          * width, so that any * configuration changes are
5516                          * properly accounted for.
5517                          */
5518                         (void) pool_list_iter(list, B_FALSE, refresh_iostat,
5519                             &cb);
5520
5521                         /*
5522                          * Iterate over all pools to determine the maximum width
5523                          * for the pool / device name column across all pools.
5524                          */
5525                         cb.cb_namewidth = 0;
5526                         (void) pool_list_iter(list, B_FALSE,
5527                             get_namewidth_iostat, &cb);
5528
5529                         if (timestamp_fmt != NODATE)
5530                                 print_timestamp(timestamp_fmt);
5531
5532                         if (cmd != NULL && cb.cb_verbose &&
5533                             !(cb.cb_flags & IOS_ANYHISTO_M)) {
5534                                 cb.vcdl = all_pools_for_each_vdev_run(argc,
5535                                     argv, cmd, g_zfs, cb.cb_vdev_names,
5536                                     cb.cb_vdev_names_count, cb.cb_name_flags);
5537                         } else {
5538                                 cb.vcdl = NULL;
5539                         }
5540
5541
5542                         /*
5543                          * Check terminal size so we can print headers
5544                          * even when terminal window has its height
5545                          * changed.
5546                          */
5547                         winheight = terminal_height();
5548                         /*
5549                          * Are we connected to TTY? If not, headers_once
5550                          * should be true, to avoid breaking scripts.
5551                          */
5552                         if (winheight < 0)
5553                                 headers_once = B_TRUE;
5554
5555                         /*
5556                          * If it's the first time and we're not skipping it,
5557                          * or either skip or verbose mode, print the header.
5558                          *
5559                          * The histogram code explicitly prints its header on
5560                          * every vdev, so skip this for histograms.
5561                          */
5562                         if (((++cb.cb_iteration == 1 && !skip) ||
5563                             (skip != verbose) ||
5564                             (!headers_once &&
5565                             (cb.cb_iteration % winheight) == 0)) &&
5566                             (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
5567                             !cb.cb_scripted)
5568                                 print_iostat_header(&cb);
5569
5570                         if (skip) {
5571                                 (void) fsleep(interval);
5572                                 continue;
5573                         }
5574
5575                         pool_list_iter(list, B_FALSE, print_iostat, &cb);
5576
5577                         /*
5578                          * If there's more than one pool, and we're not in
5579                          * verbose mode (which prints a separator for us),
5580                          * then print a separator.
5581                          *
5582                          * In addition, if we're printing specific vdevs then
5583                          * we also want an ending separator.
5584                          */
5585                         if (((npools > 1 && !verbose &&
5586                             !(cb.cb_flags & IOS_ANYHISTO_M)) ||
5587                             (!(cb.cb_flags & IOS_ANYHISTO_M) &&
5588                             cb.cb_vdev_names_count)) &&
5589                             !cb.cb_scripted) {
5590                                 print_iostat_separator(&cb);
5591                                 if (cb.vcdl != NULL)
5592                                         print_cmd_columns(cb.vcdl, 1);
5593                                 printf("\n");
5594                         }
5595
5596                         if (cb.vcdl != NULL)
5597                                 free_vdev_cmd_data_list(cb.vcdl);
5598
5599                 }
5600
5601                 /*
5602                  * Flush the output so that redirection to a file isn't buffered
5603                  * indefinitely.
5604                  */
5605                 (void) fflush(stdout);
5606
5607                 if (interval == 0)
5608                         break;
5609
5610                 if (count != 0 && --count == 0)
5611                         break;
5612
5613                 (void) fsleep(interval);
5614         }
5615
5616         pool_list_free(list);
5617
5618         return (ret);
5619 }
5620
5621 typedef struct list_cbdata {
5622         boolean_t       cb_verbose;
5623         int             cb_name_flags;
5624         int             cb_namewidth;
5625         boolean_t       cb_scripted;
5626         zprop_list_t    *cb_proplist;
5627         boolean_t       cb_literal;
5628 } list_cbdata_t;
5629
5630
5631 /*
5632  * Given a list of columns to display, output appropriate headers for each one.
5633  */
5634 static void
5635 print_header(list_cbdata_t *cb)
5636 {
5637         zprop_list_t *pl = cb->cb_proplist;
5638         char headerbuf[ZPOOL_MAXPROPLEN];
5639         const char *header;
5640         boolean_t first = B_TRUE;
5641         boolean_t right_justify;
5642         size_t width = 0;
5643
5644         for (; pl != NULL; pl = pl->pl_next) {
5645                 width = pl->pl_width;
5646                 if (first && cb->cb_verbose) {
5647                         /*
5648                          * Reset the width to accommodate the verbose listing
5649                          * of devices.
5650                          */
5651                         width = cb->cb_namewidth;
5652                 }
5653
5654                 if (!first)
5655                         (void) printf("  ");
5656                 else
5657                         first = B_FALSE;
5658
5659                 right_justify = B_FALSE;
5660                 if (pl->pl_prop != ZPROP_INVAL) {
5661                         header = zpool_prop_column_name(pl->pl_prop);
5662                         right_justify = zpool_prop_align_right(pl->pl_prop);
5663                 } else {
5664                         int i;
5665
5666                         for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
5667                                 headerbuf[i] = toupper(pl->pl_user_prop[i]);
5668                         headerbuf[i] = '\0';
5669                         header = headerbuf;
5670                 }
5671
5672                 if (pl->pl_next == NULL && !right_justify)
5673                         (void) printf("%s", header);
5674                 else if (right_justify)
5675                         (void) printf("%*s", (int)width, header);
5676                 else
5677                         (void) printf("%-*s", (int)width, header);
5678         }
5679
5680         (void) printf("\n");
5681 }
5682
5683 /*
5684  * Given a pool and a list of properties, print out all the properties according
5685  * to the described layout. Used by zpool_do_list().
5686  */
5687 static void
5688 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
5689 {
5690         zprop_list_t *pl = cb->cb_proplist;
5691         boolean_t first = B_TRUE;
5692         char property[ZPOOL_MAXPROPLEN];
5693         char *propstr;
5694         boolean_t right_justify;
5695         size_t width;
5696
5697         for (; pl != NULL; pl = pl->pl_next) {
5698
5699                 width = pl->pl_width;
5700                 if (first && cb->cb_verbose) {
5701                         /*
5702                          * Reset the width to accommodate the verbose listing
5703                          * of devices.
5704                          */
5705                         width = cb->cb_namewidth;
5706                 }
5707
5708                 if (!first) {
5709                         if (cb->cb_scripted)
5710                                 (void) printf("\t");
5711                         else
5712                                 (void) printf("  ");
5713                 } else {
5714                         first = B_FALSE;
5715                 }
5716
5717                 right_justify = B_FALSE;
5718                 if (pl->pl_prop != ZPROP_INVAL) {
5719                         if (zpool_get_prop(zhp, pl->pl_prop, property,
5720                             sizeof (property), NULL, cb->cb_literal) != 0)
5721                                 propstr = "-";
5722                         else
5723                                 propstr = property;
5724
5725                         right_justify = zpool_prop_align_right(pl->pl_prop);
5726                 } else if ((zpool_prop_feature(pl->pl_user_prop) ||
5727                     zpool_prop_unsupported(pl->pl_user_prop)) &&
5728                     zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
5729                     sizeof (property)) == 0) {
5730                         propstr = property;
5731                 } else {
5732                         propstr = "-";
5733                 }
5734
5735
5736                 /*
5737                  * If this is being called in scripted mode, or if this is the
5738                  * last column and it is left-justified, don't include a width
5739                  * format specifier.
5740                  */
5741                 if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
5742                         (void) printf("%s", propstr);
5743                 else if (right_justify)
5744                         (void) printf("%*s", (int)width, propstr);
5745                 else
5746                         (void) printf("%-*s", (int)width, propstr);
5747         }
5748
5749         (void) printf("\n");
5750 }
5751
5752 static void
5753 print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
5754     boolean_t scripted, boolean_t valid, enum zfs_nicenum_format format)
5755 {
5756         char propval[64];
5757         boolean_t fixed;
5758         size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
5759
5760         switch (prop) {
5761         case ZPOOL_PROP_EXPANDSZ:
5762         case ZPOOL_PROP_CHECKPOINT:
5763         case ZPOOL_PROP_DEDUPRATIO:
5764                 if (value == 0)
5765                         (void) strlcpy(propval, "-", sizeof (propval));
5766                 else
5767                         zfs_nicenum_format(value, propval, sizeof (propval),
5768                             format);
5769                 break;
5770         case ZPOOL_PROP_FRAGMENTATION:
5771                 if (value == ZFS_FRAG_INVALID) {
5772                         (void) strlcpy(propval, "-", sizeof (propval));
5773                 } else if (format == ZFS_NICENUM_RAW) {
5774                         (void) snprintf(propval, sizeof (propval), "%llu",
5775                             (unsigned long long)value);
5776                 } else {
5777                         (void) snprintf(propval, sizeof (propval), "%llu%%",
5778                             (unsigned long long)value);
5779                 }
5780                 break;
5781         case ZPOOL_PROP_CAPACITY:
5782                 /* capacity value is in parts-per-10,000 (aka permyriad) */
5783                 if (format == ZFS_NICENUM_RAW)
5784                         (void) snprintf(propval, sizeof (propval), "%llu",
5785                             (unsigned long long)value / 100);
5786                 else
5787                         (void) snprintf(propval, sizeof (propval),
5788                             value < 1000 ? "%1.2f%%" : value < 10000 ?
5789                             "%2.1f%%" : "%3.0f%%", value / 100.0);
5790                 break;
5791         case ZPOOL_PROP_HEALTH:
5792                 width = 8;
5793                 snprintf(propval, sizeof (propval), "%-*s", (int)width, str);
5794                 break;
5795         default:
5796                 zfs_nicenum_format(value, propval, sizeof (propval), format);
5797         }
5798
5799         if (!valid)
5800                 (void) strlcpy(propval, "-", sizeof (propval));
5801
5802         if (scripted)
5803                 (void) printf("\t%s", propval);
5804         else
5805                 (void) printf("  %*s", (int)width, propval);
5806 }
5807
5808 /*
5809  * print static default line per vdev
5810  * not compatible with '-o' <proplist> option
5811  */
5812 static void
5813 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
5814     list_cbdata_t *cb, int depth, boolean_t isspare)
5815 {
5816         nvlist_t **child;
5817         vdev_stat_t *vs;
5818         uint_t c, children;
5819         char *vname;
5820         boolean_t scripted = cb->cb_scripted;
5821         uint64_t islog = B_FALSE;
5822         char *dashes = "%-*s      -      -      -        -         "
5823             "-      -      -      -  -\n";
5824
5825         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
5826             (uint64_t **)&vs, &c) == 0);
5827
5828         if (name != NULL) {
5829                 boolean_t toplevel = (vs->vs_space != 0);
5830                 uint64_t cap;
5831                 enum zfs_nicenum_format format;
5832                 const char *state;
5833
5834                 if (cb->cb_literal)
5835                         format = ZFS_NICENUM_RAW;
5836                 else
5837                         format = ZFS_NICENUM_1024;
5838
5839                 if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
5840                         return;
5841
5842                 if (scripted)
5843                         (void) printf("\t%s", name);
5844                 else if (strlen(name) + depth > cb->cb_namewidth)
5845                         (void) printf("%*s%s", depth, "", name);
5846                 else
5847                         (void) printf("%*s%s%*s", depth, "", name,
5848                             (int)(cb->cb_namewidth - strlen(name) - depth), "");
5849
5850                 /*
5851                  * Print the properties for the individual vdevs. Some
5852                  * properties are only applicable to toplevel vdevs. The
5853                  * 'toplevel' boolean value is passed to the print_one_column()
5854                  * to indicate that the value is valid.
5855                  */
5856                 print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, NULL, scripted,
5857                     toplevel, format);
5858                 print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
5859                     scripted, toplevel, format);
5860                 print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
5861                     NULL, scripted, toplevel, format);
5862                 print_one_column(ZPOOL_PROP_CHECKPOINT,
5863                     vs->vs_checkpoint_space, NULL, scripted, toplevel, format);
5864                 print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
5865                     scripted, B_TRUE, format);
5866                 print_one_column(ZPOOL_PROP_FRAGMENTATION,
5867                     vs->vs_fragmentation, NULL, scripted,
5868                     (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
5869                     format);
5870                 cap = (vs->vs_space == 0) ? 0 :
5871                     (vs->vs_alloc * 10000 / vs->vs_space);
5872                 print_one_column(ZPOOL_PROP_CAPACITY, cap, NULL,
5873                     scripted, toplevel, format);
5874                 print_one_column(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
5875                     scripted, toplevel, format);
5876                 state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
5877                 if (isspare) {
5878                         if (vs->vs_aux == VDEV_AUX_SPARED)
5879                                 state = "INUSE";
5880                         else if (vs->vs_state == VDEV_STATE_HEALTHY)
5881                                 state = "AVAIL";
5882                 }
5883                 print_one_column(ZPOOL_PROP_HEALTH, 0, state, scripted,
5884                     B_TRUE, format);
5885                 (void) printf("\n");
5886         }
5887
5888         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
5889             &child, &children) != 0)
5890                 return;
5891
5892         /* list the normal vdevs first */
5893         for (c = 0; c < children; c++) {
5894                 uint64_t ishole = B_FALSE;
5895
5896                 if (nvlist_lookup_uint64(child[c],
5897                     ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
5898                         continue;
5899
5900                 if (nvlist_lookup_uint64(child[c],
5901                     ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
5902                         continue;
5903
5904                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
5905                         continue;
5906
5907                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
5908                     cb->cb_name_flags);
5909                 print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
5910                 free(vname);
5911         }
5912
5913         /* list the classes: 'logs', 'dedup', and 'special' */
5914         for (uint_t n = 0; n < 3; n++) {
5915                 boolean_t printed = B_FALSE;
5916
5917                 for (c = 0; c < children; c++) {
5918                         char *bias = NULL;
5919                         char *type = NULL;
5920
5921                         if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
5922                             &islog) == 0 && islog) {
5923                                 bias = VDEV_ALLOC_CLASS_LOGS;
5924                         } else {
5925                                 (void) nvlist_lookup_string(child[c],
5926                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
5927                                 (void) nvlist_lookup_string(child[c],
5928                                     ZPOOL_CONFIG_TYPE, &type);
5929                         }
5930                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
5931                                 continue;
5932                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
5933                                 continue;
5934
5935                         if (!printed) {
5936                                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5937                                 (void) printf(dashes, cb->cb_namewidth,
5938                                     class_name[n]);
5939                                 printed = B_TRUE;
5940                         }
5941                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5942                             cb->cb_name_flags);
5943                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
5944                             B_FALSE);
5945                         free(vname);
5946                 }
5947         }
5948
5949         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
5950             &child, &children) == 0 && children > 0) {
5951                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5952                 (void) printf(dashes, cb->cb_namewidth, "cache");
5953                 for (c = 0; c < children; c++) {
5954                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5955                             cb->cb_name_flags);
5956                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
5957                             B_FALSE);
5958                         free(vname);
5959                 }
5960         }
5961
5962         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
5963             &children) == 0 && children > 0) {
5964                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5965                 (void) printf(dashes, cb->cb_namewidth, "spare");
5966                 for (c = 0; c < children; c++) {
5967                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5968                             cb->cb_name_flags);
5969                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
5970                             B_TRUE);
5971                         free(vname);
5972                 }
5973         }
5974 }
5975
5976 /*
5977  * Generic callback function to list a pool.
5978  */
5979 static int
5980 list_callback(zpool_handle_t *zhp, void *data)
5981 {
5982         list_cbdata_t *cbp = data;
5983
5984         print_pool(zhp, cbp);
5985
5986         if (cbp->cb_verbose) {
5987                 nvlist_t *config, *nvroot;
5988
5989                 config = zpool_get_config(zhp, NULL);
5990                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5991                     &nvroot) == 0);
5992                 print_list_stats(zhp, NULL, nvroot, cbp, 0, B_FALSE);
5993         }
5994
5995         return (0);
5996 }
5997
5998 /*
5999  * Set the minimum pool/vdev name column width.  The width must be at least 9,
6000  * but may be as large as needed.
6001  */
6002 static int
6003 get_namewidth_list(zpool_handle_t *zhp, void *data)
6004 {
6005         list_cbdata_t *cb = data;
6006         int width;
6007
6008         width = get_namewidth(zhp, cb->cb_namewidth, cb->cb_name_flags,
6009             cb->cb_verbose);
6010
6011         if (width < 9)
6012                 width = 9;
6013
6014         cb->cb_namewidth = width;
6015
6016         return (0);
6017 }
6018
6019 /*
6020  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
6021  *
6022  *      -g      Display guid for individual vdev name.
6023  *      -H      Scripted mode.  Don't display headers, and separate properties
6024  *              by a single tab.
6025  *      -L      Follow links when resolving vdev path name.
6026  *      -o      List of properties to display.  Defaults to
6027  *              "name,size,allocated,free,expandsize,fragmentation,capacity,"
6028  *              "dedupratio,health,altroot"
6029  *      -p      Display values in parsable (exact) format.
6030  *      -P      Display full path for vdev name.
6031  *      -T      Display a timestamp in date(1) or Unix format
6032  *
6033  * List all pools in the system, whether or not they're healthy.  Output space
6034  * statistics for each one, as well as health status summary.
6035  */
6036 int
6037 zpool_do_list(int argc, char **argv)
6038 {
6039         int c;
6040         int ret = 0;
6041         list_cbdata_t cb = { 0 };
6042         static char default_props[] =
6043             "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
6044             "capacity,dedupratio,health,altroot";
6045         char *props = default_props;
6046         float interval = 0;
6047         unsigned long count = 0;
6048         zpool_list_t *list;
6049         boolean_t first = B_TRUE;
6050
6051         /* check options */
6052         while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
6053                 switch (c) {
6054                 case 'g':
6055                         cb.cb_name_flags |= VDEV_NAME_GUID;
6056                         break;
6057                 case 'H':
6058                         cb.cb_scripted = B_TRUE;
6059                         break;
6060                 case 'L':
6061                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6062                         break;
6063                 case 'o':
6064                         props = optarg;
6065                         break;
6066                 case 'P':
6067                         cb.cb_name_flags |= VDEV_NAME_PATH;
6068                         break;
6069                 case 'p':
6070                         cb.cb_literal = B_TRUE;
6071                         break;
6072                 case 'T':
6073                         get_timestamp_arg(*optarg);
6074                         break;
6075                 case 'v':
6076                         cb.cb_verbose = B_TRUE;
6077                         cb.cb_namewidth = 8;    /* 8 until precalc is avail */
6078                         break;
6079                 case ':':
6080                         (void) fprintf(stderr, gettext("missing argument for "
6081                             "'%c' option\n"), optopt);
6082                         usage(B_FALSE);
6083                         break;
6084                 case '?':
6085                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6086                             optopt);
6087                         usage(B_FALSE);
6088                 }
6089         }
6090
6091         argc -= optind;
6092         argv += optind;
6093
6094         get_interval_count(&argc, argv, &interval, &count);
6095
6096         if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
6097                 usage(B_FALSE);
6098
6099         for (;;) {
6100                 if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
6101                     &ret)) == NULL)
6102                         return (1);
6103
6104                 if (pool_list_count(list) == 0)
6105                         break;
6106
6107                 cb.cb_namewidth = 0;
6108                 (void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);
6109
6110                 if (timestamp_fmt != NODATE)
6111                         print_timestamp(timestamp_fmt);
6112
6113                 if (!cb.cb_scripted && (first || cb.cb_verbose)) {
6114                         print_header(&cb);
6115                         first = B_FALSE;
6116                 }
6117                 ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
6118
6119                 if (interval == 0)
6120                         break;
6121
6122                 if (count != 0 && --count == 0)
6123                         break;
6124
6125                 pool_list_free(list);
6126                 (void) fsleep(interval);
6127         }
6128
6129         if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
6130                 (void) printf(gettext("no pools available\n"));
6131                 ret = 0;
6132         }
6133
6134         pool_list_free(list);
6135         zprop_free_list(cb.cb_proplist);
6136         return (ret);
6137 }
6138
6139 static int
6140 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
6141 {
6142         boolean_t force = B_FALSE;
6143         boolean_t rebuild = B_FALSE;
6144         boolean_t wait = B_FALSE;
6145         int c;
6146         nvlist_t *nvroot;
6147         char *poolname, *old_disk, *new_disk;
6148         zpool_handle_t *zhp;
6149         nvlist_t *props = NULL;
6150         char *propval;
6151         int ret;
6152
6153         /* check options */
6154         while ((c = getopt(argc, argv, "fo:sw")) != -1) {
6155                 switch (c) {
6156                 case 'f':
6157                         force = B_TRUE;
6158                         break;
6159                 case 'o':
6160                         if ((propval = strchr(optarg, '=')) == NULL) {
6161                                 (void) fprintf(stderr, gettext("missing "
6162                                     "'=' for -o option\n"));
6163                                 usage(B_FALSE);
6164                         }
6165                         *propval = '\0';
6166                         propval++;
6167
6168                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
6169                             (add_prop_list(optarg, propval, &props, B_TRUE)))
6170                                 usage(B_FALSE);
6171                         break;
6172                 case 's':
6173                         rebuild = B_TRUE;
6174                         break;
6175                 case 'w':
6176                         wait = B_TRUE;
6177                         break;
6178                 case '?':
6179                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6180                             optopt);
6181                         usage(B_FALSE);
6182                 }
6183         }
6184
6185         argc -= optind;
6186         argv += optind;
6187
6188         /* get pool name and check number of arguments */
6189         if (argc < 1) {
6190                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6191                 usage(B_FALSE);
6192         }
6193
6194         poolname = argv[0];
6195
6196         if (argc < 2) {
6197                 (void) fprintf(stderr,
6198                     gettext("missing <device> specification\n"));
6199                 usage(B_FALSE);
6200         }
6201
6202         old_disk = argv[1];
6203
6204         if (argc < 3) {
6205                 if (!replacing) {
6206                         (void) fprintf(stderr,
6207                             gettext("missing <new_device> specification\n"));
6208                         usage(B_FALSE);
6209                 }
6210                 new_disk = old_disk;
6211                 argc -= 1;
6212                 argv += 1;
6213         } else {
6214                 new_disk = argv[2];
6215                 argc -= 2;
6216                 argv += 2;
6217         }
6218
6219         if (argc > 1) {
6220                 (void) fprintf(stderr, gettext("too many arguments\n"));
6221                 usage(B_FALSE);
6222         }
6223
6224         if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
6225                 nvlist_free(props);
6226                 return (1);
6227         }
6228
6229         if (zpool_get_config(zhp, NULL) == NULL) {
6230                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
6231                     poolname);
6232                 zpool_close(zhp);
6233                 nvlist_free(props);
6234                 return (1);
6235         }
6236
6237         /* unless manually specified use "ashift" pool property (if set) */
6238         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
6239                 int intval;
6240                 zprop_source_t src;
6241                 char strval[ZPOOL_MAXPROPLEN];
6242
6243                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
6244                 if (src != ZPROP_SRC_DEFAULT) {
6245                         (void) sprintf(strval, "%" PRId32, intval);
6246                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
6247                             &props, B_TRUE) == 0);
6248                 }
6249         }
6250
6251         nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
6252             argc, argv);
6253         if (nvroot == NULL) {
6254                 zpool_close(zhp);
6255                 nvlist_free(props);
6256                 return (1);
6257         }
6258
6259         ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
6260             rebuild);
6261
6262         if (ret == 0 && wait)
6263                 ret = zpool_wait(zhp,
6264                     replacing ? ZPOOL_WAIT_REPLACE : ZPOOL_WAIT_RESILVER);
6265
6266         nvlist_free(props);
6267         nvlist_free(nvroot);
6268         zpool_close(zhp);
6269
6270         return (ret);
6271 }
6272
6273 /*
6274  * zpool replace [-fsw] [-o property=value] <pool> <device> <new_device>
6275  *
6276  *      -f      Force attach, even if <new_device> appears to be in use.
6277  *      -s      Use sequential instead of healing reconstruction for resilver.
6278  *      -o      Set property=value.
6279  *      -w      Wait for replacing to complete before returning
6280  *
6281  * Replace <device> with <new_device>.
6282  */
6283 /* ARGSUSED */
6284 int
6285 zpool_do_replace(int argc, char **argv)
6286 {
6287         return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
6288 }
6289
6290 /*
6291  * zpool attach [-fsw] [-o property=value] <pool> <device> <new_device>
6292  *
6293  *      -f      Force attach, even if <new_device> appears to be in use.
6294  *      -s      Use sequential instead of healing reconstruction for resilver.
6295  *      -o      Set property=value.
6296  *      -w      Wait for resilvering to complete before returning
6297  *
6298  * Attach <new_device> to the mirror containing <device>.  If <device> is not
6299  * part of a mirror, then <device> will be transformed into a mirror of
6300  * <device> and <new_device>.  In either case, <new_device> will begin life
6301  * with a DTL of [0, now], and will immediately begin to resilver itself.
6302  */
6303 int
6304 zpool_do_attach(int argc, char **argv)
6305 {
6306         return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
6307 }
6308
6309 /*
6310  * zpool detach [-f] <pool> <device>
6311  *
6312  *      -f      Force detach of <device>, even if DTLs argue against it
6313  *              (not supported yet)
6314  *
6315  * Detach a device from a mirror.  The operation will be refused if <device>
6316  * is the last device in the mirror, or if the DTLs indicate that this device
6317  * has the only valid copy of some data.
6318  */
6319 /* ARGSUSED */
6320 int
6321 zpool_do_detach(int argc, char **argv)
6322 {
6323         int c;
6324         char *poolname, *path;
6325         zpool_handle_t *zhp;
6326         int ret;
6327
6328         /* check options */
6329         while ((c = getopt(argc, argv, "")) != -1) {
6330                 switch (c) {
6331                 case '?':
6332                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6333                             optopt);
6334                         usage(B_FALSE);
6335                 }
6336         }
6337
6338         argc -= optind;
6339         argv += optind;
6340
6341         /* get pool name and check number of arguments */
6342         if (argc < 1) {
6343                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6344                 usage(B_FALSE);
6345         }
6346
6347         if (argc < 2) {
6348                 (void) fprintf(stderr,
6349                     gettext("missing <device> specification\n"));
6350                 usage(B_FALSE);
6351         }
6352
6353         poolname = argv[0];
6354         path = argv[1];
6355
6356         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6357                 return (1);
6358
6359         ret = zpool_vdev_detach(zhp, path);
6360
6361         zpool_close(zhp);
6362
6363         return (ret);
6364 }
6365
6366 /*
6367  * zpool split [-gLnP] [-o prop=val] ...
6368  *              [-o mntopt] ...
6369  *              [-R altroot] <pool> <newpool> [<device> ...]
6370  *
6371  *      -g      Display guid for individual vdev name.
6372  *      -L      Follow links when resolving vdev path name.
6373  *      -n      Do not split the pool, but display the resulting layout if
6374  *              it were to be split.
6375  *      -o      Set property=value, or set mount options.
6376  *      -P      Display full path for vdev name.
6377  *      -R      Mount the split-off pool under an alternate root.
6378  *      -l      Load encryption keys while importing.
6379  *
6380  * Splits the named pool and gives it the new pool name.  Devices to be split
6381  * off may be listed, provided that no more than one device is specified
6382  * per top-level vdev mirror.  The newly split pool is left in an exported
6383  * state unless -R is specified.
6384  *
6385  * Restrictions: the top-level of the pool pool must only be made up of
6386  * mirrors; all devices in the pool must be healthy; no device may be
6387  * undergoing a resilvering operation.
6388  */
6389 int
6390 zpool_do_split(int argc, char **argv)
6391 {
6392         char *srcpool, *newpool, *propval;
6393         char *mntopts = NULL;
6394         splitflags_t flags;
6395         int c, ret = 0;
6396         boolean_t loadkeys = B_FALSE;
6397         zpool_handle_t *zhp;
6398         nvlist_t *config, *props = NULL;
6399
6400         flags.dryrun = B_FALSE;
6401         flags.import = B_FALSE;
6402         flags.name_flags = 0;
6403
6404         /* check options */
6405         while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
6406                 switch (c) {
6407                 case 'g':
6408                         flags.name_flags |= VDEV_NAME_GUID;
6409                         break;
6410                 case 'L':
6411                         flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
6412                         break;
6413                 case 'R':
6414                         flags.import = B_TRUE;
6415                         if (add_prop_list(
6416                             zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
6417                             &props, B_TRUE) != 0) {
6418                                 nvlist_free(props);
6419                                 usage(B_FALSE);
6420                         }
6421                         break;
6422                 case 'l':
6423                         loadkeys = B_TRUE;
6424                         break;
6425                 case 'n':
6426                         flags.dryrun = B_TRUE;
6427                         break;
6428                 case 'o':
6429                         if ((propval = strchr(optarg, '=')) != NULL) {
6430                                 *propval = '\0';
6431                                 propval++;
6432                                 if (add_prop_list(optarg, propval,
6433                                     &props, B_TRUE) != 0) {
6434                                         nvlist_free(props);
6435                                         usage(B_FALSE);
6436                                 }
6437                         } else {
6438                                 mntopts = optarg;
6439                         }
6440                         break;
6441                 case 'P':
6442                         flags.name_flags |= VDEV_NAME_PATH;
6443                         break;
6444                 case ':':
6445                         (void) fprintf(stderr, gettext("missing argument for "
6446                             "'%c' option\n"), optopt);
6447                         usage(B_FALSE);
6448                         break;
6449                 case '?':
6450                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6451                             optopt);
6452                         usage(B_FALSE);
6453                         break;
6454                 }
6455         }
6456
6457         if (!flags.import && mntopts != NULL) {
6458                 (void) fprintf(stderr, gettext("setting mntopts is only "
6459                     "valid when importing the pool\n"));
6460                 usage(B_FALSE);
6461         }
6462
6463         if (!flags.import && loadkeys) {
6464                 (void) fprintf(stderr, gettext("loading keys is only "
6465                     "valid when importing the pool\n"));
6466                 usage(B_FALSE);
6467         }
6468
6469         argc -= optind;
6470         argv += optind;
6471
6472         if (argc < 1) {
6473                 (void) fprintf(stderr, gettext("Missing pool name\n"));
6474                 usage(B_FALSE);
6475         }
6476         if (argc < 2) {
6477                 (void) fprintf(stderr, gettext("Missing new pool name\n"));
6478                 usage(B_FALSE);
6479         }
6480
6481         srcpool = argv[0];
6482         newpool = argv[1];
6483
6484         argc -= 2;
6485         argv += 2;
6486
6487         if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
6488                 nvlist_free(props);
6489                 return (1);
6490         }
6491
6492         config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
6493         if (config == NULL) {
6494                 ret = 1;
6495         } else {
6496                 if (flags.dryrun) {
6497                         (void) printf(gettext("would create '%s' with the "
6498                             "following layout:\n\n"), newpool);
6499                         print_vdev_tree(NULL, newpool, config, 0, "",
6500                             flags.name_flags);
6501                 }
6502         }
6503
6504         zpool_close(zhp);
6505
6506         if (ret != 0 || flags.dryrun || !flags.import) {
6507                 nvlist_free(config);
6508                 nvlist_free(props);
6509                 return (ret);
6510         }
6511
6512         /*
6513          * The split was successful. Now we need to open the new
6514          * pool and import it.
6515          */
6516         if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
6517                 nvlist_free(config);
6518                 nvlist_free(props);
6519                 return (1);
6520         }
6521
6522         if (loadkeys) {
6523                 ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
6524                 if (ret != 0)
6525                         ret = 1;
6526         }
6527
6528         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
6529             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
6530                 ret = 1;
6531                 (void) fprintf(stderr, gettext("Split was successful, but "
6532                     "the datasets could not all be mounted\n"));
6533                 (void) fprintf(stderr, gettext("Try doing '%s' with a "
6534                     "different altroot\n"), "zpool import");
6535         }
6536         zpool_close(zhp);
6537         nvlist_free(config);
6538         nvlist_free(props);
6539
6540         return (ret);
6541 }
6542
6543
6544
6545 /*
6546  * zpool online <pool> <device> ...
6547  */
6548 int
6549 zpool_do_online(int argc, char **argv)
6550 {
6551         int c, i;
6552         char *poolname;
6553         zpool_handle_t *zhp;
6554         int ret = 0;
6555         vdev_state_t newstate;
6556         int flags = 0;
6557
6558         /* check options */
6559         while ((c = getopt(argc, argv, "e")) != -1) {
6560                 switch (c) {
6561                 case 'e':
6562                         flags |= ZFS_ONLINE_EXPAND;
6563                         break;
6564                 case '?':
6565                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6566                             optopt);
6567                         usage(B_FALSE);
6568                 }
6569         }
6570
6571         argc -= optind;
6572         argv += optind;
6573
6574         /* get pool name and check number of arguments */
6575         if (argc < 1) {
6576                 (void) fprintf(stderr, gettext("missing pool name\n"));
6577                 usage(B_FALSE);
6578         }
6579         if (argc < 2) {
6580                 (void) fprintf(stderr, gettext("missing device name\n"));
6581                 usage(B_FALSE);
6582         }
6583
6584         poolname = argv[0];
6585
6586         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6587                 return (1);
6588
6589         for (i = 1; i < argc; i++) {
6590                 if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
6591                         if (newstate != VDEV_STATE_HEALTHY) {
6592                                 (void) printf(gettext("warning: device '%s' "
6593                                     "onlined, but remains in faulted state\n"),
6594                                     argv[i]);
6595                                 if (newstate == VDEV_STATE_FAULTED)
6596                                         (void) printf(gettext("use 'zpool "
6597                                             "clear' to restore a faulted "
6598                                             "device\n"));
6599                                 else
6600                                         (void) printf(gettext("use 'zpool "
6601                                             "replace' to replace devices "
6602                                             "that are no longer present\n"));
6603                         }
6604                 } else {
6605                         ret = 1;
6606                 }
6607         }
6608
6609         zpool_close(zhp);
6610
6611         return (ret);
6612 }
6613
6614 /*
6615  * zpool offline [-ft] <pool> <device> ...
6616  *
6617  *      -f      Force the device into a faulted state.
6618  *
6619  *      -t      Only take the device off-line temporarily.  The offline/faulted
6620  *              state will not be persistent across reboots.
6621  */
6622 /* ARGSUSED */
6623 int
6624 zpool_do_offline(int argc, char **argv)
6625 {
6626         int c, i;
6627         char *poolname;
6628         zpool_handle_t *zhp;
6629         int ret = 0;
6630         boolean_t istmp = B_FALSE;
6631         boolean_t fault = B_FALSE;
6632
6633         /* check options */
6634         while ((c = getopt(argc, argv, "ft")) != -1) {
6635                 switch (c) {
6636                 case 'f':
6637                         fault = B_TRUE;
6638                         break;
6639                 case 't':
6640                         istmp = B_TRUE;
6641                         break;
6642                 case '?':
6643                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6644                             optopt);
6645                         usage(B_FALSE);
6646                 }
6647         }
6648
6649         argc -= optind;
6650         argv += optind;
6651
6652         /* get pool name and check number of arguments */
6653         if (argc < 1) {
6654                 (void) fprintf(stderr, gettext("missing pool name\n"));
6655                 usage(B_FALSE);
6656         }
6657         if (argc < 2) {
6658                 (void) fprintf(stderr, gettext("missing device name\n"));
6659                 usage(B_FALSE);
6660         }
6661
6662         poolname = argv[0];
6663
6664         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6665                 return (1);
6666
6667         for (i = 1; i < argc; i++) {
6668                 if (fault) {
6669                         uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
6670                         vdev_aux_t aux;
6671                         if (istmp == B_FALSE) {
6672                                 /* Force the fault to persist across imports */
6673                                 aux = VDEV_AUX_EXTERNAL_PERSIST;
6674                         } else {
6675                                 aux = VDEV_AUX_EXTERNAL;
6676                         }
6677
6678                         if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
6679                                 ret = 1;
6680                 } else {
6681                         if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
6682                                 ret = 1;
6683                 }
6684         }
6685
6686         zpool_close(zhp);
6687
6688         return (ret);
6689 }
6690
6691 /*
6692  * zpool clear <pool> [device]
6693  *
6694  * Clear all errors associated with a pool or a particular device.
6695  */
6696 int
6697 zpool_do_clear(int argc, char **argv)
6698 {
6699         int c;
6700         int ret = 0;
6701         boolean_t dryrun = B_FALSE;
6702         boolean_t do_rewind = B_FALSE;
6703         boolean_t xtreme_rewind = B_FALSE;
6704         uint32_t rewind_policy = ZPOOL_NO_REWIND;
6705         nvlist_t *policy = NULL;
6706         zpool_handle_t *zhp;
6707         char *pool, *device;
6708
6709         /* check options */
6710         while ((c = getopt(argc, argv, "FnX")) != -1) {
6711                 switch (c) {
6712                 case 'F':
6713                         do_rewind = B_TRUE;
6714                         break;
6715                 case 'n':
6716                         dryrun = B_TRUE;
6717                         break;
6718                 case 'X':
6719                         xtreme_rewind = B_TRUE;
6720                         break;
6721                 case '?':
6722                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6723                             optopt);
6724                         usage(B_FALSE);
6725                 }
6726         }
6727
6728         argc -= optind;
6729         argv += optind;
6730
6731         if (argc < 1) {
6732                 (void) fprintf(stderr, gettext("missing pool name\n"));
6733                 usage(B_FALSE);
6734         }
6735
6736         if (argc > 2) {
6737                 (void) fprintf(stderr, gettext("too many arguments\n"));
6738                 usage(B_FALSE);
6739         }
6740
6741         if ((dryrun || xtreme_rewind) && !do_rewind) {
6742                 (void) fprintf(stderr,
6743                     gettext("-n or -X only meaningful with -F\n"));
6744                 usage(B_FALSE);
6745         }
6746         if (dryrun)
6747                 rewind_policy = ZPOOL_TRY_REWIND;
6748         else if (do_rewind)
6749                 rewind_policy = ZPOOL_DO_REWIND;
6750         if (xtreme_rewind)
6751                 rewind_policy |= ZPOOL_EXTREME_REWIND;
6752
6753         /* In future, further rewind policy choices can be passed along here */
6754         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
6755             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
6756             rewind_policy) != 0) {
6757                 return (1);
6758         }
6759
6760         pool = argv[0];
6761         device = argc == 2 ? argv[1] : NULL;
6762
6763         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
6764                 nvlist_free(policy);
6765                 return (1);
6766         }
6767
6768         if (zpool_clear(zhp, device, policy) != 0)
6769                 ret = 1;
6770
6771         zpool_close(zhp);
6772
6773         nvlist_free(policy);
6774
6775         return (ret);
6776 }
6777
6778 /*
6779  * zpool reguid <pool>
6780  */
6781 int
6782 zpool_do_reguid(int argc, char **argv)
6783 {
6784         int c;
6785         char *poolname;
6786         zpool_handle_t *zhp;
6787         int ret = 0;
6788
6789         /* check options */
6790         while ((c = getopt(argc, argv, "")) != -1) {
6791                 switch (c) {
6792                 case '?':
6793                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6794                             optopt);
6795                         usage(B_FALSE);
6796                 }
6797         }
6798
6799         argc -= optind;
6800         argv += optind;
6801
6802         /* get pool name and check number of arguments */
6803         if (argc < 1) {
6804                 (void) fprintf(stderr, gettext("missing pool name\n"));
6805                 usage(B_FALSE);
6806         }
6807
6808         if (argc > 1) {
6809                 (void) fprintf(stderr, gettext("too many arguments\n"));
6810                 usage(B_FALSE);
6811         }
6812
6813         poolname = argv[0];
6814         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6815                 return (1);
6816
6817         ret = zpool_reguid(zhp);
6818
6819         zpool_close(zhp);
6820         return (ret);
6821 }
6822
6823
6824 /*
6825  * zpool reopen <pool>
6826  *
6827  * Reopen the pool so that the kernel can update the sizes of all vdevs.
6828  */
6829 int
6830 zpool_do_reopen(int argc, char **argv)
6831 {
6832         int c;
6833         int ret = 0;
6834         boolean_t scrub_restart = B_TRUE;
6835
6836         /* check options */
6837         while ((c = getopt(argc, argv, "n")) != -1) {
6838                 switch (c) {
6839                 case 'n':
6840                         scrub_restart = B_FALSE;
6841                         break;
6842                 case '?':
6843                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6844                             optopt);
6845                         usage(B_FALSE);
6846                 }
6847         }
6848
6849         argc -= optind;
6850         argv += optind;
6851
6852         /* if argc == 0 we will execute zpool_reopen_one on all pools */
6853         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_reopen_one,
6854             &scrub_restart);
6855
6856         return (ret);
6857 }
6858
6859 typedef struct scrub_cbdata {
6860         int     cb_type;
6861         pool_scrub_cmd_t cb_scrub_cmd;
6862 } scrub_cbdata_t;
6863
6864 static boolean_t
6865 zpool_has_checkpoint(zpool_handle_t *zhp)
6866 {
6867         nvlist_t *config, *nvroot;
6868
6869         config = zpool_get_config(zhp, NULL);
6870
6871         if (config != NULL) {
6872                 pool_checkpoint_stat_t *pcs = NULL;
6873                 uint_t c;
6874
6875                 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6876                 (void) nvlist_lookup_uint64_array(nvroot,
6877                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
6878
6879                 if (pcs == NULL || pcs->pcs_state == CS_NONE)
6880                         return (B_FALSE);
6881
6882                 assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
6883                     pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
6884                 return (B_TRUE);
6885         }
6886
6887         return (B_FALSE);
6888 }
6889
6890 static int
6891 scrub_callback(zpool_handle_t *zhp, void *data)
6892 {
6893         scrub_cbdata_t *cb = data;
6894         int err;
6895
6896         /*
6897          * Ignore faulted pools.
6898          */
6899         if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
6900                 (void) fprintf(stderr, gettext("cannot scan '%s': pool is "
6901                     "currently unavailable\n"), zpool_get_name(zhp));
6902                 return (1);
6903         }
6904
6905         err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
6906
6907         if (err == 0 && zpool_has_checkpoint(zhp) &&
6908             cb->cb_type == POOL_SCAN_SCRUB) {
6909                 (void) printf(gettext("warning: will not scrub state that "
6910                     "belongs to the checkpoint of pool '%s'\n"),
6911                     zpool_get_name(zhp));
6912         }
6913
6914         return (err != 0);
6915 }
6916
6917 static int
6918 wait_callback(zpool_handle_t *zhp, void *data)
6919 {
6920         zpool_wait_activity_t *act = data;
6921         return (zpool_wait(zhp, *act));
6922 }
6923
6924 /*
6925  * zpool scrub [-s | -p] [-w] <pool> ...
6926  *
6927  *      -s      Stop.  Stops any in-progress scrub.
6928  *      -p      Pause. Pause in-progress scrub.
6929  *      -w      Wait.  Blocks until scrub has completed.
6930  */
6931 int
6932 zpool_do_scrub(int argc, char **argv)
6933 {
6934         int c;
6935         scrub_cbdata_t cb;
6936         boolean_t wait = B_FALSE;
6937         int error;
6938
6939         cb.cb_type = POOL_SCAN_SCRUB;
6940         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
6941
6942         /* check options */
6943         while ((c = getopt(argc, argv, "spw")) != -1) {
6944                 switch (c) {
6945                 case 's':
6946                         cb.cb_type = POOL_SCAN_NONE;
6947                         break;
6948                 case 'p':
6949                         cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
6950                         break;
6951                 case 'w':
6952                         wait = B_TRUE;
6953                         break;
6954                 case '?':
6955                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6956                             optopt);
6957                         usage(B_FALSE);
6958                 }
6959         }
6960
6961         if (cb.cb_type == POOL_SCAN_NONE &&
6962             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
6963                 (void) fprintf(stderr, gettext("invalid option combination: "
6964                     "-s and -p are mutually exclusive\n"));
6965                 usage(B_FALSE);
6966         }
6967
6968         if (wait && (cb.cb_type == POOL_SCAN_NONE ||
6969             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
6970                 (void) fprintf(stderr, gettext("invalid option combination: "
6971                     "-w cannot be used with -p or -s\n"));
6972                 usage(B_FALSE);
6973         }
6974
6975         argc -= optind;
6976         argv += optind;
6977
6978         if (argc < 1) {
6979                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6980                 usage(B_FALSE);
6981         }
6982
6983         error = for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb);
6984
6985         if (wait && !error) {
6986                 zpool_wait_activity_t act = ZPOOL_WAIT_SCRUB;
6987                 error = for_each_pool(argc, argv, B_TRUE, NULL, wait_callback,
6988                     &act);
6989         }
6990
6991         return (error);
6992 }
6993
6994 /*
6995  * zpool resilver <pool> ...
6996  *
6997  *      Restarts any in-progress resilver
6998  */
6999 int
7000 zpool_do_resilver(int argc, char **argv)
7001 {
7002         int c;
7003         scrub_cbdata_t cb;
7004
7005         cb.cb_type = POOL_SCAN_RESILVER;
7006         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7007
7008         /* check options */
7009         while ((c = getopt(argc, argv, "")) != -1) {
7010                 switch (c) {
7011                 case '?':
7012                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7013                             optopt);
7014                         usage(B_FALSE);
7015                 }
7016         }
7017
7018         argc -= optind;
7019         argv += optind;
7020
7021         if (argc < 1) {
7022                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7023                 usage(B_FALSE);
7024         }
7025
7026         return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
7027 }
7028
7029 /*
7030  * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
7031  *
7032  *      -c              Cancel. Ends any in-progress trim.
7033  *      -d              Secure trim.  Requires kernel and device support.
7034  *      -r <rate>       Sets the TRIM rate in bytes (per second). Supports
7035  *                      adding a multiplier suffix such as 'k' or 'm'.
7036  *      -s              Suspend. TRIM can then be restarted with no flags.
7037  *      -w              Wait. Blocks until trimming has completed.
7038  */
7039 int
7040 zpool_do_trim(int argc, char **argv)
7041 {
7042         struct option long_options[] = {
7043                 {"cancel",      no_argument,            NULL,   'c'},
7044                 {"secure",      no_argument,            NULL,   'd'},
7045                 {"rate",        required_argument,      NULL,   'r'},
7046                 {"suspend",     no_argument,            NULL,   's'},
7047                 {"wait",        no_argument,            NULL,   'w'},
7048                 {0, 0, 0, 0}
7049         };
7050
7051         pool_trim_func_t cmd_type = POOL_TRIM_START;
7052         uint64_t rate = 0;
7053         boolean_t secure = B_FALSE;
7054         boolean_t wait = B_FALSE;
7055
7056         int c;
7057         while ((c = getopt_long(argc, argv, "cdr:sw", long_options, NULL))
7058             != -1) {
7059                 switch (c) {
7060                 case 'c':
7061                         if (cmd_type != POOL_TRIM_START &&
7062                             cmd_type != POOL_TRIM_CANCEL) {
7063                                 (void) fprintf(stderr, gettext("-c cannot be "
7064                                     "combined with other options\n"));
7065                                 usage(B_FALSE);
7066                         }
7067                         cmd_type = POOL_TRIM_CANCEL;
7068                         break;
7069                 case 'd':
7070                         if (cmd_type != POOL_TRIM_START) {
7071                                 (void) fprintf(stderr, gettext("-d cannot be "
7072                                     "combined with the -c or -s options\n"));
7073                                 usage(B_FALSE);
7074                         }
7075                         secure = B_TRUE;
7076                         break;
7077                 case 'r':
7078                         if (cmd_type != POOL_TRIM_START) {
7079                                 (void) fprintf(stderr, gettext("-r cannot be "
7080                                     "combined with the -c or -s options\n"));
7081                                 usage(B_FALSE);
7082                         }
7083                         if (zfs_nicestrtonum(NULL, optarg, &rate) == -1) {
7084                                 (void) fprintf(stderr,
7085                                     gettext("invalid value for rate\n"));
7086                                 usage(B_FALSE);
7087                         }
7088                         break;
7089                 case 's':
7090                         if (cmd_type != POOL_TRIM_START &&
7091                             cmd_type != POOL_TRIM_SUSPEND) {
7092                                 (void) fprintf(stderr, gettext("-s cannot be "
7093                                     "combined with other options\n"));
7094                                 usage(B_FALSE);
7095                         }
7096                         cmd_type = POOL_TRIM_SUSPEND;
7097                         break;
7098                 case 'w':
7099                         wait = B_TRUE;
7100                         break;
7101                 case '?':
7102                         if (optopt != 0) {
7103                                 (void) fprintf(stderr,
7104                                     gettext("invalid option '%c'\n"), optopt);
7105                         } else {
7106                                 (void) fprintf(stderr,
7107                                     gettext("invalid option '%s'\n"),
7108                                     argv[optind - 1]);
7109                         }
7110                         usage(B_FALSE);
7111                 }
7112         }
7113
7114         argc -= optind;
7115         argv += optind;
7116
7117         if (argc < 1) {
7118                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7119                 usage(B_FALSE);
7120                 return (-1);
7121         }
7122
7123         if (wait && (cmd_type != POOL_TRIM_START)) {
7124                 (void) fprintf(stderr, gettext("-w cannot be used with -c or "
7125                     "-s\n"));
7126                 usage(B_FALSE);
7127         }
7128
7129         char *poolname = argv[0];
7130         zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
7131         if (zhp == NULL)
7132                 return (-1);
7133
7134         trimflags_t trim_flags = {
7135                 .secure = secure,
7136                 .rate = rate,
7137                 .wait = wait,
7138         };
7139
7140         nvlist_t *vdevs = fnvlist_alloc();
7141         if (argc == 1) {
7142                 /* no individual leaf vdevs specified, so add them all */
7143                 nvlist_t *config = zpool_get_config(zhp, NULL);
7144                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
7145                     ZPOOL_CONFIG_VDEV_TREE);
7146                 zpool_collect_leaves(zhp, nvroot, vdevs);
7147                 trim_flags.fullpool = B_TRUE;
7148         } else {
7149                 trim_flags.fullpool = B_FALSE;
7150                 for (int i = 1; i < argc; i++) {
7151                         fnvlist_add_boolean(vdevs, argv[i]);
7152                 }
7153         }
7154
7155         int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
7156
7157         fnvlist_free(vdevs);
7158         zpool_close(zhp);
7159
7160         return (error);
7161 }
7162
7163 /*
7164  * Converts a total number of seconds to a human readable string broken
7165  * down in to days/hours/minutes/seconds.
7166  */
7167 static void
7168 secs_to_dhms(uint64_t total, char *buf)
7169 {
7170         uint64_t days = total / 60 / 60 / 24;
7171         uint64_t hours = (total / 60 / 60) % 24;
7172         uint64_t mins = (total / 60) % 60;
7173         uint64_t secs = (total % 60);
7174
7175         if (days > 0) {
7176                 (void) sprintf(buf, "%llu days %02llu:%02llu:%02llu",
7177                     (u_longlong_t)days, (u_longlong_t)hours,
7178                     (u_longlong_t)mins, (u_longlong_t)secs);
7179         } else {
7180                 (void) sprintf(buf, "%02llu:%02llu:%02llu",
7181                     (u_longlong_t)hours, (u_longlong_t)mins,
7182                     (u_longlong_t)secs);
7183         }
7184 }
7185
7186 /*
7187  * Print out detailed scrub status.
7188  */
7189 static void
7190 print_scan_scrub_resilver_status(pool_scan_stat_t *ps)
7191 {
7192         time_t start, end, pause;
7193         uint64_t pass_scanned, scanned, pass_issued, issued, total;
7194         uint64_t elapsed, scan_rate, issue_rate;
7195         double fraction_done;
7196         char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
7197         char srate_buf[7], irate_buf[7], time_buf[32];
7198
7199         printf("  ");
7200         printf_color(ANSI_BOLD, gettext("scan:"));
7201         printf(" ");
7202
7203         /* If there's never been a scan, there's not much to say. */
7204         if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
7205             ps->pss_func >= POOL_SCAN_FUNCS) {
7206                 (void) printf(gettext("none requested\n"));
7207                 return;
7208         }
7209
7210         start = ps->pss_start_time;
7211         end = ps->pss_end_time;
7212         pause = ps->pss_pass_scrub_pause;
7213
7214         zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
7215
7216         assert(ps->pss_func == POOL_SCAN_SCRUB ||
7217             ps->pss_func == POOL_SCAN_RESILVER);
7218
7219         /* Scan is finished or canceled. */
7220         if (ps->pss_state == DSS_FINISHED) {
7221                 secs_to_dhms(end - start, time_buf);
7222
7223                 if (ps->pss_func == POOL_SCAN_SCRUB) {
7224                         (void) printf(gettext("scrub repaired %s "
7225                             "in %s with %llu errors on %s"), processed_buf,
7226                             time_buf, (u_longlong_t)ps->pss_errors,
7227                             ctime(&end));
7228                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
7229                         (void) printf(gettext("resilvered %s "
7230                             "in %s with %llu errors on %s"), processed_buf,
7231                             time_buf, (u_longlong_t)ps->pss_errors,
7232                             ctime(&end));
7233                 }
7234                 return;
7235         } else if (ps->pss_state == DSS_CANCELED) {
7236                 if (ps->pss_func == POOL_SCAN_SCRUB) {
7237                         (void) printf(gettext("scrub canceled on %s"),
7238                             ctime(&end));
7239                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
7240                         (void) printf(gettext("resilver canceled on %s"),
7241                             ctime(&end));
7242                 }
7243                 return;
7244         }
7245
7246         assert(ps->pss_state == DSS_SCANNING);
7247
7248         /* Scan is in progress. Resilvers can't be paused. */
7249         if (ps->pss_func == POOL_SCAN_SCRUB) {
7250                 if (pause == 0) {
7251                         (void) printf(gettext("scrub in progress since %s"),
7252                             ctime(&start));
7253                 } else {
7254                         (void) printf(gettext("scrub paused since %s"),
7255                             ctime(&pause));
7256                         (void) printf(gettext("\tscrub started on %s"),
7257                             ctime(&start));
7258                 }
7259         } else if (ps->pss_func == POOL_SCAN_RESILVER) {
7260                 (void) printf(gettext("resilver in progress since %s"),
7261                     ctime(&start));
7262         }
7263
7264         scanned = ps->pss_examined;
7265         pass_scanned = ps->pss_pass_exam;
7266         issued = ps->pss_issued;
7267         pass_issued = ps->pss_pass_issued;
7268         total = ps->pss_to_examine;
7269
7270         /* we are only done with a block once we have issued the IO for it */
7271         fraction_done = (double)issued / total;
7272
7273         /* elapsed time for this pass, rounding up to 1 if it's 0 */
7274         elapsed = time(NULL) - ps->pss_pass_start;
7275         elapsed -= ps->pss_pass_scrub_spent_paused;
7276         elapsed = (elapsed != 0) ? elapsed : 1;
7277
7278         scan_rate = pass_scanned / elapsed;
7279         issue_rate = pass_issued / elapsed;
7280         uint64_t total_secs_left = (issue_rate != 0 && total >= issued) ?
7281             ((total - issued) / issue_rate) : UINT64_MAX;
7282         secs_to_dhms(total_secs_left, time_buf);
7283
7284         /* format all of the numbers we will be reporting */
7285         zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
7286         zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
7287         zfs_nicebytes(total, total_buf, sizeof (total_buf));
7288         zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
7289         zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
7290
7291         /* do not print estimated time if we have a paused scrub */
7292         if (pause == 0) {
7293                 (void) printf(gettext("\t%s scanned at %s/s, "
7294                     "%s issued at %s/s, %s total\n"),
7295                     scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
7296         } else {
7297                 (void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
7298                     scanned_buf, issued_buf, total_buf);
7299         }
7300
7301         if (ps->pss_func == POOL_SCAN_RESILVER) {
7302                 (void) printf(gettext("\t%s resilvered, %.2f%% done"),
7303                     processed_buf, 100 * fraction_done);
7304         } else if (ps->pss_func == POOL_SCAN_SCRUB) {
7305                 (void) printf(gettext("\t%s repaired, %.2f%% done"),
7306                     processed_buf, 100 * fraction_done);
7307         }
7308
7309         if (pause == 0) {
7310                 if (total_secs_left != UINT64_MAX &&
7311                     issue_rate >= 10 * 1024 * 1024) {
7312                         (void) printf(gettext(", %s to go\n"), time_buf);
7313                 } else {
7314                         (void) printf(gettext(", no estimated "
7315                             "completion time\n"));
7316                 }
7317         } else {
7318                 (void) printf(gettext("\n"));
7319         }
7320 }
7321
7322 static void
7323 print_rebuild_status_impl(vdev_rebuild_stat_t *vrs, char *vdev_name)
7324 {
7325         if (vrs == NULL || vrs->vrs_state == VDEV_REBUILD_NONE)
7326                 return;
7327
7328         printf("  ");
7329         printf_color(ANSI_BOLD, gettext("scan:"));
7330         printf(" ");
7331
7332         uint64_t bytes_scanned = vrs->vrs_bytes_scanned;
7333         uint64_t bytes_issued = vrs->vrs_bytes_issued;
7334         uint64_t bytes_rebuilt = vrs->vrs_bytes_rebuilt;
7335         uint64_t bytes_est = vrs->vrs_bytes_est;
7336         uint64_t scan_rate = (vrs->vrs_pass_bytes_scanned /
7337             (vrs->vrs_pass_time_ms + 1)) * 1000;
7338         uint64_t issue_rate = (vrs->vrs_pass_bytes_issued /
7339             (vrs->vrs_pass_time_ms + 1)) * 1000;
7340         double scan_pct = MIN((double)bytes_scanned * 100 /
7341             (bytes_est + 1), 100);
7342
7343         /* Format all of the numbers we will be reporting */
7344         char bytes_scanned_buf[7], bytes_issued_buf[7];
7345         char bytes_rebuilt_buf[7], bytes_est_buf[7];
7346         char scan_rate_buf[7], issue_rate_buf[7], time_buf[32];
7347         zfs_nicebytes(bytes_scanned, bytes_scanned_buf,
7348             sizeof (bytes_scanned_buf));
7349         zfs_nicebytes(bytes_issued, bytes_issued_buf,
7350             sizeof (bytes_issued_buf));
7351         zfs_nicebytes(bytes_rebuilt, bytes_rebuilt_buf,
7352             sizeof (bytes_rebuilt_buf));
7353         zfs_nicebytes(bytes_est, bytes_est_buf, sizeof (bytes_est_buf));
7354         zfs_nicebytes(scan_rate, scan_rate_buf, sizeof (scan_rate_buf));
7355         zfs_nicebytes(issue_rate, issue_rate_buf, sizeof (issue_rate_buf));
7356
7357         time_t start = vrs->vrs_start_time;
7358         time_t end = vrs->vrs_end_time;
7359
7360         /* Rebuild is finished or canceled. */
7361         if (vrs->vrs_state == VDEV_REBUILD_COMPLETE) {
7362                 secs_to_dhms(vrs->vrs_scan_time_ms / 1000, time_buf);
7363                 (void) printf(gettext("resilvered (%s) %s in %s "
7364                     "with %llu errors on %s"), vdev_name, bytes_rebuilt_buf,
7365                     time_buf, (u_longlong_t)vrs->vrs_errors, ctime(&end));
7366                 return;
7367         } else if (vrs->vrs_state == VDEV_REBUILD_CANCELED) {
7368                 (void) printf(gettext("resilver (%s) canceled on %s"),
7369                     vdev_name, ctime(&end));
7370                 return;
7371         } else if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7372                 (void) printf(gettext("resilver (%s) in progress since %s"),
7373                     vdev_name, ctime(&start));
7374         }
7375
7376         assert(vrs->vrs_state == VDEV_REBUILD_ACTIVE);
7377
7378         secs_to_dhms(MAX((int64_t)bytes_est - (int64_t)bytes_scanned, 0) /
7379             MAX(scan_rate, 1), time_buf);
7380
7381         (void) printf(gettext("\t%s scanned at %s/s, %s issued %s/s, "
7382             "%s total\n"), bytes_scanned_buf, scan_rate_buf,
7383             bytes_issued_buf, issue_rate_buf, bytes_est_buf);
7384         (void) printf(gettext("\t%s resilvered, %.2f%% done"),
7385             bytes_rebuilt_buf, scan_pct);
7386
7387         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7388                 if (scan_rate >= 10 * 1024 * 1024) {
7389                         (void) printf(gettext(", %s to go\n"), time_buf);
7390                 } else {
7391                         (void) printf(gettext(", no estimated "
7392                             "completion time\n"));
7393                 }
7394         } else {
7395                 (void) printf(gettext("\n"));
7396         }
7397 }
7398
7399 /*
7400  * Print rebuild status for top-level vdevs.
7401  */
7402 static void
7403 print_rebuild_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7404 {
7405         nvlist_t **child;
7406         uint_t children;
7407
7408         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7409             &child, &children) != 0)
7410                 children = 0;
7411
7412         for (uint_t c = 0; c < children; c++) {
7413                 vdev_rebuild_stat_t *vrs;
7414                 uint_t i;
7415
7416                 if (nvlist_lookup_uint64_array(child[c],
7417                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7418                         char *name = zpool_vdev_name(g_zfs, zhp,
7419                             child[c], VDEV_NAME_TYPE_ID);
7420                         print_rebuild_status_impl(vrs, name);
7421                         free(name);
7422                 }
7423         }
7424 }
7425
7426 /*
7427  * As we don't scrub checkpointed blocks, we want to warn the user that we
7428  * skipped scanning some blocks if a checkpoint exists or existed at any
7429  * time during the scan.  If a sequential instead of healing reconstruction
7430  * was performed then the blocks were reconstructed.  However, their checksums
7431  * have not been verified so we still print the warning.
7432  */
7433 static void
7434 print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
7435 {
7436         if (ps == NULL || pcs == NULL)
7437                 return;
7438
7439         if (pcs->pcs_state == CS_NONE ||
7440             pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
7441                 return;
7442
7443         assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
7444
7445         if (ps->pss_state == DSS_NONE)
7446                 return;
7447
7448         if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
7449             ps->pss_end_time < pcs->pcs_start_time)
7450                 return;
7451
7452         if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
7453                 (void) printf(gettext("    scan warning: skipped blocks "
7454                     "that are only referenced by the checkpoint.\n"));
7455         } else {
7456                 assert(ps->pss_state == DSS_SCANNING);
7457                 (void) printf(gettext("    scan warning: skipping blocks "
7458                     "that are only referenced by the checkpoint.\n"));
7459         }
7460 }
7461
7462 /*
7463  * Returns B_TRUE if there is an active rebuild in progress.  Otherwise,
7464  * B_FALSE is returned and 'rebuild_end_time' is set to the end time for
7465  * the last completed (or cancelled) rebuild.
7466  */
7467 static boolean_t
7468 check_rebuilding(nvlist_t *nvroot, uint64_t *rebuild_end_time)
7469 {
7470         nvlist_t **child;
7471         uint_t children;
7472         boolean_t rebuilding = B_FALSE;
7473         uint64_t end_time = 0;
7474
7475         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7476             &child, &children) != 0)
7477                 children = 0;
7478
7479         for (uint_t c = 0; c < children; c++) {
7480                 vdev_rebuild_stat_t *vrs;
7481                 uint_t i;
7482
7483                 if (nvlist_lookup_uint64_array(child[c],
7484                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7485
7486                         if (vrs->vrs_end_time > end_time)
7487                                 end_time = vrs->vrs_end_time;
7488
7489                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7490                                 rebuilding = B_TRUE;
7491                                 end_time = 0;
7492                                 break;
7493                         }
7494                 }
7495         }
7496
7497         if (rebuild_end_time != NULL)
7498                 *rebuild_end_time = end_time;
7499
7500         return (rebuilding);
7501 }
7502
7503 /*
7504  * Print the scan status.
7505  */
7506 static void
7507 print_scan_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7508 {
7509         uint64_t rebuild_end_time = 0, resilver_end_time = 0;
7510         boolean_t have_resilver = B_FALSE, have_scrub = B_FALSE;
7511         boolean_t active_resilver = B_FALSE;
7512         pool_checkpoint_stat_t *pcs = NULL;
7513         pool_scan_stat_t *ps = NULL;
7514         uint_t c;
7515
7516         if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
7517             (uint64_t **)&ps, &c) == 0) {
7518                 if (ps->pss_func == POOL_SCAN_RESILVER) {
7519                         resilver_end_time = ps->pss_end_time;
7520                         active_resilver = (ps->pss_state == DSS_SCANNING);
7521                 }
7522
7523                 have_resilver = (ps->pss_func == POOL_SCAN_RESILVER);
7524                 have_scrub = (ps->pss_func == POOL_SCAN_SCRUB);
7525         }
7526
7527         boolean_t active_rebuild = check_rebuilding(nvroot, &rebuild_end_time);
7528         boolean_t have_rebuild = (active_rebuild || (rebuild_end_time > 0));
7529
7530         /* Always print the scrub status when available. */
7531         if (have_scrub)
7532                 print_scan_scrub_resilver_status(ps);
7533
7534         /*
7535          * When there is an active resilver or rebuild print its status.
7536          * Otherwise print the status of the last resilver or rebuild.
7537          */
7538         if (active_resilver || (!active_rebuild && have_resilver &&
7539             resilver_end_time && resilver_end_time > rebuild_end_time)) {
7540                 print_scan_scrub_resilver_status(ps);
7541         } else if (active_rebuild || (!active_resilver && have_rebuild &&
7542             rebuild_end_time && rebuild_end_time > resilver_end_time)) {
7543                 print_rebuild_status(zhp, nvroot);
7544         }
7545
7546         (void) nvlist_lookup_uint64_array(nvroot,
7547             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7548         print_checkpoint_scan_warning(ps, pcs);
7549 }
7550
7551 /*
7552  * Print out detailed removal status.
7553  */
7554 static void
7555 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
7556 {
7557         char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
7558         time_t start, end;
7559         nvlist_t *config, *nvroot;
7560         nvlist_t **child;
7561         uint_t children;
7562         char *vdev_name;
7563
7564         if (prs == NULL || prs->prs_state == DSS_NONE)
7565                 return;
7566
7567         /*
7568          * Determine name of vdev.
7569          */
7570         config = zpool_get_config(zhp, NULL);
7571         nvroot = fnvlist_lookup_nvlist(config,
7572             ZPOOL_CONFIG_VDEV_TREE);
7573         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7574             &child, &children) == 0);
7575         assert(prs->prs_removing_vdev < children);
7576         vdev_name = zpool_vdev_name(g_zfs, zhp,
7577             child[prs->prs_removing_vdev], B_TRUE);
7578
7579         (void) printf(gettext("remove: "));
7580
7581         start = prs->prs_start_time;
7582         end = prs->prs_end_time;
7583         zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
7584
7585         /*
7586          * Removal is finished or canceled.
7587          */
7588         if (prs->prs_state == DSS_FINISHED) {
7589                 uint64_t minutes_taken = (end - start) / 60;
7590
7591                 (void) printf(gettext("Removal of vdev %llu copied %s "
7592                     "in %lluh%um, completed on %s"),
7593                     (longlong_t)prs->prs_removing_vdev,
7594                     copied_buf,
7595                     (u_longlong_t)(minutes_taken / 60),
7596                     (uint_t)(minutes_taken % 60),
7597                     ctime((time_t *)&end));
7598         } else if (prs->prs_state == DSS_CANCELED) {
7599                 (void) printf(gettext("Removal of %s canceled on %s"),
7600                     vdev_name, ctime(&end));
7601         } else {
7602                 uint64_t copied, total, elapsed, mins_left, hours_left;
7603                 double fraction_done;
7604                 uint_t rate;
7605
7606                 assert(prs->prs_state == DSS_SCANNING);
7607
7608                 /*
7609                  * Removal is in progress.
7610                  */
7611                 (void) printf(gettext(
7612                     "Evacuation of %s in progress since %s"),
7613                     vdev_name, ctime(&start));
7614
7615                 copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
7616                 total = prs->prs_to_copy;
7617                 fraction_done = (double)copied / total;
7618
7619                 /* elapsed time for this pass */
7620                 elapsed = time(NULL) - prs->prs_start_time;
7621                 elapsed = elapsed > 0 ? elapsed : 1;
7622                 rate = copied / elapsed;
7623                 rate = rate > 0 ? rate : 1;
7624                 mins_left = ((total - copied) / rate) / 60;
7625                 hours_left = mins_left / 60;
7626
7627                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
7628                 zfs_nicenum(total, total_buf, sizeof (total_buf));
7629                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
7630
7631                 /*
7632                  * do not print estimated time if hours_left is more than
7633                  * 30 days
7634                  */
7635                 (void) printf(gettext("    %s copied out of %s at %s/s, "
7636                     "%.2f%% done"),
7637                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
7638                 if (hours_left < (30 * 24)) {
7639                         (void) printf(gettext(", %lluh%um to go\n"),
7640                             (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
7641                 } else {
7642                         (void) printf(gettext(
7643                             ", (copy is slow, no estimated time)\n"));
7644                 }
7645         }
7646         free(vdev_name);
7647
7648         if (prs->prs_mapping_memory > 0) {
7649                 char mem_buf[7];
7650                 zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
7651                 (void) printf(gettext("    %s memory used for "
7652                     "removed device mappings\n"),
7653                     mem_buf);
7654         }
7655 }
7656
7657 static void
7658 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
7659 {
7660         time_t start;
7661         char space_buf[7];
7662
7663         if (pcs == NULL || pcs->pcs_state == CS_NONE)
7664                 return;
7665
7666         (void) printf(gettext("checkpoint: "));
7667
7668         start = pcs->pcs_start_time;
7669         zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
7670
7671         if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
7672                 char *date = ctime(&start);
7673
7674                 /*
7675                  * ctime() adds a newline at the end of the generated
7676                  * string, thus the weird format specifier and the
7677                  * strlen() call used to chop it off from the output.
7678                  */
7679                 (void) printf(gettext("created %.*s, consumes %s\n"),
7680                     (int)(strlen(date) - 1), date, space_buf);
7681                 return;
7682         }
7683
7684         assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
7685
7686         (void) printf(gettext("discarding, %s remaining.\n"),
7687             space_buf);
7688 }
7689
7690 static void
7691 print_error_log(zpool_handle_t *zhp)
7692 {
7693         nvlist_t *nverrlist = NULL;
7694         nvpair_t *elem;
7695         char *pathname;
7696         size_t len = MAXPATHLEN * 2;
7697
7698         if (zpool_get_errlog(zhp, &nverrlist) != 0)
7699                 return;
7700
7701         (void) printf("errors: Permanent errors have been "
7702             "detected in the following files:\n\n");
7703
7704         pathname = safe_malloc(len);
7705         elem = NULL;
7706         while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
7707                 nvlist_t *nv;
7708                 uint64_t dsobj, obj;
7709
7710                 verify(nvpair_value_nvlist(elem, &nv) == 0);
7711                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
7712                     &dsobj) == 0);
7713                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
7714                     &obj) == 0);
7715                 zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
7716                 (void) printf("%7s %s\n", "", pathname);
7717         }
7718         free(pathname);
7719         nvlist_free(nverrlist);
7720 }
7721
7722 static void
7723 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
7724     uint_t nspares)
7725 {
7726         uint_t i;
7727         char *name;
7728
7729         if (nspares == 0)
7730                 return;
7731
7732         (void) printf(gettext("\tspares\n"));
7733
7734         for (i = 0; i < nspares; i++) {
7735                 name = zpool_vdev_name(g_zfs, zhp, spares[i],
7736                     cb->cb_name_flags);
7737                 print_status_config(zhp, cb, name, spares[i], 2, B_TRUE, NULL);
7738                 free(name);
7739         }
7740 }
7741
7742 static void
7743 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
7744     uint_t nl2cache)
7745 {
7746         uint_t i;
7747         char *name;
7748
7749         if (nl2cache == 0)
7750                 return;
7751
7752         (void) printf(gettext("\tcache\n"));
7753
7754         for (i = 0; i < nl2cache; i++) {
7755                 name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
7756                     cb->cb_name_flags);
7757                 print_status_config(zhp, cb, name, l2cache[i], 2,
7758                     B_FALSE, NULL);
7759                 free(name);
7760         }
7761 }
7762
7763 static void
7764 print_dedup_stats(nvlist_t *config)
7765 {
7766         ddt_histogram_t *ddh;
7767         ddt_stat_t *dds;
7768         ddt_object_t *ddo;
7769         uint_t c;
7770         char dspace[6], mspace[6];
7771
7772         /*
7773          * If the pool was faulted then we may not have been able to
7774          * obtain the config. Otherwise, if we have anything in the dedup
7775          * table continue processing the stats.
7776          */
7777         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
7778             (uint64_t **)&ddo, &c) != 0)
7779                 return;
7780
7781         (void) printf("\n");
7782         (void) printf(gettext(" dedup: "));
7783         if (ddo->ddo_count == 0) {
7784                 (void) printf(gettext("no DDT entries\n"));
7785                 return;
7786         }
7787
7788         zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
7789         zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
7790         (void) printf("DDT entries %llu, size %s on disk, %s in core\n",
7791             (u_longlong_t)ddo->ddo_count,
7792             dspace,
7793             mspace);
7794
7795         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
7796             (uint64_t **)&dds, &c) == 0);
7797         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
7798             (uint64_t **)&ddh, &c) == 0);
7799         zpool_dump_ddt(dds, ddh);
7800 }
7801
7802 /*
7803  * Display a summary of pool status.  Displays a summary such as:
7804  *
7805  *        pool: tank
7806  *      status: DEGRADED
7807  *      reason: One or more devices ...
7808  *         see: https://openzfs.github.io/openzfs-docs/msg/ZFS-xxxx-01
7809  *      config:
7810  *              mirror          DEGRADED
7811  *                c1t0d0        OK
7812  *                c2t0d0        UNAVAIL
7813  *
7814  * When given the '-v' option, we print out the complete config.  If the '-e'
7815  * option is specified, then we print out error rate information as well.
7816  */
7817 static int
7818 status_callback(zpool_handle_t *zhp, void *data)
7819 {
7820         status_cbdata_t *cbp = data;
7821         nvlist_t *config, *nvroot;
7822         char *msgid;
7823         zpool_status_t reason;
7824         zpool_errata_t errata;
7825         const char *health;
7826         uint_t c;
7827         vdev_stat_t *vs;
7828
7829         config = zpool_get_config(zhp, NULL);
7830         reason = zpool_get_status(zhp, &msgid, &errata);
7831
7832         cbp->cb_count++;
7833
7834         /*
7835          * If we were given 'zpool status -x', only report those pools with
7836          * problems.
7837          */
7838         if (cbp->cb_explain &&
7839             (reason == ZPOOL_STATUS_OK ||
7840             reason == ZPOOL_STATUS_VERSION_OLDER ||
7841             reason == ZPOOL_STATUS_FEAT_DISABLED)) {
7842                 if (!cbp->cb_allpools) {
7843                         (void) printf(gettext("pool '%s' is healthy\n"),
7844                             zpool_get_name(zhp));
7845                         if (cbp->cb_first)
7846                                 cbp->cb_first = B_FALSE;
7847                 }
7848                 return (0);
7849         }
7850
7851         if (cbp->cb_first)
7852                 cbp->cb_first = B_FALSE;
7853         else
7854                 (void) printf("\n");
7855
7856         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
7857         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
7858             (uint64_t **)&vs, &c) == 0);
7859
7860         health = zpool_get_state_str(zhp);
7861
7862         printf("  ");
7863         printf_color(ANSI_BOLD, gettext("pool:"));
7864         printf(" %s\n", zpool_get_name(zhp));
7865         printf(" ");
7866         printf_color(ANSI_BOLD, gettext("state: "));
7867
7868         printf_color(health_str_to_color(health), "%s", health);
7869
7870         printf("\n");
7871
7872         switch (reason) {
7873         case ZPOOL_STATUS_MISSING_DEV_R:
7874                 printf_color(ANSI_BOLD, gettext("status: "));
7875                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7876                     "not be opened.  Sufficient replicas exist for\n\tthe pool "
7877                     "to continue functioning in a degraded state.\n"));
7878                 printf_color(ANSI_BOLD, gettext("action: "));
7879                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
7880                     "and online it using 'zpool online'.\n"));
7881                 break;
7882
7883         case ZPOOL_STATUS_MISSING_DEV_NR:
7884                 printf_color(ANSI_BOLD, gettext("status: "));
7885                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7886                     "not be opened.  There are insufficient\n\treplicas for the"
7887                     " pool to continue functioning.\n"));
7888                 printf_color(ANSI_BOLD, gettext("action: "));
7889                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
7890                     "and online it using 'zpool online'.\n"));
7891                 break;
7892
7893         case ZPOOL_STATUS_CORRUPT_LABEL_R:
7894                 printf_color(ANSI_BOLD, gettext("status: "));
7895                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7896                     "not be used because the label is missing or\n\tinvalid.  "
7897                     "Sufficient replicas exist for the pool to continue\n\t"
7898                     "functioning in a degraded state.\n"));
7899                 printf_color(ANSI_BOLD, gettext("action: "));
7900                 printf_color(ANSI_YELLOW, gettext("Replace the device using "
7901                     "'zpool replace'.\n"));
7902                 break;
7903
7904         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
7905                 printf_color(ANSI_BOLD, gettext("status: "));
7906                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
7907                     "not be used because the label is missing \n\tor invalid.  "
7908                     "There are insufficient replicas for the pool to "
7909                     "continue\n\tfunctioning.\n"));
7910                 zpool_explain_recover(zpool_get_handle(zhp),
7911                     zpool_get_name(zhp), reason, config);
7912                 break;
7913
7914         case ZPOOL_STATUS_FAILING_DEV:
7915                 printf_color(ANSI_BOLD, gettext("status: "));
7916                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7917                     "experienced an unrecoverable error.  An\n\tattempt was "
7918                     "made to correct the error.  Applications are "
7919                     "unaffected.\n"));
7920                 printf_color(ANSI_BOLD, gettext("action: "));
7921                         printf_color(ANSI_YELLOW, gettext("Determine if the "
7922                     "device needs to be replaced, and clear the errors\n\tusing"
7923                     " 'zpool clear' or replace the device with 'zpool "
7924                     "replace'.\n"));
7925                 break;
7926
7927         case ZPOOL_STATUS_OFFLINE_DEV:
7928                 printf_color(ANSI_BOLD, gettext("status: "));
7929                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7930                     "been taken offline by the administrator.\n\tSufficient "
7931                     "replicas exist for the pool to continue functioning in "
7932                     "a\n\tdegraded state.\n"));
7933                 printf_color(ANSI_BOLD, gettext("action: "));
7934                 printf_color(ANSI_YELLOW, gettext("Online the device "
7935                     "using 'zpool online' or replace the device with\n\t'zpool "
7936                     "replace'.\n"));
7937                 break;
7938
7939         case ZPOOL_STATUS_REMOVED_DEV:
7940                 printf_color(ANSI_BOLD, gettext("status: "));
7941                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7942                     "been removed by the administrator.\n\tSufficient "
7943                     "replicas exist for the pool to continue functioning in "
7944                     "a\n\tdegraded state.\n"));
7945                 printf_color(ANSI_BOLD, gettext("action: "));
7946                 printf_color(ANSI_YELLOW, gettext("Online the device "
7947                     "using zpool online' or replace the device with\n\t'zpool "
7948                     "replace'.\n"));
7949                 break;
7950
7951         case ZPOOL_STATUS_RESILVERING:
7952         case ZPOOL_STATUS_REBUILDING:
7953                 printf_color(ANSI_BOLD, gettext("status: "));
7954                 printf_color(ANSI_YELLOW, gettext("One or more devices is "
7955                     "currently being resilvered.  The pool will\n\tcontinue "
7956                     "to function, possibly in a degraded state.\n"));
7957                 printf_color(ANSI_BOLD, gettext("action: "));
7958                 printf_color(ANSI_YELLOW, gettext("Wait for the resilver to "
7959                     "complete.\n"));
7960                 break;
7961
7962         case ZPOOL_STATUS_REBUILD_SCRUB:
7963                 printf_color(ANSI_BOLD, gettext("status: "));
7964                 printf_color(ANSI_YELLOW, gettext("One or more devices have "
7965                     "been sequentially resilvered, scrubbing\n\tthe pool "
7966                     "is recommended.\n"));
7967                 printf_color(ANSI_BOLD, gettext("action: "));
7968                 printf_color(ANSI_YELLOW, gettext("Use 'zpool scrub' to "
7969                     "verify all data checksums.\n"));
7970                 break;
7971
7972         case ZPOOL_STATUS_CORRUPT_DATA:
7973                 printf_color(ANSI_BOLD, gettext("status: "));
7974                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
7975                     "experienced an error resulting in data\n\tcorruption.  "
7976                     "Applications may be affected.\n"));
7977                 printf_color(ANSI_BOLD, gettext("action: "));
7978                 printf_color(ANSI_YELLOW, gettext("Restore the file in question"
7979                     " if possible.  Otherwise restore the\n\tentire pool from "
7980                     "backup.\n"));
7981                 break;
7982
7983         case ZPOOL_STATUS_CORRUPT_POOL:
7984                 printf_color(ANSI_BOLD, gettext("status: "));
7985                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
7986                     "corrupted and the pool cannot be opened.\n"));
7987                 zpool_explain_recover(zpool_get_handle(zhp),
7988                     zpool_get_name(zhp), reason, config);
7989                 break;
7990
7991         case ZPOOL_STATUS_VERSION_OLDER:
7992                 printf_color(ANSI_BOLD, gettext("status: "));
7993                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
7994                     "a legacy on-disk format.  The pool can\n\tstill be used, "
7995                     "but some features are unavailable.\n"));
7996                 printf_color(ANSI_BOLD, gettext("action: "));
7997                 printf_color(ANSI_YELLOW, gettext("Upgrade the pool using "
7998                     "'zpool upgrade'.  Once this is done, the\n\tpool will no "
7999                     "longer be accessible on software that does not support\n\t"
8000                     "feature flags.\n"));
8001                 break;
8002
8003         case ZPOOL_STATUS_VERSION_NEWER:
8004                 printf_color(ANSI_BOLD, gettext("status: "));
8005                 printf_color(ANSI_YELLOW, gettext("The pool has been upgraded "
8006                     "to a newer, incompatible on-disk version.\n\tThe pool "
8007                     "cannot be accessed on this system.\n"));
8008                 printf_color(ANSI_BOLD, gettext("action: "));
8009                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8010                     "system running more recent software, or\n\trestore the "
8011                     "pool from backup.\n"));
8012                 break;
8013
8014         case ZPOOL_STATUS_FEAT_DISABLED:
8015                 printf_color(ANSI_BOLD, gettext("status: "));
8016                 printf_color(ANSI_YELLOW, gettext("Some supported features are "
8017                     "not enabled on the pool. The pool can\n\tstill be used, "
8018                     "but some features are unavailable.\n"));
8019                 printf_color(ANSI_BOLD, gettext("action: "));
8020                 printf_color(ANSI_YELLOW, gettext("Enable all features using "
8021                     "'zpool upgrade'. Once this is done,\n\tthe pool may no "
8022                     "longer be accessible by software that does not support\n\t"
8023                     "the features. See zpool-features(5) for details.\n"));
8024                 break;
8025
8026         case ZPOOL_STATUS_UNSUP_FEAT_READ:
8027                 printf_color(ANSI_BOLD, gettext("status: "));
8028                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8029                     "on this system because it uses the\n\tfollowing feature(s)"
8030                     " not supported on this system:\n"));
8031                 zpool_print_unsup_feat(config);
8032                 (void) printf("\n");
8033                 printf_color(ANSI_BOLD, gettext("action: "));
8034                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8035                     "system that supports the required feature(s),\n\tor "
8036                     "restore the pool from backup.\n"));
8037                 break;
8038
8039         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
8040                 printf_color(ANSI_BOLD, gettext("status: "));
8041                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
8042                     "accessed in read-only mode on this system. It\n\tcannot be"
8043                     " accessed in read-write mode because it uses the "
8044                     "following\n\tfeature(s) not supported on this system:\n"));
8045                 zpool_print_unsup_feat(config);
8046                 (void) printf("\n");
8047                 printf_color(ANSI_BOLD, gettext("action: "));
8048                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8049                     "in read-write mode. Import the pool with\n"
8050                     "\t\"-o readonly=on\", access the pool from a system that "
8051                     "supports the\n\trequired feature(s), or restore the "
8052                     "pool from backup.\n"));
8053                 break;
8054
8055         case ZPOOL_STATUS_FAULTED_DEV_R:
8056                 printf_color(ANSI_BOLD, gettext("status: "));
8057                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8058                     "faulted in response to persistent errors.\n\tSufficient "
8059                     "replicas exist for the pool to continue functioning "
8060                     "in a\n\tdegraded state.\n"));
8061                 printf_color(ANSI_BOLD, gettext("action: "));
8062                 printf_color(ANSI_YELLOW, gettext("Replace the faulted device, "
8063                     "or use 'zpool clear' to mark the device\n\trepaired.\n"));
8064                 break;
8065
8066         case ZPOOL_STATUS_FAULTED_DEV_NR:
8067                 printf_color(ANSI_BOLD, gettext("status: "));
8068                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8069                     "faulted in response to persistent errors.  There are "
8070                     "insufficient replicas for the pool to\n\tcontinue "
8071                     "functioning.\n"));
8072                 printf_color(ANSI_BOLD, gettext("action: "));
8073                 printf_color(ANSI_YELLOW, gettext("Destroy and re-create the "
8074                     "pool from a backup source.  Manually marking the device\n"
8075                     "\trepaired using 'zpool clear' may allow some data "
8076                     "to be recovered.\n"));
8077                 break;
8078
8079         case ZPOOL_STATUS_IO_FAILURE_MMP:
8080                 printf_color(ANSI_BOLD, gettext("status: "));
8081                 printf_color(ANSI_YELLOW, gettext("The pool is suspended "
8082                     "because multihost writes failed or were delayed;\n\t"
8083                     "another system could import the pool undetected.\n"));
8084                 printf_color(ANSI_BOLD, gettext("action: "));
8085                 printf_color(ANSI_YELLOW, gettext("Make sure the pool's devices"
8086                     " are connected, then reboot your system and\n\timport the "
8087                     "pool.\n"));
8088                 break;
8089
8090         case ZPOOL_STATUS_IO_FAILURE_WAIT:
8091         case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
8092                 printf_color(ANSI_BOLD, gettext("status: "));
8093                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8094                     "faulted in response to IO failures.\n"));
8095                 printf_color(ANSI_BOLD, gettext("action: "));
8096                 printf_color(ANSI_YELLOW, gettext("Make sure the affected "
8097                     "devices are connected, then run 'zpool clear'.\n"));
8098                 break;
8099
8100         case ZPOOL_STATUS_BAD_LOG:
8101                 printf_color(ANSI_BOLD, gettext("status: "));
8102                 printf_color(ANSI_YELLOW, gettext("An intent log record "
8103                     "could not be read.\n"
8104                     "\tWaiting for administrator intervention to fix the "
8105                     "faulted pool.\n"));
8106                 printf_color(ANSI_BOLD, gettext("action: "));
8107                 printf_color(ANSI_YELLOW, gettext("Either restore the affected "
8108                     "device(s) and run 'zpool online',\n"
8109                     "\tor ignore the intent log records by running "
8110                     "'zpool clear'.\n"));
8111                 break;
8112
8113         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
8114                 (void) printf(gettext("status: One or more devices are "
8115                     "configured to use a non-native block size.\n"
8116                     "\tExpect reduced performance.\n"));
8117                 (void) printf(gettext("action: Replace affected devices with "
8118                     "devices that support the\n\tconfigured block size, or "
8119                     "migrate data to a properly configured\n\tpool.\n"));
8120                 break;
8121
8122         case ZPOOL_STATUS_HOSTID_MISMATCH:
8123                 printf_color(ANSI_BOLD, gettext("status: "));
8124                 printf_color(ANSI_YELLOW, gettext("Mismatch between pool hostid"
8125                     " and system hostid on imported pool.\n\tThis pool was "
8126                     "previously imported into a system with a different "
8127                     "hostid,\n\tand then was verbatim imported into this "
8128                     "system.\n"));
8129                 printf_color(ANSI_BOLD, gettext("action: "));
8130                 printf_color(ANSI_YELLOW, gettext("Export this pool on all "
8131                     "systems on which it is imported.\n"
8132                     "\tThen import it to correct the mismatch.\n"));
8133                 break;
8134
8135         case ZPOOL_STATUS_ERRATA:
8136                 printf_color(ANSI_BOLD, gettext("status: "));
8137                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
8138                     errata);
8139
8140                 switch (errata) {
8141                 case ZPOOL_ERRATA_NONE:
8142                         break;
8143
8144                 case ZPOOL_ERRATA_ZOL_2094_SCRUB:
8145                         printf_color(ANSI_BOLD, gettext("action: "));
8146                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
8147                             " run 'zpool scrub'.\n"));
8148                         break;
8149
8150                 case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
8151                         (void) printf(gettext("\tExisting encrypted datasets "
8152                             "contain an on-disk incompatibility\n\twhich "
8153                             "needs to be corrected.\n"));
8154                         printf_color(ANSI_BOLD, gettext("action: "));
8155                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
8156                             " backup existing encrypted datasets to new\n\t"
8157                             "encrypted datasets and destroy the old ones. "
8158                             "'zfs mount -o ro' can\n\tbe used to temporarily "
8159                             "mount existing encrypted datasets readonly.\n"));
8160                         break;
8161
8162                 case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
8163                         (void) printf(gettext("\tExisting encrypted snapshots "
8164                             "and bookmarks contain an on-disk\n\tincompat"
8165                             "ibility. This may cause on-disk corruption if "
8166                             "they are used\n\twith 'zfs recv'.\n"));
8167                         printf_color(ANSI_BOLD, gettext("action: "));
8168                         printf_color(ANSI_YELLOW, gettext("To correct the"
8169                             "issue, enable the bookmark_v2 feature. No "
8170                             "additional\n\taction is needed if there are no "
8171                             "encrypted snapshots or bookmarks.\n\tIf preserving"
8172                             "the encrypted snapshots and bookmarks is required,"
8173                             " use\n\ta non-raw send to backup and restore them."
8174                             " Alternately, they may be\n\tremoved to resolve "
8175                             "the incompatibility.\n"));
8176                         break;
8177
8178                 default:
8179                         /*
8180                          * All errata which allow the pool to be imported
8181                          * must contain an action message.
8182                          */
8183                         assert(0);
8184                 }
8185                 break;
8186
8187         default:
8188                 /*
8189                  * The remaining errors can't actually be generated, yet.
8190                  */
8191                 assert(reason == ZPOOL_STATUS_OK);
8192         }
8193
8194         if (msgid != NULL) {
8195                 printf("   ");
8196                 printf_color(ANSI_BOLD, gettext("see:"));
8197                 printf(gettext(
8198                     " https://openzfs.github.io/openzfs-docs/msg/%s\n"),
8199                     msgid);
8200         }
8201
8202         if (config != NULL) {
8203                 uint64_t nerr;
8204                 nvlist_t **spares, **l2cache;
8205                 uint_t nspares, nl2cache;
8206                 pool_checkpoint_stat_t *pcs = NULL;
8207                 pool_removal_stat_t *prs = NULL;
8208
8209                 print_scan_status(zhp, nvroot);
8210
8211                 (void) nvlist_lookup_uint64_array(nvroot,
8212                     ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
8213                 print_removal_status(zhp, prs);
8214
8215                 (void) nvlist_lookup_uint64_array(nvroot,
8216                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
8217                 print_checkpoint_status(pcs);
8218
8219                 cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
8220                     cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
8221                 if (cbp->cb_namewidth < 10)
8222                         cbp->cb_namewidth = 10;
8223
8224                 color_start(ANSI_BOLD);
8225                 (void) printf(gettext("config:\n\n"));
8226                 (void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
8227                     cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
8228                     "CKSUM");
8229                 color_end();
8230
8231                 if (cbp->cb_print_slow_ios) {
8232                         printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
8233                 }
8234
8235                 if (cbp->vcdl != NULL)
8236                         print_cmd_columns(cbp->vcdl, 0);
8237
8238                 printf("\n");
8239
8240                 print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
8241                     B_FALSE, NULL);
8242
8243                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
8244                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
8245                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
8246
8247                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
8248                     &l2cache, &nl2cache) == 0)
8249                         print_l2cache(zhp, cbp, l2cache, nl2cache);
8250
8251                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
8252                     &spares, &nspares) == 0)
8253                         print_spares(zhp, cbp, spares, nspares);
8254
8255                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
8256                     &nerr) == 0) {
8257                         nvlist_t *nverrlist = NULL;
8258
8259                         /*
8260                          * If the approximate error count is small, get a
8261                          * precise count by fetching the entire log and
8262                          * uniquifying the results.
8263                          */
8264                         if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
8265                             zpool_get_errlog(zhp, &nverrlist) == 0) {
8266                                 nvpair_t *elem;
8267
8268                                 elem = NULL;
8269                                 nerr = 0;
8270                                 while ((elem = nvlist_next_nvpair(nverrlist,
8271                                     elem)) != NULL) {
8272                                         nerr++;
8273                                 }
8274                         }
8275                         nvlist_free(nverrlist);
8276
8277                         (void) printf("\n");
8278
8279                         if (nerr == 0)
8280                                 (void) printf(gettext("errors: No known data "
8281                                     "errors\n"));
8282                         else if (!cbp->cb_verbose)
8283                                 (void) printf(gettext("errors: %llu data "
8284                                     "errors, use '-v' for a list\n"),
8285                                     (u_longlong_t)nerr);
8286                         else
8287                                 print_error_log(zhp);
8288                 }
8289
8290                 if (cbp->cb_dedup_stats)
8291                         print_dedup_stats(config);
8292         } else {
8293                 (void) printf(gettext("config: The configuration cannot be "
8294                     "determined.\n"));
8295         }
8296
8297         return (0);
8298 }
8299
8300 /*
8301  * zpool status [-c [script1,script2,...]] [-igLpPstvx] [-T d|u] [pool] ...
8302  *              [interval [count]]
8303  *
8304  *      -c CMD  For each vdev, run command CMD
8305  *      -i      Display vdev initialization status.
8306  *      -g      Display guid for individual vdev name.
8307  *      -L      Follow links when resolving vdev path name.
8308  *      -p      Display values in parsable (exact) format.
8309  *      -P      Display full path for vdev name.
8310  *      -s      Display slow IOs column.
8311  *      -v      Display complete error logs
8312  *      -x      Display only pools with potential problems
8313  *      -D      Display dedup status (undocumented)
8314  *      -t      Display vdev TRIM status.
8315  *      -T      Display a timestamp in date(1) or Unix format
8316  *
8317  * Describes the health status of all pools or some subset.
8318  */
8319 int
8320 zpool_do_status(int argc, char **argv)
8321 {
8322         int c;
8323         int ret;
8324         float interval = 0;
8325         unsigned long count = 0;
8326         status_cbdata_t cb = { 0 };
8327         char *cmd = NULL;
8328
8329         /* check options */
8330         while ((c = getopt(argc, argv, "c:igLpPsvxDtT:")) != -1) {
8331                 switch (c) {
8332                 case 'c':
8333                         if (cmd != NULL) {
8334                                 fprintf(stderr,
8335                                     gettext("Can't set -c flag twice\n"));
8336                                 exit(1);
8337                         }
8338
8339                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
8340                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
8341                                 fprintf(stderr, gettext(
8342                                     "Can't run -c, disabled by "
8343                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
8344                                 exit(1);
8345                         }
8346
8347                         if ((getuid() <= 0 || geteuid() <= 0) &&
8348                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
8349                                 fprintf(stderr, gettext(
8350                                     "Can't run -c with root privileges "
8351                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
8352                                 exit(1);
8353                         }
8354                         cmd = optarg;
8355                         break;
8356                 case 'i':
8357                         cb.cb_print_vdev_init = B_TRUE;
8358                         break;
8359                 case 'g':
8360                         cb.cb_name_flags |= VDEV_NAME_GUID;
8361                         break;
8362                 case 'L':
8363                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
8364                         break;
8365                 case 'p':
8366                         cb.cb_literal = B_TRUE;
8367                         break;
8368                 case 'P':
8369                         cb.cb_name_flags |= VDEV_NAME_PATH;
8370                         break;
8371                 case 's':
8372                         cb.cb_print_slow_ios = B_TRUE;
8373                         break;
8374                 case 'v':
8375                         cb.cb_verbose = B_TRUE;
8376                         break;
8377                 case 'x':
8378                         cb.cb_explain = B_TRUE;
8379                         break;
8380                 case 'D':
8381                         cb.cb_dedup_stats = B_TRUE;
8382                         break;
8383                 case 't':
8384                         cb.cb_print_vdev_trim = B_TRUE;
8385                         break;
8386                 case 'T':
8387                         get_timestamp_arg(*optarg);
8388                         break;
8389                 case '?':
8390                         if (optopt == 'c') {
8391                                 print_zpool_script_list("status");
8392                                 exit(0);
8393                         } else {
8394                                 fprintf(stderr,
8395                                     gettext("invalid option '%c'\n"), optopt);
8396                         }
8397                         usage(B_FALSE);
8398                 }
8399         }
8400
8401         argc -= optind;
8402         argv += optind;
8403
8404         get_interval_count(&argc, argv, &interval, &count);
8405
8406         if (argc == 0)
8407                 cb.cb_allpools = B_TRUE;
8408
8409         cb.cb_first = B_TRUE;
8410         cb.cb_print_status = B_TRUE;
8411
8412         for (;;) {
8413                 if (timestamp_fmt != NODATE)
8414                         print_timestamp(timestamp_fmt);
8415
8416                 if (cmd != NULL)
8417                         cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
8418                             NULL, NULL, 0, 0);
8419
8420                 ret = for_each_pool(argc, argv, B_TRUE, NULL,
8421                     status_callback, &cb);
8422
8423                 if (cb.vcdl != NULL)
8424                         free_vdev_cmd_data_list(cb.vcdl);
8425
8426                 if (argc == 0 && cb.cb_count == 0)
8427                         (void) fprintf(stderr, gettext("no pools available\n"));
8428                 else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
8429                         (void) printf(gettext("all pools are healthy\n"));
8430
8431                 if (ret != 0)
8432                         return (ret);
8433
8434                 if (interval == 0)
8435                         break;
8436
8437                 if (count != 0 && --count == 0)
8438                         break;
8439
8440                 (void) fsleep(interval);
8441         }
8442
8443         return (0);
8444 }
8445
8446 typedef struct upgrade_cbdata {
8447         int     cb_first;
8448         int     cb_argc;
8449         uint64_t cb_version;
8450         char    **cb_argv;
8451 } upgrade_cbdata_t;
8452
8453 static int
8454 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
8455 {
8456         int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
8457         int *count = (int *)unsupp_fs;
8458
8459         if (zfs_version > ZPL_VERSION) {
8460                 (void) printf(gettext("%s (v%d) is not supported by this "
8461                     "implementation of ZFS.\n"),
8462                     zfs_get_name(zhp), zfs_version);
8463                 (*count)++;
8464         }
8465
8466         zfs_iter_filesystems(zhp, check_unsupp_fs, unsupp_fs);
8467
8468         zfs_close(zhp);
8469
8470         return (0);
8471 }
8472
8473 static int
8474 upgrade_version(zpool_handle_t *zhp, uint64_t version)
8475 {
8476         int ret;
8477         nvlist_t *config;
8478         uint64_t oldversion;
8479         int unsupp_fs = 0;
8480
8481         config = zpool_get_config(zhp, NULL);
8482         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8483             &oldversion) == 0);
8484
8485         assert(SPA_VERSION_IS_SUPPORTED(oldversion));
8486         assert(oldversion < version);
8487
8488         ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
8489         if (ret != 0)
8490                 return (ret);
8491
8492         if (unsupp_fs) {
8493                 (void) fprintf(stderr, gettext("Upgrade not performed due "
8494                     "to %d unsupported filesystems (max v%d).\n"),
8495                     unsupp_fs, (int)ZPL_VERSION);
8496                 return (1);
8497         }
8498
8499         ret = zpool_upgrade(zhp, version);
8500         if (ret != 0)
8501                 return (ret);
8502
8503         if (version >= SPA_VERSION_FEATURES) {
8504                 (void) printf(gettext("Successfully upgraded "
8505                     "'%s' from version %llu to feature flags.\n"),
8506                     zpool_get_name(zhp), (u_longlong_t)oldversion);
8507         } else {
8508                 (void) printf(gettext("Successfully upgraded "
8509                     "'%s' from version %llu to version %llu.\n"),
8510                     zpool_get_name(zhp), (u_longlong_t)oldversion,
8511                     (u_longlong_t)version);
8512         }
8513
8514         return (0);
8515 }
8516
8517 static int
8518 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
8519 {
8520         int i, ret, count;
8521         boolean_t firstff = B_TRUE;
8522         nvlist_t *enabled = zpool_get_features(zhp);
8523
8524         count = 0;
8525         for (i = 0; i < SPA_FEATURES; i++) {
8526                 const char *fname = spa_feature_table[i].fi_uname;
8527                 const char *fguid = spa_feature_table[i].fi_guid;
8528                 if (!nvlist_exists(enabled, fguid)) {
8529                         char *propname;
8530                         verify(-1 != asprintf(&propname, "feature@%s", fname));
8531                         ret = zpool_set_prop(zhp, propname,
8532                             ZFS_FEATURE_ENABLED);
8533                         if (ret != 0) {
8534                                 free(propname);
8535                                 return (ret);
8536                         }
8537                         count++;
8538
8539                         if (firstff) {
8540                                 (void) printf(gettext("Enabled the "
8541                                     "following features on '%s':\n"),
8542                                     zpool_get_name(zhp));
8543                                 firstff = B_FALSE;
8544                         }
8545                         (void) printf(gettext("  %s\n"), fname);
8546                         free(propname);
8547                 }
8548         }
8549
8550         if (countp != NULL)
8551                 *countp = count;
8552         return (0);
8553 }
8554
8555 static int
8556 upgrade_cb(zpool_handle_t *zhp, void *arg)
8557 {
8558         upgrade_cbdata_t *cbp = arg;
8559         nvlist_t *config;
8560         uint64_t version;
8561         boolean_t printnl = B_FALSE;
8562         int ret;
8563
8564         config = zpool_get_config(zhp, NULL);
8565         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8566             &version) == 0);
8567
8568         assert(SPA_VERSION_IS_SUPPORTED(version));
8569
8570         if (version < cbp->cb_version) {
8571                 cbp->cb_first = B_FALSE;
8572                 ret = upgrade_version(zhp, cbp->cb_version);
8573                 if (ret != 0)
8574                         return (ret);
8575                 printnl = B_TRUE;
8576
8577                 /*
8578                  * If they did "zpool upgrade -a", then we could
8579                  * be doing ioctls to different pools.  We need
8580                  * to log this history once to each pool, and bypass
8581                  * the normal history logging that happens in main().
8582                  */
8583                 (void) zpool_log_history(g_zfs, history_str);
8584                 log_history = B_FALSE;
8585         }
8586
8587         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
8588                 int count;
8589                 ret = upgrade_enable_all(zhp, &count);
8590                 if (ret != 0)
8591                         return (ret);
8592
8593                 if (count > 0) {
8594                         cbp->cb_first = B_FALSE;
8595                         printnl = B_TRUE;
8596                 }
8597         }
8598
8599         if (printnl) {
8600                 (void) printf(gettext("\n"));
8601         }
8602
8603         return (0);
8604 }
8605
8606 static int
8607 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
8608 {
8609         upgrade_cbdata_t *cbp = arg;
8610         nvlist_t *config;
8611         uint64_t version;
8612
8613         config = zpool_get_config(zhp, NULL);
8614         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8615             &version) == 0);
8616
8617         assert(SPA_VERSION_IS_SUPPORTED(version));
8618
8619         if (version < SPA_VERSION_FEATURES) {
8620                 if (cbp->cb_first) {
8621                         (void) printf(gettext("The following pools are "
8622                             "formatted with legacy version numbers and can\n"
8623                             "be upgraded to use feature flags.  After "
8624                             "being upgraded, these pools\nwill no "
8625                             "longer be accessible by software that does not "
8626                             "support feature\nflags.\n\n"));
8627                         (void) printf(gettext("VER  POOL\n"));
8628                         (void) printf(gettext("---  ------------\n"));
8629                         cbp->cb_first = B_FALSE;
8630                 }
8631
8632                 (void) printf("%2llu   %s\n", (u_longlong_t)version,
8633                     zpool_get_name(zhp));
8634         }
8635
8636         return (0);
8637 }
8638
8639 static int
8640 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
8641 {
8642         upgrade_cbdata_t *cbp = arg;
8643         nvlist_t *config;
8644         uint64_t version;
8645
8646         config = zpool_get_config(zhp, NULL);
8647         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
8648             &version) == 0);
8649
8650         if (version >= SPA_VERSION_FEATURES) {
8651                 int i;
8652                 boolean_t poolfirst = B_TRUE;
8653                 nvlist_t *enabled = zpool_get_features(zhp);
8654
8655                 for (i = 0; i < SPA_FEATURES; i++) {
8656                         const char *fguid = spa_feature_table[i].fi_guid;
8657                         const char *fname = spa_feature_table[i].fi_uname;
8658                         if (!nvlist_exists(enabled, fguid)) {
8659                                 if (cbp->cb_first) {
8660                                         (void) printf(gettext("\nSome "
8661                                             "supported features are not "
8662                                             "enabled on the following pools. "
8663                                             "Once a\nfeature is enabled the "
8664                                             "pool may become incompatible with "
8665                                             "software\nthat does not support "
8666                                             "the feature. See "
8667                                             "zpool-features(5) for "
8668                                             "details.\n\n"));
8669                                         (void) printf(gettext("POOL  "
8670                                             "FEATURE\n"));
8671                                         (void) printf(gettext("------"
8672                                             "---------\n"));
8673                                         cbp->cb_first = B_FALSE;
8674                                 }
8675
8676                                 if (poolfirst) {
8677                                         (void) printf(gettext("%s\n"),
8678                                             zpool_get_name(zhp));
8679                                         poolfirst = B_FALSE;
8680                                 }
8681
8682                                 (void) printf(gettext("      %s\n"), fname);
8683                         }
8684                         /*
8685                          * If they did "zpool upgrade -a", then we could
8686                          * be doing ioctls to different pools.  We need
8687                          * to log this history once to each pool, and bypass
8688                          * the normal history logging that happens in main().
8689                          */
8690                         (void) zpool_log_history(g_zfs, history_str);
8691                         log_history = B_FALSE;
8692                 }
8693         }
8694
8695         return (0);
8696 }
8697
8698 /* ARGSUSED */
8699 static int
8700 upgrade_one(zpool_handle_t *zhp, void *data)
8701 {
8702         boolean_t printnl = B_FALSE;
8703         upgrade_cbdata_t *cbp = data;
8704         uint64_t cur_version;
8705         int ret;
8706
8707         if (strcmp("log", zpool_get_name(zhp)) == 0) {
8708                 (void) fprintf(stderr, gettext("'log' is now a reserved word\n"
8709                     "Pool 'log' must be renamed using export and import"
8710                     " to upgrade.\n"));
8711                 return (1);
8712         }
8713
8714         cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
8715         if (cur_version > cbp->cb_version) {
8716                 (void) printf(gettext("Pool '%s' is already formatted "
8717                     "using more current version '%llu'.\n\n"),
8718                     zpool_get_name(zhp), (u_longlong_t)cur_version);
8719                 return (0);
8720         }
8721
8722         if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
8723                 (void) printf(gettext("Pool '%s' is already formatted "
8724                     "using version %llu.\n\n"), zpool_get_name(zhp),
8725                     (u_longlong_t)cbp->cb_version);
8726                 return (0);
8727         }
8728
8729         if (cur_version != cbp->cb_version) {
8730                 printnl = B_TRUE;
8731                 ret = upgrade_version(zhp, cbp->cb_version);
8732                 if (ret != 0)
8733                         return (ret);
8734         }
8735
8736         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
8737                 int count = 0;
8738                 ret = upgrade_enable_all(zhp, &count);
8739                 if (ret != 0)
8740                         return (ret);
8741
8742                 if (count != 0) {
8743                         printnl = B_TRUE;
8744                 } else if (cur_version == SPA_VERSION) {
8745                         (void) printf(gettext("Pool '%s' already has all "
8746                             "supported features enabled.\n"),
8747                             zpool_get_name(zhp));
8748                 }
8749         }
8750
8751         if (printnl) {
8752                 (void) printf(gettext("\n"));
8753         }
8754
8755         return (0);
8756 }
8757
8758 /*
8759  * zpool upgrade
8760  * zpool upgrade -v
8761  * zpool upgrade [-V version] <-a | pool ...>
8762  *
8763  * With no arguments, display downrev'd ZFS pool available for upgrade.
8764  * Individual pools can be upgraded by specifying the pool, and '-a' will
8765  * upgrade all pools.
8766  */
8767 int
8768 zpool_do_upgrade(int argc, char **argv)
8769 {
8770         int c;
8771         upgrade_cbdata_t cb = { 0 };
8772         int ret = 0;
8773         boolean_t showversions = B_FALSE;
8774         boolean_t upgradeall = B_FALSE;
8775         char *end;
8776
8777
8778         /* check options */
8779         while ((c = getopt(argc, argv, ":avV:")) != -1) {
8780                 switch (c) {
8781                 case 'a':
8782                         upgradeall = B_TRUE;
8783                         break;
8784                 case 'v':
8785                         showversions = B_TRUE;
8786                         break;
8787                 case 'V':
8788                         cb.cb_version = strtoll(optarg, &end, 10);
8789                         if (*end != '\0' ||
8790                             !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
8791                                 (void) fprintf(stderr,
8792                                     gettext("invalid version '%s'\n"), optarg);
8793                                 usage(B_FALSE);
8794                         }
8795                         break;
8796                 case ':':
8797                         (void) fprintf(stderr, gettext("missing argument for "
8798                             "'%c' option\n"), optopt);
8799                         usage(B_FALSE);
8800                         break;
8801                 case '?':
8802                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
8803                             optopt);
8804                         usage(B_FALSE);
8805                 }
8806         }
8807
8808         cb.cb_argc = argc;
8809         cb.cb_argv = argv;
8810         argc -= optind;
8811         argv += optind;
8812
8813         if (cb.cb_version == 0) {
8814                 cb.cb_version = SPA_VERSION;
8815         } else if (!upgradeall && argc == 0) {
8816                 (void) fprintf(stderr, gettext("-V option is "
8817                     "incompatible with other arguments\n"));
8818                 usage(B_FALSE);
8819         }
8820
8821         if (showversions) {
8822                 if (upgradeall || argc != 0) {
8823                         (void) fprintf(stderr, gettext("-v option is "
8824                             "incompatible with other arguments\n"));
8825                         usage(B_FALSE);
8826                 }
8827         } else if (upgradeall) {
8828                 if (argc != 0) {
8829                         (void) fprintf(stderr, gettext("-a option should not "
8830                             "be used along with a pool name\n"));
8831                         usage(B_FALSE);
8832                 }
8833         }
8834
8835         (void) printf(gettext("This system supports ZFS pool feature "
8836             "flags.\n\n"));
8837         if (showversions) {
8838                 int i;
8839
8840                 (void) printf(gettext("The following features are "
8841                     "supported:\n\n"));
8842                 (void) printf(gettext("FEAT DESCRIPTION\n"));
8843                 (void) printf("----------------------------------------------"
8844                     "---------------\n");
8845                 for (i = 0; i < SPA_FEATURES; i++) {
8846                         zfeature_info_t *fi = &spa_feature_table[i];
8847                         const char *ro =
8848                             (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
8849                             " (read-only compatible)" : "";
8850
8851                         (void) printf("%-37s%s\n", fi->fi_uname, ro);
8852                         (void) printf("     %s\n", fi->fi_desc);
8853                 }
8854                 (void) printf("\n");
8855
8856                 (void) printf(gettext("The following legacy versions are also "
8857                     "supported:\n\n"));
8858                 (void) printf(gettext("VER  DESCRIPTION\n"));
8859                 (void) printf("---  -----------------------------------------"
8860                     "---------------\n");
8861                 (void) printf(gettext(" 1   Initial ZFS version\n"));
8862                 (void) printf(gettext(" 2   Ditto blocks "
8863                     "(replicated metadata)\n"));
8864                 (void) printf(gettext(" 3   Hot spares and double parity "
8865                     "RAID-Z\n"));
8866                 (void) printf(gettext(" 4   zpool history\n"));
8867                 (void) printf(gettext(" 5   Compression using the gzip "
8868                     "algorithm\n"));
8869                 (void) printf(gettext(" 6   bootfs pool property\n"));
8870                 (void) printf(gettext(" 7   Separate intent log devices\n"));
8871                 (void) printf(gettext(" 8   Delegated administration\n"));
8872                 (void) printf(gettext(" 9   refquota and refreservation "
8873                     "properties\n"));
8874                 (void) printf(gettext(" 10  Cache devices\n"));
8875                 (void) printf(gettext(" 11  Improved scrub performance\n"));
8876                 (void) printf(gettext(" 12  Snapshot properties\n"));
8877                 (void) printf(gettext(" 13  snapused property\n"));
8878                 (void) printf(gettext(" 14  passthrough-x aclinherit\n"));
8879                 (void) printf(gettext(" 15  user/group space accounting\n"));
8880                 (void) printf(gettext(" 16  stmf property support\n"));
8881                 (void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
8882                 (void) printf(gettext(" 18  Snapshot user holds\n"));
8883                 (void) printf(gettext(" 19  Log device removal\n"));
8884                 (void) printf(gettext(" 20  Compression using zle "
8885                     "(zero-length encoding)\n"));
8886                 (void) printf(gettext(" 21  Deduplication\n"));
8887                 (void) printf(gettext(" 22  Received properties\n"));
8888                 (void) printf(gettext(" 23  Slim ZIL\n"));
8889                 (void) printf(gettext(" 24  System attributes\n"));
8890                 (void) printf(gettext(" 25  Improved scrub stats\n"));
8891                 (void) printf(gettext(" 26  Improved snapshot deletion "
8892                     "performance\n"));
8893                 (void) printf(gettext(" 27  Improved snapshot creation "
8894                     "performance\n"));
8895                 (void) printf(gettext(" 28  Multiple vdev replacements\n"));
8896                 (void) printf(gettext("\nFor more information on a particular "
8897                     "version, including supported releases,\n"));
8898                 (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
8899         } else if (argc == 0 && upgradeall) {
8900                 cb.cb_first = B_TRUE;
8901                 ret = zpool_iter(g_zfs, upgrade_cb, &cb);
8902                 if (ret == 0 && cb.cb_first) {
8903                         if (cb.cb_version == SPA_VERSION) {
8904                                 (void) printf(gettext("All pools are already "
8905                                     "formatted using feature flags.\n\n"));
8906                                 (void) printf(gettext("Every feature flags "
8907                                     "pool already has all supported features "
8908                                     "enabled.\n"));
8909                         } else {
8910                                 (void) printf(gettext("All pools are already "
8911                                     "formatted with version %llu or higher.\n"),
8912                                     (u_longlong_t)cb.cb_version);
8913                         }
8914                 }
8915         } else if (argc == 0) {
8916                 cb.cb_first = B_TRUE;
8917                 ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
8918                 assert(ret == 0);
8919
8920                 if (cb.cb_first) {
8921                         (void) printf(gettext("All pools are formatted "
8922                             "using feature flags.\n\n"));
8923                 } else {
8924                         (void) printf(gettext("\nUse 'zpool upgrade -v' "
8925                             "for a list of available legacy versions.\n"));
8926                 }
8927
8928                 cb.cb_first = B_TRUE;
8929                 ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
8930                 assert(ret == 0);
8931
8932                 if (cb.cb_first) {
8933                         (void) printf(gettext("Every feature flags pool has "
8934                             "all supported features enabled.\n"));
8935                 } else {
8936                         (void) printf(gettext("\n"));
8937                 }
8938         } else {
8939                 ret = for_each_pool(argc, argv, B_FALSE, NULL,
8940                     upgrade_one, &cb);
8941         }
8942
8943         return (ret);
8944 }
8945
8946 typedef struct hist_cbdata {
8947         boolean_t first;
8948         boolean_t longfmt;
8949         boolean_t internal;
8950 } hist_cbdata_t;
8951
8952 static void
8953 print_history_records(nvlist_t *nvhis, hist_cbdata_t *cb)
8954 {
8955         nvlist_t **records;
8956         uint_t numrecords;
8957         int i;
8958
8959         verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
8960             &records, &numrecords) == 0);
8961         for (i = 0; i < numrecords; i++) {
8962                 nvlist_t *rec = records[i];
8963                 char tbuf[30] = "";
8964
8965                 if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
8966                         time_t tsec;
8967                         struct tm t;
8968
8969                         tsec = fnvlist_lookup_uint64(records[i],
8970                             ZPOOL_HIST_TIME);
8971                         (void) localtime_r(&tsec, &t);
8972                         (void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
8973                 }
8974
8975                 if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
8976                         (void) printf("%s %s", tbuf,
8977                             fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
8978                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
8979                         int ievent =
8980                             fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
8981                         if (!cb->internal)
8982                                 continue;
8983                         if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
8984                                 (void) printf("%s unrecognized record:\n",
8985                                     tbuf);
8986                                 dump_nvlist(rec, 4);
8987                                 continue;
8988                         }
8989                         (void) printf("%s [internal %s txg:%lld] %s", tbuf,
8990                             zfs_history_event_names[ievent],
8991                             (longlong_t)fnvlist_lookup_uint64(
8992                             rec, ZPOOL_HIST_TXG),
8993                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
8994                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
8995                         if (!cb->internal)
8996                                 continue;
8997                         (void) printf("%s [txg:%lld] %s", tbuf,
8998                             (longlong_t)fnvlist_lookup_uint64(
8999                             rec, ZPOOL_HIST_TXG),
9000                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
9001                         if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
9002                                 (void) printf(" %s (%llu)",
9003                                     fnvlist_lookup_string(rec,
9004                                     ZPOOL_HIST_DSNAME),
9005                                     (u_longlong_t)fnvlist_lookup_uint64(rec,
9006                                     ZPOOL_HIST_DSID));
9007                         }
9008                         (void) printf(" %s", fnvlist_lookup_string(rec,
9009                             ZPOOL_HIST_INT_STR));
9010                 } else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
9011                         if (!cb->internal)
9012                                 continue;
9013                         (void) printf("%s ioctl %s\n", tbuf,
9014                             fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
9015                         if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
9016                                 (void) printf("    input:\n");
9017                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9018                                     ZPOOL_HIST_INPUT_NVL), 8);
9019                         }
9020                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
9021                                 (void) printf("    output:\n");
9022                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9023                                     ZPOOL_HIST_OUTPUT_NVL), 8);
9024                         }
9025                         if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
9026                                 (void) printf("    errno: %lld\n",
9027                                     (longlong_t)fnvlist_lookup_int64(rec,
9028                                     ZPOOL_HIST_ERRNO));
9029                         }
9030                 } else {
9031                         if (!cb->internal)
9032                                 continue;
9033                         (void) printf("%s unrecognized record:\n", tbuf);
9034                         dump_nvlist(rec, 4);
9035                 }
9036
9037                 if (!cb->longfmt) {
9038                         (void) printf("\n");
9039                         continue;
9040                 }
9041                 (void) printf(" [");
9042                 if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
9043                         uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
9044                         struct passwd *pwd = getpwuid(who);
9045                         (void) printf("user %d ", (int)who);
9046                         if (pwd != NULL)
9047                                 (void) printf("(%s) ", pwd->pw_name);
9048                 }
9049                 if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
9050                         (void) printf("on %s",
9051                             fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
9052                 }
9053                 if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
9054                         (void) printf(":%s",
9055                             fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
9056                 }
9057
9058                 (void) printf("]");
9059                 (void) printf("\n");
9060         }
9061 }
9062
9063 /*
9064  * Print out the command history for a specific pool.
9065  */
9066 static int
9067 get_history_one(zpool_handle_t *zhp, void *data)
9068 {
9069         nvlist_t *nvhis;
9070         int ret;
9071         hist_cbdata_t *cb = (hist_cbdata_t *)data;
9072         uint64_t off = 0;
9073         boolean_t eof = B_FALSE;
9074
9075         cb->first = B_FALSE;
9076
9077         (void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
9078
9079         while (!eof) {
9080                 if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
9081                         return (ret);
9082
9083                 print_history_records(nvhis, cb);
9084                 nvlist_free(nvhis);
9085         }
9086         (void) printf("\n");
9087
9088         return (ret);
9089 }
9090
9091 /*
9092  * zpool history <pool>
9093  *
9094  * Displays the history of commands that modified pools.
9095  */
9096 int
9097 zpool_do_history(int argc, char **argv)
9098 {
9099         hist_cbdata_t cbdata = { 0 };
9100         int ret;
9101         int c;
9102
9103         cbdata.first = B_TRUE;
9104         /* check options */
9105         while ((c = getopt(argc, argv, "li")) != -1) {
9106                 switch (c) {
9107                 case 'l':
9108                         cbdata.longfmt = B_TRUE;
9109                         break;
9110                 case 'i':
9111                         cbdata.internal = B_TRUE;
9112                         break;
9113                 case '?':
9114                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9115                             optopt);
9116                         usage(B_FALSE);
9117                 }
9118         }
9119         argc -= optind;
9120         argv += optind;
9121
9122         ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
9123             &cbdata);
9124
9125         if (argc == 0 && cbdata.first == B_TRUE) {
9126                 (void) fprintf(stderr, gettext("no pools available\n"));
9127                 return (0);
9128         }
9129
9130         return (ret);
9131 }
9132
9133 typedef struct ev_opts {
9134         int verbose;
9135         int scripted;
9136         int follow;
9137         int clear;
9138         char poolname[ZFS_MAX_DATASET_NAME_LEN];
9139 } ev_opts_t;
9140
9141 static void
9142 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
9143 {
9144         char ctime_str[26], str[32], *ptr;
9145         int64_t *tv;
9146         uint_t n;
9147
9148         verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
9149         memset(str, ' ', 32);
9150         (void) ctime_r((const time_t *)&tv[0], ctime_str);
9151         (void) memcpy(str, ctime_str+4,  6);            /* 'Jun 30' */
9152         (void) memcpy(str+7, ctime_str+20, 4);          /* '1993' */
9153         (void) memcpy(str+12, ctime_str+11, 8);         /* '21:49:08' */
9154         (void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
9155         if (opts->scripted)
9156                 (void) printf(gettext("%s\t"), str);
9157         else
9158                 (void) printf(gettext("%s "), str);
9159
9160         verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
9161         (void) printf(gettext("%s\n"), ptr);
9162 }
9163
9164 static void
9165 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
9166 {
9167         nvpair_t *nvp;
9168
9169         for (nvp = nvlist_next_nvpair(nvl, NULL);
9170             nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
9171
9172                 data_type_t type = nvpair_type(nvp);
9173                 const char *name = nvpair_name(nvp);
9174
9175                 boolean_t b;
9176                 uint8_t i8;
9177                 uint16_t i16;
9178                 uint32_t i32;
9179                 uint64_t i64;
9180                 char *str;
9181                 nvlist_t *cnv;
9182
9183                 printf(gettext("%*s%s = "), depth, "", name);
9184
9185                 switch (type) {
9186                 case DATA_TYPE_BOOLEAN:
9187                         printf(gettext("%s"), "1");
9188                         break;
9189
9190                 case DATA_TYPE_BOOLEAN_VALUE:
9191                         (void) nvpair_value_boolean_value(nvp, &b);
9192                         printf(gettext("%s"), b ? "1" : "0");
9193                         break;
9194
9195                 case DATA_TYPE_BYTE:
9196                         (void) nvpair_value_byte(nvp, &i8);
9197                         printf(gettext("0x%x"), i8);
9198                         break;
9199
9200                 case DATA_TYPE_INT8:
9201                         (void) nvpair_value_int8(nvp, (void *)&i8);
9202                         printf(gettext("0x%x"), i8);
9203                         break;
9204
9205                 case DATA_TYPE_UINT8:
9206                         (void) nvpair_value_uint8(nvp, &i8);
9207                         printf(gettext("0x%x"), i8);
9208                         break;
9209
9210                 case DATA_TYPE_INT16:
9211                         (void) nvpair_value_int16(nvp, (void *)&i16);
9212                         printf(gettext("0x%x"), i16);
9213                         break;
9214
9215                 case DATA_TYPE_UINT16:
9216                         (void) nvpair_value_uint16(nvp, &i16);
9217                         printf(gettext("0x%x"), i16);
9218                         break;
9219
9220                 case DATA_TYPE_INT32:
9221                         (void) nvpair_value_int32(nvp, (void *)&i32);
9222                         printf(gettext("0x%x"), i32);
9223                         break;
9224
9225                 case DATA_TYPE_UINT32:
9226                         (void) nvpair_value_uint32(nvp, &i32);
9227                         printf(gettext("0x%x"), i32);
9228                         break;
9229
9230                 case DATA_TYPE_INT64:
9231                         (void) nvpair_value_int64(nvp, (void *)&i64);
9232                         printf(gettext("0x%llx"), (u_longlong_t)i64);
9233                         break;
9234
9235                 case DATA_TYPE_UINT64:
9236                         (void) nvpair_value_uint64(nvp, &i64);
9237                         /*
9238                          * translate vdev state values to readable
9239                          * strings to aide zpool events consumers
9240                          */
9241                         if (strcmp(name,
9242                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
9243                             strcmp(name,
9244                             FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
9245                                 printf(gettext("\"%s\" (0x%llx)"),
9246                                     zpool_state_to_name(i64, VDEV_AUX_NONE),
9247                                     (u_longlong_t)i64);
9248                         } else {
9249                                 printf(gettext("0x%llx"), (u_longlong_t)i64);
9250                         }
9251                         break;
9252
9253                 case DATA_TYPE_HRTIME:
9254                         (void) nvpair_value_hrtime(nvp, (void *)&i64);
9255                         printf(gettext("0x%llx"), (u_longlong_t)i64);
9256                         break;
9257
9258                 case DATA_TYPE_STRING:
9259                         (void) nvpair_value_string(nvp, &str);
9260                         printf(gettext("\"%s\""), str ? str : "<NULL>");
9261                         break;
9262
9263                 case DATA_TYPE_NVLIST:
9264                         printf(gettext("(embedded nvlist)\n"));
9265                         (void) nvpair_value_nvlist(nvp, &cnv);
9266                         zpool_do_events_nvprint(cnv, depth + 8);
9267                         printf(gettext("%*s(end %s)"), depth, "", name);
9268                         break;
9269
9270                 case DATA_TYPE_NVLIST_ARRAY: {
9271                         nvlist_t **val;
9272                         uint_t i, nelem;
9273
9274                         (void) nvpair_value_nvlist_array(nvp, &val, &nelem);
9275                         printf(gettext("(%d embedded nvlists)\n"), nelem);
9276                         for (i = 0; i < nelem; i++) {
9277                                 printf(gettext("%*s%s[%d] = %s\n"),
9278                                     depth, "", name, i, "(embedded nvlist)");
9279                                 zpool_do_events_nvprint(val[i], depth + 8);
9280                                 printf(gettext("%*s(end %s[%i])\n"),
9281                                     depth, "", name, i);
9282                         }
9283                         printf(gettext("%*s(end %s)\n"), depth, "", name);
9284                         }
9285                         break;
9286
9287                 case DATA_TYPE_INT8_ARRAY: {
9288                         int8_t *val;
9289                         uint_t i, nelem;
9290
9291                         (void) nvpair_value_int8_array(nvp, &val, &nelem);
9292                         for (i = 0; i < nelem; i++)
9293                                 printf(gettext("0x%x "), val[i]);
9294
9295                         break;
9296                         }
9297
9298                 case DATA_TYPE_UINT8_ARRAY: {
9299                         uint8_t *val;
9300                         uint_t i, nelem;
9301
9302                         (void) nvpair_value_uint8_array(nvp, &val, &nelem);
9303                         for (i = 0; i < nelem; i++)
9304                                 printf(gettext("0x%x "), val[i]);
9305
9306                         break;
9307                         }
9308
9309                 case DATA_TYPE_INT16_ARRAY: {
9310                         int16_t *val;
9311                         uint_t i, nelem;
9312
9313                         (void) nvpair_value_int16_array(nvp, &val, &nelem);
9314                         for (i = 0; i < nelem; i++)
9315                                 printf(gettext("0x%x "), val[i]);
9316
9317                         break;
9318                         }
9319
9320                 case DATA_TYPE_UINT16_ARRAY: {
9321                         uint16_t *val;
9322                         uint_t i, nelem;
9323
9324                         (void) nvpair_value_uint16_array(nvp, &val, &nelem);
9325                         for (i = 0; i < nelem; i++)
9326                                 printf(gettext("0x%x "), val[i]);
9327
9328                         break;
9329                         }
9330
9331                 case DATA_TYPE_INT32_ARRAY: {
9332                         int32_t *val;
9333                         uint_t i, nelem;
9334
9335                         (void) nvpair_value_int32_array(nvp, &val, &nelem);
9336                         for (i = 0; i < nelem; i++)
9337                                 printf(gettext("0x%x "), val[i]);
9338
9339                         break;
9340                         }
9341
9342                 case DATA_TYPE_UINT32_ARRAY: {
9343                         uint32_t *val;
9344                         uint_t i, nelem;
9345
9346                         (void) nvpair_value_uint32_array(nvp, &val, &nelem);
9347                         for (i = 0; i < nelem; i++)
9348                                 printf(gettext("0x%x "), val[i]);
9349
9350                         break;
9351                         }
9352
9353                 case DATA_TYPE_INT64_ARRAY: {
9354                         int64_t *val;
9355                         uint_t i, nelem;
9356
9357                         (void) nvpair_value_int64_array(nvp, &val, &nelem);
9358                         for (i = 0; i < nelem; i++)
9359                                 printf(gettext("0x%llx "),
9360                                     (u_longlong_t)val[i]);
9361
9362                         break;
9363                         }
9364
9365                 case DATA_TYPE_UINT64_ARRAY: {
9366                         uint64_t *val;
9367                         uint_t i, nelem;
9368
9369                         (void) nvpair_value_uint64_array(nvp, &val, &nelem);
9370                         for (i = 0; i < nelem; i++)
9371                                 printf(gettext("0x%llx "),
9372                                     (u_longlong_t)val[i]);
9373
9374                         break;
9375                         }
9376
9377                 case DATA_TYPE_STRING_ARRAY: {
9378                         char **str;
9379                         uint_t i, nelem;
9380
9381                         (void) nvpair_value_string_array(nvp, &str, &nelem);
9382                         for (i = 0; i < nelem; i++)
9383                                 printf(gettext("\"%s\" "),
9384                                     str[i] ? str[i] : "<NULL>");
9385
9386                         break;
9387                         }
9388
9389                 case DATA_TYPE_BOOLEAN_ARRAY:
9390                 case DATA_TYPE_BYTE_ARRAY:
9391                 case DATA_TYPE_DOUBLE:
9392                 case DATA_TYPE_DONTCARE:
9393                 case DATA_TYPE_UNKNOWN:
9394                         printf(gettext("<unknown>"));
9395                         break;
9396                 }
9397
9398                 printf(gettext("\n"));
9399         }
9400 }
9401
9402 static int
9403 zpool_do_events_next(ev_opts_t *opts)
9404 {
9405         nvlist_t *nvl;
9406         int zevent_fd, ret, dropped;
9407         char *pool;
9408
9409         zevent_fd = open(ZFS_DEV, O_RDWR);
9410         VERIFY(zevent_fd >= 0);
9411
9412         if (!opts->scripted)
9413                 (void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
9414
9415         while (1) {
9416                 ret = zpool_events_next(g_zfs, &nvl, &dropped,
9417                     (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
9418                 if (ret || nvl == NULL)
9419                         break;
9420
9421                 if (dropped > 0)
9422                         (void) printf(gettext("dropped %d events\n"), dropped);
9423
9424                 if (strlen(opts->poolname) > 0 &&
9425                     nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
9426                     strcmp(opts->poolname, pool) != 0)
9427                         continue;
9428
9429                 zpool_do_events_short(nvl, opts);
9430
9431                 if (opts->verbose) {
9432                         zpool_do_events_nvprint(nvl, 8);
9433                         printf(gettext("\n"));
9434                 }
9435                 (void) fflush(stdout);
9436
9437                 nvlist_free(nvl);
9438         }
9439
9440         VERIFY(0 == close(zevent_fd));
9441
9442         return (ret);
9443 }
9444
9445 static int
9446 zpool_do_events_clear(ev_opts_t *opts)
9447 {
9448         int count, ret;
9449
9450         ret = zpool_events_clear(g_zfs, &count);
9451         if (!ret)
9452                 (void) printf(gettext("cleared %d events\n"), count);
9453
9454         return (ret);
9455 }
9456
9457 /*
9458  * zpool events [-vHf [pool] | -c]
9459  *
9460  * Displays events logs by ZFS.
9461  */
9462 int
9463 zpool_do_events(int argc, char **argv)
9464 {
9465         ev_opts_t opts = { 0 };
9466         int ret;
9467         int c;
9468
9469         /* check options */
9470         while ((c = getopt(argc, argv, "vHfc")) != -1) {
9471                 switch (c) {
9472                 case 'v':
9473                         opts.verbose = 1;
9474                         break;
9475                 case 'H':
9476                         opts.scripted = 1;
9477                         break;
9478                 case 'f':
9479                         opts.follow = 1;
9480                         break;
9481                 case 'c':
9482                         opts.clear = 1;
9483                         break;
9484                 case '?':
9485                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9486                             optopt);
9487                         usage(B_FALSE);
9488                 }
9489         }
9490         argc -= optind;
9491         argv += optind;
9492
9493         if (argc > 1) {
9494                 (void) fprintf(stderr, gettext("too many arguments\n"));
9495                 usage(B_FALSE);
9496         } else if (argc == 1) {
9497                 (void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
9498                 if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
9499                         (void) fprintf(stderr,
9500                             gettext("invalid pool name '%s'\n"), opts.poolname);
9501                         usage(B_FALSE);
9502                 }
9503         }
9504
9505         if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
9506             opts.clear) {
9507                 (void) fprintf(stderr,
9508                     gettext("invalid options combined with -c\n"));
9509                 usage(B_FALSE);
9510         }
9511
9512         if (opts.clear)
9513                 ret = zpool_do_events_clear(&opts);
9514         else
9515                 ret = zpool_do_events_next(&opts);
9516
9517         return (ret);
9518 }
9519
9520 static int
9521 get_callback(zpool_handle_t *zhp, void *data)
9522 {
9523         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
9524         char value[MAXNAMELEN];
9525         zprop_source_t srctype;
9526         zprop_list_t *pl;
9527
9528         for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
9529
9530                 /*
9531                  * Skip the special fake placeholder. This will also skip
9532                  * over the name property when 'all' is specified.
9533                  */
9534                 if (pl->pl_prop == ZPOOL_PROP_NAME &&
9535                     pl == cbp->cb_proplist)
9536                         continue;
9537
9538                 if (pl->pl_prop == ZPROP_INVAL &&
9539                     (zpool_prop_feature(pl->pl_user_prop) ||
9540                     zpool_prop_unsupported(pl->pl_user_prop))) {
9541                         srctype = ZPROP_SRC_LOCAL;
9542
9543                         if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
9544                             value, sizeof (value)) == 0) {
9545                                 zprop_print_one_property(zpool_get_name(zhp),
9546                                     cbp, pl->pl_user_prop, value, srctype,
9547                                     NULL, NULL);
9548                         }
9549                 } else {
9550                         if (zpool_get_prop(zhp, pl->pl_prop, value,
9551                             sizeof (value), &srctype, cbp->cb_literal) != 0)
9552                                 continue;
9553
9554                         zprop_print_one_property(zpool_get_name(zhp), cbp,
9555                             zpool_prop_to_name(pl->pl_prop), value, srctype,
9556                             NULL, NULL);
9557                 }
9558         }
9559         return (0);
9560 }
9561
9562 /*
9563  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
9564  *
9565  *      -H      Scripted mode.  Don't display headers, and separate properties
9566  *              by a single tab.
9567  *      -o      List of columns to display.  Defaults to
9568  *              "name,property,value,source".
9569  *      -p      Display values in parsable (exact) format.
9570  *
9571  * Get properties of pools in the system. Output space statistics
9572  * for each one as well as other attributes.
9573  */
9574 int
9575 zpool_do_get(int argc, char **argv)
9576 {
9577         zprop_get_cbdata_t cb = { 0 };
9578         zprop_list_t fake_name = { 0 };
9579         int ret;
9580         int c, i;
9581         char *value;
9582
9583         cb.cb_first = B_TRUE;
9584
9585         /*
9586          * Set up default columns and sources.
9587          */
9588         cb.cb_sources = ZPROP_SRC_ALL;
9589         cb.cb_columns[0] = GET_COL_NAME;
9590         cb.cb_columns[1] = GET_COL_PROPERTY;
9591         cb.cb_columns[2] = GET_COL_VALUE;
9592         cb.cb_columns[3] = GET_COL_SOURCE;
9593         cb.cb_type = ZFS_TYPE_POOL;
9594
9595         /* check options */
9596         while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
9597                 switch (c) {
9598                 case 'p':
9599                         cb.cb_literal = B_TRUE;
9600                         break;
9601                 case 'H':
9602                         cb.cb_scripted = B_TRUE;
9603                         break;
9604                 case 'o':
9605                         bzero(&cb.cb_columns, sizeof (cb.cb_columns));
9606                         i = 0;
9607                         while (*optarg != '\0') {
9608                                 static char *col_subopts[] =
9609                                 { "name", "property", "value", "source",
9610                                 "all", NULL };
9611
9612                                 if (i == ZFS_GET_NCOLS) {
9613                                         (void) fprintf(stderr, gettext("too "
9614                                         "many fields given to -o "
9615                                         "option\n"));
9616                                         usage(B_FALSE);
9617                                 }
9618
9619                                 switch (getsubopt(&optarg, col_subopts,
9620                                     &value)) {
9621                                 case 0:
9622                                         cb.cb_columns[i++] = GET_COL_NAME;
9623                                         break;
9624                                 case 1:
9625                                         cb.cb_columns[i++] = GET_COL_PROPERTY;
9626                                         break;
9627                                 case 2:
9628                                         cb.cb_columns[i++] = GET_COL_VALUE;
9629                                         break;
9630                                 case 3:
9631                                         cb.cb_columns[i++] = GET_COL_SOURCE;
9632                                         break;
9633                                 case 4:
9634                                         if (i > 0) {
9635                                                 (void) fprintf(stderr,
9636                                                     gettext("\"all\" conflicts "
9637                                                     "with specific fields "
9638                                                     "given to -o option\n"));
9639                                                 usage(B_FALSE);
9640                                         }
9641                                         cb.cb_columns[0] = GET_COL_NAME;
9642                                         cb.cb_columns[1] = GET_COL_PROPERTY;
9643                                         cb.cb_columns[2] = GET_COL_VALUE;
9644                                         cb.cb_columns[3] = GET_COL_SOURCE;
9645                                         i = ZFS_GET_NCOLS;
9646                                         break;
9647                                 default:
9648                                         (void) fprintf(stderr,
9649                                             gettext("invalid column name "
9650                                             "'%s'\n"), value);
9651                                         usage(B_FALSE);
9652                                 }
9653                         }
9654                         break;
9655                 case '?':
9656                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9657                             optopt);
9658                         usage(B_FALSE);
9659                 }
9660         }
9661
9662         argc -= optind;
9663         argv += optind;
9664
9665         if (argc < 1) {
9666                 (void) fprintf(stderr, gettext("missing property "
9667                     "argument\n"));
9668                 usage(B_FALSE);
9669         }
9670
9671         if (zprop_get_list(g_zfs, argv[0], &cb.cb_proplist,
9672             ZFS_TYPE_POOL) != 0)
9673                 usage(B_FALSE);
9674
9675         argc--;
9676         argv++;
9677
9678         if (cb.cb_proplist != NULL) {
9679                 fake_name.pl_prop = ZPOOL_PROP_NAME;
9680                 fake_name.pl_width = strlen(gettext("NAME"));
9681                 fake_name.pl_next = cb.cb_proplist;
9682                 cb.cb_proplist = &fake_name;
9683         }
9684
9685         ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
9686             get_callback, &cb);
9687
9688         if (cb.cb_proplist == &fake_name)
9689                 zprop_free_list(fake_name.pl_next);
9690         else
9691                 zprop_free_list(cb.cb_proplist);
9692
9693         return (ret);
9694 }
9695
9696 typedef struct set_cbdata {
9697         char *cb_propname;
9698         char *cb_value;
9699         boolean_t cb_any_successful;
9700 } set_cbdata_t;
9701
9702 static int
9703 set_callback(zpool_handle_t *zhp, void *data)
9704 {
9705         int error;
9706         set_cbdata_t *cb = (set_cbdata_t *)data;
9707
9708         error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
9709
9710         if (!error)
9711                 cb->cb_any_successful = B_TRUE;
9712
9713         return (error);
9714 }
9715
9716 int
9717 zpool_do_set(int argc, char **argv)
9718 {
9719         set_cbdata_t cb = { 0 };
9720         int error;
9721
9722         if (argc > 1 && argv[1][0] == '-') {
9723                 (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9724                     argv[1][1]);
9725                 usage(B_FALSE);
9726         }
9727
9728         if (argc < 2) {
9729                 (void) fprintf(stderr, gettext("missing property=value "
9730                     "argument\n"));
9731                 usage(B_FALSE);
9732         }
9733
9734         if (argc < 3) {
9735                 (void) fprintf(stderr, gettext("missing pool name\n"));
9736                 usage(B_FALSE);
9737         }
9738
9739         if (argc > 3) {
9740                 (void) fprintf(stderr, gettext("too many pool names\n"));
9741                 usage(B_FALSE);
9742         }
9743
9744         cb.cb_propname = argv[1];
9745         cb.cb_value = strchr(cb.cb_propname, '=');
9746         if (cb.cb_value == NULL) {
9747                 (void) fprintf(stderr, gettext("missing value in "
9748                     "property=value argument\n"));
9749                 usage(B_FALSE);
9750         }
9751
9752         *(cb.cb_value) = '\0';
9753         cb.cb_value++;
9754
9755         error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
9756             set_callback, &cb);
9757
9758         return (error);
9759 }
9760
9761 /* Add up the total number of bytes left to initialize/trim across all vdevs */
9762 static uint64_t
9763 vdev_activity_remaining(nvlist_t *nv, zpool_wait_activity_t activity)
9764 {
9765         uint64_t bytes_remaining;
9766         nvlist_t **child;
9767         uint_t c, children;
9768         vdev_stat_t *vs;
9769
9770         assert(activity == ZPOOL_WAIT_INITIALIZE ||
9771             activity == ZPOOL_WAIT_TRIM);
9772
9773         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
9774             (uint64_t **)&vs, &c) == 0);
9775
9776         if (activity == ZPOOL_WAIT_INITIALIZE &&
9777             vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
9778                 bytes_remaining = vs->vs_initialize_bytes_est -
9779                     vs->vs_initialize_bytes_done;
9780         else if (activity == ZPOOL_WAIT_TRIM &&
9781             vs->vs_trim_state == VDEV_TRIM_ACTIVE)
9782                 bytes_remaining = vs->vs_trim_bytes_est -
9783                     vs->vs_trim_bytes_done;
9784         else
9785                 bytes_remaining = 0;
9786
9787         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
9788             &child, &children) != 0)
9789                 children = 0;
9790
9791         for (c = 0; c < children; c++)
9792                 bytes_remaining += vdev_activity_remaining(child[c], activity);
9793
9794         return (bytes_remaining);
9795 }
9796
9797 /* Add up the total number of bytes left to rebuild across top-level vdevs */
9798 static uint64_t
9799 vdev_activity_top_remaining(nvlist_t *nv)
9800 {
9801         uint64_t bytes_remaining = 0;
9802         nvlist_t **child;
9803         uint_t children;
9804         int error;
9805
9806         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
9807             &child, &children) != 0)
9808                 children = 0;
9809
9810         for (uint_t c = 0; c < children; c++) {
9811                 vdev_rebuild_stat_t *vrs;
9812                 uint_t i;
9813
9814                 error = nvlist_lookup_uint64_array(child[c],
9815                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i);
9816                 if (error == 0) {
9817                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
9818                                 bytes_remaining += (vrs->vrs_bytes_est -
9819                                     vrs->vrs_bytes_rebuilt);
9820                         }
9821                 }
9822         }
9823
9824         return (bytes_remaining);
9825 }
9826
9827 /* Whether any vdevs are 'spare' or 'replacing' vdevs */
9828 static boolean_t
9829 vdev_any_spare_replacing(nvlist_t *nv)
9830 {
9831         nvlist_t **child;
9832         uint_t c, children;
9833         char *vdev_type;
9834
9835         (void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &vdev_type);
9836
9837         if (strcmp(vdev_type, VDEV_TYPE_REPLACING) == 0 ||
9838             strcmp(vdev_type, VDEV_TYPE_SPARE) == 0) {
9839                 return (B_TRUE);
9840         }
9841
9842         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
9843             &child, &children) != 0)
9844                 children = 0;
9845
9846         for (c = 0; c < children; c++) {
9847                 if (vdev_any_spare_replacing(child[c]))
9848                         return (B_TRUE);
9849         }
9850
9851         return (B_FALSE);
9852 }
9853
9854 typedef struct wait_data {
9855         char *wd_poolname;
9856         boolean_t wd_scripted;
9857         boolean_t wd_exact;
9858         boolean_t wd_headers_once;
9859         boolean_t wd_should_exit;
9860         /* Which activities to wait for */
9861         boolean_t wd_enabled[ZPOOL_WAIT_NUM_ACTIVITIES];
9862         float wd_interval;
9863         pthread_cond_t wd_cv;
9864         pthread_mutex_t wd_mutex;
9865 } wait_data_t;
9866
9867 /*
9868  * Print to stdout a single line, containing one column for each activity that
9869  * we are waiting for specifying how many bytes of work are left for that
9870  * activity.
9871  */
9872 static void
9873 print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
9874 {
9875         nvlist_t *config, *nvroot;
9876         uint_t c;
9877         int i;
9878         pool_checkpoint_stat_t *pcs = NULL;
9879         pool_scan_stat_t *pss = NULL;
9880         pool_removal_stat_t *prs = NULL;
9881         char *headers[] = {"DISCARD", "FREE", "INITIALIZE", "REPLACE",
9882             "REMOVE", "RESILVER", "SCRUB", "TRIM"};
9883         int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];
9884
9885         /* Calculate the width of each column */
9886         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
9887                 /*
9888                  * Make sure we have enough space in the col for pretty-printed
9889                  * numbers and for the column header, and then leave a couple
9890                  * spaces between cols for readability.
9891                  */
9892                 col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
9893         }
9894
9895         /* Print header if appropriate */
9896         int term_height = terminal_height();
9897         boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
9898             row % (term_height-1) == 0);
9899         if (!wd->wd_scripted && (row == 0 || reprint_header)) {
9900                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
9901                         if (wd->wd_enabled[i])
9902                                 (void) printf("%*s", col_widths[i], headers[i]);
9903                 }
9904                 (void) printf("\n");
9905         }
9906
9907         /* Bytes of work remaining in each activity */
9908         int64_t bytes_rem[ZPOOL_WAIT_NUM_ACTIVITIES] = {0};
9909
9910         bytes_rem[ZPOOL_WAIT_FREE] =
9911             zpool_get_prop_int(zhp, ZPOOL_PROP_FREEING, NULL);
9912
9913         config = zpool_get_config(zhp, NULL);
9914         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
9915
9916         (void) nvlist_lookup_uint64_array(nvroot,
9917             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
9918         if (pcs != NULL && pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
9919                 bytes_rem[ZPOOL_WAIT_CKPT_DISCARD] = pcs->pcs_space;
9920
9921         (void) nvlist_lookup_uint64_array(nvroot,
9922             ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
9923         if (prs != NULL && prs->prs_state == DSS_SCANNING)
9924                 bytes_rem[ZPOOL_WAIT_REMOVE] = prs->prs_to_copy -
9925                     prs->prs_copied;
9926
9927         (void) nvlist_lookup_uint64_array(nvroot,
9928             ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&pss, &c);
9929         if (pss != NULL && pss->pss_state == DSS_SCANNING &&
9930             pss->pss_pass_scrub_pause == 0) {
9931                 int64_t rem = pss->pss_to_examine - pss->pss_issued;
9932                 if (pss->pss_func == POOL_SCAN_SCRUB)
9933                         bytes_rem[ZPOOL_WAIT_SCRUB] = rem;
9934                 else
9935                         bytes_rem[ZPOOL_WAIT_RESILVER] = rem;
9936         } else if (check_rebuilding(nvroot, NULL)) {
9937                 bytes_rem[ZPOOL_WAIT_RESILVER] =
9938                     vdev_activity_top_remaining(nvroot);
9939         }
9940
9941         bytes_rem[ZPOOL_WAIT_INITIALIZE] =
9942             vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
9943         bytes_rem[ZPOOL_WAIT_TRIM] =
9944             vdev_activity_remaining(nvroot, ZPOOL_WAIT_TRIM);
9945
9946         /*
9947          * A replace finishes after resilvering finishes, so the amount of work
9948          * left for a replace is the same as for resilvering.
9949          *
9950          * It isn't quite correct to say that if we have any 'spare' or
9951          * 'replacing' vdevs and a resilver is happening, then a replace is in
9952          * progress, like we do here. When a hot spare is used, the faulted vdev
9953          * is not removed after the hot spare is resilvered, so parent 'spare'
9954          * vdev is not removed either. So we could have a 'spare' vdev, but be
9955          * resilvering for a different reason. However, we use it as a heuristic
9956          * because we don't have access to the DTLs, which could tell us whether
9957          * or not we have really finished resilvering a hot spare.
9958          */
9959         if (vdev_any_spare_replacing(nvroot))
9960                 bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
9961
9962         if (timestamp_fmt != NODATE)
9963                 print_timestamp(timestamp_fmt);
9964
9965         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
9966                 char buf[64];
9967                 if (!wd->wd_enabled[i])
9968                         continue;
9969
9970                 if (wd->wd_exact)
9971                         (void) snprintf(buf, sizeof (buf), "%" PRIi64,
9972                             bytes_rem[i]);
9973                 else
9974                         zfs_nicenum(bytes_rem[i], buf, sizeof (buf));
9975
9976                 if (wd->wd_scripted)
9977                         (void) printf(i == 0 ? "%s" : "\t%s", buf);
9978                 else
9979                         (void) printf(" %*s", col_widths[i] - 1, buf);
9980         }
9981         (void) printf("\n");
9982         (void) fflush(stdout);
9983 }
9984
9985 static void *
9986 wait_status_thread(void *arg)
9987 {
9988         wait_data_t *wd = (wait_data_t *)arg;
9989         zpool_handle_t *zhp;
9990
9991         if ((zhp = zpool_open(g_zfs, wd->wd_poolname)) == NULL)
9992                 return (void *)(1);
9993
9994         for (int row = 0; ; row++) {
9995                 boolean_t missing;
9996                 struct timespec timeout;
9997                 int ret = 0;
9998                 (void) clock_gettime(CLOCK_REALTIME, &timeout);
9999
10000                 if (zpool_refresh_stats(zhp, &missing) != 0 || missing ||
10001                     zpool_props_refresh(zhp) != 0) {
10002                         zpool_close(zhp);
10003                         return (void *)(uintptr_t)(missing ? 0 : 1);
10004                 }
10005
10006                 print_wait_status_row(wd, zhp, row);
10007
10008                 timeout.tv_sec += floor(wd->wd_interval);
10009                 long nanos = timeout.tv_nsec +
10010                     (wd->wd_interval - floor(wd->wd_interval)) * NANOSEC;
10011                 if (nanos >= NANOSEC) {
10012                         timeout.tv_sec++;
10013                         timeout.tv_nsec = nanos - NANOSEC;
10014                 } else {
10015                         timeout.tv_nsec = nanos;
10016                 }
10017                 pthread_mutex_lock(&wd->wd_mutex);
10018                 if (!wd->wd_should_exit)
10019                         ret = pthread_cond_timedwait(&wd->wd_cv, &wd->wd_mutex,
10020                             &timeout);
10021                 pthread_mutex_unlock(&wd->wd_mutex);
10022                 if (ret == 0) {
10023                         break; /* signaled by main thread */
10024                 } else if (ret != ETIMEDOUT) {
10025                         (void) fprintf(stderr, gettext("pthread_cond_timedwait "
10026                             "failed: %s\n"), strerror(ret));
10027                         zpool_close(zhp);
10028                         return (void *)(uintptr_t)(1);
10029                 }
10030         }
10031
10032         zpool_close(zhp);
10033         return (void *)(0);
10034 }
10035
10036 int
10037 zpool_do_wait(int argc, char **argv)
10038 {
10039         boolean_t verbose = B_FALSE;
10040         char c;
10041         char *value;
10042         int i;
10043         unsigned long count;
10044         pthread_t status_thr;
10045         int error = 0;
10046         zpool_handle_t *zhp;
10047
10048         wait_data_t wd;
10049         wd.wd_scripted = B_FALSE;
10050         wd.wd_exact = B_FALSE;
10051         wd.wd_headers_once = B_FALSE;
10052         wd.wd_should_exit = B_FALSE;
10053
10054         pthread_mutex_init(&wd.wd_mutex, NULL);
10055         pthread_cond_init(&wd.wd_cv, NULL);
10056
10057         /* By default, wait for all types of activity. */
10058         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++)
10059                 wd.wd_enabled[i] = B_TRUE;
10060
10061         while ((c = getopt(argc, argv, "HpT:t:")) != -1) {
10062                 switch (c) {
10063                 case 'H':
10064                         wd.wd_scripted = B_TRUE;
10065                         break;
10066                 case 'n':
10067                         wd.wd_headers_once = B_TRUE;
10068                         break;
10069                 case 'p':
10070                         wd.wd_exact = B_TRUE;
10071                         break;
10072                 case 'T':
10073                         get_timestamp_arg(*optarg);
10074                         break;
10075                 case 't':
10076                 {
10077                         static char *col_subopts[] = { "discard", "free",
10078                             "initialize", "replace", "remove", "resilver",
10079                             "scrub", "trim", NULL };
10080
10081                         /* Reset activities array */
10082                         bzero(&wd.wd_enabled, sizeof (wd.wd_enabled));
10083                         while (*optarg != '\0') {
10084                                 int activity = getsubopt(&optarg, col_subopts,
10085                                     &value);
10086
10087                                 if (activity < 0) {
10088                                         (void) fprintf(stderr,
10089                                             gettext("invalid activity '%s'\n"),
10090                                             value);
10091                                         usage(B_FALSE);
10092                                 }
10093
10094                                 wd.wd_enabled[activity] = B_TRUE;
10095                         }
10096                         break;
10097                 }
10098                 case '?':
10099                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10100                             optopt);
10101                         usage(B_FALSE);
10102                 }
10103         }
10104
10105         argc -= optind;
10106         argv += optind;
10107
10108         get_interval_count(&argc, argv, &wd.wd_interval, &count);
10109         if (count != 0) {
10110                 /* This subcmd only accepts an interval, not a count */
10111                 (void) fprintf(stderr, gettext("too many arguments\n"));
10112                 usage(B_FALSE);
10113         }
10114
10115         if (wd.wd_interval != 0)
10116                 verbose = B_TRUE;
10117
10118         if (argc < 1) {
10119                 (void) fprintf(stderr, gettext("missing 'pool' argument\n"));
10120                 usage(B_FALSE);
10121         }
10122         if (argc > 1) {
10123                 (void) fprintf(stderr, gettext("too many arguments\n"));
10124                 usage(B_FALSE);
10125         }
10126
10127         wd.wd_poolname = argv[0];
10128
10129         if ((zhp = zpool_open(g_zfs, wd.wd_poolname)) == NULL)
10130                 return (1);
10131
10132         if (verbose) {
10133                 /*
10134                  * We use a separate thread for printing status updates because
10135                  * the main thread will call lzc_wait(), which blocks as long
10136                  * as an activity is in progress, which can be a long time.
10137                  */
10138                 if (pthread_create(&status_thr, NULL, wait_status_thread, &wd)
10139                     != 0) {
10140                         (void) fprintf(stderr, gettext("failed to create status"
10141                             "thread: %s\n"), strerror(errno));
10142                         zpool_close(zhp);
10143                         return (1);
10144                 }
10145         }
10146
10147         /*
10148          * Loop over all activities that we are supposed to wait for until none
10149          * of them are in progress. Note that this means we can end up waiting
10150          * for more activities to complete than just those that were in progress
10151          * when we began waiting; if an activity we are interested in begins
10152          * while we are waiting for another activity, we will wait for both to
10153          * complete before exiting.
10154          */
10155         for (;;) {
10156                 boolean_t missing = B_FALSE;
10157                 boolean_t any_waited = B_FALSE;
10158
10159                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10160                         boolean_t waited;
10161
10162                         if (!wd.wd_enabled[i])
10163                                 continue;
10164
10165                         error = zpool_wait_status(zhp, i, &missing, &waited);
10166                         if (error != 0 || missing)
10167                                 break;
10168
10169                         any_waited = (any_waited || waited);
10170                 }
10171
10172                 if (error != 0 || missing || !any_waited)
10173                         break;
10174         }
10175
10176         zpool_close(zhp);
10177
10178         if (verbose) {
10179                 uintptr_t status;
10180                 pthread_mutex_lock(&wd.wd_mutex);
10181                 wd.wd_should_exit = B_TRUE;
10182                 pthread_cond_signal(&wd.wd_cv);
10183                 pthread_mutex_unlock(&wd.wd_mutex);
10184                 (void) pthread_join(status_thr, (void *)&status);
10185                 if (status != 0)
10186                         error = status;
10187         }
10188
10189         pthread_mutex_destroy(&wd.wd_mutex);
10190         pthread_cond_destroy(&wd.wd_cv);
10191         return (error);
10192 }
10193
10194 static int
10195 find_command_idx(char *command, int *idx)
10196 {
10197         int i;
10198
10199         for (i = 0; i < NCOMMAND; i++) {
10200                 if (command_table[i].name == NULL)
10201                         continue;
10202
10203                 if (strcmp(command, command_table[i].name) == 0) {
10204                         *idx = i;
10205                         return (0);
10206                 }
10207         }
10208         return (1);
10209 }
10210
10211 /*
10212  * Display version message
10213  */
10214 static int
10215 zpool_do_version(int argc, char **argv)
10216 {
10217         if (zfs_version_print() == -1)
10218                 return (1);
10219
10220         return (0);
10221 }
10222
10223 int
10224 main(int argc, char **argv)
10225 {
10226         int ret = 0;
10227         int i = 0;
10228         char *cmdname;
10229         char **newargv;
10230
10231         (void) setlocale(LC_ALL, "");
10232         (void) textdomain(TEXT_DOMAIN);
10233         srand(time(NULL));
10234
10235         opterr = 0;
10236
10237         /*
10238          * Make sure the user has specified some command.
10239          */
10240         if (argc < 2) {
10241                 (void) fprintf(stderr, gettext("missing command\n"));
10242                 usage(B_FALSE);
10243         }
10244
10245         cmdname = argv[1];
10246
10247         /*
10248          * Special case '-?'
10249          */
10250         if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
10251                 usage(B_TRUE);
10252
10253         /*
10254          * Special case '-V|--version'
10255          */
10256         if ((strcmp(cmdname, "-V") == 0) || (strcmp(cmdname, "--version") == 0))
10257                 return (zpool_do_version(argc, argv));
10258
10259         if ((g_zfs = libzfs_init()) == NULL) {
10260                 (void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
10261                 return (1);
10262         }
10263
10264         libzfs_print_on_error(g_zfs, B_TRUE);
10265
10266         zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
10267
10268         /*
10269          * Many commands modify input strings for string parsing reasons.
10270          * We create a copy to protect the original argv.
10271          */
10272         newargv = malloc((argc + 1) * sizeof (newargv[0]));
10273         for (i = 0; i < argc; i++)
10274                 newargv[i] = strdup(argv[i]);
10275         newargv[argc] = NULL;
10276
10277         /*
10278          * Run the appropriate command.
10279          */
10280         if (find_command_idx(cmdname, &i) == 0) {
10281                 current_command = &command_table[i];
10282                 ret = command_table[i].func(argc - 1, newargv + 1);
10283         } else if (strchr(cmdname, '=')) {
10284                 verify(find_command_idx("set", &i) == 0);
10285                 current_command = &command_table[i];
10286                 ret = command_table[i].func(argc, newargv);
10287         } else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
10288                 /*
10289                  * 'freeze' is a vile debugging abomination, so we treat
10290                  * it as such.
10291                  */
10292                 zfs_cmd_t zc = {"\0"};
10293
10294                 (void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
10295                 ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
10296                 if (ret != 0) {
10297                         (void) fprintf(stderr,
10298                         gettext("failed to freeze pool: %d\n"), errno);
10299                         ret = 1;
10300                 }
10301
10302                 log_history = 0;
10303         } else {
10304                 (void) fprintf(stderr, gettext("unrecognized "
10305                     "command '%s'\n"), cmdname);
10306                 usage(B_FALSE);
10307                 ret = 1;
10308         }
10309
10310         for (i = 0; i < argc; i++)
10311                 free(newargv[i]);
10312         free(newargv);
10313
10314         if (ret == 0 && log_history)
10315                 (void) zpool_log_history(g_zfs, history_str);
10316
10317         libzfs_fini(g_zfs);
10318
10319         /*
10320          * The 'ZFS_ABORT' environment variable causes us to dump core on exit
10321          * for the purposes of running ::findleaks.
10322          */
10323         if (getenv("ZFS_ABORT") != NULL) {
10324                 (void) printf("dumping core by request\n");
10325                 abort();
10326         }
10327
10328         return (ret);
10329 }