cmd/zpool/zpool_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  30  * Copyright (c) 2017 Datto Inc.
  31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  32  * Copyright (c) 2017, Intel Corporation.
  33  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
  34  * Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
  35  * Copyright (c) 2021, Klara Inc.
  36  * Copyright [2021] Hewlett Packard Enterprise Development LP
  37  */
  38
  39 #include <assert.h>
  40 #include <ctype.h>
  41 #include <dirent.h>
  42 #include <errno.h>
  43 #include <fcntl.h>
  44 #include <getopt.h>
  45 #include <libgen.h>
  46 #include <libintl.h>
  47 #include <libuutil.h>
  48 #include <locale.h>
  49 #include <pthread.h>
  50 #include <stdio.h>
  51 #include <stdlib.h>
  52 #include <string.h>
  53 #include <time.h>
  54 #include <unistd.h>
  55 #include <pwd.h>
  56 #include <zone.h>
  57 #include <sys/wait.h>
  58 #include <zfs_prop.h>
  59 #include <sys/fs/zfs.h>
  60 #include <sys/stat.h>
  61 #include <sys/systeminfo.h>
  62 #include <sys/fm/fs/zfs.h>
  63 #include <sys/fm/util.h>
  64 #include <sys/fm/protocol.h>
  65 #include <sys/zfs_ioctl.h>
  66 #include <sys/mount.h>
  67 #include <sys/sysmacros.h>
  68
  69 #include <math.h>
  70
  71 #include <libzfs.h>
  72 #include <libzutil.h>
  73
  74 #include "zpool_util.h"
  75 #include "zfs_comutil.h"
  76 #include "zfeature_common.h"
  77
  78 #include "statcommon.h"
  79
  80 libzfs_handle_t *g_zfs;
  81
  82 static int zpool_do_create(int, char **);
  83 static int zpool_do_destroy(int, char **);
  84
  85 static int zpool_do_add(int, char **);
  86 static int zpool_do_remove(int, char **);
  87 static int zpool_do_labelclear(int, char **);
  88
  89 static int zpool_do_checkpoint(int, char **);
  90
  91 static int zpool_do_list(int, char **);
  92 static int zpool_do_iostat(int, char **);
  93 static int zpool_do_status(int, char **);
  94
  95 static int zpool_do_online(int, char **);
  96 static int zpool_do_offline(int, char **);
  97 static int zpool_do_clear(int, char **);
  98 static int zpool_do_reopen(int, char **);
  99
 100 static int zpool_do_reguid(int, char **);
 101
 102 static int zpool_do_attach(int, char **);
 103 static int zpool_do_detach(int, char **);
 104 static int zpool_do_replace(int, char **);
 105 static int zpool_do_split(int, char **);
 106
 107 static int zpool_do_initialize(int, char **);
 108 static int zpool_do_scrub(int, char **);
 109 static int zpool_do_resilver(int, char **);
 110 static int zpool_do_trim(int, char **);
 111
 112 static int zpool_do_import(int, char **);
 113 static int zpool_do_export(int, char **);
 114
 115 static int zpool_do_upgrade(int, char **);
 116
 117 static int zpool_do_history(int, char **);
 118 static int zpool_do_events(int, char **);
 119
 120 static int zpool_do_get(int, char **);
 121 static int zpool_do_set(int, char **);
 122
 123 static int zpool_do_sync(int, char **);
 124
 125 static int zpool_do_version(int, char **);
 126
 127 static int zpool_do_wait(int, char **);
 128
 129 static int zpool_do_help(int argc, char **argv);
 130
 131 static zpool_compat_status_t zpool_do_load_compat(
 132     const char *, boolean_t *);
 133
 134 /*
 135  * These libumem hooks provide a reasonable set of defaults for the allocator's
 136  * debugging facilities.
 137  */
 138
 139 #ifdef DEBUG
 140 const char *
 141 _umem_debug_init(void)
 142 {
 143         return ("default,verbose"); /* $UMEM_DEBUG setting */
 144 }
 145
 146 const char *
 147 _umem_logging_init(void)
 148 {
 149         return ("fail,contents"); /* $UMEM_LOGGING setting */
 150 }
 151 #endif
 152
 153 typedef enum {
 154         HELP_ADD,
 155         HELP_ATTACH,
 156         HELP_CLEAR,
 157         HELP_CREATE,
 158         HELP_CHECKPOINT,
 159         HELP_DESTROY,
 160         HELP_DETACH,
 161         HELP_EXPORT,
 162         HELP_HISTORY,
 163         HELP_IMPORT,
 164         HELP_IOSTAT,
 165         HELP_LABELCLEAR,
 166         HELP_LIST,
 167         HELP_OFFLINE,
 168         HELP_ONLINE,
 169         HELP_REPLACE,
 170         HELP_REMOVE,
 171         HELP_INITIALIZE,
 172         HELP_SCRUB,
 173         HELP_RESILVER,
 174         HELP_TRIM,
 175         HELP_STATUS,
 176         HELP_UPGRADE,
 177         HELP_EVENTS,
 178         HELP_GET,
 179         HELP_SET,
 180         HELP_SPLIT,
 181         HELP_SYNC,
 182         HELP_REGUID,
 183         HELP_REOPEN,
 184         HELP_VERSION,
 185         HELP_WAIT
 186 } zpool_help_t;
 187
 188
 189 /*
 190  * Flags for stats to display with "zpool iostats"
 191  */
 192 enum iostat_type {
 193         IOS_DEFAULT = 0,
 194         IOS_LATENCY = 1,
 195         IOS_QUEUES = 2,
 196         IOS_L_HISTO = 3,
 197         IOS_RQ_HISTO = 4,
 198         IOS_COUNT,      /* always last element */
 199 };
 200
 201 /* iostat_type entries as bitmasks */
 202 #define IOS_DEFAULT_M   (1ULL << IOS_DEFAULT)
 203 #define IOS_LATENCY_M   (1ULL << IOS_LATENCY)
 204 #define IOS_QUEUES_M    (1ULL << IOS_QUEUES)
 205 #define IOS_L_HISTO_M   (1ULL << IOS_L_HISTO)
 206 #define IOS_RQ_HISTO_M  (1ULL << IOS_RQ_HISTO)
 207
 208 /* Mask of all the histo bits */
 209 #define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
 210
 211 /*
 212  * Lookup table for iostat flags to nvlist names.  Basically a list
 213  * of all the nvlists a flag requires.  Also specifies the order in
 214  * which data gets printed in zpool iostat.
 215  */
 216 static const char *vsx_type_to_nvlist[IOS_COUNT][15] = {
 217         [IOS_L_HISTO] = {
 218             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 219             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 220             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 221             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 222             ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
 223             ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
 224             ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
 225             ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
 226             ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
 227             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 228             ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
 229             NULL},
 230         [IOS_LATENCY] = {
 231             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 232             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 233             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 234             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 235             ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
 236             ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
 237             NULL},
 238         [IOS_QUEUES] = {
 239             ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
 240             ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
 241             ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
 242             ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
 243             ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
 244             ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
 245             ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
 246             NULL},
 247         [IOS_RQ_HISTO] = {
 248             ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
 249             ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
 250             ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
 251             ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
 252             ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
 253             ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
 254             ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
 255             ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
 256             ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
 257             ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
 258             ZPOOL_CONFIG_VDEV_IND_TRIM_HISTO,
 259             ZPOOL_CONFIG_VDEV_AGG_TRIM_HISTO,
 260             ZPOOL_CONFIG_VDEV_IND_REBUILD_HISTO,
 261             ZPOOL_CONFIG_VDEV_AGG_REBUILD_HISTO,
 262             NULL},
 263 };
 264
 265
 266 /*
 267  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
 268  * Right now, only one histo bit is ever set at one time, so we can
 269  * just do a highbit64(a)
 270  */
 271 #define IOS_HISTO_IDX(a)        (highbit64(a & IOS_ANYHISTO_M) - 1)
 272
 273 typedef struct zpool_command {
 274         const char      *name;
 275         int             (*func)(int, char **);
 276         zpool_help_t    usage;
 277 } zpool_command_t;
 278
 279 /*
 280  * Master command table.  Each ZFS command has a name, associated function, and
 281  * usage message.  The usage messages need to be internationalized, so we have
 282  * to have a function to return the usage message based on a command index.
 283  *
 284  * These commands are organized according to how they are displayed in the usage
 285  * message.  An empty command (one with a NULL name) indicates an empty line in
 286  * the generic usage message.
 287  */
 288 static zpool_command_t command_table[] = {
 289         { "version",    zpool_do_version,       HELP_VERSION            },
 290         { NULL },
 291         { "create",     zpool_do_create,        HELP_CREATE             },
 292         { "destroy",    zpool_do_destroy,       HELP_DESTROY            },
 293         { NULL },
 294         { "add",        zpool_do_add,           HELP_ADD                },
 295         { "remove",     zpool_do_remove,        HELP_REMOVE             },
 296         { NULL },
 297         { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
 298         { NULL },
 299         { "checkpoint", zpool_do_checkpoint,    HELP_CHECKPOINT         },
 300         { NULL },
 301         { "list",       zpool_do_list,          HELP_LIST               },
 302         { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
 303         { "status",     zpool_do_status,        HELP_STATUS             },
 304         { NULL },
 305         { "online",     zpool_do_online,        HELP_ONLINE             },
 306         { "offline",    zpool_do_offline,       HELP_OFFLINE            },
 307         { "clear",      zpool_do_clear,         HELP_CLEAR              },
 308         { "reopen",     zpool_do_reopen,        HELP_REOPEN             },
 309         { NULL },
 310         { "attach",     zpool_do_attach,        HELP_ATTACH             },
 311         { "detach",     zpool_do_detach,        HELP_DETACH             },
 312         { "replace",    zpool_do_replace,       HELP_REPLACE            },
 313         { "split",      zpool_do_split,         HELP_SPLIT              },
 314         { NULL },
 315         { "initialize", zpool_do_initialize,    HELP_INITIALIZE         },
 316         { "resilver",   zpool_do_resilver,      HELP_RESILVER           },
 317         { "scrub",      zpool_do_scrub,         HELP_SCRUB              },
 318         { "trim",       zpool_do_trim,          HELP_TRIM               },
 319         { NULL },
 320         { "import",     zpool_do_import,        HELP_IMPORT             },
 321         { "export",     zpool_do_export,        HELP_EXPORT             },
 322         { "upgrade",    zpool_do_upgrade,       HELP_UPGRADE            },
 323         { "reguid",     zpool_do_reguid,        HELP_REGUID             },
 324         { NULL },
 325         { "history",    zpool_do_history,       HELP_HISTORY            },
 326         { "events",     zpool_do_events,        HELP_EVENTS             },
 327         { NULL },
 328         { "get",        zpool_do_get,           HELP_GET                },
 329         { "set",        zpool_do_set,           HELP_SET                },
 330         { "sync",       zpool_do_sync,          HELP_SYNC               },
 331         { NULL },
 332         { "wait",       zpool_do_wait,          HELP_WAIT               },
 333 };
 334
 335 #define NCOMMAND        (ARRAY_SIZE(command_table))
 336
 337 #define VDEV_ALLOC_CLASS_LOGS   "logs"
 338
 339 static zpool_command_t *current_command;
 340 static zfs_type_t current_prop_type = (ZFS_TYPE_POOL | ZFS_TYPE_VDEV);
 341 static char history_str[HIS_MAX_RECORD_LEN];
 342 static boolean_t log_history = B_TRUE;
 343 static uint_t timestamp_fmt = NODATE;
 344
 345 static const char *
 346 get_usage(zpool_help_t idx)
 347 {
 348         switch (idx) {
 349         case HELP_ADD:
 350                 return (gettext("\tadd [-fgLnP] [-o property=value] "
 351                     "<pool> <vdev> ...\n"));
 352         case HELP_ATTACH:
 353                 return (gettext("\tattach [-fsw] [-o property=value] "
 354                     "<pool> <device> <new-device>\n"));
 355         case HELP_CLEAR:
 356                 return (gettext("\tclear [-nF] <pool> [device]\n"));
 357         case HELP_CREATE:
 358                 return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 359                     "\t    [-O file-system-property=value] ... \n"
 360                     "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 361         case HELP_CHECKPOINT:
 362                 return (gettext("\tcheckpoint [-d [-w]] <pool> ...\n"));
 363         case HELP_DESTROY:
 364                 return (gettext("\tdestroy [-f] <pool>\n"));
 365         case HELP_DETACH:
 366                 return (gettext("\tdetach <pool> <device>\n"));
 367         case HELP_EXPORT:
 368                 return (gettext("\texport [-af] <pool> ...\n"));
 369         case HELP_HISTORY:
 370                 return (gettext("\thistory [-il] [<pool>] ...\n"));
 371         case HELP_IMPORT:
 372                 return (gettext("\timport [-d dir] [-D]\n"
 373                     "\timport [-o mntopts] [-o property=value] ... \n"
 374                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 375                     "[-R root] [-F [-n]] -a\n"
 376                     "\timport [-o mntopts] [-o property=value] ... \n"
 377                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 378                     "[-R root] [-F [-n]]\n"
 379                     "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
 380         case HELP_IOSTAT:
 381                 return (gettext("\tiostat [[[-c [script1,script2,...]"
 382                     "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
 383                     "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
 384                     " [[-n] interval [count]]\n"));
 385         case HELP_LABELCLEAR:
 386                 return (gettext("\tlabelclear [-f] <vdev>\n"));
 387         case HELP_LIST:
 388                 return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 389                     "[-T d|u] [pool] ... \n"
 390                     "\t    [interval [count]]\n"));
 391         case HELP_OFFLINE:
 392                 return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
 393         case HELP_ONLINE:
 394                 return (gettext("\tonline [-e] <pool> <device> ...\n"));
 395         case HELP_REPLACE:
 396                 return (gettext("\treplace [-fsw] [-o property=value] "
 397                     "<pool> <device> [new-device]\n"));
 398         case HELP_REMOVE:
 399                 return (gettext("\tremove [-npsw] <pool> <device> ...\n"));
 400         case HELP_REOPEN:
 401                 return (gettext("\treopen [-n] <pool>\n"));
 402         case HELP_INITIALIZE:
 403                 return (gettext("\tinitialize [-c | -s | -u] [-w] <pool> "
 404                     "[<device> ...]\n"));
 405         case HELP_SCRUB:
 406                 return (gettext("\tscrub [-s | -p] [-w] [-e] <pool> ...\n"));
 407         case HELP_RESILVER:
 408                 return (gettext("\tresilver <pool> ...\n"));
 409         case HELP_TRIM:
 410                 return (gettext("\ttrim [-dw] [-r <rate>] [-c | -s] <pool> "
 411                     "[<device> ...]\n"));
 412         case HELP_STATUS:
 413                 return (gettext("\tstatus [-c [script1,script2,...]] "
 414                     "[-igLpPstvxD]  [-T d|u] [pool] ... \n"
 415                     "\t    [interval [count]]\n"));
 416         case HELP_UPGRADE:
 417                 return (gettext("\tupgrade\n"
 418                     "\tupgrade -v\n"
 419                     "\tupgrade [-V version] <-a | pool ...>\n"));
 420         case HELP_EVENTS:
 421                 return (gettext("\tevents [-vHf [pool] | -c]\n"));
 422         case HELP_GET:
 423                 return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
 424                     "<\"all\" | property[,...]> <pool> ...\n"));
 425         case HELP_SET:
 426                 return (gettext("\tset <property=value> <pool>\n"
 427                     "\tset <vdev_property=value> <pool> <vdev>\n"));
 428         case HELP_SPLIT:
 429                 return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
 430                     "\t    [-o property=value] <pool> <newpool> "
 431                     "[<device> ...]\n"));
 432         case HELP_REGUID:
 433                 return (gettext("\treguid <pool>\n"));
 434         case HELP_SYNC:
 435                 return (gettext("\tsync [pool] ...\n"));
 436         case HELP_VERSION:
 437                 return (gettext("\tversion\n"));
 438         case HELP_WAIT:
 439                 return (gettext("\twait [-Hp] [-T d|u] [-t <activity>[,...]] "
 440                     "<pool> [interval]\n"));
 441         default:
 442                 __builtin_unreachable();
 443         }
 444 }
 445
 446 static void
 447 zpool_collect_leaves(zpool_handle_t *zhp, nvlist_t *nvroot, nvlist_t *res)
 448 {
 449         uint_t children = 0;
 450         nvlist_t **child;
 451         uint_t i;
 452
 453         (void) nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
 454             &child, &children);
 455
 456         if (children == 0) {
 457                 char *path = zpool_vdev_name(g_zfs, zhp, nvroot,
 458                     VDEV_NAME_PATH);
 459
 460                 if (strcmp(path, VDEV_TYPE_INDIRECT) != 0 &&
 461                     strcmp(path, VDEV_TYPE_HOLE) != 0)
 462                         fnvlist_add_boolean(res, path);
 463
 464                 free(path);
 465                 return;
 466         }
 467
 468         for (i = 0; i < children; i++) {
 469                 zpool_collect_leaves(zhp, child[i], res);
 470         }
 471 }
 472
 473 /*
 474  * Callback routine that will print out a pool property value.
 475  */
 476 static int
 477 print_pool_prop_cb(int prop, void *cb)
 478 {
 479         FILE *fp = cb;
 480
 481         (void) fprintf(fp, "\t%-19s  ", zpool_prop_to_name(prop));
 482
 483         if (zpool_prop_readonly(prop))
 484                 (void) fprintf(fp, "  NO   ");
 485         else
 486                 (void) fprintf(fp, " YES   ");
 487
 488         if (zpool_prop_values(prop) == NULL)
 489                 (void) fprintf(fp, "-\n");
 490         else
 491                 (void) fprintf(fp, "%s\n", zpool_prop_values(prop));
 492
 493         return (ZPROP_CONT);
 494 }
 495
 496 /*
 497  * Callback routine that will print out a vdev property value.
 498  */
 499 static int
 500 print_vdev_prop_cb(int prop, void *cb)
 501 {
 502         FILE *fp = cb;
 503
 504         (void) fprintf(fp, "\t%-19s  ", vdev_prop_to_name(prop));
 505
 506         if (vdev_prop_readonly(prop))
 507                 (void) fprintf(fp, "  NO   ");
 508         else
 509                 (void) fprintf(fp, " YES   ");
 510
 511         if (vdev_prop_values(prop) == NULL)
 512                 (void) fprintf(fp, "-\n");
 513         else
 514                 (void) fprintf(fp, "%s\n", vdev_prop_values(prop));
 515
 516         return (ZPROP_CONT);
 517 }
 518
 519 /*
 520  * Display usage message.  If we're inside a command, display only the usage for
 521  * that command.  Otherwise, iterate over the entire command table and display
 522  * a complete usage message.
 523  */
 524 static __attribute__((noreturn)) void
 525 usage(boolean_t requested)
 526 {
 527         FILE *fp = requested ? stdout : stderr;
 528
 529         if (current_command == NULL) {
 530                 int i;
 531
 532                 (void) fprintf(fp, gettext("usage: zpool command args ...\n"));
 533                 (void) fprintf(fp,
 534                     gettext("where 'command' is one of the following:\n\n"));
 535
 536                 for (i = 0; i < NCOMMAND; i++) {
 537                         if (command_table[i].name == NULL)
 538                                 (void) fprintf(fp, "\n");
 539                         else
 540                                 (void) fprintf(fp, "%s",
 541                                     get_usage(command_table[i].usage));
 542                 }
 543
 544                 (void) fprintf(fp,
 545                     gettext("\nFor further help on a command or topic, "
 546                     "run: %s\n"), "zpool help [<topic>]");
 547         } else {
 548                 (void) fprintf(fp, gettext("usage:\n"));
 549                 (void) fprintf(fp, "%s", get_usage(current_command->usage));
 550         }
 551
 552         if (current_command != NULL &&
 553             current_prop_type != (ZFS_TYPE_POOL | ZFS_TYPE_VDEV) &&
 554             ((strcmp(current_command->name, "set") == 0) ||
 555             (strcmp(current_command->name, "get") == 0) ||
 556             (strcmp(current_command->name, "list") == 0))) {
 557
 558                 (void) fprintf(fp, "%s",
 559                     gettext("\nthe following properties are supported:\n"));
 560
 561                 (void) fprintf(fp, "\n\t%-19s  %s   %s\n\n",
 562                     "PROPERTY", "EDIT", "VALUES");
 563
 564                 /* Iterate over all properties */
 565                 if (current_prop_type == ZFS_TYPE_POOL) {
 566                         (void) zprop_iter(print_pool_prop_cb, fp, B_FALSE,
 567                             B_TRUE, current_prop_type);
 568
 569                         (void) fprintf(fp, "\t%-19s   ", "feature@...");
 570                         (void) fprintf(fp, "YES   "
 571                             "disabled | enabled | active\n");
 572
 573                         (void) fprintf(fp, gettext("\nThe feature@ properties "
 574                             "must be appended with a feature name.\n"
 575                             "See zpool-features(7).\n"));
 576                 } else if (current_prop_type == ZFS_TYPE_VDEV) {
 577                         (void) zprop_iter(print_vdev_prop_cb, fp, B_FALSE,
 578                             B_TRUE, current_prop_type);
 579                 }
 580         }
 581
 582         /*
 583          * See comments at end of main().
 584          */
 585         if (getenv("ZFS_ABORT") != NULL) {
 586                 (void) printf("dumping core by request\n");
 587                 abort();
 588         }
 589
 590         exit(requested ? 0 : 2);
 591 }
 592
 593 /*
 594  * zpool initialize [-c | -s | -u] [-w] <pool> [<vdev> ...]
 595  * Initialize all unused blocks in the specified vdevs, or all vdevs in the pool
 596  * if none specified.
 597  *
 598  *      -c      Cancel. Ends active initializing.
 599  *      -s      Suspend. Initializing can then be restarted with no flags.
 600  *      -u      Uninitialize. Clears initialization state.
 601  *      -w      Wait. Blocks until initializing has completed.
 602  */
 603 int
 604 zpool_do_initialize(int argc, char **argv)
 605 {
 606         int c;
 607         char *poolname;
 608         zpool_handle_t *zhp;
 609         nvlist_t *vdevs;
 610         int err = 0;
 611         boolean_t wait = B_FALSE;
 612
 613         struct option long_options[] = {
 614                 {"cancel",      no_argument,            NULL, 'c'},
 615                 {"suspend",     no_argument,            NULL, 's'},
 616                 {"uninit",      no_argument,            NULL, 'u'},
 617                 {"wait",        no_argument,            NULL, 'w'},
 618                 {0, 0, 0, 0}
 619         };
 620
 621         pool_initialize_func_t cmd_type = POOL_INITIALIZE_START;
 622         while ((c = getopt_long(argc, argv, "csuw", long_options,
 623             NULL)) != -1) {
 624                 switch (c) {
 625                 case 'c':
 626                         if (cmd_type != POOL_INITIALIZE_START &&
 627                             cmd_type != POOL_INITIALIZE_CANCEL) {
 628                                 (void) fprintf(stderr, gettext("-c cannot be "
 629                                     "combined with other options\n"));
 630                                 usage(B_FALSE);
 631                         }
 632                         cmd_type = POOL_INITIALIZE_CANCEL;
 633                         break;
 634                 case 's':
 635                         if (cmd_type != POOL_INITIALIZE_START &&
 636                             cmd_type != POOL_INITIALIZE_SUSPEND) {
 637                                 (void) fprintf(stderr, gettext("-s cannot be "
 638                                     "combined with other options\n"));
 639                                 usage(B_FALSE);
 640                         }
 641                         cmd_type = POOL_INITIALIZE_SUSPEND;
 642                         break;
 643                 case 'u':
 644                         if (cmd_type != POOL_INITIALIZE_START &&
 645                             cmd_type != POOL_INITIALIZE_UNINIT) {
 646                                 (void) fprintf(stderr, gettext("-u cannot be "
 647                                     "combined with other options\n"));
 648                                 usage(B_FALSE);
 649                         }
 650                         cmd_type = POOL_INITIALIZE_UNINIT;
 651                         break;
 652                 case 'w':
 653                         wait = B_TRUE;
 654                         break;
 655                 case '?':
 656                         if (optopt != 0) {
 657                                 (void) fprintf(stderr,
 658                                     gettext("invalid option '%c'\n"), optopt);
 659                         } else {
 660                                 (void) fprintf(stderr,
 661                                     gettext("invalid option '%s'\n"),
 662                                     argv[optind - 1]);
 663                         }
 664                         usage(B_FALSE);
 665                 }
 666         }
 667
 668         argc -= optind;
 669         argv += optind;
 670
 671         if (argc < 1) {
 672                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 673                 usage(B_FALSE);
 674                 return (-1);
 675         }
 676
 677         if (wait && (cmd_type != POOL_INITIALIZE_START)) {
 678                 (void) fprintf(stderr, gettext("-w cannot be used with -c, -s"
 679                     "or -u\n"));
 680                 usage(B_FALSE);
 681         }
 682
 683         poolname = argv[0];
 684         zhp = zpool_open(g_zfs, poolname);
 685         if (zhp == NULL)
 686                 return (-1);
 687
 688         vdevs = fnvlist_alloc();
 689         if (argc == 1) {
 690                 /* no individual leaf vdevs specified, so add them all */
 691                 nvlist_t *config = zpool_get_config(zhp, NULL);
 692                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
 693                     ZPOOL_CONFIG_VDEV_TREE);
 694                 zpool_collect_leaves(zhp, nvroot, vdevs);
 695         } else {
 696                 for (int i = 1; i < argc; i++) {
 697                         fnvlist_add_boolean(vdevs, argv[i]);
 698                 }
 699         }
 700
 701         if (wait)
 702                 err = zpool_initialize_wait(zhp, cmd_type, vdevs);
 703         else
 704                 err = zpool_initialize(zhp, cmd_type, vdevs);
 705
 706         fnvlist_free(vdevs);
 707         zpool_close(zhp);
 708
 709         return (err);
 710 }
 711
 712 /*
 713  * print a pool vdev config for dry runs
 714  */
 715 static void
 716 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 717     const char *match, int name_flags)
 718 {
 719         nvlist_t **child;
 720         uint_t c, children;
 721         char *vname;
 722         boolean_t printed = B_FALSE;
 723
 724         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 725             &child, &children) != 0) {
 726                 if (name != NULL)
 727                         (void) printf("\t%*s%s\n", indent, "", name);
 728                 return;
 729         }
 730
 731         for (c = 0; c < children; c++) {
 732                 uint64_t is_log = B_FALSE, is_hole = B_FALSE;
 733                 const char *class = "";
 734
 735                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
 736                     &is_hole);
 737
 738                 if (is_hole == B_TRUE) {
 739                         continue;
 740                 }
 741
 742                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 743                     &is_log);
 744                 if (is_log)
 745                         class = VDEV_ALLOC_BIAS_LOG;
 746                 (void) nvlist_lookup_string(child[c],
 747                     ZPOOL_CONFIG_ALLOCATION_BIAS, &class);
 748                 if (strcmp(match, class) != 0)
 749                         continue;
 750
 751                 if (!printed && name != NULL) {
 752                         (void) printf("\t%*s%s\n", indent, "", name);
 753                         printed = B_TRUE;
 754                 }
 755                 vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
 756                 print_vdev_tree(zhp, vname, child[c], indent + 2, "",
 757                     name_flags);
 758                 free(vname);
 759         }
 760 }
 761
 762 /*
 763  * Print the list of l2cache devices for dry runs.
 764  */
 765 static void
 766 print_cache_list(nvlist_t *nv, int indent)
 767 {
 768         nvlist_t **child;
 769         uint_t c, children;
 770
 771         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
 772             &child, &children) == 0 && children > 0) {
 773                 (void) printf("\t%*s%s\n", indent, "", "cache");
 774         } else {
 775                 return;
 776         }
 777         for (c = 0; c < children; c++) {
 778                 char *vname;
 779
 780                 vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
 781                 (void) printf("\t%*s%s\n", indent + 2, "", vname);
 782                 free(vname);
 783         }
 784 }
 785
 786 /*
 787  * Print the list of spares for dry runs.
 788  */
 789 static void
 790 print_spare_list(nvlist_t *nv, int indent)
 791 {
 792         nvlist_t **child;
 793         uint_t c, children;
 794
 795         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
 796             &child, &children) == 0 && children > 0) {
 797                 (void) printf("\t%*s%s\n", indent, "", "spares");
 798         } else {
 799                 return;
 800         }
 801         for (c = 0; c < children; c++) {
 802                 char *vname;
 803
 804                 vname = zpool_vdev_name(g_zfs, NULL, child[c], 0);
 805                 (void) printf("\t%*s%s\n", indent + 2, "", vname);
 806                 free(vname);
 807         }
 808 }
 809
 810 static boolean_t
 811 prop_list_contains_feature(nvlist_t *proplist)
 812 {
 813         nvpair_t *nvp;
 814         for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
 815             nvp = nvlist_next_nvpair(proplist, nvp)) {
 816                 if (zpool_prop_feature(nvpair_name(nvp)))
 817                         return (B_TRUE);
 818         }
 819         return (B_FALSE);
 820 }
 821
 822 /*
 823  * Add a property pair (name, string-value) into a property nvlist.
 824  */
 825 static int
 826 add_prop_list(const char *propname, const char *propval, nvlist_t **props,
 827     boolean_t poolprop)
 828 {
 829         zpool_prop_t prop = ZPOOL_PROP_INVAL;
 830         nvlist_t *proplist;
 831         const char *normnm;
 832         const char *strval;
 833
 834         if (*props == NULL &&
 835             nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
 836                 (void) fprintf(stderr,
 837                     gettext("internal error: out of memory\n"));
 838                 return (1);
 839         }
 840
 841         proplist = *props;
 842
 843         if (poolprop) {
 844                 const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 845                 const char *cname =
 846                     zpool_prop_to_name(ZPOOL_PROP_COMPATIBILITY);
 847
 848                 if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 849                     (!zpool_prop_feature(propname) &&
 850                     !zpool_prop_vdev(propname))) {
 851                         (void) fprintf(stderr, gettext("property '%s' is "
 852                             "not a valid pool or vdev property\n"), propname);
 853                         return (2);
 854                 }
 855
 856                 /*
 857                  * feature@ properties and version should not be specified
 858                  * at the same time.
 859                  */
 860                 if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
 861                     nvlist_exists(proplist, vname)) ||
 862                     (prop == ZPOOL_PROP_VERSION &&
 863                     prop_list_contains_feature(proplist))) {
 864                         (void) fprintf(stderr, gettext("'feature@' and "
 865                             "'version' properties cannot be specified "
 866                             "together\n"));
 867                         return (2);
 868                 }
 869
 870                 /*
 871                  * if version is specified, only "legacy" compatibility
 872                  * may be requested
 873                  */
 874                 if ((prop == ZPOOL_PROP_COMPATIBILITY &&
 875                     strcmp(propval, ZPOOL_COMPAT_LEGACY) != 0 &&
 876                     nvlist_exists(proplist, vname)) ||
 877                     (prop == ZPOOL_PROP_VERSION &&
 878                     nvlist_exists(proplist, cname) &&
 879                     strcmp(fnvlist_lookup_string(proplist, cname),
 880                     ZPOOL_COMPAT_LEGACY) != 0)) {
 881                         (void) fprintf(stderr, gettext("when 'version' is "
 882                             "specified, the 'compatibility' feature may only "
 883                             "be set to '" ZPOOL_COMPAT_LEGACY "'\n"));
 884                         return (2);
 885                 }
 886
 887                 if (zpool_prop_feature(propname) || zpool_prop_vdev(propname))
 888                         normnm = propname;
 889                 else
 890                         normnm = zpool_prop_to_name(prop);
 891         } else {
 892                 zfs_prop_t fsprop = zfs_name_to_prop(propname);
 893
 894                 if (zfs_prop_valid_for_type(fsprop, ZFS_TYPE_FILESYSTEM,
 895                     B_FALSE)) {
 896                         normnm = zfs_prop_to_name(fsprop);
 897                 } else if (zfs_prop_user(propname) ||
 898                     zfs_prop_userquota(propname)) {
 899                         normnm = propname;
 900                 } else {
 901                         (void) fprintf(stderr, gettext("property '%s' is "
 902                             "not a valid filesystem property\n"), propname);
 903                         return (2);
 904                 }
 905         }
 906
 907         if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
 908             prop != ZPOOL_PROP_CACHEFILE) {
 909                 (void) fprintf(stderr, gettext("property '%s' "
 910                     "specified multiple times\n"), propname);
 911                 return (2);
 912         }
 913
 914         if (nvlist_add_string(proplist, normnm, propval) != 0) {
 915                 (void) fprintf(stderr, gettext("internal "
 916                     "error: out of memory\n"));
 917                 return (1);
 918         }
 919
 920         return (0);
 921 }
 922
 923 /*
 924  * Set a default property pair (name, string-value) in a property nvlist
 925  */
 926 static int
 927 add_prop_list_default(const char *propname, const char *propval,
 928     nvlist_t **props)
 929 {
 930         const char *pval;
 931
 932         if (nvlist_lookup_string(*props, propname, &pval) == 0)
 933                 return (0);
 934
 935         return (add_prop_list(propname, propval, props, B_TRUE));
 936 }
 937
 938 /*
 939  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
 940  *
 941  *      -f      Force addition of devices, even if they appear in use
 942  *      -g      Display guid for individual vdev name.
 943  *      -L      Follow links when resolving vdev path name.
 944  *      -n      Do not add the devices, but display the resulting layout if
 945  *              they were to be added.
 946  *      -o      Set property=value.
 947  *      -P      Display full path for vdev name.
 948  *
 949  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
 950  * handled by make_root_vdev(), which constructs the nvlist needed to pass to
 951  * libzfs.
 952  */
 953 int
 954 zpool_do_add(int argc, char **argv)
 955 {
 956         boolean_t force = B_FALSE;
 957         boolean_t dryrun = B_FALSE;
 958         int name_flags = 0;
 959         int c;
 960         nvlist_t *nvroot;
 961         char *poolname;
 962         int ret;
 963         zpool_handle_t *zhp;
 964         nvlist_t *config;
 965         nvlist_t *props = NULL;
 966         char *propval;
 967
 968         /* check options */
 969         while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
 970                 switch (c) {
 971                 case 'f':
 972                         force = B_TRUE;
 973                         break;
 974                 case 'g':
 975                         name_flags |= VDEV_NAME_GUID;
 976                         break;
 977                 case 'L':
 978                         name_flags |= VDEV_NAME_FOLLOW_LINKS;
 979                         break;
 980                 case 'n':
 981                         dryrun = B_TRUE;
 982                         break;
 983                 case 'o':
 984                         if ((propval = strchr(optarg, '=')) == NULL) {
 985                                 (void) fprintf(stderr, gettext("missing "
 986                                     "'=' for -o option\n"));
 987                                 usage(B_FALSE);
 988                         }
 989                         *propval = '\0';
 990                         propval++;
 991
 992                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
 993                             (add_prop_list(optarg, propval, &props, B_TRUE)))
 994                                 usage(B_FALSE);
 995                         break;
 996                 case 'P':
 997                         name_flags |= VDEV_NAME_PATH;
 998                         break;
 999                 case '?':
1000                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1001                             optopt);
1002                         usage(B_FALSE);
1003                 }
1004         }
1005
1006         argc -= optind;
1007         argv += optind;
1008
1009         /* get pool name and check number of arguments */
1010         if (argc < 1) {
1011                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1012                 usage(B_FALSE);
1013         }
1014         if (argc < 2) {
1015                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1016                 usage(B_FALSE);
1017         }
1018
1019         poolname = argv[0];
1020
1021         argc--;
1022         argv++;
1023
1024         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1025                 return (1);
1026
1027         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
1028                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
1029                     poolname);
1030                 zpool_close(zhp);
1031                 return (1);
1032         }
1033
1034         /* unless manually specified use "ashift" pool property (if set) */
1035         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
1036                 int intval;
1037                 zprop_source_t src;
1038                 char strval[ZPOOL_MAXPROPLEN];
1039
1040                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
1041                 if (src != ZPROP_SRC_DEFAULT) {
1042                         (void) sprintf(strval, "%" PRId32, intval);
1043                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
1044                             &props, B_TRUE) == 0);
1045                 }
1046         }
1047
1048         /* pass off to make_root_vdev for processing */
1049         nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
1050             argc, argv);
1051         if (nvroot == NULL) {
1052                 zpool_close(zhp);
1053                 return (1);
1054         }
1055
1056         if (dryrun) {
1057                 nvlist_t *poolnvroot;
1058                 nvlist_t **l2child, **sparechild;
1059                 uint_t l2children, sparechildren, c;
1060                 char *vname;
1061                 boolean_t hadcache = B_FALSE, hadspare = B_FALSE;
1062
1063                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1064                     &poolnvroot) == 0);
1065
1066                 (void) printf(gettext("would update '%s' to the following "
1067                     "configuration:\n\n"), zpool_get_name(zhp));
1068
1069                 /* print original main pool and new tree */
1070                 print_vdev_tree(zhp, poolname, poolnvroot, 0, "",
1071                     name_flags | VDEV_NAME_TYPE_ID);
1072                 print_vdev_tree(zhp, NULL, nvroot, 0, "", name_flags);
1073
1074                 /* print other classes: 'dedup', 'special', and 'log' */
1075                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_DEDUP)) {
1076                         print_vdev_tree(zhp, "dedup", poolnvroot, 0,
1077                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
1078                         print_vdev_tree(zhp, NULL, nvroot, 0,
1079                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
1080                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_DEDUP)) {
1081                         print_vdev_tree(zhp, "dedup", nvroot, 0,
1082                             VDEV_ALLOC_BIAS_DEDUP, name_flags);
1083                 }
1084
1085                 if (zfs_special_devs(poolnvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
1086                         print_vdev_tree(zhp, "special", poolnvroot, 0,
1087                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1088                         print_vdev_tree(zhp, NULL, nvroot, 0,
1089                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1090                 } else if (zfs_special_devs(nvroot, VDEV_ALLOC_BIAS_SPECIAL)) {
1091                         print_vdev_tree(zhp, "special", nvroot, 0,
1092                             VDEV_ALLOC_BIAS_SPECIAL, name_flags);
1093                 }
1094
1095                 if (num_logs(poolnvroot) > 0) {
1096                         print_vdev_tree(zhp, "logs", poolnvroot, 0,
1097                             VDEV_ALLOC_BIAS_LOG, name_flags);
1098                         print_vdev_tree(zhp, NULL, nvroot, 0,
1099                             VDEV_ALLOC_BIAS_LOG, name_flags);
1100                 } else if (num_logs(nvroot) > 0) {
1101                         print_vdev_tree(zhp, "logs", nvroot, 0,
1102                             VDEV_ALLOC_BIAS_LOG, name_flags);
1103                 }
1104
1105                 /* Do the same for the caches */
1106                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
1107                     &l2child, &l2children) == 0 && l2children) {
1108                         hadcache = B_TRUE;
1109                         (void) printf(gettext("\tcache\n"));
1110                         for (c = 0; c < l2children; c++) {
1111                                 vname = zpool_vdev_name(g_zfs, NULL,
1112                                     l2child[c], name_flags);
1113                                 (void) printf("\t  %s\n", vname);
1114                                 free(vname);
1115                         }
1116                 }
1117                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
1118                     &l2child, &l2children) == 0 && l2children) {
1119                         if (!hadcache)
1120                                 (void) printf(gettext("\tcache\n"));
1121                         for (c = 0; c < l2children; c++) {
1122                                 vname = zpool_vdev_name(g_zfs, NULL,
1123                                     l2child[c], name_flags);
1124                                 (void) printf("\t  %s\n", vname);
1125                                 free(vname);
1126                         }
1127                 }
1128                 /* And finally the spares */
1129                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_SPARES,
1130                     &sparechild, &sparechildren) == 0 && sparechildren > 0) {
1131                         hadspare = B_TRUE;
1132                         (void) printf(gettext("\tspares\n"));
1133                         for (c = 0; c < sparechildren; c++) {
1134                                 vname = zpool_vdev_name(g_zfs, NULL,
1135                                     sparechild[c], name_flags);
1136                                 (void) printf("\t  %s\n", vname);
1137                                 free(vname);
1138                         }
1139                 }
1140                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
1141                     &sparechild, &sparechildren) == 0 && sparechildren > 0) {
1142                         if (!hadspare)
1143                                 (void) printf(gettext("\tspares\n"));
1144                         for (c = 0; c < sparechildren; c++) {
1145                                 vname = zpool_vdev_name(g_zfs, NULL,
1146                                     sparechild[c], name_flags);
1147                                 (void) printf("\t  %s\n", vname);
1148                                 free(vname);
1149                         }
1150                 }
1151
1152                 ret = 0;
1153         } else {
1154                 ret = (zpool_add(zhp, nvroot) != 0);
1155         }
1156
1157         nvlist_free(props);
1158         nvlist_free(nvroot);
1159         zpool_close(zhp);
1160
1161         return (ret);
1162 }
1163
1164 /*
1165  * zpool remove [-npsw] <pool> <vdev> ...
1166  *
1167  * Removes the given vdev from the pool.
1168  */
1169 int
1170 zpool_do_remove(int argc, char **argv)
1171 {
1172         char *poolname;
1173         int i, ret = 0;
1174         zpool_handle_t *zhp = NULL;
1175         boolean_t stop = B_FALSE;
1176         int c;
1177         boolean_t noop = B_FALSE;
1178         boolean_t parsable = B_FALSE;
1179         boolean_t wait = B_FALSE;
1180
1181         /* check options */
1182         while ((c = getopt(argc, argv, "npsw")) != -1) {
1183                 switch (c) {
1184                 case 'n':
1185                         noop = B_TRUE;
1186                         break;
1187                 case 'p':
1188                         parsable = B_TRUE;
1189                         break;
1190                 case 's':
1191                         stop = B_TRUE;
1192                         break;
1193                 case 'w':
1194                         wait = B_TRUE;
1195                         break;
1196                 case '?':
1197                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1198                             optopt);
1199                         usage(B_FALSE);
1200                 }
1201         }
1202
1203         argc -= optind;
1204         argv += optind;
1205
1206         /* get pool name and check number of arguments */
1207         if (argc < 1) {
1208                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1209                 usage(B_FALSE);
1210         }
1211
1212         poolname = argv[0];
1213
1214         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
1215                 return (1);
1216
1217         if (stop && noop) {
1218                 zpool_close(zhp);
1219                 (void) fprintf(stderr, gettext("stop request ignored\n"));
1220                 return (0);
1221         }
1222
1223         if (stop) {
1224                 if (argc > 1) {
1225                         (void) fprintf(stderr, gettext("too many arguments\n"));
1226                         usage(B_FALSE);
1227                 }
1228                 if (zpool_vdev_remove_cancel(zhp) != 0)
1229                         ret = 1;
1230                 if (wait) {
1231                         (void) fprintf(stderr, gettext("invalid option "
1232                             "combination: -w cannot be used with -s\n"));
1233                         usage(B_FALSE);
1234                 }
1235         } else {
1236                 if (argc < 2) {
1237                         (void) fprintf(stderr, gettext("missing device\n"));
1238                         usage(B_FALSE);
1239                 }
1240
1241                 for (i = 1; i < argc; i++) {
1242                         if (noop) {
1243                                 uint64_t size;
1244
1245                                 if (zpool_vdev_indirect_size(zhp, argv[i],
1246                                     &size) != 0) {
1247                                         ret = 1;
1248                                         break;
1249                                 }
1250                                 if (parsable) {
1251                                         (void) printf("%s %llu\n",
1252                                             argv[i], (unsigned long long)size);
1253                                 } else {
1254                                         char valstr[32];
1255                                         zfs_nicenum(size, valstr,
1256                                             sizeof (valstr));
1257                                         (void) printf("Memory that will be "
1258                                             "used after removing %s: %s\n",
1259                                             argv[i], valstr);
1260                                 }
1261                         } else {
1262                                 if (zpool_vdev_remove(zhp, argv[i]) != 0)
1263                                         ret = 1;
1264                         }
1265                 }
1266
1267                 if (ret == 0 && wait)
1268                         ret = zpool_wait(zhp, ZPOOL_WAIT_REMOVE);
1269         }
1270         zpool_close(zhp);
1271
1272         return (ret);
1273 }
1274
1275 /*
1276  * Return 1 if a vdev is active (being used in a pool)
1277  * Return 0 if a vdev is inactive (offlined or faulted, or not in active pool)
1278  *
1279  * This is useful for checking if a disk in an active pool is offlined or
1280  * faulted.
1281  */
1282 static int
1283 vdev_is_active(char *vdev_path)
1284 {
1285         int fd;
1286         fd = open(vdev_path, O_EXCL);
1287         if (fd < 0) {
1288                 return (1);   /* cant open O_EXCL - disk is active */
1289         }
1290
1291         close(fd);
1292         return (0);   /* disk is inactive in the pool */
1293 }
1294
1295 /*
1296  * zpool labelclear [-f] <vdev>
1297  *
1298  *      -f      Force clearing the label for the vdevs which are members of
1299  *              the exported or foreign pools.
1300  *
1301  * Verifies that the vdev is not active and zeros out the label information
1302  * on the device.
1303  */
1304 int
1305 zpool_do_labelclear(int argc, char **argv)
1306 {
1307         char vdev[MAXPATHLEN];
1308         char *name = NULL;
1309         int c, fd = -1, ret = 0;
1310         nvlist_t *config;
1311         pool_state_t state;
1312         boolean_t inuse = B_FALSE;
1313         boolean_t force = B_FALSE;
1314
1315         /* check options */
1316         while ((c = getopt(argc, argv, "f")) != -1) {
1317                 switch (c) {
1318                 case 'f':
1319                         force = B_TRUE;
1320                         break;
1321                 default:
1322                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1323                             optopt);
1324                         usage(B_FALSE);
1325                 }
1326         }
1327
1328         argc -= optind;
1329         argv += optind;
1330
1331         /* get vdev name */
1332         if (argc < 1) {
1333                 (void) fprintf(stderr, gettext("missing vdev name\n"));
1334                 usage(B_FALSE);
1335         }
1336         if (argc > 1) {
1337                 (void) fprintf(stderr, gettext("too many arguments\n"));
1338                 usage(B_FALSE);
1339         }
1340
1341         (void) strlcpy(vdev, argv[0], sizeof (vdev));
1342
1343         /*
1344          * If we cannot open an absolute path, we quit.
1345          * Otherwise if the provided vdev name doesn't point to a file,
1346          * try prepending expected disk paths and partition numbers.
1347          */
1348         if ((fd = open(vdev, O_RDWR)) < 0) {
1349                 int error;
1350                 if (vdev[0] == '/') {
1351                         (void) fprintf(stderr, gettext("failed to open "
1352                             "%s: %s\n"), vdev, strerror(errno));
1353                         return (1);
1354                 }
1355
1356                 error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
1357                 if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
1358                         if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
1359                                 error = ENOENT;
1360                 }
1361
1362                 if (error || ((fd = open(vdev, O_RDWR)) < 0)) {
1363                         if (errno == ENOENT) {
1364                                 (void) fprintf(stderr, gettext(
1365                                     "failed to find device %s, try "
1366                                     "specifying absolute path instead\n"),
1367                                     argv[0]);
1368                                 return (1);
1369                         }
1370
1371                         (void) fprintf(stderr, gettext("failed to open %s:"
1372                             " %s\n"), vdev, strerror(errno));
1373                         return (1);
1374                 }
1375         }
1376
1377         /*
1378          * Flush all dirty pages for the block device.  This should not be
1379          * fatal when the device does not support BLKFLSBUF as would be the
1380          * case for a file vdev.
1381          */
1382         if ((zfs_dev_flush(fd) != 0) && (errno != ENOTTY))
1383                 (void) fprintf(stderr, gettext("failed to invalidate "
1384                     "cache for %s: %s\n"), vdev, strerror(errno));
1385
1386         if (zpool_read_label(fd, &config, NULL) != 0) {
1387                 (void) fprintf(stderr,
1388                     gettext("failed to read label from %s\n"), vdev);
1389                 ret = 1;
1390                 goto errout;
1391         }
1392         nvlist_free(config);
1393
1394         ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
1395         if (ret != 0) {
1396                 (void) fprintf(stderr,
1397                     gettext("failed to check state for %s\n"), vdev);
1398                 ret = 1;
1399                 goto errout;
1400         }
1401
1402         if (!inuse)
1403                 goto wipe_label;
1404
1405         switch (state) {
1406         default:
1407         case POOL_STATE_ACTIVE:
1408         case POOL_STATE_SPARE:
1409         case POOL_STATE_L2CACHE:
1410                 /*
1411                  * We allow the user to call 'zpool offline -f'
1412                  * on an offlined disk in an active pool. We can check if
1413                  * the disk is online by calling vdev_is_active().
1414                  */
1415                 if (force && !vdev_is_active(vdev))
1416                         break;
1417
1418                 (void) fprintf(stderr, gettext(
1419                     "%s is a member (%s) of pool \"%s\""),
1420                     vdev, zpool_pool_state_to_name(state), name);
1421
1422                 if (force) {
1423                         (void) fprintf(stderr, gettext(
1424                             ". Offline the disk first to clear its label."));
1425                 }
1426                 printf("\n");
1427                 ret = 1;
1428                 goto errout;
1429
1430         case POOL_STATE_EXPORTED:
1431                 if (force)
1432                         break;
1433                 (void) fprintf(stderr, gettext(
1434                     "use '-f' to override the following error:\n"
1435                     "%s is a member of exported pool \"%s\"\n"),
1436                     vdev, name);
1437                 ret = 1;
1438                 goto errout;
1439
1440         case POOL_STATE_POTENTIALLY_ACTIVE:
1441                 if (force)
1442                         break;
1443                 (void) fprintf(stderr, gettext(
1444                     "use '-f' to override the following error:\n"
1445                     "%s is a member of potentially active pool \"%s\"\n"),
1446                     vdev, name);
1447                 ret = 1;
1448                 goto errout;
1449
1450         case POOL_STATE_DESTROYED:
1451                 /* inuse should never be set for a destroyed pool */
1452                 assert(0);
1453                 break;
1454         }
1455
1456 wipe_label:
1457         ret = zpool_clear_label(fd);
1458         if (ret != 0) {
1459                 (void) fprintf(stderr,
1460                     gettext("failed to clear label for %s\n"), vdev);
1461         }
1462
1463 errout:
1464         free(name);
1465         (void) close(fd);
1466
1467         return (ret);
1468 }
1469
1470 /*
1471  * zpool create [-fnd] [-o property=value] ...
1472  *              [-O file-system-property=value] ...
1473  *              [-R root] [-m mountpoint] <pool> <dev> ...
1474  *
1475  *      -f      Force creation, even if devices appear in use
1476  *      -n      Do not create the pool, but display the resulting layout if it
1477  *              were to be created.
1478  *      -R      Create a pool under an alternate root
1479  *      -m      Set default mountpoint for the root dataset.  By default it's
1480  *              '/<pool>'
1481  *      -o      Set property=value.
1482  *      -o      Set feature@feature=enabled|disabled.
1483  *      -d      Don't automatically enable all supported pool features
1484  *              (individual features can be enabled with -o).
1485  *      -O      Set fsproperty=value in the pool's root file system
1486  *
1487  * Creates the named pool according to the given vdev specification.  The
1488  * bulk of the vdev processing is done in make_root_vdev() in zpool_vdev.c.
1489  * Once we get the nvlist back from make_root_vdev(), we either print out the
1490  * contents (if '-n' was specified), or pass it to libzfs to do the creation.
1491  */
1492 int
1493 zpool_do_create(int argc, char **argv)
1494 {
1495         boolean_t force = B_FALSE;
1496         boolean_t dryrun = B_FALSE;
1497         boolean_t enable_pool_features = B_TRUE;
1498
1499         int c;
1500         nvlist_t *nvroot = NULL;
1501         char *poolname;
1502         char *tname = NULL;
1503         int ret = 1;
1504         char *altroot = NULL;
1505         char *compat = NULL;
1506         char *mountpoint = NULL;
1507         nvlist_t *fsprops = NULL;
1508         nvlist_t *props = NULL;
1509         char *propval;
1510
1511         /* check options */
1512         while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1513                 switch (c) {
1514                 case 'f':
1515                         force = B_TRUE;
1516                         break;
1517                 case 'n':
1518                         dryrun = B_TRUE;
1519                         break;
1520                 case 'd':
1521                         enable_pool_features = B_FALSE;
1522                         break;
1523                 case 'R':
1524                         altroot = optarg;
1525                         if (add_prop_list(zpool_prop_to_name(
1526                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1527                                 goto errout;
1528                         if (add_prop_list_default(zpool_prop_to_name(
1529                             ZPOOL_PROP_CACHEFILE), "none", &props))
1530                                 goto errout;
1531                         break;
1532                 case 'm':
1533                         /* Equivalent to -O mountpoint=optarg */
1534                         mountpoint = optarg;
1535                         break;
1536                 case 'o':
1537                         if ((propval = strchr(optarg, '=')) == NULL) {
1538                                 (void) fprintf(stderr, gettext("missing "
1539                                     "'=' for -o option\n"));
1540                                 goto errout;
1541                         }
1542                         *propval = '\0';
1543                         propval++;
1544
1545                         if (add_prop_list(optarg, propval, &props, B_TRUE))
1546                                 goto errout;
1547
1548                         /*
1549                          * If the user is creating a pool that doesn't support
1550                          * feature flags, don't enable any features.
1551                          */
1552                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1553                                 char *end;
1554                                 u_longlong_t ver;
1555
1556                                 ver = strtoull(propval, &end, 10);
1557                                 if (*end == '\0' &&
1558                                     ver < SPA_VERSION_FEATURES) {
1559                                         enable_pool_features = B_FALSE;
1560                                 }
1561                         }
1562                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1563                                 altroot = propval;
1564                         if (zpool_name_to_prop(optarg) ==
1565                             ZPOOL_PROP_COMPATIBILITY)
1566                                 compat = propval;
1567                         break;
1568                 case 'O':
1569                         if ((propval = strchr(optarg, '=')) == NULL) {
1570                                 (void) fprintf(stderr, gettext("missing "
1571                                     "'=' for -O option\n"));
1572                                 goto errout;
1573                         }
1574                         *propval = '\0';
1575                         propval++;
1576
1577                         /*
1578                          * Mountpoints are checked and then added later.
1579                          * Uniquely among properties, they can be specified
1580                          * more than once, to avoid conflict with -m.
1581                          */
1582                         if (0 == strcmp(optarg,
1583                             zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1584                                 mountpoint = propval;
1585                         } else if (add_prop_list(optarg, propval, &fsprops,
1586                             B_FALSE)) {
1587                                 goto errout;
1588                         }
1589                         break;
1590                 case 't':
1591                         /*
1592                          * Sanity check temporary pool name.
1593                          */
1594                         if (strchr(optarg, '/') != NULL) {
1595                                 (void) fprintf(stderr, gettext("cannot create "
1596                                     "'%s': invalid character '/' in temporary "
1597                                     "name\n"), optarg);
1598                                 (void) fprintf(stderr, gettext("use 'zfs "
1599                                     "create' to create a dataset\n"));
1600                                 goto errout;
1601                         }
1602
1603                         if (add_prop_list(zpool_prop_to_name(
1604                             ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1605                                 goto errout;
1606                         if (add_prop_list_default(zpool_prop_to_name(
1607                             ZPOOL_PROP_CACHEFILE), "none", &props))
1608                                 goto errout;
1609                         tname = optarg;
1610                         break;
1611                 case ':':
1612                         (void) fprintf(stderr, gettext("missing argument for "
1613                             "'%c' option\n"), optopt);
1614                         goto badusage;
1615                 case '?':
1616                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1617                             optopt);
1618                         goto badusage;
1619                 }
1620         }
1621
1622         argc -= optind;
1623         argv += optind;
1624
1625         /* get pool name and check number of arguments */
1626         if (argc < 1) {
1627                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1628                 goto badusage;
1629         }
1630         if (argc < 2) {
1631                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1632                 goto badusage;
1633         }
1634
1635         poolname = argv[0];
1636
1637         /*
1638          * As a special case, check for use of '/' in the name, and direct the
1639          * user to use 'zfs create' instead.
1640          */
1641         if (strchr(poolname, '/') != NULL) {
1642                 (void) fprintf(stderr, gettext("cannot create '%s': invalid "
1643                     "character '/' in pool name\n"), poolname);
1644                 (void) fprintf(stderr, gettext("use 'zfs create' to "
1645                     "create a dataset\n"));
1646                 goto errout;
1647         }
1648
1649         /* pass off to make_root_vdev for bulk processing */
1650         nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1651             argc - 1, argv + 1);
1652         if (nvroot == NULL)
1653                 goto errout;
1654
1655         /* make_root_vdev() allows 0 toplevel children if there are spares */
1656         if (!zfs_allocatable_devs(nvroot)) {
1657                 (void) fprintf(stderr, gettext("invalid vdev "
1658                     "specification: at least one toplevel vdev must be "
1659                     "specified\n"));
1660                 goto errout;
1661         }
1662
1663         if (altroot != NULL && altroot[0] != '/') {
1664                 (void) fprintf(stderr, gettext("invalid alternate root '%s': "
1665                     "must be an absolute path\n"), altroot);
1666                 goto errout;
1667         }
1668
1669         /*
1670          * Check the validity of the mountpoint and direct the user to use the
1671          * '-m' mountpoint option if it looks like its in use.
1672          */
1673         if (mountpoint == NULL ||
1674             (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1675             strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1676                 char buf[MAXPATHLEN];
1677                 DIR *dirp;
1678
1679                 if (mountpoint && mountpoint[0] != '/') {
1680                         (void) fprintf(stderr, gettext("invalid mountpoint "
1681                             "'%s': must be an absolute path, 'legacy', or "
1682                             "'none'\n"), mountpoint);
1683                         goto errout;
1684                 }
1685
1686                 if (mountpoint == NULL) {
1687                         if (altroot != NULL)
1688                                 (void) snprintf(buf, sizeof (buf), "%s/%s",
1689                                     altroot, poolname);
1690                         else
1691                                 (void) snprintf(buf, sizeof (buf), "/%s",
1692                                     poolname);
1693                 } else {
1694                         if (altroot != NULL)
1695                                 (void) snprintf(buf, sizeof (buf), "%s%s",
1696                                     altroot, mountpoint);
1697                         else
1698                                 (void) snprintf(buf, sizeof (buf), "%s",
1699                                     mountpoint);
1700                 }
1701
1702                 if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1703                         (void) fprintf(stderr, gettext("mountpoint '%s' : "
1704                             "%s\n"), buf, strerror(errno));
1705                         (void) fprintf(stderr, gettext("use '-m' "
1706                             "option to provide a different default\n"));
1707                         goto errout;
1708                 } else if (dirp) {
1709                         int count = 0;
1710
1711                         while (count < 3 && readdir(dirp) != NULL)
1712                                 count++;
1713                         (void) closedir(dirp);
1714
1715                         if (count > 2) {
1716                                 (void) fprintf(stderr, gettext("mountpoint "
1717                                     "'%s' exists and is not empty\n"), buf);
1718                                 (void) fprintf(stderr, gettext("use '-m' "
1719                                     "option to provide a "
1720                                     "different default\n"));
1721                                 goto errout;
1722                         }
1723                 }
1724         }
1725
1726         /*
1727          * Now that the mountpoint's validity has been checked, ensure that
1728          * the property is set appropriately prior to creating the pool.
1729          */
1730         if (mountpoint != NULL) {
1731                 ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1732                     mountpoint, &fsprops, B_FALSE);
1733                 if (ret != 0)
1734                         goto errout;
1735         }
1736
1737         ret = 1;
1738         if (dryrun) {
1739                 /*
1740                  * For a dry run invocation, print out a basic message and run
1741                  * through all the vdevs in the list and print out in an
1742                  * appropriate hierarchy.
1743                  */
1744                 (void) printf(gettext("would create '%s' with the "
1745                     "following layout:\n\n"), poolname);
1746
1747                 print_vdev_tree(NULL, poolname, nvroot, 0, "", 0);
1748                 print_vdev_tree(NULL, "dedup", nvroot, 0,
1749                     VDEV_ALLOC_BIAS_DEDUP, 0);
1750                 print_vdev_tree(NULL, "special", nvroot, 0,
1751                     VDEV_ALLOC_BIAS_SPECIAL, 0);
1752                 print_vdev_tree(NULL, "logs", nvroot, 0,
1753                     VDEV_ALLOC_BIAS_LOG, 0);
1754                 print_cache_list(nvroot, 0);
1755                 print_spare_list(nvroot, 0);
1756
1757                 ret = 0;
1758         } else {
1759                 /*
1760                  * Load in feature set.
1761                  * Note: if compatibility property not given, we'll have
1762                  * NULL, which means 'all features'.
1763                  */
1764                 boolean_t requested_features[SPA_FEATURES];
1765                 if (zpool_do_load_compat(compat, requested_features) !=
1766                     ZPOOL_COMPATIBILITY_OK)
1767                         goto errout;
1768
1769                 /*
1770                  * props contains list of features to enable.
1771                  * For each feature:
1772                  *  - remove it if feature@name=disabled
1773                  *  - leave it there if feature@name=enabled
1774                  *  - add it if:
1775                  *    - enable_pool_features (ie: no '-d' or '-o version')
1776                  *    - it's supported by the kernel module
1777                  *    - it's in the requested feature set
1778                  *  - warn if it's enabled but not in compat
1779                  */
1780                 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) {
1781                         char propname[MAXPATHLEN];
1782                         const char *propval;
1783                         zfeature_info_t *feat = &spa_feature_table[i];
1784
1785                         (void) snprintf(propname, sizeof (propname),
1786                             "feature@%s", feat->fi_uname);
1787
1788                         if (!nvlist_lookup_string(props, propname, &propval)) {
1789                                 if (strcmp(propval,
1790                                     ZFS_FEATURE_DISABLED) == 0) {
1791                                         (void) nvlist_remove_all(props,
1792                                             propname);
1793                                 } else if (strcmp(propval,
1794                                     ZFS_FEATURE_ENABLED) == 0 &&
1795                                     !requested_features[i]) {
1796                                         (void) fprintf(stderr, gettext(
1797                                             "Warning: feature \"%s\" enabled "
1798                                             "but is not in specified "
1799                                             "'compatibility' feature set.\n"),
1800                                             feat->fi_uname);
1801                                 }
1802                         } else if (
1803                             enable_pool_features &&
1804                             feat->fi_zfs_mod_supported &&
1805                             requested_features[i]) {
1806                                 ret = add_prop_list(propname,
1807                                     ZFS_FEATURE_ENABLED, &props, B_TRUE);
1808                                 if (ret != 0)
1809                                         goto errout;
1810                         }
1811                 }
1812
1813                 ret = 1;
1814                 if (zpool_create(g_zfs, poolname,
1815                     nvroot, props, fsprops) == 0) {
1816                         zfs_handle_t *pool = zfs_open(g_zfs,
1817                             tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1818                         if (pool != NULL) {
1819                                 if (zfs_mount(pool, NULL, 0) == 0) {
1820                                         ret = zfs_share(pool, NULL);
1821                                         zfs_commit_shares(NULL);
1822                                 }
1823                                 zfs_close(pool);
1824                         }
1825                 } else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1826                         (void) fprintf(stderr, gettext("pool name may have "
1827                             "been omitted\n"));
1828                 }
1829         }
1830
1831 errout:
1832         nvlist_free(nvroot);
1833         nvlist_free(fsprops);
1834         nvlist_free(props);
1835         return (ret);
1836 badusage:
1837         nvlist_free(fsprops);
1838         nvlist_free(props);
1839         usage(B_FALSE);
1840         return (2);
1841 }
1842
1843 /*
1844  * zpool destroy <pool>
1845  *
1846  *      -f      Forcefully unmount any datasets
1847  *
1848  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1849  */
1850 int
1851 zpool_do_destroy(int argc, char **argv)
1852 {
1853         boolean_t force = B_FALSE;
1854         int c;
1855         char *pool;
1856         zpool_handle_t *zhp;
1857         int ret;
1858
1859         /* check options */
1860         while ((c = getopt(argc, argv, "f")) != -1) {
1861                 switch (c) {
1862                 case 'f':
1863                         force = B_TRUE;
1864                         break;
1865                 case '?':
1866                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1867                             optopt);
1868                         usage(B_FALSE);
1869                 }
1870         }
1871
1872         argc -= optind;
1873         argv += optind;
1874
1875         /* check arguments */
1876         if (argc < 1) {
1877                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1878                 usage(B_FALSE);
1879         }
1880         if (argc > 1) {
1881                 (void) fprintf(stderr, gettext("too many arguments\n"));
1882                 usage(B_FALSE);
1883         }
1884
1885         pool = argv[0];
1886
1887         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1888                 /*
1889                  * As a special case, check for use of '/' in the name, and
1890                  * direct the user to use 'zfs destroy' instead.
1891                  */
1892                 if (strchr(pool, '/') != NULL)
1893                         (void) fprintf(stderr, gettext("use 'zfs destroy' to "
1894                             "destroy a dataset\n"));
1895                 return (1);
1896         }
1897
1898         if (zpool_disable_datasets(zhp, force) != 0) {
1899                 (void) fprintf(stderr, gettext("could not destroy '%s': "
1900                     "could not unmount datasets\n"), zpool_get_name(zhp));
1901                 zpool_close(zhp);
1902                 return (1);
1903         }
1904
1905         /* The history must be logged as part of the export */
1906         log_history = B_FALSE;
1907
1908         ret = (zpool_destroy(zhp, history_str) != 0);
1909
1910         zpool_close(zhp);
1911
1912         return (ret);
1913 }
1914
1915 typedef struct export_cbdata {
1916         boolean_t force;
1917         boolean_t hardforce;
1918 } export_cbdata_t;
1919
1920 /*
1921  * Export one pool
1922  */
1923 static int
1924 zpool_export_one(zpool_handle_t *zhp, void *data)
1925 {
1926         export_cbdata_t *cb = data;
1927
1928         if (zpool_disable_datasets(zhp, cb->force) != 0)
1929                 return (1);
1930
1931         /* The history must be logged as part of the export */
1932         log_history = B_FALSE;
1933
1934         if (cb->hardforce) {
1935                 if (zpool_export_force(zhp, history_str) != 0)
1936                         return (1);
1937         } else if (zpool_export(zhp, cb->force, history_str) != 0) {
1938                 return (1);
1939         }
1940
1941         return (0);
1942 }
1943
1944 /*
1945  * zpool export [-f] <pool> ...
1946  *
1947  *      -a      Export all pools
1948  *      -f      Forcefully unmount datasets
1949  *
1950  * Export the given pools.  By default, the command will attempt to cleanly
1951  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1952  * then the datasets will be forcefully unmounted.
1953  */
1954 int
1955 zpool_do_export(int argc, char **argv)
1956 {
1957         export_cbdata_t cb;
1958         boolean_t do_all = B_FALSE;
1959         boolean_t force = B_FALSE;
1960         boolean_t hardforce = B_FALSE;
1961         int c, ret;
1962
1963         /* check options */
1964         while ((c = getopt(argc, argv, "afF")) != -1) {
1965                 switch (c) {
1966                 case 'a':
1967                         do_all = B_TRUE;
1968                         break;
1969                 case 'f':
1970                         force = B_TRUE;
1971                         break;
1972                 case 'F':
1973                         hardforce = B_TRUE;
1974                         break;
1975                 case '?':
1976                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1977                             optopt);
1978                         usage(B_FALSE);
1979                 }
1980         }
1981
1982         cb.force = force;
1983         cb.hardforce = hardforce;
1984         argc -= optind;
1985         argv += optind;
1986
1987         if (do_all) {
1988                 if (argc != 0) {
1989                         (void) fprintf(stderr, gettext("too many arguments\n"));
1990                         usage(B_FALSE);
1991                 }
1992
1993                 return (for_each_pool(argc, argv, B_TRUE, NULL,
1994                     ZFS_TYPE_POOL, B_FALSE, zpool_export_one, &cb));
1995         }
1996
1997         /* check arguments */
1998         if (argc < 1) {
1999                 (void) fprintf(stderr, gettext("missing pool argument\n"));
2000                 usage(B_FALSE);
2001         }
2002
2003         ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
2004             B_FALSE, zpool_export_one, &cb);
2005
2006         return (ret);
2007 }
2008
2009 /*
2010  * Given a vdev configuration, determine the maximum width needed for the device
2011  * name column.
2012  */
2013 static int
2014 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
2015     int name_flags)
2016 {
2017         static const char *const subtypes[] =
2018             {ZPOOL_CONFIG_SPARES, ZPOOL_CONFIG_L2CACHE, ZPOOL_CONFIG_CHILDREN};
2019
2020         char *name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
2021         max = MAX(strlen(name) + depth, max);
2022         free(name);
2023
2024         nvlist_t **child;
2025         uint_t children;
2026         for (size_t i = 0; i < ARRAY_SIZE(subtypes); ++i)
2027                 if (nvlist_lookup_nvlist_array(nv, subtypes[i],
2028                     &child, &children) == 0)
2029                         for (uint_t c = 0; c < children; ++c)
2030                                 max = MAX(max_width(zhp, child[c], depth + 2,
2031                                     max, name_flags), max);
2032
2033         return (max);
2034 }
2035
2036 typedef struct spare_cbdata {
2037         uint64_t        cb_guid;
2038         zpool_handle_t  *cb_zhp;
2039 } spare_cbdata_t;
2040
2041 static boolean_t
2042 find_vdev(nvlist_t *nv, uint64_t search)
2043 {
2044         uint64_t guid;
2045         nvlist_t **child;
2046         uint_t c, children;
2047
2048         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
2049             search == guid)
2050                 return (B_TRUE);
2051
2052         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2053             &child, &children) == 0) {
2054                 for (c = 0; c < children; c++)
2055                         if (find_vdev(child[c], search))
2056                                 return (B_TRUE);
2057         }
2058
2059         return (B_FALSE);
2060 }
2061
2062 static int
2063 find_spare(zpool_handle_t *zhp, void *data)
2064 {
2065         spare_cbdata_t *cbp = data;
2066         nvlist_t *config, *nvroot;
2067
2068         config = zpool_get_config(zhp, NULL);
2069         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2070             &nvroot) == 0);
2071
2072         if (find_vdev(nvroot, cbp->cb_guid)) {
2073                 cbp->cb_zhp = zhp;
2074                 return (1);
2075         }
2076
2077         zpool_close(zhp);
2078         return (0);
2079 }
2080
2081 typedef struct status_cbdata {
2082         int             cb_count;
2083         int             cb_name_flags;
2084         int             cb_namewidth;
2085         boolean_t       cb_allpools;
2086         boolean_t       cb_verbose;
2087         boolean_t       cb_literal;
2088         boolean_t       cb_explain;
2089         boolean_t       cb_first;
2090         boolean_t       cb_dedup_stats;
2091         boolean_t       cb_print_status;
2092         boolean_t       cb_print_slow_ios;
2093         boolean_t       cb_print_vdev_init;
2094         boolean_t       cb_print_vdev_trim;
2095         vdev_cmd_data_list_t    *vcdl;
2096 } status_cbdata_t;
2097
2098 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
2099 static boolean_t
2100 is_blank_str(const char *str)
2101 {
2102         for (; str != NULL && *str != '\0'; ++str)
2103                 if (!isblank(*str))
2104                         return (B_FALSE);
2105         return (B_TRUE);
2106 }
2107
2108 /* Print command output lines for specific vdev in a specific pool */
2109 static void
2110 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, const char *path)
2111 {
2112         vdev_cmd_data_t *data;
2113         int i, j;
2114         const char *val;
2115
2116         for (i = 0; i < vcdl->count; i++) {
2117                 if ((strcmp(vcdl->data[i].path, path) != 0) ||
2118                     (strcmp(vcdl->data[i].pool, pool) != 0)) {
2119                         /* Not the vdev we're looking for */
2120                         continue;
2121                 }
2122
2123                 data = &vcdl->data[i];
2124                 /* Print out all the output values for this vdev */
2125                 for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
2126                         val = NULL;
2127                         /* Does this vdev have values for this column? */
2128                         for (int k = 0; k < data->cols_cnt; k++) {
2129                                 if (strcmp(data->cols[k],
2130                                     vcdl->uniq_cols[j]) == 0) {
2131                                         /* yes it does, record the value */
2132                                         val = data->lines[k];
2133                                         break;
2134                                 }
2135                         }
2136                         /*
2137                          * Mark empty values with dashes to make output
2138                          * awk-able.
2139                          */
2140                         if (val == NULL || is_blank_str(val))
2141                                 val = "-";
2142
2143                         printf("%*s", vcdl->uniq_cols_width[j], val);
2144                         if (j < vcdl->uniq_cols_cnt - 1)
2145                                 fputs("  ", stdout);
2146                 }
2147
2148                 /* Print out any values that aren't in a column at the end */
2149                 for (j = data->cols_cnt; j < data->lines_cnt; j++) {
2150                         /* Did we have any columns?  If so print a spacer. */
2151                         if (vcdl->uniq_cols_cnt > 0)
2152                                 fputs("  ", stdout);
2153
2154                         val = data->lines[j];
2155                         fputs(val ?: "", stdout);
2156                 }
2157                 break;
2158         }
2159 }
2160
2161 /*
2162  * Print vdev initialization status for leaves
2163  */
2164 static void
2165 print_status_initialize(vdev_stat_t *vs, boolean_t verbose)
2166 {
2167         if (verbose) {
2168                 if ((vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE ||
2169                     vs->vs_initialize_state == VDEV_INITIALIZE_SUSPENDED ||
2170                     vs->vs_initialize_state == VDEV_INITIALIZE_COMPLETE) &&
2171                     !vs->vs_scan_removing) {
2172                         char zbuf[1024];
2173                         char tbuf[256];
2174                         struct tm zaction_ts;
2175
2176                         time_t t = vs->vs_initialize_action_time;
2177                         int initialize_pct = 100;
2178                         if (vs->vs_initialize_state !=
2179                             VDEV_INITIALIZE_COMPLETE) {
2180                                 initialize_pct = (vs->vs_initialize_bytes_done *
2181                                     100 / (vs->vs_initialize_bytes_est + 1));
2182                         }
2183
2184                         (void) localtime_r(&t, &zaction_ts);
2185                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2186
2187                         switch (vs->vs_initialize_state) {
2188                         case VDEV_INITIALIZE_SUSPENDED:
2189                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2190                                     gettext("suspended, started at"), tbuf);
2191                                 break;
2192                         case VDEV_INITIALIZE_ACTIVE:
2193                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2194                                     gettext("started at"), tbuf);
2195                                 break;
2196                         case VDEV_INITIALIZE_COMPLETE:
2197                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2198                                     gettext("completed at"), tbuf);
2199                                 break;
2200                         }
2201
2202                         (void) printf(gettext("  (%d%% initialized%s)"),
2203                             initialize_pct, zbuf);
2204                 } else {
2205                         (void) printf(gettext("  (uninitialized)"));
2206                 }
2207         } else if (vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE) {
2208                 (void) printf(gettext("  (initializing)"));
2209         }
2210 }
2211
2212 /*
2213  * Print vdev TRIM status for leaves
2214  */
2215 static void
2216 print_status_trim(vdev_stat_t *vs, boolean_t verbose)
2217 {
2218         if (verbose) {
2219                 if ((vs->vs_trim_state == VDEV_TRIM_ACTIVE ||
2220                     vs->vs_trim_state == VDEV_TRIM_SUSPENDED ||
2221                     vs->vs_trim_state == VDEV_TRIM_COMPLETE) &&
2222                     !vs->vs_scan_removing) {
2223                         char zbuf[1024];
2224                         char tbuf[256];
2225                         struct tm zaction_ts;
2226
2227                         time_t t = vs->vs_trim_action_time;
2228                         int trim_pct = 100;
2229                         if (vs->vs_trim_state != VDEV_TRIM_COMPLETE) {
2230                                 trim_pct = (vs->vs_trim_bytes_done *
2231                                     100 / (vs->vs_trim_bytes_est + 1));
2232                         }
2233
2234                         (void) localtime_r(&t, &zaction_ts);
2235                         (void) strftime(tbuf, sizeof (tbuf), "%c", &zaction_ts);
2236
2237                         switch (vs->vs_trim_state) {
2238                         case VDEV_TRIM_SUSPENDED:
2239                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2240                                     gettext("suspended, started at"), tbuf);
2241                                 break;
2242                         case VDEV_TRIM_ACTIVE:
2243                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2244                                     gettext("started at"), tbuf);
2245                                 break;
2246                         case VDEV_TRIM_COMPLETE:
2247                                 (void) snprintf(zbuf, sizeof (zbuf), ", %s %s",
2248                                     gettext("completed at"), tbuf);
2249                                 break;
2250                         }
2251
2252                         (void) printf(gettext("  (%d%% trimmed%s)"),
2253                             trim_pct, zbuf);
2254                 } else if (vs->vs_trim_notsup) {
2255                         (void) printf(gettext("  (trim unsupported)"));
2256                 } else {
2257                         (void) printf(gettext("  (untrimmed)"));
2258                 }
2259         } else if (vs->vs_trim_state == VDEV_TRIM_ACTIVE) {
2260                 (void) printf(gettext("  (trimming)"));
2261         }
2262 }
2263
2264 /*
2265  * Return the color associated with a health string.  This includes returning
2266  * NULL for no color change.
2267  */
2268 static const char *
2269 health_str_to_color(const char *health)
2270 {
2271         if (strcmp(health, gettext("FAULTED")) == 0 ||
2272             strcmp(health, gettext("SUSPENDED")) == 0 ||
2273             strcmp(health, gettext("UNAVAIL")) == 0) {
2274                 return (ANSI_RED);
2275         }
2276
2277         if (strcmp(health, gettext("OFFLINE")) == 0 ||
2278             strcmp(health, gettext("DEGRADED")) == 0 ||
2279             strcmp(health, gettext("REMOVED")) == 0) {
2280                 return (ANSI_YELLOW);
2281         }
2282
2283         return (NULL);
2284 }
2285
2286 /*
2287  * Print out configuration state as requested by status_callback.
2288  */
2289 static void
2290 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
2291     nvlist_t *nv, int depth, boolean_t isspare, vdev_rebuild_stat_t *vrs)
2292 {
2293         nvlist_t **child, *root;
2294         uint_t c, i, vsc, children;
2295         pool_scan_stat_t *ps = NULL;
2296         vdev_stat_t *vs;
2297         char rbuf[6], wbuf[6], cbuf[6];
2298         char *vname;
2299         uint64_t notpresent;
2300         spare_cbdata_t spare_cb;
2301         const char *state;
2302         const char *type;
2303         const char *path = NULL;
2304         const char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL;
2305
2306         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2307             &child, &children) != 0)
2308                 children = 0;
2309
2310         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2311             (uint64_t **)&vs, &vsc) == 0);
2312
2313         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2314
2315         if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2316                 return;
2317
2318         state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2319
2320         if (isspare) {
2321                 /*
2322                  * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
2323                  * online drives.
2324                  */
2325                 if (vs->vs_aux == VDEV_AUX_SPARED)
2326                         state = gettext("INUSE");
2327                 else if (vs->vs_state == VDEV_STATE_HEALTHY)
2328                         state = gettext("AVAIL");
2329         }
2330
2331         printf_color(health_str_to_color(state),
2332             "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
2333             name, state);
2334
2335         if (!isspare) {
2336                 if (vs->vs_read_errors)
2337                         rcolor = ANSI_RED;
2338
2339                 if (vs->vs_write_errors)
2340                         wcolor = ANSI_RED;
2341
2342                 if (vs->vs_checksum_errors)
2343                         ccolor = ANSI_RED;
2344
2345                 if (cb->cb_literal) {
2346                         fputc(' ', stdout);
2347                         printf_color(rcolor, "%5llu",
2348                             (u_longlong_t)vs->vs_read_errors);
2349                         fputc(' ', stdout);
2350                         printf_color(wcolor, "%5llu",
2351                             (u_longlong_t)vs->vs_write_errors);
2352                         fputc(' ', stdout);
2353                         printf_color(ccolor, "%5llu",
2354                             (u_longlong_t)vs->vs_checksum_errors);
2355                 } else {
2356                         zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
2357                         zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
2358                         zfs_nicenum(vs->vs_checksum_errors, cbuf,
2359                             sizeof (cbuf));
2360                         fputc(' ', stdout);
2361                         printf_color(rcolor, "%5s", rbuf);
2362                         fputc(' ', stdout);
2363                         printf_color(wcolor, "%5s", wbuf);
2364                         fputc(' ', stdout);
2365                         printf_color(ccolor, "%5s", cbuf);
2366                 }
2367                 if (cb->cb_print_slow_ios) {
2368                         if (children == 0)  {
2369                                 /* Only leafs vdevs have slow IOs */
2370                                 zfs_nicenum(vs->vs_slow_ios, rbuf,
2371                                     sizeof (rbuf));
2372                         } else {
2373                                 snprintf(rbuf, sizeof (rbuf), "-");
2374                         }
2375
2376                         if (cb->cb_literal)
2377                                 printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
2378                         else
2379                                 printf(" %5s", rbuf);
2380                 }
2381         }
2382
2383         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
2384             &notpresent) == 0) {
2385                 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
2386                 (void) printf("  %s %s", gettext("was"), path);
2387         } else if (vs->vs_aux != 0) {
2388                 (void) printf("  ");
2389                 color_start(ANSI_RED);
2390                 switch (vs->vs_aux) {
2391                 case VDEV_AUX_OPEN_FAILED:
2392                         (void) printf(gettext("cannot open"));
2393                         break;
2394
2395                 case VDEV_AUX_BAD_GUID_SUM:
2396                         (void) printf(gettext("missing device"));
2397                         break;
2398
2399                 case VDEV_AUX_NO_REPLICAS:
2400                         (void) printf(gettext("insufficient replicas"));
2401                         break;
2402
2403                 case VDEV_AUX_VERSION_NEWER:
2404                         (void) printf(gettext("newer version"));
2405                         break;
2406
2407                 case VDEV_AUX_UNSUP_FEAT:
2408                         (void) printf(gettext("unsupported feature(s)"));
2409                         break;
2410
2411                 case VDEV_AUX_ASHIFT_TOO_BIG:
2412                         (void) printf(gettext("unsupported minimum blocksize"));
2413                         break;
2414
2415                 case VDEV_AUX_SPARED:
2416                         verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
2417                             &spare_cb.cb_guid) == 0);
2418                         if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
2419                                 if (strcmp(zpool_get_name(spare_cb.cb_zhp),
2420                                     zpool_get_name(zhp)) == 0)
2421                                         (void) printf(gettext("currently in "
2422                                             "use"));
2423                                 else
2424                                         (void) printf(gettext("in use by "
2425                                             "pool '%s'"),
2426                                             zpool_get_name(spare_cb.cb_zhp));
2427                                 zpool_close(spare_cb.cb_zhp);
2428                         } else {
2429                                 (void) printf(gettext("currently in use"));
2430                         }
2431                         break;
2432
2433                 case VDEV_AUX_ERR_EXCEEDED:
2434                         (void) printf(gettext("too many errors"));
2435                         break;
2436
2437                 case VDEV_AUX_IO_FAILURE:
2438                         (void) printf(gettext("experienced I/O failures"));
2439                         break;
2440
2441                 case VDEV_AUX_BAD_LOG:
2442                         (void) printf(gettext("bad intent log"));
2443                         break;
2444
2445                 case VDEV_AUX_EXTERNAL:
2446                         (void) printf(gettext("external device fault"));
2447                         break;
2448
2449                 case VDEV_AUX_SPLIT_POOL:
2450                         (void) printf(gettext("split into new pool"));
2451                         break;
2452
2453                 case VDEV_AUX_ACTIVE:
2454                         (void) printf(gettext("currently in use"));
2455                         break;
2456
2457                 case VDEV_AUX_CHILDREN_OFFLINE:
2458                         (void) printf(gettext("all children offline"));
2459                         break;
2460
2461                 case VDEV_AUX_BAD_LABEL:
2462                         (void) printf(gettext("invalid label"));
2463                         break;
2464
2465                 default:
2466                         (void) printf(gettext("corrupted data"));
2467                         break;
2468                 }
2469                 color_end();
2470         } else if (children == 0 && !isspare &&
2471             getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
2472             VDEV_STAT_VALID(vs_physical_ashift, vsc) &&
2473             vs->vs_configured_ashift < vs->vs_physical_ashift) {
2474                 (void) printf(
2475                     gettext("  block size: %dB configured, %dB native"),
2476                     1 << vs->vs_configured_ashift, 1 << vs->vs_physical_ashift);
2477         }
2478
2479         if (vs->vs_scan_removing != 0) {
2480                 (void) printf(gettext("  (removing)"));
2481         } else if (VDEV_STAT_VALID(vs_noalloc, vsc) && vs->vs_noalloc != 0) {
2482                 (void) printf(gettext("  (non-allocating)"));
2483         }
2484
2485         /* The root vdev has the scrub/resilver stats */
2486         root = fnvlist_lookup_nvlist(zpool_get_config(zhp, NULL),
2487             ZPOOL_CONFIG_VDEV_TREE);
2488         (void) nvlist_lookup_uint64_array(root, ZPOOL_CONFIG_SCAN_STATS,
2489             (uint64_t **)&ps, &c);
2490
2491         /*
2492          * If you force fault a drive that's resilvering, its scan stats can
2493          * get frozen in time, giving the false impression that it's
2494          * being resilvered.  That's why we check the state to see if the vdev
2495          * is healthy before reporting "resilvering" or "repairing".
2496          */
2497         if (ps != NULL && ps->pss_state == DSS_SCANNING && children == 0 &&
2498             vs->vs_state == VDEV_STATE_HEALTHY) {
2499                 if (vs->vs_scan_processed != 0) {
2500                         (void) printf(gettext("  (%s)"),
2501                             (ps->pss_func == POOL_SCAN_RESILVER) ?
2502                             "resilvering" : "repairing");
2503                 } else if (vs->vs_resilver_deferred) {
2504                         (void) printf(gettext("  (awaiting resilver)"));
2505                 }
2506         }
2507
2508         /* The top-level vdevs have the rebuild stats */
2509         if (vrs != NULL && vrs->vrs_state == VDEV_REBUILD_ACTIVE &&
2510             children == 0 && vs->vs_state == VDEV_STATE_HEALTHY) {
2511                 if (vs->vs_rebuild_processed != 0) {
2512                         (void) printf(gettext("  (resilvering)"));
2513                 }
2514         }
2515
2516         if (cb->vcdl != NULL) {
2517                 if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
2518                         printf("  ");
2519                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
2520                 }
2521         }
2522
2523         /* Display vdev initialization and trim status for leaves. */
2524         if (children == 0) {
2525                 print_status_initialize(vs, cb->cb_print_vdev_init);
2526                 print_status_trim(vs, cb->cb_print_vdev_trim);
2527         }
2528
2529         (void) printf("\n");
2530
2531         for (c = 0; c < children; c++) {
2532                 uint64_t islog = B_FALSE, ishole = B_FALSE;
2533
2534                 /* Don't print logs or holes here */
2535                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2536                     &islog);
2537                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
2538                     &ishole);
2539                 if (islog || ishole)
2540                         continue;
2541                 /* Only print normal classes here */
2542                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2543                         continue;
2544
2545                 /* Provide vdev_rebuild_stats to children if available */
2546                 if (vrs == NULL) {
2547                         (void) nvlist_lookup_uint64_array(nv,
2548                             ZPOOL_CONFIG_REBUILD_STATS,
2549                             (uint64_t **)&vrs, &i);
2550                 }
2551
2552                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
2553                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2554                 print_status_config(zhp, cb, vname, child[c], depth + 2,
2555                     isspare, vrs);
2556                 free(vname);
2557         }
2558 }
2559
2560 /*
2561  * Print the configuration of an exported pool.  Iterate over all vdevs in the
2562  * pool, printing out the name and status for each one.
2563  */
2564 static void
2565 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
2566     int depth)
2567 {
2568         nvlist_t **child;
2569         uint_t c, children;
2570         vdev_stat_t *vs;
2571         const char *type;
2572         char *vname;
2573
2574         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
2575         if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
2576             strcmp(type, VDEV_TYPE_HOLE) == 0)
2577                 return;
2578
2579         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
2580             (uint64_t **)&vs, &c) == 0);
2581
2582         (void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
2583         (void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
2584
2585         if (vs->vs_aux != 0) {
2586                 (void) printf("  ");
2587
2588                 switch (vs->vs_aux) {
2589                 case VDEV_AUX_OPEN_FAILED:
2590                         (void) printf(gettext("cannot open"));
2591                         break;
2592
2593                 case VDEV_AUX_BAD_GUID_SUM:
2594                         (void) printf(gettext("missing device"));
2595                         break;
2596
2597                 case VDEV_AUX_NO_REPLICAS:
2598                         (void) printf(gettext("insufficient replicas"));
2599                         break;
2600
2601                 case VDEV_AUX_VERSION_NEWER:
2602                         (void) printf(gettext("newer version"));
2603                         break;
2604
2605                 case VDEV_AUX_UNSUP_FEAT:
2606                         (void) printf(gettext("unsupported feature(s)"));
2607                         break;
2608
2609                 case VDEV_AUX_ERR_EXCEEDED:
2610                         (void) printf(gettext("too many errors"));
2611                         break;
2612
2613                 case VDEV_AUX_ACTIVE:
2614                         (void) printf(gettext("currently in use"));
2615                         break;
2616
2617                 case VDEV_AUX_CHILDREN_OFFLINE:
2618                         (void) printf(gettext("all children offline"));
2619                         break;
2620
2621                 case VDEV_AUX_BAD_LABEL:
2622                         (void) printf(gettext("invalid label"));
2623                         break;
2624
2625                 default:
2626                         (void) printf(gettext("corrupted data"));
2627                         break;
2628                 }
2629         }
2630         (void) printf("\n");
2631
2632         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
2633             &child, &children) != 0)
2634                 return;
2635
2636         for (c = 0; c < children; c++) {
2637                 uint64_t is_log = B_FALSE;
2638
2639                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2640                     &is_log);
2641                 if (is_log)
2642                         continue;
2643                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
2644                         continue;
2645
2646                 vname = zpool_vdev_name(g_zfs, NULL, child[c],
2647                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2648                 print_import_config(cb, vname, child[c], depth + 2);
2649                 free(vname);
2650         }
2651
2652         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
2653             &child, &children) == 0) {
2654                 (void) printf(gettext("\tcache\n"));
2655                 for (c = 0; c < children; c++) {
2656                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2657                             cb->cb_name_flags);
2658                         (void) printf("\t  %s\n", vname);
2659                         free(vname);
2660                 }
2661         }
2662
2663         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
2664             &child, &children) == 0) {
2665                 (void) printf(gettext("\tspares\n"));
2666                 for (c = 0; c < children; c++) {
2667                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
2668                             cb->cb_name_flags);
2669                         (void) printf("\t  %s\n", vname);
2670                         free(vname);
2671                 }
2672         }
2673 }
2674
2675 /*
2676  * Print specialized class vdevs.
2677  *
2678  * These are recorded as top level vdevs in the main pool child array
2679  * but with "is_log" set to 1 or an "alloc_bias" string. We use either
2680  * print_status_config() or print_import_config() to print the top level
2681  * class vdevs then any of their children (eg mirrored slogs) are printed
2682  * recursively - which works because only the top level vdev is marked.
2683  */
2684 static void
2685 print_class_vdevs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
2686     const char *class)
2687 {
2688         uint_t c, children;
2689         nvlist_t **child;
2690         boolean_t printed = B_FALSE;
2691
2692         assert(zhp != NULL || !cb->cb_verbose);
2693
2694         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
2695             &children) != 0)
2696                 return;
2697
2698         for (c = 0; c < children; c++) {
2699                 uint64_t is_log = B_FALSE;
2700                 const char *bias = NULL;
2701                 const char *type = NULL;
2702
2703                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2704                     &is_log);
2705
2706                 if (is_log) {
2707                         bias = (char *)VDEV_ALLOC_CLASS_LOGS;
2708                 } else {
2709                         (void) nvlist_lookup_string(child[c],
2710                             ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
2711                         (void) nvlist_lookup_string(child[c],
2712                             ZPOOL_CONFIG_TYPE, &type);
2713                 }
2714
2715                 if (bias == NULL || strcmp(bias, class) != 0)
2716                         continue;
2717                 if (!is_log && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
2718                         continue;
2719
2720                 if (!printed) {
2721                         (void) printf("\t%s\t\n", gettext(class));
2722                         printed = B_TRUE;
2723                 }
2724
2725                 char *name = zpool_vdev_name(g_zfs, zhp, child[c],
2726                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2727                 if (cb->cb_print_status)
2728                         print_status_config(zhp, cb, name, child[c], 2,
2729                             B_FALSE, NULL);
2730                 else
2731                         print_import_config(cb, name, child[c], 2);
2732                 free(name);
2733         }
2734 }
2735
2736 /*
2737  * Display the status for the given pool.
2738  */
2739 static int
2740 show_import(nvlist_t *config, boolean_t report_error)
2741 {
2742         uint64_t pool_state;
2743         vdev_stat_t *vs;
2744         const char *name;
2745         uint64_t guid;
2746         uint64_t hostid = 0;
2747         const char *msgid;
2748         const char *hostname = "unknown";
2749         nvlist_t *nvroot, *nvinfo;
2750         zpool_status_t reason;
2751         zpool_errata_t errata;
2752         const char *health;
2753         uint_t vsc;
2754         const char *comment;
2755         status_cbdata_t cb = { 0 };
2756
2757         verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2758             &name) == 0);
2759         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2760             &guid) == 0);
2761         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2762             &pool_state) == 0);
2763         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2764             &nvroot) == 0);
2765
2766         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2767             (uint64_t **)&vs, &vsc) == 0);
2768         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2769
2770         reason = zpool_import_status(config, &msgid, &errata);
2771
2772         /*
2773          * If we're importing using a cachefile, then we won't report any
2774          * errors unless we are in the scan phase of the import.
2775          */
2776         if (reason != ZPOOL_STATUS_OK && !report_error)
2777                 return (reason);
2778
2779         (void) printf(gettext("   pool: %s\n"), name);
2780         (void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2781         (void) printf(gettext("  state: %s"), health);
2782         if (pool_state == POOL_STATE_DESTROYED)
2783                 (void) printf(gettext(" (DESTROYED)"));
2784         (void) printf("\n");
2785
2786         switch (reason) {
2787         case ZPOOL_STATUS_MISSING_DEV_R:
2788         case ZPOOL_STATUS_MISSING_DEV_NR:
2789         case ZPOOL_STATUS_BAD_GUID_SUM:
2790                 printf_color(ANSI_BOLD, gettext("status: "));
2791                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2792                     "missing from the system.\n"));
2793                 break;
2794
2795         case ZPOOL_STATUS_CORRUPT_LABEL_R:
2796         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2797                 printf_color(ANSI_BOLD, gettext("status: "));
2798                 printf_color(ANSI_YELLOW, gettext("One or more devices contains"
2799                     " corrupted data.\n"));
2800                 break;
2801
2802         case ZPOOL_STATUS_CORRUPT_DATA:
2803                 (void) printf(
2804                     gettext(" status: The pool data is corrupted.\n"));
2805                 break;
2806
2807         case ZPOOL_STATUS_OFFLINE_DEV:
2808                 printf_color(ANSI_BOLD, gettext("status: "));
2809                 printf_color(ANSI_YELLOW, gettext("One or more devices "
2810                     "are offlined.\n"));
2811                 break;
2812
2813         case ZPOOL_STATUS_CORRUPT_POOL:
2814                 printf_color(ANSI_BOLD, gettext("status: "));
2815                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
2816                     "corrupted.\n"));
2817                 break;
2818
2819         case ZPOOL_STATUS_VERSION_OLDER:
2820                 printf_color(ANSI_BOLD, gettext("status: "));
2821                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2822                     "a legacy on-disk version.\n"));
2823                 break;
2824
2825         case ZPOOL_STATUS_VERSION_NEWER:
2826                 printf_color(ANSI_BOLD, gettext("status: "));
2827                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
2828                     "an incompatible version.\n"));
2829                 break;
2830
2831         case ZPOOL_STATUS_FEAT_DISABLED:
2832                 printf_color(ANSI_BOLD, gettext("status: "));
2833                 printf_color(ANSI_YELLOW, gettext("Some supported "
2834                     "features are not enabled on the pool.\n\t"
2835                     "(Note that they may be intentionally disabled "
2836                     "if the\n\t'compatibility' property is set.)\n"));
2837                 break;
2838
2839         case ZPOOL_STATUS_COMPATIBILITY_ERR:
2840                 printf_color(ANSI_BOLD, gettext("status: "));
2841                 printf_color(ANSI_YELLOW, gettext("Error reading or parsing "
2842                     "the file(s) indicated by the 'compatibility'\n"
2843                     "property.\n"));
2844                 break;
2845
2846         case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
2847                 printf_color(ANSI_BOLD, gettext("status: "));
2848                 printf_color(ANSI_YELLOW, gettext("One or more features "
2849                     "are enabled on the pool despite not being\n"
2850                     "requested by the 'compatibility' property.\n"));
2851                 break;
2852
2853         case ZPOOL_STATUS_UNSUP_FEAT_READ:
2854                 printf_color(ANSI_BOLD, gettext("status: "));
2855                 printf_color(ANSI_YELLOW, gettext("The pool uses the following "
2856                     "feature(s) not supported on this system:\n"));
2857                 color_start(ANSI_YELLOW);
2858                 zpool_print_unsup_feat(config);
2859                 color_end();
2860                 break;
2861
2862         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2863                 printf_color(ANSI_BOLD, gettext("status: "));
2864                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
2865                     "accessed in read-only mode on this system. It\n\tcannot be"
2866                     " accessed in read-write mode because it uses the "
2867                     "following\n\tfeature(s) not supported on this system:\n"));
2868                 color_start(ANSI_YELLOW);
2869                 zpool_print_unsup_feat(config);
2870                 color_end();
2871                 break;
2872
2873         case ZPOOL_STATUS_HOSTID_ACTIVE:
2874                 printf_color(ANSI_BOLD, gettext("status: "));
2875                 printf_color(ANSI_YELLOW, gettext("The pool is currently "
2876                     "imported by another system.\n"));
2877                 break;
2878
2879         case ZPOOL_STATUS_HOSTID_REQUIRED:
2880                 printf_color(ANSI_BOLD, gettext("status: "));
2881                 printf_color(ANSI_YELLOW, gettext("The pool has the "
2882                     "multihost property on.  It cannot\n\tbe safely imported "
2883                     "when the system hostid is not set.\n"));
2884                 break;
2885
2886         case ZPOOL_STATUS_HOSTID_MISMATCH:
2887                 printf_color(ANSI_BOLD, gettext("status: "));
2888                 printf_color(ANSI_YELLOW, gettext("The pool was last accessed "
2889                     "by another system.\n"));
2890                 break;
2891
2892         case ZPOOL_STATUS_FAULTED_DEV_R:
2893         case ZPOOL_STATUS_FAULTED_DEV_NR:
2894                 printf_color(ANSI_BOLD, gettext("status: "));
2895                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2896                     "faulted.\n"));
2897                 break;
2898
2899         case ZPOOL_STATUS_BAD_LOG:
2900                 printf_color(ANSI_BOLD, gettext("status: "));
2901                 printf_color(ANSI_YELLOW, gettext("An intent log record cannot "
2902                     "be read.\n"));
2903                 break;
2904
2905         case ZPOOL_STATUS_RESILVERING:
2906         case ZPOOL_STATUS_REBUILDING:
2907                 printf_color(ANSI_BOLD, gettext("status: "));
2908                 printf_color(ANSI_YELLOW, gettext("One or more devices were "
2909                     "being resilvered.\n"));
2910                 break;
2911
2912         case ZPOOL_STATUS_ERRATA:
2913                 printf_color(ANSI_BOLD, gettext("status: "));
2914                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
2915                     errata);
2916                 break;
2917
2918         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
2919                 printf_color(ANSI_BOLD, gettext("status: "));
2920                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
2921                     "configured to use a non-native block size.\n"
2922                     "\tExpect reduced performance.\n"));
2923                 break;
2924
2925         default:
2926                 /*
2927                  * No other status can be seen when importing pools.
2928                  */
2929                 assert(reason == ZPOOL_STATUS_OK);
2930         }
2931
2932         /*
2933          * Print out an action according to the overall state of the pool.
2934          */
2935         if (vs->vs_state == VDEV_STATE_HEALTHY) {
2936                 if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2937                     reason == ZPOOL_STATUS_FEAT_DISABLED) {
2938                         (void) printf(gettext(" action: The pool can be "
2939                             "imported using its name or numeric identifier, "
2940                             "though\n\tsome features will not be available "
2941                             "without an explicit 'zpool upgrade'.\n"));
2942                 } else if (reason == ZPOOL_STATUS_COMPATIBILITY_ERR) {
2943                         (void) printf(gettext(" action: The pool can be "
2944                             "imported using its name or numeric\n\tidentifier, "
2945                             "though the file(s) indicated by its "
2946                             "'compatibility'\n\tproperty cannot be parsed at "
2947                             "this time.\n"));
2948                 } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2949                         (void) printf(gettext(" action: The pool can be "
2950                             "imported using its name or numeric "
2951                             "identifier and\n\tthe '-f' flag.\n"));
2952                 } else if (reason == ZPOOL_STATUS_ERRATA) {
2953                         switch (errata) {
2954                         case ZPOOL_ERRATA_NONE:
2955                                 break;
2956
2957                         case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2958                                 (void) printf(gettext(" action: The pool can "
2959                                     "be imported using its name or numeric "
2960                                     "identifier,\n\thowever there is a compat"
2961                                     "ibility issue which should be corrected"
2962                                     "\n\tby running 'zpool scrub'\n"));
2963                                 break;
2964
2965                         case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2966                                 (void) printf(gettext(" action: The pool can"
2967                                     "not be imported with this version of ZFS "
2968                                     "due to\n\tan active asynchronous destroy. "
2969                                     "Revert to an earlier version\n\tand "
2970                                     "allow the destroy to complete before "
2971                                     "updating.\n"));
2972                                 break;
2973
2974                         case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2975                                 (void) printf(gettext(" action: Existing "
2976                                     "encrypted datasets contain an on-disk "
2977                                     "incompatibility, which\n\tneeds to be "
2978                                     "corrected. Backup these datasets to new "
2979                                     "encrypted datasets\n\tand destroy the "
2980                                     "old ones.\n"));
2981                                 break;
2982
2983                         case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
2984                                 (void) printf(gettext(" action: Existing "
2985                                     "encrypted snapshots and bookmarks contain "
2986                                     "an on-disk\n\tincompatibility. This may "
2987                                     "cause on-disk corruption if they are used"
2988                                     "\n\twith 'zfs recv'. To correct the "
2989                                     "issue, enable the bookmark_v2 feature.\n\t"
2990                                     "No additional action is needed if there "
2991                                     "are no encrypted snapshots or\n\t"
2992                                     "bookmarks. If preserving the encrypted "
2993                                     "snapshots and bookmarks is\n\trequired, "
2994                                     "use a non-raw send to backup and restore "
2995                                     "them. Alternately,\n\tthey may be removed"
2996                                     " to resolve the incompatibility.\n"));
2997                                 break;
2998                         default:
2999                                 /*
3000                                  * All errata must contain an action message.
3001                                  */
3002                                 assert(0);
3003                         }
3004                 } else {
3005                         (void) printf(gettext(" action: The pool can be "
3006                             "imported using its name or numeric "
3007                             "identifier.\n"));
3008                 }
3009         } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
3010                 (void) printf(gettext(" action: The pool can be imported "
3011                     "despite missing or damaged devices.  The\n\tfault "
3012                     "tolerance of the pool may be compromised if imported.\n"));
3013         } else {
3014                 switch (reason) {
3015                 case ZPOOL_STATUS_VERSION_NEWER:
3016                         (void) printf(gettext(" action: The pool cannot be "
3017                             "imported.  Access the pool on a system running "
3018                             "newer\n\tsoftware, or recreate the pool from "
3019                             "backup.\n"));
3020                         break;
3021                 case ZPOOL_STATUS_UNSUP_FEAT_READ:
3022                         printf_color(ANSI_BOLD, gettext("action: "));
3023                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
3024                             "imported. Access the pool on a system that "
3025                             "supports\n\tthe required feature(s), or recreate "
3026                             "the pool from backup.\n"));
3027                         break;
3028                 case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
3029                         printf_color(ANSI_BOLD, gettext("action: "));
3030                         printf_color(ANSI_YELLOW, gettext("The pool cannot be "
3031                             "imported in read-write mode. Import the pool "
3032                             "with\n"
3033                             "\t\"-o readonly=on\", access the pool on a system "
3034                             "that supports the\n\trequired feature(s), or "
3035                             "recreate the pool from backup.\n"));
3036                         break;
3037                 case ZPOOL_STATUS_MISSING_DEV_R:
3038                 case ZPOOL_STATUS_MISSING_DEV_NR:
3039                 case ZPOOL_STATUS_BAD_GUID_SUM:
3040                         (void) printf(gettext(" action: The pool cannot be "
3041                             "imported. Attach the missing\n\tdevices and try "
3042                             "again.\n"));
3043                         break;
3044                 case ZPOOL_STATUS_HOSTID_ACTIVE:
3045                         VERIFY0(nvlist_lookup_nvlist(config,
3046                             ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
3047
3048                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
3049                                 hostname = fnvlist_lookup_string(nvinfo,
3050                                     ZPOOL_CONFIG_MMP_HOSTNAME);
3051
3052                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
3053                                 hostid = fnvlist_lookup_uint64(nvinfo,
3054                                     ZPOOL_CONFIG_MMP_HOSTID);
3055
3056                         (void) printf(gettext(" action: The pool must be "
3057                             "exported from %s (hostid=%"PRIx64")\n\tbefore it "
3058                             "can be safely imported.\n"), hostname, hostid);
3059                         break;
3060                 case ZPOOL_STATUS_HOSTID_REQUIRED:
3061                         (void) printf(gettext(" action: Set a unique system "
3062                             "hostid with the zgenhostid(8) command.\n"));
3063                         break;
3064                 default:
3065                         (void) printf(gettext(" action: The pool cannot be "
3066                             "imported due to damaged devices or data.\n"));
3067                 }
3068         }
3069
3070         /* Print the comment attached to the pool. */
3071         if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
3072                 (void) printf(gettext("comment: %s\n"), comment);
3073
3074         /*
3075          * If the state is "closed" or "can't open", and the aux state
3076          * is "corrupt data":
3077          */
3078         if (((vs->vs_state == VDEV_STATE_CLOSED) ||
3079             (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
3080             (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
3081                 if (pool_state == POOL_STATE_DESTROYED)
3082                         (void) printf(gettext("\tThe pool was destroyed, "
3083                             "but can be imported using the '-Df' flags.\n"));
3084                 else if (pool_state != POOL_STATE_EXPORTED)
3085                         (void) printf(gettext("\tThe pool may be active on "
3086                             "another system, but can be imported using\n\t"
3087                             "the '-f' flag.\n"));
3088         }
3089
3090         if (msgid != NULL) {
3091                 (void) printf(gettext(
3092                     "   see: https://openzfs.github.io/openzfs-docs/msg/%s\n"),
3093                     msgid);
3094         }
3095
3096         (void) printf(gettext(" config:\n\n"));
3097
3098         cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
3099             VDEV_NAME_TYPE_ID);
3100         if (cb.cb_namewidth < 10)
3101                 cb.cb_namewidth = 10;
3102
3103         print_import_config(&cb, name, nvroot, 0);
3104
3105         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_DEDUP);
3106         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
3107         print_class_vdevs(NULL, &cb, nvroot, VDEV_ALLOC_CLASS_LOGS);
3108
3109         if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
3110                 (void) printf(gettext("\n\tAdditional devices are known to "
3111                     "be part of this pool, though their\n\texact "
3112                     "configuration cannot be determined.\n"));
3113         }
3114         return (0);
3115 }
3116
3117 static boolean_t
3118 zfs_force_import_required(nvlist_t *config)
3119 {
3120         uint64_t state;
3121         uint64_t hostid = 0;
3122         nvlist_t *nvinfo;
3123
3124         state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
3125         nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3126
3127         /*
3128          * The hostid on LOAD_INFO comes from the MOS label via
3129          * spa_tryimport(). If its not there then we're likely talking to an
3130          * older kernel, so use the top one, which will be from the label
3131          * discovered in zpool_find_import(), or if a cachefile is in use, the
3132          * local hostid.
3133          */
3134         if (nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_HOSTID, &hostid) != 0)
3135                 nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
3136
3137         if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
3138                 return (B_TRUE);
3139
3140         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
3141                 mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
3142                     ZPOOL_CONFIG_MMP_STATE);
3143
3144                 if (mmp_state != MMP_STATE_INACTIVE)
3145                         return (B_TRUE);
3146         }
3147
3148         return (B_FALSE);
3149 }
3150
3151 /*
3152  * Perform the import for the given configuration.  This passes the heavy
3153  * lifting off to zpool_import_props(), and then mounts the datasets contained
3154  * within the pool.
3155  */
3156 static int
3157 do_import(nvlist_t *config, const char *newname, const char *mntopts,
3158     nvlist_t *props, int flags)
3159 {
3160         int ret = 0;
3161         int ms_status = 0;
3162         zpool_handle_t *zhp;
3163         const char *name;
3164         uint64_t version;
3165
3166         name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
3167         version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
3168
3169         if (!SPA_VERSION_IS_SUPPORTED(version)) {
3170                 (void) fprintf(stderr, gettext("cannot import '%s': pool "
3171                     "is formatted using an unsupported ZFS version\n"), name);
3172                 return (1);
3173         } else if (zfs_force_import_required(config) &&
3174             !(flags & ZFS_IMPORT_ANY_HOST)) {
3175                 mmp_state_t mmp_state = MMP_STATE_INACTIVE;
3176                 nvlist_t *nvinfo;
3177
3178                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
3179                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
3180                         mmp_state = fnvlist_lookup_uint64(nvinfo,
3181                             ZPOOL_CONFIG_MMP_STATE);
3182
3183                 if (mmp_state == MMP_STATE_ACTIVE) {
3184                         const char *hostname = "<unknown>";
3185                         uint64_t hostid = 0;
3186
3187                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
3188                                 hostname = fnvlist_lookup_string(nvinfo,
3189                                     ZPOOL_CONFIG_MMP_HOSTNAME);
3190
3191                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
3192                                 hostid = fnvlist_lookup_uint64(nvinfo,
3193                                     ZPOOL_CONFIG_MMP_HOSTID);
3194
3195                         (void) fprintf(stderr, gettext("cannot import '%s': "
3196                             "pool is imported on %s (hostid: "
3197                             "0x%"PRIx64")\nExport the pool on the other "
3198                             "system, then run 'zpool import'.\n"),
3199                             name, hostname, hostid);
3200                 } else if (mmp_state == MMP_STATE_NO_HOSTID) {
3201                         (void) fprintf(stderr, gettext("Cannot import '%s': "
3202                             "pool has the multihost property on and the\n"
3203                             "system's hostid is not set. Set a unique hostid "
3204                             "with the zgenhostid(8) command.\n"), name);
3205                 } else {
3206                         const char *hostname = "<unknown>";
3207                         time_t timestamp = 0;
3208                         uint64_t hostid = 0;
3209
3210                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTNAME))
3211                                 hostname = fnvlist_lookup_string(nvinfo,
3212                                     ZPOOL_CONFIG_HOSTNAME);
3213                         else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
3214                                 hostname = fnvlist_lookup_string(config,
3215                                     ZPOOL_CONFIG_HOSTNAME);
3216
3217                         if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
3218                                 timestamp = fnvlist_lookup_uint64(config,
3219                                     ZPOOL_CONFIG_TIMESTAMP);
3220
3221                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_HOSTID))
3222                                 hostid = fnvlist_lookup_uint64(nvinfo,
3223                                     ZPOOL_CONFIG_HOSTID);
3224                         else if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
3225                                 hostid = fnvlist_lookup_uint64(config,
3226                                     ZPOOL_CONFIG_HOSTID);
3227
3228                         (void) fprintf(stderr, gettext("cannot import '%s': "
3229                             "pool was previously in use from another system.\n"
3230                             "Last accessed by %s (hostid=%"PRIx64") at %s"
3231                             "The pool can be imported, use 'zpool import -f' "
3232                             "to import the pool.\n"), name, hostname,
3233                             hostid, ctime(&timestamp));
3234                 }
3235
3236                 return (1);
3237         }
3238
3239         if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
3240                 return (1);
3241
3242         if (newname != NULL)
3243                 name = newname;
3244
3245         if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
3246                 return (1);
3247
3248         /*
3249          * Loading keys is best effort. We don't want to return immediately
3250          * if it fails but we do want to give the error to the caller.
3251          */
3252         if (flags & ZFS_IMPORT_LOAD_KEYS &&
3253             zfs_crypto_attempt_load_keys(g_zfs, name) != 0)
3254                         ret = 1;
3255
3256         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
3257             !(flags & ZFS_IMPORT_ONLY)) {
3258                 ms_status = zpool_enable_datasets(zhp, mntopts, 0);
3259                 if (ms_status == EZFS_SHAREFAILED) {
3260                         (void) fprintf(stderr, gettext("Import was "
3261                             "successful, but unable to share some datasets"));
3262                 } else if (ms_status == EZFS_MOUNTFAILED) {
3263                         (void) fprintf(stderr, gettext("Import was "
3264                             "successful, but unable to mount some datasets"));
3265                 }
3266         }
3267
3268         zpool_close(zhp);
3269         return (ret);
3270 }
3271
3272 static int
3273 import_pools(nvlist_t *pools, nvlist_t *props, char *mntopts, int flags,
3274     char *orig_name, char *new_name,
3275     boolean_t do_destroyed, boolean_t pool_specified, boolean_t do_all,
3276     importargs_t *import)
3277 {
3278         nvlist_t *config = NULL;
3279         nvlist_t *found_config = NULL;
3280         uint64_t pool_state;
3281
3282         /*
3283          * At this point we have a list of import candidate configs. Even if
3284          * we were searching by pool name or guid, we still need to
3285          * post-process the list to deal with pool state and possible
3286          * duplicate names.
3287          */
3288         int err = 0;
3289         nvpair_t *elem = NULL;
3290         boolean_t first = B_TRUE;
3291         while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
3292
3293                 verify(nvpair_value_nvlist(elem, &config) == 0);
3294
3295                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
3296                     &pool_state) == 0);
3297                 if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
3298                         continue;
3299                 if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
3300                         continue;
3301
3302                 verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
3303                     import->policy) == 0);
3304
3305                 if (!pool_specified) {
3306                         if (first)
3307                                 first = B_FALSE;
3308                         else if (!do_all)
3309                                 (void) fputc('\n', stdout);
3310
3311                         if (do_all) {
3312                                 err |= do_import(config, NULL, mntopts,
3313                                     props, flags);
3314                         } else {
3315                                 /*
3316                                  * If we're importing from cachefile, then
3317                                  * we don't want to report errors until we
3318                                  * are in the scan phase of the import. If
3319                                  * we get an error, then we return that error
3320                                  * to invoke the scan phase.
3321                                  */
3322                                 if (import->cachefile && !import->scan)
3323                                         err = show_import(config, B_FALSE);
3324                                 else
3325                                         (void) show_import(config, B_TRUE);
3326                         }
3327                 } else if (import->poolname != NULL) {
3328                         const char *name;
3329
3330                         /*
3331                          * We are searching for a pool based on name.
3332                          */
3333                         verify(nvlist_lookup_string(config,
3334                             ZPOOL_CONFIG_POOL_NAME, &name) == 0);
3335
3336                         if (strcmp(name, import->poolname) == 0) {
3337                                 if (found_config != NULL) {
3338                                         (void) fprintf(stderr, gettext(
3339                                             "cannot import '%s': more than "
3340                                             "one matching pool\n"),
3341                                             import->poolname);
3342                                         (void) fprintf(stderr, gettext(
3343                                             "import by numeric ID instead\n"));
3344                                         err = B_TRUE;
3345                                 }
3346                                 found_config = config;
3347                         }
3348                 } else {
3349                         uint64_t guid;
3350
3351                         /*
3352                          * Search for a pool by guid.
3353                          */
3354                         verify(nvlist_lookup_uint64(config,
3355                             ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
3356
3357                         if (guid == import->guid)
3358                                 found_config = config;
3359                 }
3360         }
3361
3362         /*
3363          * If we were searching for a specific pool, verify that we found a
3364          * pool, and then do the import.
3365          */
3366         if (pool_specified && err == 0) {
3367                 if (found_config == NULL) {
3368                         (void) fprintf(stderr, gettext("cannot import '%s': "
3369                             "no such pool available\n"), orig_name);
3370                         err = B_TRUE;
3371                 } else {
3372                         err |= do_import(found_config, new_name,
3373                             mntopts, props, flags);
3374                 }
3375         }
3376
3377         /*
3378          * If we were just looking for pools, report an error if none were
3379          * found.
3380          */
3381         if (!pool_specified && first)
3382                 (void) fprintf(stderr,
3383                     gettext("no pools available to import\n"));
3384         return (err);
3385 }
3386
3387 typedef struct target_exists_args {
3388         const char      *poolname;
3389         uint64_t        poolguid;
3390 } target_exists_args_t;
3391
3392 static int
3393 name_or_guid_exists(zpool_handle_t *zhp, void *data)
3394 {
3395         target_exists_args_t *args = data;
3396         nvlist_t *config = zpool_get_config(zhp, NULL);
3397         int found = 0;
3398
3399         if (config == NULL)
3400                 return (0);
3401
3402         if (args->poolname != NULL) {
3403                 const char *pool_name;
3404
3405                 verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
3406                     &pool_name) == 0);
3407                 if (strcmp(pool_name, args->poolname) == 0)
3408                         found = 1;
3409         } else {
3410                 uint64_t pool_guid;
3411
3412                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
3413                     &pool_guid) == 0);
3414                 if (pool_guid == args->poolguid)
3415                         found = 1;
3416         }
3417         zpool_close(zhp);
3418
3419         return (found);
3420 }
3421 /*
3422  * zpool checkpoint <pool>
3423  *       checkpoint --discard <pool>
3424  *
3425  *       -d         Discard the checkpoint from a checkpointed
3426  *       --discard  pool.
3427  *
3428  *       -w         Wait for discarding a checkpoint to complete.
3429  *       --wait
3430  *
3431  * Checkpoints the specified pool, by taking a "snapshot" of its
3432  * current state. A pool can only have one checkpoint at a time.
3433  */
3434 int
3435 zpool_do_checkpoint(int argc, char **argv)
3436 {
3437         boolean_t discard, wait;
3438         char *pool;
3439         zpool_handle_t *zhp;
3440         int c, err;
3441
3442         struct option long_options[] = {
3443                 {"discard", no_argument, NULL, 'd'},
3444                 {"wait", no_argument, NULL, 'w'},
3445                 {0, 0, 0, 0}
3446         };
3447
3448         discard = B_FALSE;
3449         wait = B_FALSE;
3450         while ((c = getopt_long(argc, argv, ":dw", long_options, NULL)) != -1) {
3451                 switch (c) {
3452                 case 'd':
3453                         discard = B_TRUE;
3454                         break;
3455                 case 'w':
3456                         wait = B_TRUE;
3457                         break;
3458                 case '?':
3459                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3460                             optopt);
3461                         usage(B_FALSE);
3462                 }
3463         }
3464
3465         if (wait && !discard) {
3466                 (void) fprintf(stderr, gettext("--wait only valid when "
3467                     "--discard also specified\n"));
3468                 usage(B_FALSE);
3469         }
3470
3471         argc -= optind;
3472         argv += optind;
3473
3474         if (argc < 1) {
3475                 (void) fprintf(stderr, gettext("missing pool argument\n"));
3476                 usage(B_FALSE);
3477         }
3478
3479         if (argc > 1) {
3480                 (void) fprintf(stderr, gettext("too many arguments\n"));
3481                 usage(B_FALSE);
3482         }
3483
3484         pool = argv[0];
3485
3486         if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
3487                 /* As a special case, check for use of '/' in the name */
3488                 if (strchr(pool, '/') != NULL)
3489                         (void) fprintf(stderr, gettext("'zpool checkpoint' "
3490                             "doesn't work on datasets. To save the state "
3491                             "of a dataset from a specific point in time "
3492                             "please use 'zfs snapshot'\n"));
3493                 return (1);
3494         }
3495
3496         if (discard) {
3497                 err = (zpool_discard_checkpoint(zhp) != 0);
3498                 if (err == 0 && wait)
3499                         err = zpool_wait(zhp, ZPOOL_WAIT_CKPT_DISCARD);
3500         } else {
3501                 err = (zpool_checkpoint(zhp) != 0);
3502         }
3503
3504         zpool_close(zhp);
3505
3506         return (err);
3507 }
3508
3509 #define CHECKPOINT_OPT  1024
3510
3511 /*
3512  * zpool import [-d dir] [-D]
3513  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3514  *              [-d dir | -c cachefile | -s] [-f] -a
3515  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
3516  *              [-d dir | -c cachefile | -s] [-f] [-n] [-F] <pool | id>
3517  *              [newpool]
3518  *
3519  *      -c      Read pool information from a cachefile instead of searching
3520  *              devices. If importing from a cachefile config fails, then
3521  *              fallback to searching for devices only in the directories that
3522  *              exist in the cachefile.
3523  *
3524  *      -d      Scan in a specific directory, other than /dev/.  More than
3525  *              one directory can be specified using multiple '-d' options.
3526  *
3527  *      -D      Scan for previously destroyed pools or import all or only
3528  *              specified destroyed pools.
3529  *
3530  *      -R      Temporarily import the pool, with all mountpoints relative to
3531  *              the given root.  The pool will remain exported when the machine
3532  *              is rebooted.
3533  *
3534  *      -V      Import even in the presence of faulted vdevs.  This is an
3535  *              intentionally undocumented option for testing purposes, and
3536  *              treats the pool configuration as complete, leaving any bad
3537  *              vdevs in the FAULTED state. In other words, it does verbatim
3538  *              import.
3539  *
3540  *      -f      Force import, even if it appears that the pool is active.
3541  *
3542  *      -F      Attempt rewind if necessary.
3543  *
3544  *      -n      See if rewind would work, but don't actually rewind.
3545  *
3546  *      -N      Import the pool but don't mount datasets.
3547  *
3548  *      -T      Specify a starting txg to use for import. This option is
3549  *              intentionally undocumented option for testing purposes.
3550  *
3551  *      -a      Import all pools found.
3552  *
3553  *      -l      Load encryption keys while importing.
3554  *
3555  *      -o      Set property=value and/or temporary mount options (without '=').
3556  *
3557  *      -s      Scan using the default search path, the libblkid cache will
3558  *              not be consulted.
3559  *
3560  *      --rewind-to-checkpoint
3561  *              Import the pool and revert back to the checkpoint.
3562  *
3563  * The import command scans for pools to import, and import pools based on pool
3564  * name and GUID.  The pool can also be renamed as part of the import process.
3565  */
3566 int
3567 zpool_do_import(int argc, char **argv)
3568 {
3569         char **searchdirs = NULL;
3570         char *env, *envdup = NULL;
3571         int nsearch = 0;
3572         int c;
3573         int err = 0;
3574         nvlist_t *pools = NULL;
3575         boolean_t do_all = B_FALSE;
3576         boolean_t do_destroyed = B_FALSE;
3577         char *mntopts = NULL;
3578         uint64_t searchguid = 0;
3579         char *searchname = NULL;
3580         char *propval;
3581         nvlist_t *policy = NULL;
3582         nvlist_t *props = NULL;
3583         int flags = ZFS_IMPORT_NORMAL;
3584         uint32_t rewind_policy = ZPOOL_NO_REWIND;
3585         boolean_t dryrun = B_FALSE;
3586         boolean_t do_rewind = B_FALSE;
3587         boolean_t xtreme_rewind = B_FALSE;
3588         boolean_t do_scan = B_FALSE;
3589         boolean_t pool_exists = B_FALSE;
3590         boolean_t pool_specified = B_FALSE;
3591         uint64_t txg = -1ULL;
3592         char *cachefile = NULL;
3593         importargs_t idata = { 0 };
3594         char *endptr;
3595
3596         struct option long_options[] = {
3597                 {"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
3598                 {0, 0, 0, 0}
3599         };
3600
3601         /* check options */
3602         while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
3603             long_options, NULL)) != -1) {
3604                 switch (c) {
3605                 case 'a':
3606                         do_all = B_TRUE;
3607                         break;
3608                 case 'c':
3609                         cachefile = optarg;
3610                         break;
3611                 case 'd':
3612                         searchdirs = safe_realloc(searchdirs,
3613                             (nsearch + 1) * sizeof (char *));
3614                         searchdirs[nsearch++] = optarg;
3615                         break;
3616                 case 'D':
3617                         do_destroyed = B_TRUE;
3618                         break;
3619                 case 'f':
3620                         flags |= ZFS_IMPORT_ANY_HOST;
3621                         break;
3622                 case 'F':
3623                         do_rewind = B_TRUE;
3624                         break;
3625                 case 'l':
3626                         flags |= ZFS_IMPORT_LOAD_KEYS;
3627                         break;
3628                 case 'm':
3629                         flags |= ZFS_IMPORT_MISSING_LOG;
3630                         break;
3631                 case 'n':
3632                         dryrun = B_TRUE;
3633                         break;
3634                 case 'N':
3635                         flags |= ZFS_IMPORT_ONLY;
3636                         break;
3637                 case 'o':
3638                         if ((propval = strchr(optarg, '=')) != NULL) {
3639                                 *propval = '\0';
3640                                 propval++;
3641                                 if (add_prop_list(optarg, propval,
3642                                     &props, B_TRUE))
3643                                         goto error;
3644                         } else {
3645                                 mntopts = optarg;
3646                         }
3647                         break;
3648                 case 'R':
3649                         if (add_prop_list(zpool_prop_to_name(
3650                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
3651                                 goto error;
3652                         if (add_prop_list_default(zpool_prop_to_name(
3653                             ZPOOL_PROP_CACHEFILE), "none", &props))
3654                                 goto error;
3655                         break;
3656                 case 's':
3657                         do_scan = B_TRUE;
3658                         break;
3659                 case 't':
3660                         flags |= ZFS_IMPORT_TEMP_NAME;
3661                         if (add_prop_list_default(zpool_prop_to_name(
3662                             ZPOOL_PROP_CACHEFILE), "none", &props))
3663                                 goto error;
3664                         break;
3665
3666                 case 'T':
3667                         errno = 0;
3668                         txg = strtoull(optarg, &endptr, 0);
3669                         if (errno != 0 || *endptr != '\0') {
3670                                 (void) fprintf(stderr,
3671                                     gettext("invalid txg value\n"));
3672                                 usage(B_FALSE);
3673                         }
3674                         rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
3675                         break;
3676                 case 'V':
3677                         flags |= ZFS_IMPORT_VERBATIM;
3678                         break;
3679                 case 'X':
3680                         xtreme_rewind = B_TRUE;
3681                         break;
3682                 case CHECKPOINT_OPT:
3683                         flags |= ZFS_IMPORT_CHECKPOINT;
3684                         break;
3685                 case ':':
3686                         (void) fprintf(stderr, gettext("missing argument for "
3687                             "'%c' option\n"), optopt);
3688                         usage(B_FALSE);
3689                         break;
3690                 case '?':
3691                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3692                             optopt);
3693                         usage(B_FALSE);
3694                 }
3695         }
3696
3697         argc -= optind;
3698         argv += optind;
3699
3700         if (cachefile && nsearch != 0) {
3701                 (void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
3702                 usage(B_FALSE);
3703         }
3704
3705         if (cachefile && do_scan) {
3706                 (void) fprintf(stderr, gettext("-c is incompatible with -s\n"));
3707                 usage(B_FALSE);
3708         }
3709
3710         if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
3711                 (void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
3712                 usage(B_FALSE);
3713         }
3714
3715         if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
3716                 (void) fprintf(stderr, gettext("-l is only meaningful during "
3717                     "an import\n"));
3718                 usage(B_FALSE);
3719         }
3720
3721         if ((dryrun || xtreme_rewind) && !do_rewind) {
3722                 (void) fprintf(stderr,
3723                     gettext("-n or -X only meaningful with -F\n"));
3724                 usage(B_FALSE);
3725         }
3726         if (dryrun)
3727                 rewind_policy = ZPOOL_TRY_REWIND;
3728         else if (do_rewind)
3729                 rewind_policy = ZPOOL_DO_REWIND;
3730         if (xtreme_rewind)
3731                 rewind_policy |= ZPOOL_EXTREME_REWIND;
3732
3733         /* In the future, we can capture further policy and include it here */
3734         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
3735             nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
3736             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
3737             rewind_policy) != 0)
3738                 goto error;
3739
3740         /* check argument count */
3741         if (do_all) {
3742                 if (argc != 0) {
3743                         (void) fprintf(stderr, gettext("too many arguments\n"));
3744                         usage(B_FALSE);
3745                 }
3746         } else {
3747                 if (argc > 2) {
3748                         (void) fprintf(stderr, gettext("too many arguments\n"));
3749                         usage(B_FALSE);
3750                 }
3751         }
3752
3753         /*
3754          * Check for the effective uid.  We do this explicitly here because
3755          * otherwise any attempt to discover pools will silently fail.
3756          */
3757         if (argc == 0 && geteuid() != 0) {
3758                 (void) fprintf(stderr, gettext("cannot "
3759                     "discover pools: permission denied\n"));
3760
3761                 free(searchdirs);
3762                 nvlist_free(props);
3763                 nvlist_free(policy);
3764                 return (1);
3765         }
3766
3767         /*
3768          * Depending on the arguments given, we do one of the following:
3769          *
3770          *      <none>  Iterate through all pools and display information about
3771          *              each one.
3772          *
3773          *      -a      Iterate through all pools and try to import each one.
3774          *
3775          *      <id>    Find the pool that corresponds to the given GUID/pool
3776          *              name and import that one.
3777          *
3778          *      -D      Above options applies only to destroyed pools.
3779          */
3780         if (argc != 0) {
3781                 char *endptr;
3782
3783                 errno = 0;
3784                 searchguid = strtoull(argv[0], &endptr, 10);
3785                 if (errno != 0 || *endptr != '\0') {
3786                         searchname = argv[0];
3787                         searchguid = 0;
3788                 }
3789                 pool_specified = B_TRUE;
3790
3791                 /*
3792                  * User specified a name or guid.  Ensure it's unique.
3793                  */
3794                 target_exists_args_t search = {searchname, searchguid};
3795                 pool_exists = zpool_iter(g_zfs, name_or_guid_exists, &search);
3796         }
3797
3798         /*
3799          * Check the environment for the preferred search path.
3800          */
3801         if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
3802                 char *dir, *tmp = NULL;
3803
3804                 envdup = strdup(env);
3805
3806                 for (dir = strtok_r(envdup, ":", &tmp);
3807                     dir != NULL;
3808                     dir = strtok_r(NULL, ":", &tmp)) {
3809                         searchdirs = safe_realloc(searchdirs,
3810                             (nsearch + 1) * sizeof (char *));
3811                         searchdirs[nsearch++] = dir;
3812                 }
3813         }
3814
3815         idata.path = searchdirs;
3816         idata.paths = nsearch;
3817         idata.poolname = searchname;
3818         idata.guid = searchguid;
3819         idata.cachefile = cachefile;
3820         idata.scan = do_scan;
3821         idata.policy = policy;
3822
3823         libpc_handle_t lpch = {
3824                 .lpc_lib_handle = g_zfs,
3825                 .lpc_ops = &libzfs_config_ops,
3826                 .lpc_printerr = B_TRUE
3827         };
3828         pools = zpool_search_import(&lpch, &idata);
3829
3830         if (pools != NULL && pool_exists &&
3831             (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
3832                 (void) fprintf(stderr, gettext("cannot import '%s': "
3833                     "a pool with that name already exists\n"),
3834                     argv[0]);
3835                 (void) fprintf(stderr, gettext("use the form '%s "
3836                     "<pool | id> <newpool>' to give it a new name\n"),
3837                     "zpool import");
3838                 err = 1;
3839         } else if (pools == NULL && pool_exists) {
3840                 (void) fprintf(stderr, gettext("cannot import '%s': "
3841                     "a pool with that name is already created/imported,\n"),
3842                     argv[0]);
3843                 (void) fprintf(stderr, gettext("and no additional pools "
3844                     "with that name were found\n"));
3845                 err = 1;
3846         } else if (pools == NULL) {
3847                 if (argc != 0) {
3848                         (void) fprintf(stderr, gettext("cannot import '%s': "
3849                             "no such pool available\n"), argv[0]);
3850                 }
3851                 err = 1;
3852         }
3853
3854         if (err == 1) {
3855                 free(searchdirs);
3856                 free(envdup);
3857                 nvlist_free(policy);
3858                 nvlist_free(pools);
3859                 nvlist_free(props);
3860                 return (1);
3861         }
3862
3863         err = import_pools(pools, props, mntopts, flags,
3864             argc >= 1 ? argv[0] : NULL,
3865             argc >= 2 ? argv[1] : NULL,
3866             do_destroyed, pool_specified, do_all, &idata);
3867
3868         /*
3869          * If we're using the cachefile and we failed to import, then
3870          * fallback to scanning the directory for pools that match
3871          * those in the cachefile.
3872          */
3873         if (err != 0 && cachefile != NULL) {
3874                 (void) printf(gettext("cachefile import failed, retrying\n"));
3875
3876                 /*
3877                  * We use the scan flag to gather the directories that exist
3878                  * in the cachefile. If we need to fallback to searching for
3879                  * the pool config, we will only search devices in these
3880                  * directories.
3881                  */
3882                 idata.scan = B_TRUE;
3883                 nvlist_free(pools);
3884                 pools = zpool_search_import(&lpch, &idata);
3885
3886                 err = import_pools(pools, props, mntopts, flags,
3887                     argc >= 1 ? argv[0] : NULL,
3888                     argc >= 2 ? argv[1] : NULL,
3889                     do_destroyed, pool_specified, do_all, &idata);
3890         }
3891
3892 error:
3893         nvlist_free(props);
3894         nvlist_free(pools);
3895         nvlist_free(policy);
3896         free(searchdirs);
3897         free(envdup);
3898
3899         return (err ? 1 : 0);
3900 }
3901
3902 /*
3903  * zpool sync [-f] [pool] ...
3904  *
3905  * -f (undocumented) force uberblock (and config including zpool cache file)
3906  *    update.
3907  *
3908  * Sync the specified pool(s).
3909  * Without arguments "zpool sync" will sync all pools.
3910  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
3911  *
3912  */
3913 static int
3914 zpool_do_sync(int argc, char **argv)
3915 {
3916         int ret;
3917         boolean_t force = B_FALSE;
3918
3919         /* check options */
3920         while ((ret  = getopt(argc, argv, "f")) != -1) {
3921                 switch (ret) {
3922                 case 'f':
3923                         force = B_TRUE;
3924                         break;
3925                 case '?':
3926                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
3927                             optopt);
3928                         usage(B_FALSE);
3929                 }
3930         }
3931
3932         argc -= optind;
3933         argv += optind;
3934
3935         /* if argc == 0 we will execute zpool_sync_one on all pools */
3936         ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
3937             B_FALSE, zpool_sync_one, &force);
3938
3939         return (ret);
3940 }
3941
3942 typedef struct iostat_cbdata {
3943         uint64_t cb_flags;
3944         int cb_namewidth;
3945         int cb_iteration;
3946         boolean_t cb_verbose;
3947         boolean_t cb_literal;
3948         boolean_t cb_scripted;
3949         zpool_list_t *cb_list;
3950         vdev_cmd_data_list_t *vcdl;
3951         vdev_cbdata_t cb_vdevs;
3952 } iostat_cbdata_t;
3953
3954 /*  iostat labels */
3955 typedef struct name_and_columns {
3956         const char *name;       /* Column name */
3957         unsigned int columns;   /* Center name to this number of columns */
3958 } name_and_columns_t;
3959
3960 #define IOSTAT_MAX_LABELS       15      /* Max number of labels on one line */
3961
3962 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
3963 {
3964         [IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
3965             {NULL}},
3966         [IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3967             {"asyncq_wait", 2}, {"scrub", 1}, {"trim", 1}, {"rebuild", 1},
3968             {NULL}},
3969         [IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
3970             {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
3971             {"trimq_write", 2}, {"rebuildq_write", 2}, {NULL}},
3972         [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
3973             {"asyncq_wait", 2}, {NULL}},
3974         [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
3975             {"async_read", 2}, {"async_write", 2}, {"scrub", 2},
3976             {"trim", 2}, {"rebuild", 2}, {NULL}},
3977 };
3978
3979 /* Shorthand - if "columns" field not set, default to 1 column */
3980 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
3981 {
3982         [IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
3983             {"write"}, {NULL}},
3984         [IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3985             {"write"}, {"read"}, {"write"}, {"wait"}, {"wait"}, {"wait"},
3986             {NULL}},
3987         [IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
3988             {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"},
3989             {"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}},
3990         [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
3991             {"write"}, {"read"}, {"write"}, {"scrub"}, {"trim"}, {"rebuild"},
3992             {NULL}},
3993         [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3994             {"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
3995             {"ind"}, {"agg"}, {NULL}},
3996 };
3997
3998 static const char *histo_to_title[] = {
3999         [IOS_L_HISTO] = "latency",
4000         [IOS_RQ_HISTO] = "req_size",
4001 };
4002
4003 /*
4004  * Return the number of labels in a null-terminated name_and_columns_t
4005  * array.
4006  *
4007  */
4008 static unsigned int
4009 label_array_len(const name_and_columns_t *labels)
4010 {
4011         int i = 0;
4012
4013         while (labels[i].name)
4014                 i++;
4015
4016         return (i);
4017 }
4018
4019 /*
4020  * Return the number of strings in a null-terminated string array.
4021  * For example:
4022  *
4023  *     const char foo[] = {"bar", "baz", NULL}
4024  *
4025  * returns 2
4026  */
4027 static uint64_t
4028 str_array_len(const char *array[])
4029 {
4030         uint64_t i = 0;
4031         while (array[i])
4032                 i++;
4033
4034         return (i);
4035 }
4036
4037
4038 /*
4039  * Return a default column width for default/latency/queue columns. This does
4040  * not include histograms, which have their columns autosized.
4041  */
4042 static unsigned int
4043 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
4044 {
4045         unsigned long column_width = 5; /* Normal niceprint */
4046         static unsigned long widths[] = {
4047                 /*
4048                  * Choose some sane default column sizes for printing the
4049                  * raw numbers.
4050                  */
4051                 [IOS_DEFAULT] = 15, /* 1PB capacity */
4052                 [IOS_LATENCY] = 10, /* 1B ns = 10sec */
4053                 [IOS_QUEUES] = 6,   /* 1M queue entries */
4054                 [IOS_L_HISTO] = 10, /* 1B ns = 10sec */
4055                 [IOS_RQ_HISTO] = 6, /* 1M queue entries */
4056         };
4057
4058         if (cb->cb_literal)
4059                 column_width = widths[type];
4060
4061         return (column_width);
4062 }
4063
4064 /*
4065  * Print the column labels, i.e:
4066  *
4067  *   capacity     operations     bandwidth
4068  * alloc   free   read  write   read  write  ...
4069  *
4070  * If force_column_width is set, use it for the column width.  If not set, use
4071  * the default column width.
4072  */
4073 static void
4074 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
4075     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
4076 {
4077         int i, idx, s;
4078         int text_start, rw_column_width, spaces_to_end;
4079         uint64_t flags = cb->cb_flags;
4080         uint64_t f;
4081         unsigned int column_width = force_column_width;
4082
4083         /* For each bit set in flags */
4084         for (f = flags; f; f &= ~(1ULL << idx)) {
4085                 idx = lowbit64(f) - 1;
4086                 if (!force_column_width)
4087                         column_width = default_column_width(cb, idx);
4088                 /* Print our top labels centered over "read  write" label. */
4089                 for (i = 0; i < label_array_len(labels[idx]); i++) {
4090                         const char *name = labels[idx][i].name;
4091                         /*
4092                          * We treat labels[][].columns == 0 as shorthand
4093                          * for one column.  It makes writing out the label
4094                          * tables more concise.
4095                          */
4096                         unsigned int columns = MAX(1, labels[idx][i].columns);
4097                         unsigned int slen = strlen(name);
4098
4099                         rw_column_width = (column_width * columns) +
4100                             (2 * (columns - 1));
4101
4102                         text_start = (int)((rw_column_width) / columns -
4103                             slen / columns);
4104                         if (text_start < 0)
4105                                 text_start = 0;
4106
4107                         printf("  ");   /* Two spaces between columns */
4108
4109                         /* Space from beginning of column to label */
4110                         for (s = 0; s < text_start; s++)
4111                                 printf(" ");
4112
4113                         printf("%s", name);
4114
4115                         /* Print space after label to end of column */
4116                         spaces_to_end = rw_column_width - text_start - slen;
4117                         if (spaces_to_end < 0)
4118                                 spaces_to_end = 0;
4119
4120                         for (s = 0; s < spaces_to_end; s++)
4121                                 printf(" ");
4122                 }
4123         }
4124 }
4125
4126
4127 /*
4128  * print_cmd_columns - Print custom column titles from -c
4129  *
4130  * If the user specified the "zpool status|iostat -c" then print their custom
4131  * column titles in the header.  For example, print_cmd_columns() would print
4132  * the "  col1  col2" part of this:
4133  *
4134  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
4135  * ...
4136  *            capacity     operations     bandwidth
4137  * pool        alloc   free   read  write   read  write  col1  col2
4138  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
4139  * mypool       269K  1008M      0      0    107    946
4140  *   mirror     269K  1008M      0      0    107    946
4141  *     sdb         -      -      0      0    102    473  val1  val2
4142  *     sdc         -      -      0      0      5    473  val1  val2
4143  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
4144  */
4145 static void
4146 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
4147 {
4148         int i, j;
4149         vdev_cmd_data_t *data = &vcdl->data[0];
4150
4151         if (vcdl->count == 0 || data == NULL)
4152                 return;
4153
4154         /*
4155          * Each vdev cmd should have the same column names unless the user did
4156          * something weird with their cmd.  Just take the column names from the
4157          * first vdev and assume it works for all of them.
4158          */
4159         for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
4160                 printf("  ");
4161                 if (use_dashes) {
4162                         for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
4163                                 printf("-");
4164                 } else {
4165                         printf_color(ANSI_BOLD, "%*s", vcdl->uniq_cols_width[i],
4166                             vcdl->uniq_cols[i]);
4167                 }
4168         }
4169 }
4170
4171
4172 /*
4173  * Utility function to print out a line of dashes like:
4174  *
4175  *      --------------------------------  -----  -----  -----  -----  -----
4176  *
4177  * ...or a dashed named-row line like:
4178  *
4179  *      logs                                  -      -      -      -      -
4180  *
4181  * @cb:                         iostat data
4182  *
4183  * @force_column_width          If non-zero, use the value as the column width.
4184  *                              Otherwise use the default column widths.
4185  *
4186  * @name:                       Print a dashed named-row line starting
4187  *                              with @name.  Otherwise, print a regular
4188  *                              dashed line.
4189  */
4190 static void
4191 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
4192     const char *name)
4193 {
4194         int i;
4195         unsigned int namewidth;
4196         uint64_t flags = cb->cb_flags;
4197         uint64_t f;
4198         int idx;
4199         const name_and_columns_t *labels;
4200         const char *title;
4201
4202
4203         if (cb->cb_flags & IOS_ANYHISTO_M) {
4204                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
4205         } else if (cb->cb_vdevs.cb_names_count) {
4206                 title = "vdev";
4207         } else  {
4208                 title = "pool";
4209         }
4210
4211         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
4212             name ? strlen(name) : 0);
4213
4214
4215         if (name) {
4216                 printf("%-*s", namewidth, name);
4217         } else {
4218                 for (i = 0; i < namewidth; i++)
4219                         (void) printf("-");
4220         }
4221
4222         /* For each bit in flags */
4223         for (f = flags; f; f &= ~(1ULL << idx)) {
4224                 unsigned int column_width;
4225                 idx = lowbit64(f) - 1;
4226                 if (force_column_width)
4227                         column_width = force_column_width;
4228                 else
4229                         column_width = default_column_width(cb, idx);
4230
4231                 labels = iostat_bottom_labels[idx];
4232                 for (i = 0; i < label_array_len(labels); i++) {
4233                         if (name)
4234                                 printf("  %*s-", column_width - 1, " ");
4235                         else
4236                                 printf("  %.*s", column_width,
4237                                     "--------------------");
4238                 }
4239         }
4240 }
4241
4242
4243 static void
4244 print_iostat_separator_impl(iostat_cbdata_t *cb,
4245     unsigned int force_column_width)
4246 {
4247         print_iostat_dashes(cb, force_column_width, NULL);
4248 }
4249
4250 static void
4251 print_iostat_separator(iostat_cbdata_t *cb)
4252 {
4253         print_iostat_separator_impl(cb, 0);
4254 }
4255
4256 static void
4257 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
4258     const char *histo_vdev_name)
4259 {
4260         unsigned int namewidth;
4261         const char *title;
4262
4263         color_start(ANSI_BOLD);
4264
4265         if (cb->cb_flags & IOS_ANYHISTO_M) {
4266                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
4267         } else if (cb->cb_vdevs.cb_names_count) {
4268                 title = "vdev";
4269         } else  {
4270                 title = "pool";
4271         }
4272
4273         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
4274             histo_vdev_name ? strlen(histo_vdev_name) : 0);
4275
4276         if (histo_vdev_name)
4277                 printf("%-*s", namewidth, histo_vdev_name);
4278         else
4279                 printf("%*s", namewidth, "");
4280
4281
4282         print_iostat_labels(cb, force_column_width, iostat_top_labels);
4283         printf("\n");
4284
4285         printf("%-*s", namewidth, title);
4286
4287         print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
4288         if (cb->vcdl != NULL)
4289                 print_cmd_columns(cb->vcdl, 0);
4290
4291         printf("\n");
4292
4293         print_iostat_separator_impl(cb, force_column_width);
4294
4295         if (cb->vcdl != NULL)
4296                 print_cmd_columns(cb->vcdl, 1);
4297
4298         color_end();
4299
4300         printf("\n");
4301 }
4302
4303 static void
4304 print_iostat_header(iostat_cbdata_t *cb)
4305 {
4306         print_iostat_header_impl(cb, 0, NULL);
4307 }
4308
4309 /*
4310  * Prints a size string (i.e. 120M) with the suffix ("M") colored
4311  * by order of magnitude. Uses column_size to add padding.
4312  */
4313 static void
4314 print_stat_color(const char *statbuf, unsigned int column_size)
4315 {
4316         fputs("  ", stdout);
4317         size_t len = strlen(statbuf);
4318         while (len < column_size) {
4319                 fputc(' ', stdout);
4320                 column_size--;
4321         }
4322         if (*statbuf == '0') {
4323                 color_start(ANSI_GRAY);
4324                 fputc('0', stdout);
4325         } else {
4326                 for (; *statbuf; statbuf++) {
4327                         if (*statbuf == 'K') color_start(ANSI_GREEN);
4328                         else if (*statbuf == 'M') color_start(ANSI_YELLOW);
4329                         else if (*statbuf == 'G') color_start(ANSI_RED);
4330                         else if (*statbuf == 'T') color_start(ANSI_BOLD_BLUE);
4331                         else if (*statbuf == 'P') color_start(ANSI_MAGENTA);
4332                         else if (*statbuf == 'E') color_start(ANSI_CYAN);
4333                         fputc(*statbuf, stdout);
4334                         if (--column_size <= 0)
4335                                 break;
4336                 }
4337         }
4338         color_end();
4339 }
4340
4341 /*
4342  * Display a single statistic.
4343  */
4344 static void
4345 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
4346     unsigned int column_size, boolean_t scripted)
4347 {
4348         char buf[64];
4349
4350         zfs_nicenum_format(value, buf, sizeof (buf), format);
4351
4352         if (scripted)
4353                 printf("\t%s", buf);
4354         else
4355                 print_stat_color(buf, column_size);
4356 }
4357
4358 /*
4359  * Calculate the default vdev stats
4360  *
4361  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
4362  * stats into calcvs.
4363  */
4364 static void
4365 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
4366     vdev_stat_t *calcvs)
4367 {
4368         int i;
4369
4370         memcpy(calcvs, newvs, sizeof (*calcvs));
4371         for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
4372                 calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
4373
4374         for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
4375                 calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
4376 }
4377
4378 /*
4379  * Internal representation of the extended iostats data.
4380  *
4381  * The extended iostat stats are exported in nvlists as either uint64_t arrays
4382  * or single uint64_t's.  We make both look like arrays to make them easier
4383  * to process.  In order to make single uint64_t's look like arrays, we set
4384  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
4385  * we can just use *data and count.
4386  */
4387 struct stat_array {
4388         uint64_t *data;
4389         uint_t count;   /* Number of entries in data[] */
4390         uint64_t __data; /* Only used when data is a single uint64_t */
4391 };
4392
4393 static uint64_t
4394 stat_histo_max(struct stat_array *nva, unsigned int len)
4395 {
4396         uint64_t max = 0;
4397         int i;
4398         for (i = 0; i < len; i++)
4399                 max = MAX(max, array64_max(nva[i].data, nva[i].count));
4400
4401         return (max);
4402 }
4403
4404 /*
4405  * Helper function to lookup a uint64_t array or uint64_t value and store its
4406  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
4407  * it look like a one element array to make it easier to process.
4408  */
4409 static int
4410 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
4411     struct stat_array *nva)
4412 {
4413         nvpair_t *tmp;
4414         int ret;
4415
4416         verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
4417         switch (nvpair_type(tmp)) {
4418         case DATA_TYPE_UINT64_ARRAY:
4419                 ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
4420                 break;
4421         case DATA_TYPE_UINT64:
4422                 ret = nvpair_value_uint64(tmp, &nva->__data);
4423                 nva->data = &nva->__data;
4424                 nva->count = 1;
4425                 break;
4426         default:
4427                 /* Not a uint64_t */
4428                 ret = EINVAL;
4429                 break;
4430         }
4431
4432         return (ret);
4433 }
4434
4435 /*
4436  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
4437  * subtract them, and return the results in a newly allocated stat_array.
4438  * You must free the returned array after you are done with it with
4439  * free_calc_stats().
4440  *
4441  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
4442  * values.
4443  */
4444 static struct stat_array *
4445 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
4446     nvlist_t *newnv)
4447 {
4448         nvlist_t *oldnvx = NULL, *newnvx;
4449         struct stat_array *oldnva, *newnva, *calcnva;
4450         int i, j;
4451         unsigned int alloc_size = (sizeof (struct stat_array)) * len;
4452
4453         /* Extract our extended stats nvlist from the main list */
4454         verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4455             &newnvx) == 0);
4456         if (oldnv) {
4457                 verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
4458                     &oldnvx) == 0);
4459         }
4460
4461         newnva = safe_malloc(alloc_size);
4462         oldnva = safe_malloc(alloc_size);
4463         calcnva = safe_malloc(alloc_size);
4464
4465         for (j = 0; j < len; j++) {
4466                 verify(nvpair64_to_stat_array(newnvx, names[j],
4467                     &newnva[j]) == 0);
4468                 calcnva[j].count = newnva[j].count;
4469                 alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
4470                 calcnva[j].data = safe_malloc(alloc_size);
4471                 memcpy(calcnva[j].data, newnva[j].data, alloc_size);
4472
4473                 if (oldnvx) {
4474                         verify(nvpair64_to_stat_array(oldnvx, names[j],
4475                             &oldnva[j]) == 0);
4476                         for (i = 0; i < oldnva[j].count; i++)
4477                                 calcnva[j].data[i] -= oldnva[j].data[i];
4478                 }
4479         }
4480         free(newnva);
4481         free(oldnva);
4482         return (calcnva);
4483 }
4484
4485 static void
4486 free_calc_stats(struct stat_array *nva, unsigned int len)
4487 {
4488         int i;
4489         for (i = 0; i < len; i++)
4490                 free(nva[i].data);
4491
4492         free(nva);
4493 }
4494
4495 static void
4496 print_iostat_histo(struct stat_array *nva, unsigned int len,
4497     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
4498     double scale)
4499 {
4500         int i, j;
4501         char buf[6];
4502         uint64_t val;
4503         enum zfs_nicenum_format format;
4504         unsigned int buckets;
4505         unsigned int start_bucket;
4506
4507         if (cb->cb_literal)
4508                 format = ZFS_NICENUM_RAW;
4509         else
4510                 format = ZFS_NICENUM_1024;
4511
4512         /* All these histos are the same size, so just use nva[0].count */
4513         buckets = nva[0].count;
4514
4515         if (cb->cb_flags & IOS_RQ_HISTO_M) {
4516                 /* Start at 512 - req size should never be lower than this */
4517                 start_bucket = 9;
4518         } else {
4519                 start_bucket = 0;
4520         }
4521
4522         for (j = start_bucket; j < buckets; j++) {
4523                 /* Print histogram bucket label */
4524                 if (cb->cb_flags & IOS_L_HISTO_M) {
4525                         /* Ending range of this bucket */
4526                         val = (1UL << (j + 1)) - 1;
4527                         zfs_nicetime(val, buf, sizeof (buf));
4528                 } else {
4529                         /* Request size (starting range of bucket) */
4530                         val = (1UL << j);
4531                         zfs_nicenum(val, buf, sizeof (buf));
4532                 }
4533
4534                 if (cb->cb_scripted)
4535                         printf("%llu", (u_longlong_t)val);
4536                 else
4537                         printf("%-*s", namewidth, buf);
4538
4539                 /* Print the values on the line */
4540                 for (i = 0; i < len; i++) {
4541                         print_one_stat(nva[i].data[j] * scale, format,
4542                             column_width, cb->cb_scripted);
4543                 }
4544                 printf("\n");
4545         }
4546 }
4547
4548 static void
4549 print_solid_separator(unsigned int length)
4550 {
4551         while (length--)
4552                 printf("-");
4553         printf("\n");
4554 }
4555
4556 static void
4557 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
4558     nvlist_t *newnv, double scale, const char *name)
4559 {
4560         unsigned int column_width;
4561         unsigned int namewidth;
4562         unsigned int entire_width;
4563         enum iostat_type type;
4564         struct stat_array *nva;
4565         const char **names;
4566         unsigned int names_len;
4567
4568         /* What type of histo are we? */
4569         type = IOS_HISTO_IDX(cb->cb_flags);
4570
4571         /* Get NULL-terminated array of nvlist names for our histo */
4572         names = vsx_type_to_nvlist[type];
4573         names_len = str_array_len(names); /* num of names */
4574
4575         nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
4576
4577         if (cb->cb_literal) {
4578                 column_width = MAX(5,
4579                     (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
4580         } else {
4581                 column_width = 5;
4582         }
4583
4584         namewidth = MAX(cb->cb_namewidth,
4585             strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
4586
4587         /*
4588          * Calculate the entire line width of what we're printing.  The
4589          * +2 is for the two spaces between columns:
4590          */
4591         /*       read  write                            */
4592         /*      -----  -----                            */
4593         /*      |___|  <---------- column_width         */
4594         /*                                              */
4595         /*      |__________|  <--- entire_width         */
4596         /*                                              */
4597         entire_width = namewidth + (column_width + 2) *
4598             label_array_len(iostat_bottom_labels[type]);
4599
4600         if (cb->cb_scripted)
4601                 printf("%s\n", name);
4602         else
4603                 print_iostat_header_impl(cb, column_width, name);
4604
4605         print_iostat_histo(nva, names_len, cb, column_width,
4606             namewidth, scale);
4607
4608         free_calc_stats(nva, names_len);
4609         if (!cb->cb_scripted)
4610                 print_solid_separator(entire_width);
4611 }
4612
4613 /*
4614  * Calculate the average latency of a power-of-two latency histogram
4615  */
4616 static uint64_t
4617 single_histo_average(uint64_t *histo, unsigned int buckets)
4618 {
4619         int i;
4620         uint64_t count = 0, total = 0;
4621
4622         for (i = 0; i < buckets; i++) {
4623                 /*
4624                  * Our buckets are power-of-two latency ranges.  Use the
4625                  * midpoint latency of each bucket to calculate the average.
4626                  * For example:
4627                  *
4628                  * Bucket          Midpoint
4629                  * 8ns-15ns:       12ns
4630                  * 16ns-31ns:      24ns
4631                  * ...
4632                  */
4633                 if (histo[i] != 0) {
4634                         total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
4635                         count += histo[i];
4636                 }
4637         }
4638
4639         /* Prevent divide by zero */
4640         return (count == 0 ? 0 : total / count);
4641 }
4642
4643 static void
4644 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *newnv)
4645 {
4646         const char *names[] = {
4647                 ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
4648                 ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
4649                 ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
4650                 ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
4651                 ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
4652                 ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
4653                 ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
4654                 ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
4655                 ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
4656                 ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
4657                 ZPOOL_CONFIG_VDEV_TRIM_PEND_QUEUE,
4658                 ZPOOL_CONFIG_VDEV_TRIM_ACTIVE_QUEUE,
4659                 ZPOOL_CONFIG_VDEV_REBUILD_PEND_QUEUE,
4660                 ZPOOL_CONFIG_VDEV_REBUILD_ACTIVE_QUEUE,
4661         };
4662
4663         struct stat_array *nva;
4664
4665         unsigned int column_width = default_column_width(cb, IOS_QUEUES);
4666         enum zfs_nicenum_format format;
4667
4668         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
4669
4670         if (cb->cb_literal)
4671                 format = ZFS_NICENUM_RAW;
4672         else
4673                 format = ZFS_NICENUM_1024;
4674
4675         for (int i = 0; i < ARRAY_SIZE(names); i++) {
4676                 uint64_t val = nva[i].data[0];
4677                 print_one_stat(val, format, column_width, cb->cb_scripted);
4678         }
4679
4680         free_calc_stats(nva, ARRAY_SIZE(names));
4681 }
4682
4683 static void
4684 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
4685     nvlist_t *newnv)
4686 {
4687         int i;
4688         uint64_t val;
4689         const char *names[] = {
4690                 ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
4691                 ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
4692                 ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
4693                 ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
4694                 ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
4695                 ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
4696                 ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
4697                 ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
4698                 ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
4699                 ZPOOL_CONFIG_VDEV_TRIM_LAT_HISTO,
4700                 ZPOOL_CONFIG_VDEV_REBUILD_LAT_HISTO,
4701         };
4702         struct stat_array *nva;
4703
4704         unsigned int column_width = default_column_width(cb, IOS_LATENCY);
4705         enum zfs_nicenum_format format;
4706
4707         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
4708
4709         if (cb->cb_literal)
4710                 format = ZFS_NICENUM_RAWTIME;
4711         else
4712                 format = ZFS_NICENUM_TIME;
4713
4714         /* Print our avg latencies on the line */
4715         for (i = 0; i < ARRAY_SIZE(names); i++) {
4716                 /* Compute average latency for a latency histo */
4717                 val = single_histo_average(nva[i].data, nva[i].count);
4718                 print_one_stat(val, format, column_width, cb->cb_scripted);
4719         }
4720         free_calc_stats(nva, ARRAY_SIZE(names));
4721 }
4722
4723 /*
4724  * Print default statistics (capacity/operations/bandwidth)
4725  */
4726 static void
4727 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
4728 {
4729         unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
4730         enum zfs_nicenum_format format;
4731         char na;        /* char to print for "not applicable" values */
4732
4733         if (cb->cb_literal) {
4734                 format = ZFS_NICENUM_RAW;
4735                 na = '0';
4736         } else {
4737                 format = ZFS_NICENUM_1024;
4738                 na = '-';
4739         }
4740
4741         /* only toplevel vdevs have capacity stats */
4742         if (vs->vs_space == 0) {
4743                 if (cb->cb_scripted)
4744                         printf("\t%c\t%c", na, na);
4745                 else
4746                         printf("  %*c  %*c", column_width, na, column_width,
4747                             na);
4748         } else {
4749                 print_one_stat(vs->vs_alloc, format, column_width,
4750                     cb->cb_scripted);
4751                 print_one_stat(vs->vs_space - vs->vs_alloc, format,
4752                     column_width, cb->cb_scripted);
4753         }
4754
4755         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
4756             format, column_width, cb->cb_scripted);
4757         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
4758             format, column_width, cb->cb_scripted);
4759         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
4760             format, column_width, cb->cb_scripted);
4761         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
4762             format, column_width, cb->cb_scripted);
4763 }
4764
4765 static const char *const class_name[] = {
4766         VDEV_ALLOC_BIAS_DEDUP,
4767         VDEV_ALLOC_BIAS_SPECIAL,
4768         VDEV_ALLOC_CLASS_LOGS
4769 };
4770
4771 /*
4772  * Print out all the statistics for the given vdev.  This can either be the
4773  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
4774  * is a verbose output, and we don't want to display the toplevel pool stats.
4775  *
4776  * Returns the number of stat lines printed.
4777  */
4778 static unsigned int
4779 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
4780     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
4781 {
4782         nvlist_t **oldchild, **newchild;
4783         uint_t c, children, oldchildren;
4784         vdev_stat_t *oldvs, *newvs, *calcvs;
4785         vdev_stat_t zerovs = { 0 };
4786         char *vname;
4787         int i;
4788         int ret = 0;
4789         uint64_t tdelta;
4790         double scale;
4791
4792         if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
4793                 return (ret);
4794
4795         calcvs = safe_malloc(sizeof (*calcvs));
4796
4797         if (oldnv != NULL) {
4798                 verify(nvlist_lookup_uint64_array(oldnv,
4799                     ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
4800         } else {
4801                 oldvs = &zerovs;
4802         }
4803
4804         /* Do we only want to see a specific vdev? */
4805         for (i = 0; i < cb->cb_vdevs.cb_names_count; i++) {
4806                 /* Yes we do.  Is this the vdev? */
4807                 if (strcmp(name, cb->cb_vdevs.cb_names[i]) == 0) {
4808                         /*
4809                          * This is our vdev.  Since it is the only vdev we
4810                          * will be displaying, make depth = 0 so that it
4811                          * doesn't get indented.
4812                          */
4813                         depth = 0;
4814                         break;
4815                 }
4816         }
4817
4818         if (cb->cb_vdevs.cb_names_count && (i == cb->cb_vdevs.cb_names_count)) {
4819                 /* Couldn't match the name */
4820                 goto children;
4821         }
4822
4823
4824         verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
4825             (uint64_t **)&newvs, &c) == 0);
4826
4827         /*
4828          * Print the vdev name unless it's is a histogram.  Histograms
4829          * display the vdev name in the header itself.
4830          */
4831         if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
4832                 if (cb->cb_scripted) {
4833                         printf("%s", name);
4834                 } else {
4835                         if (strlen(name) + depth > cb->cb_namewidth)
4836                                 (void) printf("%*s%s", depth, "", name);
4837                         else
4838                                 (void) printf("%*s%s%*s", depth, "", name,
4839                                     (int)(cb->cb_namewidth - strlen(name) -
4840                                     depth), "");
4841                 }
4842         }
4843
4844         /* Calculate our scaling factor */
4845         tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
4846         if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
4847                 /*
4848                  * If we specify printing histograms with no time interval, then
4849                  * print the histogram numbers over the entire lifetime of the
4850                  * vdev.
4851                  */
4852                 scale = 1;
4853         } else {
4854                 if (tdelta == 0)
4855                         scale = 1.0;
4856                 else
4857                         scale = (double)NANOSEC / tdelta;
4858         }
4859
4860         if (cb->cb_flags & IOS_DEFAULT_M) {
4861                 calc_default_iostats(oldvs, newvs, calcvs);
4862                 print_iostat_default(calcvs, cb, scale);
4863         }
4864         if (cb->cb_flags & IOS_LATENCY_M)
4865                 print_iostat_latency(cb, oldnv, newnv);
4866         if (cb->cb_flags & IOS_QUEUES_M)
4867                 print_iostat_queues(cb, newnv);
4868         if (cb->cb_flags & IOS_ANYHISTO_M) {
4869                 printf("\n");
4870                 print_iostat_histos(cb, oldnv, newnv, scale, name);
4871         }
4872
4873         if (cb->vcdl != NULL) {
4874                 const char *path;
4875                 if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
4876                     &path) == 0) {
4877                         printf("  ");
4878                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
4879                 }
4880         }
4881
4882         if (!(cb->cb_flags & IOS_ANYHISTO_M))
4883                 printf("\n");
4884
4885         ret++;
4886
4887 children:
4888
4889         free(calcvs);
4890
4891         if (!cb->cb_verbose)
4892                 return (ret);
4893
4894         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
4895             &newchild, &children) != 0)
4896                 return (ret);
4897
4898         if (oldnv) {
4899                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
4900                     &oldchild, &oldchildren) != 0)
4901                         return (ret);
4902
4903                 children = MIN(oldchildren, children);
4904         }
4905
4906         /*
4907          * print normal top-level devices
4908          */
4909         for (c = 0; c < children; c++) {
4910                 uint64_t ishole = B_FALSE, islog = B_FALSE;
4911
4912                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
4913                     &ishole);
4914
4915                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
4916                     &islog);
4917
4918                 if (ishole || islog)
4919                         continue;
4920
4921                 if (nvlist_exists(newchild[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
4922                         continue;
4923
4924                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4925                     cb->cb_vdevs.cb_name_flags | VDEV_NAME_TYPE_ID);
4926                 ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
4927                     newchild[c], cb, depth + 2);
4928                 free(vname);
4929         }
4930
4931         /*
4932          * print all other top-level devices
4933          */
4934         for (uint_t n = 0; n < ARRAY_SIZE(class_name); n++) {
4935                 boolean_t printed = B_FALSE;
4936
4937                 for (c = 0; c < children; c++) {
4938                         uint64_t islog = B_FALSE;
4939                         const char *bias = NULL;
4940                         const char *type = NULL;
4941
4942                         (void) nvlist_lookup_uint64(newchild[c],
4943                             ZPOOL_CONFIG_IS_LOG, &islog);
4944                         if (islog) {
4945                                 bias = VDEV_ALLOC_CLASS_LOGS;
4946                         } else {
4947                                 (void) nvlist_lookup_string(newchild[c],
4948                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
4949                                 (void) nvlist_lookup_string(newchild[c],
4950                                     ZPOOL_CONFIG_TYPE, &type);
4951                         }
4952                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
4953                                 continue;
4954                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
4955                                 continue;
4956
4957                         if (!printed) {
4958                                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) &&
4959                                     !cb->cb_scripted &&
4960                                     !cb->cb_vdevs.cb_names) {
4961                                         print_iostat_dashes(cb, 0,
4962                                             class_name[n]);
4963                                 }
4964                                 printf("\n");
4965                                 printed = B_TRUE;
4966                         }
4967
4968                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
4969                             cb->cb_vdevs.cb_name_flags | VDEV_NAME_TYPE_ID);
4970                         ret += print_vdev_stats(zhp, vname, oldnv ?
4971                             oldchild[c] : NULL, newchild[c], cb, depth + 2);
4972                         free(vname);
4973                 }
4974         }
4975
4976         /*
4977          * Include level 2 ARC devices in iostat output
4978          */
4979         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
4980             &newchild, &children) != 0)
4981                 return (ret);
4982
4983         if (oldnv) {
4984                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
4985                     &oldchild, &oldchildren) != 0)
4986                         return (ret);
4987
4988                 children = MIN(oldchildren, children);
4989         }
4990
4991         if (children > 0) {
4992                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
4993                     !cb->cb_vdevs.cb_names) {
4994                         print_iostat_dashes(cb, 0, "cache");
4995                 }
4996                 printf("\n");
4997
4998                 for (c = 0; c < children; c++) {
4999                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
5000                             cb->cb_vdevs.cb_name_flags);
5001                         ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
5002                             : NULL, newchild[c], cb, depth + 2);
5003                         free(vname);
5004                 }
5005         }
5006
5007         return (ret);
5008 }
5009
5010 static int
5011 refresh_iostat(zpool_handle_t *zhp, void *data)
5012 {
5013         iostat_cbdata_t *cb = data;
5014         boolean_t missing;
5015
5016         /*
5017          * If the pool has disappeared, remove it from the list and continue.
5018          */
5019         if (zpool_refresh_stats(zhp, &missing) != 0)
5020                 return (-1);
5021
5022         if (missing)
5023                 pool_list_remove(cb->cb_list, zhp);
5024
5025         return (0);
5026 }
5027
5028 /*
5029  * Callback to print out the iostats for the given pool.
5030  */
5031 static int
5032 print_iostat(zpool_handle_t *zhp, void *data)
5033 {
5034         iostat_cbdata_t *cb = data;
5035         nvlist_t *oldconfig, *newconfig;
5036         nvlist_t *oldnvroot, *newnvroot;
5037         int ret;
5038
5039         newconfig = zpool_get_config(zhp, &oldconfig);
5040
5041         if (cb->cb_iteration == 1)
5042                 oldconfig = NULL;
5043
5044         verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
5045             &newnvroot) == 0);
5046
5047         if (oldconfig == NULL)
5048                 oldnvroot = NULL;
5049         else
5050                 verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
5051                     &oldnvroot) == 0);
5052
5053         ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
5054             cb, 0);
5055         if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
5056             !cb->cb_scripted && cb->cb_verbose &&
5057             !cb->cb_vdevs.cb_names_count) {
5058                 print_iostat_separator(cb);
5059                 if (cb->vcdl != NULL) {
5060                         print_cmd_columns(cb->vcdl, 1);
5061                 }
5062                 printf("\n");
5063         }
5064
5065         return (ret);
5066 }
5067
5068 static int
5069 get_columns(void)
5070 {
5071         struct winsize ws;
5072         int columns = 80;
5073         int error;
5074
5075         if (isatty(STDOUT_FILENO)) {
5076                 error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
5077                 if (error == 0)
5078                         columns = ws.ws_col;
5079         } else {
5080                 columns = 999;
5081         }
5082
5083         return (columns);
5084 }
5085
5086 /*
5087  * Return the required length of the pool/vdev name column.  The minimum
5088  * allowed width and output formatting flags must be provided.
5089  */
5090 static int
5091 get_namewidth(zpool_handle_t *zhp, int min_width, int flags, boolean_t verbose)
5092 {
5093         nvlist_t *config, *nvroot;
5094         int width = min_width;
5095
5096         if ((config = zpool_get_config(zhp, NULL)) != NULL) {
5097                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5098                     &nvroot) == 0);
5099                 size_t poolname_len = strlen(zpool_get_name(zhp));
5100                 if (verbose == B_FALSE) {
5101                         width = MAX(poolname_len, min_width);
5102                 } else {
5103                         width = MAX(poolname_len,
5104                             max_width(zhp, nvroot, 0, min_width, flags));
5105                 }
5106         }
5107
5108         return (width);
5109 }
5110
5111 /*
5112  * Parse the input string, get the 'interval' and 'count' value if there is one.
5113  */
5114 static void
5115 get_interval_count(int *argcp, char **argv, float *iv,
5116     unsigned long *cnt)
5117 {
5118         float interval = 0;
5119         unsigned long count = 0;
5120         int argc = *argcp;
5121
5122         /*
5123          * Determine if the last argument is an integer or a pool name
5124          */
5125         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
5126                 char *end;
5127
5128                 errno = 0;
5129                 interval = strtof(argv[argc - 1], &end);
5130
5131                 if (*end == '\0' && errno == 0) {
5132                         if (interval == 0) {
5133                                 (void) fprintf(stderr, gettext(
5134                                     "interval cannot be zero\n"));
5135                                 usage(B_FALSE);
5136                         }
5137                         /*
5138                          * Ignore the last parameter
5139                          */
5140                         argc--;
5141                 } else {
5142                         /*
5143                          * If this is not a valid number, just plow on.  The
5144                          * user will get a more informative error message later
5145                          * on.
5146                          */
5147                         interval = 0;
5148                 }
5149         }
5150
5151         /*
5152          * If the last argument is also an integer, then we have both a count
5153          * and an interval.
5154          */
5155         if (argc > 0 && zfs_isnumber(argv[argc - 1])) {
5156                 char *end;
5157
5158                 errno = 0;
5159                 count = interval;
5160                 interval = strtof(argv[argc - 1], &end);
5161
5162                 if (*end == '\0' && errno == 0) {
5163                         if (interval == 0) {
5164                                 (void) fprintf(stderr, gettext(
5165                                     "interval cannot be zero\n"));
5166                                 usage(B_FALSE);
5167                         }
5168
5169                         /*
5170                          * Ignore the last parameter
5171                          */
5172                         argc--;
5173                 } else {
5174                         interval = 0;
5175                 }
5176         }
5177
5178         *iv = interval;
5179         *cnt = count;
5180         *argcp = argc;
5181 }
5182
5183 static void
5184 get_timestamp_arg(char c)
5185 {
5186         if (c == 'u')
5187                 timestamp_fmt = UDATE;
5188         else if (c == 'd')
5189                 timestamp_fmt = DDATE;
5190         else
5191                 usage(B_FALSE);
5192 }
5193
5194 /*
5195  * Return stat flags that are supported by all pools by both the module and
5196  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
5197  * It will get ANDed down until only the flags that are supported on all pools
5198  * remain.
5199  */
5200 static int
5201 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
5202 {
5203         uint64_t *mask = data;
5204         nvlist_t *config, *nvroot, *nvx;
5205         uint64_t flags = 0;
5206         int i, j;
5207
5208         config = zpool_get_config(zhp, NULL);
5209         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5210             &nvroot) == 0);
5211
5212         /* Default stats are always supported, but for completeness.. */
5213         if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
5214                 flags |= IOS_DEFAULT_M;
5215
5216         /* Get our extended stats nvlist from the main list */
5217         if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
5218             &nvx) != 0) {
5219                 /*
5220                  * No extended stats; they're probably running an older
5221                  * module.  No big deal, we support that too.
5222                  */
5223                 goto end;
5224         }
5225
5226         /* For each extended stat, make sure all its nvpairs are supported */
5227         for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
5228                 if (!vsx_type_to_nvlist[j][0])
5229                         continue;
5230
5231                 /* Start off by assuming the flag is supported, then check */
5232                 flags |= (1ULL << j);
5233                 for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
5234                         if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
5235                                 /* flag isn't supported */
5236                                 flags = flags & ~(1ULL  << j);
5237                                 break;
5238                         }
5239                 }
5240         }
5241 end:
5242         *mask = *mask & flags;
5243         return (0);
5244 }
5245
5246 /*
5247  * Return a bitmask of stats that are supported on all pools by both the module
5248  * and zpool iostat.
5249  */
5250 static uint64_t
5251 get_stat_flags(zpool_list_t *list)
5252 {
5253         uint64_t mask = -1;
5254
5255         /*
5256          * get_stat_flags_cb() will lop off bits from "mask" until only the
5257          * flags that are supported on all pools remain.
5258          */
5259         pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
5260         return (mask);
5261 }
5262
5263 /*
5264  * Return 1 if cb_data->cb_names[0] is this vdev's name, 0 otherwise.
5265  */
5266 static int
5267 is_vdev_cb(void *zhp_data, nvlist_t *nv, void *cb_data)
5268 {
5269         uint64_t guid;
5270         vdev_cbdata_t *cb = cb_data;
5271         zpool_handle_t *zhp = zhp_data;
5272
5273         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) != 0)
5274                 return (0);
5275
5276         return (guid == zpool_vdev_path_to_guid(zhp, cb->cb_names[0]));
5277 }
5278
5279 /*
5280  * Returns 1 if cb_data->cb_names[0] is a vdev name, 0 otherwise.
5281  */
5282 static int
5283 is_vdev(zpool_handle_t *zhp, void *cb_data)
5284 {
5285         return (for_each_vdev(zhp, is_vdev_cb, cb_data));
5286 }
5287
5288 /*
5289  * Check if vdevs are in a pool
5290  *
5291  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
5292  * return 0.  If pool_name is NULL, then search all pools.
5293  */
5294 static int
5295 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
5296     vdev_cbdata_t *cb)
5297 {
5298         char **tmp_name;
5299         int ret = 0;
5300         int i;
5301         int pool_count = 0;
5302
5303         if ((argc == 0) || !*argv)
5304                 return (0);
5305
5306         if (pool_name)
5307                 pool_count = 1;
5308
5309         /* Temporarily hijack cb_names for a second... */
5310         tmp_name = cb->cb_names;
5311
5312         /* Go though our list of prospective vdev names */
5313         for (i = 0; i < argc; i++) {
5314                 cb->cb_names = argv + i;
5315
5316                 /* Is this name a vdev in our pools? */
5317                 ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
5318                     ZFS_TYPE_POOL, B_FALSE, is_vdev, cb);
5319                 if (!ret) {
5320                         /* No match */
5321                         break;
5322                 }
5323         }
5324
5325         cb->cb_names = tmp_name;
5326
5327         return (ret);
5328 }
5329
5330 static int
5331 is_pool_cb(zpool_handle_t *zhp, void *data)
5332 {
5333         char *name = data;
5334         if (strcmp(name, zpool_get_name(zhp)) == 0)
5335                 return (1);
5336
5337         return (0);
5338 }
5339
5340 /*
5341  * Do we have a pool named *name?  If so, return 1, otherwise 0.
5342  */
5343 static int
5344 is_pool(char *name)
5345 {
5346         return (for_each_pool(0, NULL, B_TRUE, NULL, ZFS_TYPE_POOL, B_FALSE,
5347             is_pool_cb, name));
5348 }
5349
5350 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
5351 static int
5352 are_all_pools(int argc, char **argv)
5353 {
5354         if ((argc == 0) || !*argv)
5355                 return (0);
5356
5357         while (--argc >= 0)
5358                 if (!is_pool(argv[argc]))
5359                         return (0);
5360
5361         return (1);
5362 }
5363
5364 /*
5365  * Helper function to print out vdev/pool names we can't resolve.  Used for an
5366  * error message.
5367  */
5368 static void
5369 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
5370     vdev_cbdata_t *cb)
5371 {
5372         int i;
5373         char *name;
5374         char *str;
5375         for (i = 0; i < argc; i++) {
5376                 name = argv[i];
5377
5378                 if (is_pool(name))
5379                         str = gettext("pool");
5380                 else if (are_vdevs_in_pool(1, &name, pool_name, cb))
5381                         str = gettext("vdev in this pool");
5382                 else if (are_vdevs_in_pool(1, &name, NULL, cb))
5383                         str = gettext("vdev in another pool");
5384                 else
5385                         str = gettext("unknown");
5386
5387                 fprintf(stderr, "\t%s (%s)\n", name, str);
5388         }
5389 }
5390
5391 /*
5392  * Same as get_interval_count(), but with additional checks to not misinterpret
5393  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
5394  * cb.cb_vdevs.cb_name_flags.
5395  */
5396 static void
5397 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
5398     unsigned long *count, iostat_cbdata_t *cb)
5399 {
5400         char **tmpargv = argv;
5401         int argc_for_interval = 0;
5402
5403         /* Is the last arg an interval value?  Or a guid? */
5404         if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL,
5405             &cb->cb_vdevs)) {
5406                 /*
5407                  * The last arg is not a guid, so it's probably an
5408                  * interval value.
5409                  */
5410                 argc_for_interval++;
5411
5412                 if (*argc >= 2 &&
5413                     !are_vdevs_in_pool(1, &argv[*argc - 2], NULL,
5414                     &cb->cb_vdevs)) {
5415                         /*
5416                          * The 2nd to last arg is not a guid, so it's probably
5417                          * an interval value.
5418                          */
5419                         argc_for_interval++;
5420                 }
5421         }
5422
5423         /* Point to our list of possible intervals */
5424         tmpargv = &argv[*argc - argc_for_interval];
5425
5426         *argc = *argc - argc_for_interval;
5427         get_interval_count(&argc_for_interval, tmpargv,
5428             interval, count);
5429 }
5430
5431 /*
5432  * Floating point sleep().  Allows you to pass in a floating point value for
5433  * seconds.
5434  */
5435 static void
5436 fsleep(float sec)
5437 {
5438         struct timespec req;
5439         req.tv_sec = floor(sec);
5440         req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
5441         nanosleep(&req, NULL);
5442 }
5443
5444 /*
5445  * Terminal height, in rows. Returns -1 if stdout is not connected to a TTY or
5446  * if we were unable to determine its size.
5447  */
5448 static int
5449 terminal_height(void)
5450 {
5451         struct winsize win;
5452
5453         if (isatty(STDOUT_FILENO) == 0)
5454                 return (-1);
5455
5456         if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) != -1 && win.ws_row > 0)
5457                 return (win.ws_row);
5458
5459         return (-1);
5460 }
5461
5462 /*
5463  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
5464  * print the result.
5465  *
5466  * name:        Short name of the script ('iostat').
5467  * path:        Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
5468  */
5469 static void
5470 print_zpool_script_help(char *name, char *path)
5471 {
5472         char *argv[] = {path, (char *)"-h", NULL};
5473         char **lines = NULL;
5474         int lines_cnt = 0;
5475         int rc;
5476
5477         rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
5478             &lines_cnt);
5479         if (rc != 0 || lines == NULL || lines_cnt <= 0) {
5480                 if (lines != NULL)
5481                         libzfs_free_str_array(lines, lines_cnt);
5482                 return;
5483         }
5484
5485         for (int i = 0; i < lines_cnt; i++)
5486                 if (!is_blank_str(lines[i]))
5487                         printf("  %-14s  %s\n", name, lines[i]);
5488
5489         libzfs_free_str_array(lines, lines_cnt);
5490 }
5491
5492 /*
5493  * Go though the zpool status/iostat -c scripts in the user's path, run their
5494  * help option (-h), and print out the results.
5495  */
5496 static void
5497 print_zpool_dir_scripts(char *dirpath)
5498 {
5499         DIR *dir;
5500         struct dirent *ent;
5501         char fullpath[MAXPATHLEN];
5502         struct stat dir_stat;
5503
5504         if ((dir = opendir(dirpath)) != NULL) {
5505                 /* print all the files and directories within directory */
5506                 while ((ent = readdir(dir)) != NULL) {
5507                         if (snprintf(fullpath, sizeof (fullpath), "%s/%s",
5508                             dirpath, ent->d_name) >= sizeof (fullpath)) {
5509                                 (void) fprintf(stderr,
5510                                     gettext("internal error: "
5511                                     "ZPOOL_SCRIPTS_PATH too large.\n"));
5512                                 exit(1);
5513                         }
5514
5515                         /* Print the scripts */
5516                         if (stat(fullpath, &dir_stat) == 0)
5517                                 if (dir_stat.st_mode & S_IXUSR &&
5518                                     S_ISREG(dir_stat.st_mode))
5519                                         print_zpool_script_help(ent->d_name,
5520                                             fullpath);
5521                 }
5522                 closedir(dir);
5523         }
5524 }
5525
5526 /*
5527  * Print out help text for all zpool status/iostat -c scripts.
5528  */
5529 static void
5530 print_zpool_script_list(const char *subcommand)
5531 {
5532         char *dir, *sp, *tmp;
5533
5534         printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
5535
5536         sp = zpool_get_cmd_search_path();
5537         if (sp == NULL)
5538                 return;
5539
5540         for (dir = strtok_r(sp, ":", &tmp);
5541             dir != NULL;
5542             dir = strtok_r(NULL, ":", &tmp))
5543                 print_zpool_dir_scripts(dir);
5544
5545         free(sp);
5546 }
5547
5548 /*
5549  * Set the minimum pool/vdev name column width.  The width must be at least 10,
5550  * but may be as large as the column width - 42 so it still fits on one line.
5551  * NOTE: 42 is the width of the default capacity/operations/bandwidth output
5552  */
5553 static int
5554 get_namewidth_iostat(zpool_handle_t *zhp, void *data)
5555 {
5556         iostat_cbdata_t *cb = data;
5557         int width, available_width;
5558
5559         /*
5560          * get_namewidth() returns the maximum width of any name in that column
5561          * for any pool/vdev/device line that will be output.
5562          */
5563         width = get_namewidth(zhp, cb->cb_namewidth,
5564             cb->cb_vdevs.cb_name_flags | VDEV_NAME_TYPE_ID, cb->cb_verbose);
5565
5566         /*
5567          * The width we are calculating is the width of the header and also the
5568          * padding width for names that are less than maximum width.  The stats
5569          * take up 42 characters, so the width available for names is:
5570          */
5571         available_width = get_columns() - 42;
5572
5573         /*
5574          * If the maximum width fits on a screen, then great!  Make everything
5575          * line up by justifying all lines to the same width.  If that max
5576          * width is larger than what's available, the name plus stats won't fit
5577          * on one line, and justifying to that width would cause every line to
5578          * wrap on the screen.  We only want lines with long names to wrap.
5579          * Limit the padding to what won't wrap.
5580          */
5581         if (width > available_width)
5582                 width = available_width;
5583
5584         /*
5585          * And regardless of whatever the screen width is (get_columns can
5586          * return 0 if the width is not known or less than 42 for a narrow
5587          * terminal) have the width be a minimum of 10.
5588          */
5589         if (width < 10)
5590                 width = 10;
5591
5592         /* Save the calculated width */
5593         cb->cb_namewidth = width;
5594
5595         return (0);
5596 }
5597
5598 /*
5599  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
5600  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
5601  *              [interval [count]]
5602  *
5603  *      -c CMD  For each vdev, run command CMD
5604  *      -g      Display guid for individual vdev name.
5605  *      -L      Follow links when resolving vdev path name.
5606  *      -P      Display full path for vdev name.
5607  *      -v      Display statistics for individual vdevs
5608  *      -h      Display help
5609  *      -p      Display values in parsable (exact) format.
5610  *      -H      Scripted mode.  Don't display headers, and separate properties
5611  *              by a single tab.
5612  *      -l      Display average latency
5613  *      -q      Display queue depths
5614  *      -w      Display latency histograms
5615  *      -r      Display request size histogram
5616  *      -T      Display a timestamp in date(1) or Unix format
5617  *      -n      Only print headers once
5618  *
5619  * This command can be tricky because we want to be able to deal with pool
5620  * creation/destruction as well as vdev configuration changes.  The bulk of this
5621  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
5622  * on pool_list_update() to detect the addition of new pools.  Configuration
5623  * changes are all handled within libzfs.
5624  */
5625 int
5626 zpool_do_iostat(int argc, char **argv)
5627 {
5628         int c;
5629         int ret;
5630         int npools;
5631         float interval = 0;
5632         unsigned long count = 0;
5633         int winheight = 24;
5634         zpool_list_t *list;
5635         boolean_t verbose = B_FALSE;
5636         boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
5637         boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
5638         boolean_t omit_since_boot = B_FALSE;
5639         boolean_t guid = B_FALSE;
5640         boolean_t follow_links = B_FALSE;
5641         boolean_t full_name = B_FALSE;
5642         boolean_t headers_once = B_FALSE;
5643         iostat_cbdata_t cb = { 0 };
5644         char *cmd = NULL;
5645
5646         /* Used for printing error message */
5647         const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
5648             [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
5649
5650         uint64_t unsupported_flags;
5651
5652         /* check options */
5653         while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwnH")) != -1) {
5654                 switch (c) {
5655                 case 'c':
5656                         if (cmd != NULL) {
5657                                 fprintf(stderr,
5658                                     gettext("Can't set -c flag twice\n"));
5659                                 exit(1);
5660                         }
5661
5662                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
5663                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
5664                                 fprintf(stderr, gettext(
5665                                     "Can't run -c, disabled by "
5666                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
5667                                 exit(1);
5668                         }
5669
5670                         if ((getuid() <= 0 || geteuid() <= 0) &&
5671                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
5672                                 fprintf(stderr, gettext(
5673                                     "Can't run -c with root privileges "
5674                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
5675                                 exit(1);
5676                         }
5677                         cmd = optarg;
5678                         verbose = B_TRUE;
5679                         break;
5680                 case 'g':
5681                         guid = B_TRUE;
5682                         break;
5683                 case 'L':
5684                         follow_links = B_TRUE;
5685                         break;
5686                 case 'P':
5687                         full_name = B_TRUE;
5688                         break;
5689                 case 'T':
5690                         get_timestamp_arg(*optarg);
5691                         break;
5692                 case 'v':
5693                         verbose = B_TRUE;
5694                         break;
5695                 case 'p':
5696                         parsable = B_TRUE;
5697                         break;
5698                 case 'l':
5699                         latency = B_TRUE;
5700                         break;
5701                 case 'q':
5702                         queues = B_TRUE;
5703                         break;
5704                 case 'H':
5705                         scripted = B_TRUE;
5706                         break;
5707                 case 'w':
5708                         l_histo = B_TRUE;
5709                         break;
5710                 case 'r':
5711                         rq_histo = B_TRUE;
5712                         break;
5713                 case 'y':
5714                         omit_since_boot = B_TRUE;
5715                         break;
5716                 case 'n':
5717                         headers_once = B_TRUE;
5718                         break;
5719                 case 'h':
5720                         usage(B_FALSE);
5721                         break;
5722                 case '?':
5723                         if (optopt == 'c') {
5724                                 print_zpool_script_list("iostat");
5725                                 exit(0);
5726                         } else {
5727                                 fprintf(stderr,
5728                                     gettext("invalid option '%c'\n"), optopt);
5729                         }
5730                         usage(B_FALSE);
5731                 }
5732         }
5733
5734         argc -= optind;
5735         argv += optind;
5736
5737         cb.cb_literal = parsable;
5738         cb.cb_scripted = scripted;
5739
5740         if (guid)
5741                 cb.cb_vdevs.cb_name_flags |= VDEV_NAME_GUID;
5742         if (follow_links)
5743                 cb.cb_vdevs.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5744         if (full_name)
5745                 cb.cb_vdevs.cb_name_flags |= VDEV_NAME_PATH;
5746         cb.cb_iteration = 0;
5747         cb.cb_namewidth = 0;
5748         cb.cb_verbose = verbose;
5749
5750         /* Get our interval and count values (if any) */
5751         if (guid) {
5752                 get_interval_count_filter_guids(&argc, argv, &interval,
5753                     &count, &cb);
5754         } else {
5755                 get_interval_count(&argc, argv, &interval, &count);
5756         }
5757
5758         if (argc == 0) {
5759                 /* No args, so just print the defaults. */
5760         } else if (are_all_pools(argc, argv)) {
5761                 /* All the args are pool names */
5762         } else if (are_vdevs_in_pool(argc, argv, NULL, &cb.cb_vdevs)) {
5763                 /* All the args are vdevs */
5764                 cb.cb_vdevs.cb_names = argv;
5765                 cb.cb_vdevs.cb_names_count = argc;
5766                 argc = 0; /* No pools to process */
5767         } else if (are_all_pools(1, argv)) {
5768                 /* The first arg is a pool name */
5769                 if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
5770                     &cb.cb_vdevs)) {
5771                         /* ...and the rest are vdev names */
5772                         cb.cb_vdevs.cb_names = argv + 1;
5773                         cb.cb_vdevs.cb_names_count = argc - 1;
5774                         argc = 1; /* One pool to process */
5775                 } else {
5776                         fprintf(stderr, gettext("Expected either a list of "));
5777                         fprintf(stderr, gettext("pools, or list of vdevs in"));
5778                         fprintf(stderr, " \"%s\", ", argv[0]);
5779                         fprintf(stderr, gettext("but got:\n"));
5780                         error_list_unresolved_vdevs(argc - 1, argv + 1,
5781                             argv[0], &cb.cb_vdevs);
5782                         fprintf(stderr, "\n");
5783                         usage(B_FALSE);
5784                         return (1);
5785                 }
5786         } else {
5787                 /*
5788                  * The args don't make sense. The first arg isn't a pool name,
5789                  * nor are all the args vdevs.
5790                  */
5791                 fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
5792                 fprintf(stderr, "\n");
5793                 return (1);
5794         }
5795
5796         if (cb.cb_vdevs.cb_names_count != 0) {
5797                 /*
5798                  * If user specified vdevs, it implies verbose.
5799                  */
5800                 cb.cb_verbose = B_TRUE;
5801         }
5802
5803         /*
5804          * Construct the list of all interesting pools.
5805          */
5806         ret = 0;
5807         if ((list = pool_list_get(argc, argv, NULL, ZFS_TYPE_POOL, parsable,
5808             &ret)) == NULL)
5809                 return (1);
5810
5811         if (pool_list_count(list) == 0 && argc != 0) {
5812                 pool_list_free(list);
5813                 return (1);
5814         }
5815
5816         if (pool_list_count(list) == 0 && interval == 0) {
5817                 pool_list_free(list);
5818                 (void) fprintf(stderr, gettext("no pools available\n"));
5819                 return (1);
5820         }
5821
5822         if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
5823                 pool_list_free(list);
5824                 (void) fprintf(stderr,
5825                     gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
5826                 usage(B_FALSE);
5827                 return (1);
5828         }
5829
5830         if (l_histo && rq_histo) {
5831                 pool_list_free(list);
5832                 (void) fprintf(stderr,
5833                     gettext("Only one of [-r|-w] can be passed at a time\n"));
5834                 usage(B_FALSE);
5835                 return (1);
5836         }
5837
5838         /*
5839          * Enter the main iostat loop.
5840          */
5841         cb.cb_list = list;
5842
5843         if (l_histo) {
5844                 /*
5845                  * Histograms tables look out of place when you try to display
5846                  * them with the other stats, so make a rule that you can only
5847                  * print histograms by themselves.
5848                  */
5849                 cb.cb_flags = IOS_L_HISTO_M;
5850         } else if (rq_histo) {
5851                 cb.cb_flags = IOS_RQ_HISTO_M;
5852         } else {
5853                 cb.cb_flags = IOS_DEFAULT_M;
5854                 if (latency)
5855                         cb.cb_flags |= IOS_LATENCY_M;
5856                 if (queues)
5857                         cb.cb_flags |= IOS_QUEUES_M;
5858         }
5859
5860         /*
5861          * See if the module supports all the stats we want to display.
5862          */
5863         unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
5864         if (unsupported_flags) {
5865                 uint64_t f;
5866                 int idx;
5867                 fprintf(stderr,
5868                     gettext("The loaded zfs module doesn't support:"));
5869
5870                 /* for each bit set in unsupported_flags */
5871                 for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
5872                         idx = lowbit64(f) - 1;
5873                         fprintf(stderr, " -%c", flag_to_arg[idx]);
5874                 }
5875
5876                 fprintf(stderr, ".  Try running a newer module.\n");
5877                 pool_list_free(list);
5878
5879                 return (1);
5880         }
5881
5882         for (;;) {
5883                 if ((npools = pool_list_count(list)) == 0)
5884                         (void) fprintf(stderr, gettext("no pools available\n"));
5885                 else {
5886                         /*
5887                          * If this is the first iteration and -y was supplied
5888                          * we skip any printing.
5889                          */
5890                         boolean_t skip = (omit_since_boot &&
5891                             cb.cb_iteration == 0);
5892
5893                         /*
5894                          * Refresh all statistics.  This is done as an
5895                          * explicit step before calculating the maximum name
5896                          * width, so that any * configuration changes are
5897                          * properly accounted for.
5898                          */
5899                         (void) pool_list_iter(list, B_FALSE, refresh_iostat,
5900                             &cb);
5901
5902                         /*
5903                          * Iterate over all pools to determine the maximum width
5904                          * for the pool / device name column across all pools.
5905                          */
5906                         cb.cb_namewidth = 0;
5907                         (void) pool_list_iter(list, B_FALSE,
5908                             get_namewidth_iostat, &cb);
5909
5910                         if (timestamp_fmt != NODATE)
5911                                 print_timestamp(timestamp_fmt);
5912
5913                         if (cmd != NULL && cb.cb_verbose &&
5914                             !(cb.cb_flags & IOS_ANYHISTO_M)) {
5915                                 cb.vcdl = all_pools_for_each_vdev_run(argc,
5916                                     argv, cmd, g_zfs, cb.cb_vdevs.cb_names,
5917                                     cb.cb_vdevs.cb_names_count,
5918                                     cb.cb_vdevs.cb_name_flags);
5919                         } else {
5920                                 cb.vcdl = NULL;
5921                         }
5922
5923
5924                         /*
5925                          * Check terminal size so we can print headers
5926                          * even when terminal window has its height
5927                          * changed.
5928                          */
5929                         winheight = terminal_height();
5930                         /*
5931                          * Are we connected to TTY? If not, headers_once
5932                          * should be true, to avoid breaking scripts.
5933                          */
5934                         if (winheight < 0)
5935                                 headers_once = B_TRUE;
5936
5937                         /*
5938                          * If it's the first time and we're not skipping it,
5939                          * or either skip or verbose mode, print the header.
5940                          *
5941                          * The histogram code explicitly prints its header on
5942                          * every vdev, so skip this for histograms.
5943                          */
5944                         if (((++cb.cb_iteration == 1 && !skip) ||
5945                             (skip != verbose) ||
5946                             (!headers_once &&
5947                             (cb.cb_iteration % winheight) == 0)) &&
5948                             (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
5949                             !cb.cb_scripted)
5950                                 print_iostat_header(&cb);
5951
5952                         if (skip) {
5953                                 (void) fflush(stdout);
5954                                 (void) fsleep(interval);
5955                                 continue;
5956                         }
5957
5958                         pool_list_iter(list, B_FALSE, print_iostat, &cb);
5959
5960                         /*
5961                          * If there's more than one pool, and we're not in
5962                          * verbose mode (which prints a separator for us),
5963                          * then print a separator.
5964                          *
5965                          * In addition, if we're printing specific vdevs then
5966                          * we also want an ending separator.
5967                          */
5968                         if (((npools > 1 && !verbose &&
5969                             !(cb.cb_flags & IOS_ANYHISTO_M)) ||
5970                             (!(cb.cb_flags & IOS_ANYHISTO_M) &&
5971                             cb.cb_vdevs.cb_names_count)) &&
5972                             !cb.cb_scripted) {
5973                                 print_iostat_separator(&cb);
5974                                 if (cb.vcdl != NULL)
5975                                         print_cmd_columns(cb.vcdl, 1);
5976                                 printf("\n");
5977                         }
5978
5979                         if (cb.vcdl != NULL)
5980                                 free_vdev_cmd_data_list(cb.vcdl);
5981
5982                 }
5983
5984                 if (interval == 0)
5985                         break;
5986
5987                 if (count != 0 && --count == 0)
5988                         break;
5989
5990                 (void) fflush(stdout);
5991                 (void) fsleep(interval);
5992         }
5993
5994         pool_list_free(list);
5995
5996         return (ret);
5997 }
5998
5999 typedef struct list_cbdata {
6000         boolean_t       cb_verbose;
6001         int             cb_name_flags;
6002         int             cb_namewidth;
6003         boolean_t       cb_scripted;
6004         zprop_list_t    *cb_proplist;
6005         boolean_t       cb_literal;
6006 } list_cbdata_t;
6007
6008
6009 /*
6010  * Given a list of columns to display, output appropriate headers for each one.
6011  */
6012 static void
6013 print_header(list_cbdata_t *cb)
6014 {
6015         zprop_list_t *pl = cb->cb_proplist;
6016         char headerbuf[ZPOOL_MAXPROPLEN];
6017         const char *header;
6018         boolean_t first = B_TRUE;
6019         boolean_t right_justify;
6020         size_t width = 0;
6021
6022         for (; pl != NULL; pl = pl->pl_next) {
6023                 width = pl->pl_width;
6024                 if (first && cb->cb_verbose) {
6025                         /*
6026                          * Reset the width to accommodate the verbose listing
6027                          * of devices.
6028                          */
6029                         width = cb->cb_namewidth;
6030                 }
6031
6032                 if (!first)
6033                         (void) fputs("  ", stdout);
6034                 else
6035                         first = B_FALSE;
6036
6037                 right_justify = B_FALSE;
6038                 if (pl->pl_prop != ZPROP_USERPROP) {
6039                         header = zpool_prop_column_name(pl->pl_prop);
6040                         right_justify = zpool_prop_align_right(pl->pl_prop);
6041                 } else {
6042                         int i;
6043
6044                         for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
6045                                 headerbuf[i] = toupper(pl->pl_user_prop[i]);
6046                         headerbuf[i] = '\0';
6047                         header = headerbuf;
6048                 }
6049
6050                 if (pl->pl_next == NULL && !right_justify)
6051                         (void) fputs(header, stdout);
6052                 else if (right_justify)
6053                         (void) printf("%*s", (int)width, header);
6054                 else
6055                         (void) printf("%-*s", (int)width, header);
6056         }
6057
6058         (void) fputc('\n', stdout);
6059 }
6060
6061 /*
6062  * Given a pool and a list of properties, print out all the properties according
6063  * to the described layout. Used by zpool_do_list().
6064  */
6065 static void
6066 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
6067 {
6068         zprop_list_t *pl = cb->cb_proplist;
6069         boolean_t first = B_TRUE;
6070         char property[ZPOOL_MAXPROPLEN];
6071         const char *propstr;
6072         boolean_t right_justify;
6073         size_t width;
6074
6075         for (; pl != NULL; pl = pl->pl_next) {
6076
6077                 width = pl->pl_width;
6078                 if (first && cb->cb_verbose) {
6079                         /*
6080                          * Reset the width to accommodate the verbose listing
6081                          * of devices.
6082                          */
6083                         width = cb->cb_namewidth;
6084                 }
6085
6086                 if (!first) {
6087                         if (cb->cb_scripted)
6088                                 (void) fputc('\t', stdout);
6089                         else
6090                                 (void) fputs("  ", stdout);
6091                 } else {
6092                         first = B_FALSE;
6093                 }
6094
6095                 right_justify = B_FALSE;
6096                 if (pl->pl_prop != ZPROP_USERPROP) {
6097                         if (zpool_get_prop(zhp, pl->pl_prop, property,
6098                             sizeof (property), NULL, cb->cb_literal) != 0)
6099                                 propstr = "-";
6100                         else
6101                                 propstr = property;
6102
6103                         right_justify = zpool_prop_align_right(pl->pl_prop);
6104                 } else if ((zpool_prop_feature(pl->pl_user_prop) ||
6105                     zpool_prop_unsupported(pl->pl_user_prop)) &&
6106                     zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
6107                     sizeof (property)) == 0) {
6108                         propstr = property;
6109                 } else if (zfs_prop_user(pl->pl_user_prop) &&
6110                     zpool_get_userprop(zhp, pl->pl_user_prop, property,
6111                     sizeof (property), NULL) == 0) {
6112                         propstr = property;
6113                 } else {
6114                         propstr = "-";
6115                 }
6116
6117                 /*
6118                  * If this is being called in scripted mode, or if this is the
6119                  * last column and it is left-justified, don't include a width
6120                  * format specifier.
6121                  */
6122                 if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
6123                         (void) fputs(propstr, stdout);
6124                 else if (right_justify)
6125                         (void) printf("%*s", (int)width, propstr);
6126                 else
6127                         (void) printf("%-*s", (int)width, propstr);
6128         }
6129
6130         (void) fputc('\n', stdout);
6131 }
6132
6133 static void
6134 print_one_column(zpool_prop_t prop, uint64_t value, const char *str,
6135     boolean_t scripted, boolean_t valid, enum zfs_nicenum_format format)
6136 {
6137         char propval[64];
6138         boolean_t fixed;
6139         size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
6140
6141         switch (prop) {
6142         case ZPOOL_PROP_SIZE:
6143         case ZPOOL_PROP_EXPANDSZ:
6144         case ZPOOL_PROP_CHECKPOINT:
6145         case ZPOOL_PROP_DEDUPRATIO:
6146                 if (value == 0)
6147                         (void) strlcpy(propval, "-", sizeof (propval));
6148                 else
6149                         zfs_nicenum_format(value, propval, sizeof (propval),
6150                             format);
6151                 break;
6152         case ZPOOL_PROP_FRAGMENTATION:
6153                 if (value == ZFS_FRAG_INVALID) {
6154                         (void) strlcpy(propval, "-", sizeof (propval));
6155                 } else if (format == ZFS_NICENUM_RAW) {
6156                         (void) snprintf(propval, sizeof (propval), "%llu",
6157                             (unsigned long long)value);
6158                 } else {
6159                         (void) snprintf(propval, sizeof (propval), "%llu%%",
6160                             (unsigned long long)value);
6161                 }
6162                 break;
6163         case ZPOOL_PROP_CAPACITY:
6164                 /* capacity value is in parts-per-10,000 (aka permyriad) */
6165                 if (format == ZFS_NICENUM_RAW)
6166                         (void) snprintf(propval, sizeof (propval), "%llu",
6167                             (unsigned long long)value / 100);
6168                 else
6169                         (void) snprintf(propval, sizeof (propval),
6170                             value < 1000 ? "%1.2f%%" : value < 10000 ?
6171                             "%2.1f%%" : "%3.0f%%", value / 100.0);
6172                 break;
6173         case ZPOOL_PROP_HEALTH:
6174                 width = 8;
6175                 (void) strlcpy(propval, str, sizeof (propval));
6176                 break;
6177         default:
6178                 zfs_nicenum_format(value, propval, sizeof (propval), format);
6179         }
6180
6181         if (!valid)
6182                 (void) strlcpy(propval, "-", sizeof (propval));
6183
6184         if (scripted)
6185                 (void) printf("\t%s", propval);
6186         else
6187                 (void) printf("  %*s", (int)width, propval);
6188 }
6189
6190 /*
6191  * print static default line per vdev
6192  * not compatible with '-o' <proplist> option
6193  */
6194 static void
6195 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
6196     list_cbdata_t *cb, int depth, boolean_t isspare)
6197 {
6198         nvlist_t **child;
6199         vdev_stat_t *vs;
6200         uint_t c, children;
6201         char *vname;
6202         boolean_t scripted = cb->cb_scripted;
6203         uint64_t islog = B_FALSE;
6204         const char *dashes = "%-*s      -      -      -        -         "
6205             "-      -      -      -         -\n";
6206
6207         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
6208             (uint64_t **)&vs, &c) == 0);
6209
6210         if (name != NULL) {
6211                 boolean_t toplevel = (vs->vs_space != 0);
6212                 uint64_t cap;
6213                 enum zfs_nicenum_format format;
6214                 const char *state;
6215
6216                 if (cb->cb_literal)
6217                         format = ZFS_NICENUM_RAW;
6218                 else
6219                         format = ZFS_NICENUM_1024;
6220
6221                 if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
6222                         return;
6223
6224                 if (scripted)
6225                         (void) printf("\t%s", name);
6226                 else if (strlen(name) + depth > cb->cb_namewidth)
6227                         (void) printf("%*s%s", depth, "", name);
6228                 else
6229                         (void) printf("%*s%s%*s", depth, "", name,
6230                             (int)(cb->cb_namewidth - strlen(name) - depth), "");
6231
6232                 /*
6233                  * Print the properties for the individual vdevs. Some
6234                  * properties are only applicable to toplevel vdevs. The
6235                  * 'toplevel' boolean value is passed to the print_one_column()
6236                  * to indicate that the value is valid.
6237                  */
6238                 if (VDEV_STAT_VALID(vs_pspace, c) && vs->vs_pspace)
6239                         print_one_column(ZPOOL_PROP_SIZE, vs->vs_pspace, NULL,
6240                             scripted, B_TRUE, format);
6241                 else
6242                         print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, NULL,
6243                             scripted, toplevel, format);
6244                 print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, NULL,
6245                     scripted, toplevel, format);
6246                 print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
6247                     NULL, scripted, toplevel, format);
6248                 print_one_column(ZPOOL_PROP_CHECKPOINT,
6249                     vs->vs_checkpoint_space, NULL, scripted, toplevel, format);
6250                 print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, NULL,
6251                     scripted, B_TRUE, format);
6252                 print_one_column(ZPOOL_PROP_FRAGMENTATION,
6253                     vs->vs_fragmentation, NULL, scripted,
6254                     (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
6255                     format);
6256                 cap = (vs->vs_space == 0) ? 0 :
6257                     (vs->vs_alloc * 10000 / vs->vs_space);
6258                 print_one_column(ZPOOL_PROP_CAPACITY, cap, NULL,
6259                     scripted, toplevel, format);
6260                 print_one_column(ZPOOL_PROP_DEDUPRATIO, 0, NULL,
6261                     scripted, toplevel, format);
6262                 state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
6263                 if (isspare) {
6264                         if (vs->vs_aux == VDEV_AUX_SPARED)
6265                                 state = "INUSE";
6266                         else if (vs->vs_state == VDEV_STATE_HEALTHY)
6267                                 state = "AVAIL";
6268                 }
6269                 print_one_column(ZPOOL_PROP_HEALTH, 0, state, scripted,
6270                     B_TRUE, format);
6271                 (void) fputc('\n', stdout);
6272         }
6273
6274         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
6275             &child, &children) != 0)
6276                 return;
6277
6278         /* list the normal vdevs first */
6279         for (c = 0; c < children; c++) {
6280                 uint64_t ishole = B_FALSE;
6281
6282                 if (nvlist_lookup_uint64(child[c],
6283                     ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
6284                         continue;
6285
6286                 if (nvlist_lookup_uint64(child[c],
6287                     ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog)
6288                         continue;
6289
6290                 if (nvlist_exists(child[c], ZPOOL_CONFIG_ALLOCATION_BIAS))
6291                         continue;
6292
6293                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
6294                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
6295                 print_list_stats(zhp, vname, child[c], cb, depth + 2, B_FALSE);
6296                 free(vname);
6297         }
6298
6299         /* list the classes: 'logs', 'dedup', and 'special' */
6300         for (uint_t n = 0; n < ARRAY_SIZE(class_name); n++) {
6301                 boolean_t printed = B_FALSE;
6302
6303                 for (c = 0; c < children; c++) {
6304                         const char *bias = NULL;
6305                         const char *type = NULL;
6306
6307                         if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
6308                             &islog) == 0 && islog) {
6309                                 bias = VDEV_ALLOC_CLASS_LOGS;
6310                         } else {
6311                                 (void) nvlist_lookup_string(child[c],
6312                                     ZPOOL_CONFIG_ALLOCATION_BIAS, &bias);
6313                                 (void) nvlist_lookup_string(child[c],
6314                                     ZPOOL_CONFIG_TYPE, &type);
6315                         }
6316                         if (bias == NULL || strcmp(bias, class_name[n]) != 0)
6317                                 continue;
6318                         if (!islog && strcmp(type, VDEV_TYPE_INDIRECT) == 0)
6319                                 continue;
6320
6321                         if (!printed) {
6322                                 /* LINTED E_SEC_PRINTF_VAR_FMT */
6323                                 (void) printf(dashes, cb->cb_namewidth,
6324                                     class_name[n]);
6325                                 printed = B_TRUE;
6326                         }
6327                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
6328                             cb->cb_name_flags | VDEV_NAME_TYPE_ID);
6329                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
6330                             B_FALSE);
6331                         free(vname);
6332                 }
6333         }
6334
6335         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
6336             &child, &children) == 0 && children > 0) {
6337                 /* LINTED E_SEC_PRINTF_VAR_FMT */
6338                 (void) printf(dashes, cb->cb_namewidth, "cache");
6339                 for (c = 0; c < children; c++) {
6340                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
6341                             cb->cb_name_flags);
6342                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
6343                             B_FALSE);
6344                         free(vname);
6345                 }
6346         }
6347
6348         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
6349             &children) == 0 && children > 0) {
6350                 /* LINTED E_SEC_PRINTF_VAR_FMT */
6351                 (void) printf(dashes, cb->cb_namewidth, "spare");
6352                 for (c = 0; c < children; c++) {
6353                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
6354                             cb->cb_name_flags);
6355                         print_list_stats(zhp, vname, child[c], cb, depth + 2,
6356                             B_TRUE);
6357                         free(vname);
6358                 }
6359         }
6360 }
6361
6362 /*
6363  * Generic callback function to list a pool.
6364  */
6365 static int
6366 list_callback(zpool_handle_t *zhp, void *data)
6367 {
6368         list_cbdata_t *cbp = data;
6369
6370         print_pool(zhp, cbp);
6371
6372         if (cbp->cb_verbose) {
6373                 nvlist_t *config, *nvroot;
6374
6375                 config = zpool_get_config(zhp, NULL);
6376                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
6377                     &nvroot) == 0);
6378                 print_list_stats(zhp, NULL, nvroot, cbp, 0, B_FALSE);
6379         }
6380
6381         return (0);
6382 }
6383
6384 /*
6385  * Set the minimum pool/vdev name column width.  The width must be at least 9,
6386  * but may be as large as needed.
6387  */
6388 static int
6389 get_namewidth_list(zpool_handle_t *zhp, void *data)
6390 {
6391         list_cbdata_t *cb = data;
6392         int width;
6393
6394         width = get_namewidth(zhp, cb->cb_namewidth,
6395             cb->cb_name_flags | VDEV_NAME_TYPE_ID, cb->cb_verbose);
6396
6397         if (width < 9)
6398                 width = 9;
6399
6400         cb->cb_namewidth = width;
6401
6402         return (0);
6403 }
6404
6405 /*
6406  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
6407  *
6408  *      -g      Display guid for individual vdev name.
6409  *      -H      Scripted mode.  Don't display headers, and separate properties
6410  *              by a single tab.
6411  *      -L      Follow links when resolving vdev path name.
6412  *      -o      List of properties to display.  Defaults to
6413  *              "name,size,allocated,free,expandsize,fragmentation,capacity,"
6414  *              "dedupratio,health,altroot"
6415  *      -p      Display values in parsable (exact) format.
6416  *      -P      Display full path for vdev name.
6417  *      -T      Display a timestamp in date(1) or Unix format
6418  *
6419  * List all pools in the system, whether or not they're healthy.  Output space
6420  * statistics for each one, as well as health status summary.
6421  */
6422 int
6423 zpool_do_list(int argc, char **argv)
6424 {
6425         int c;
6426         int ret = 0;
6427         list_cbdata_t cb = { 0 };
6428         static char default_props[] =
6429             "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
6430             "capacity,dedupratio,health,altroot";
6431         char *props = default_props;
6432         float interval = 0;
6433         unsigned long count = 0;
6434         zpool_list_t *list;
6435         boolean_t first = B_TRUE;
6436         current_prop_type = ZFS_TYPE_POOL;
6437
6438         /* check options */
6439         while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
6440                 switch (c) {
6441                 case 'g':
6442                         cb.cb_name_flags |= VDEV_NAME_GUID;
6443                         break;
6444                 case 'H':
6445                         cb.cb_scripted = B_TRUE;
6446                         break;
6447                 case 'L':
6448                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6449                         break;
6450                 case 'o':
6451                         props = optarg;
6452                         break;
6453                 case 'P':
6454                         cb.cb_name_flags |= VDEV_NAME_PATH;
6455                         break;
6456                 case 'p':
6457                         cb.cb_literal = B_TRUE;
6458                         break;
6459                 case 'T':
6460                         get_timestamp_arg(*optarg);
6461                         break;
6462                 case 'v':
6463                         cb.cb_verbose = B_TRUE;
6464                         cb.cb_namewidth = 8;    /* 8 until precalc is avail */
6465                         break;
6466                 case ':':
6467                         (void) fprintf(stderr, gettext("missing argument for "
6468                             "'%c' option\n"), optopt);
6469                         usage(B_FALSE);
6470                         break;
6471                 case '?':
6472                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6473                             optopt);
6474                         usage(B_FALSE);
6475                 }
6476         }
6477
6478         argc -= optind;
6479         argv += optind;
6480
6481         get_interval_count(&argc, argv, &interval, &count);
6482
6483         if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
6484                 usage(B_FALSE);
6485
6486         for (;;) {
6487                 if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
6488                     ZFS_TYPE_POOL, cb.cb_literal, &ret)) == NULL)
6489                         return (1);
6490
6491                 if (pool_list_count(list) == 0)
6492                         break;
6493
6494                 cb.cb_namewidth = 0;
6495                 (void) pool_list_iter(list, B_FALSE, get_namewidth_list, &cb);
6496
6497                 if (timestamp_fmt != NODATE)
6498                         print_timestamp(timestamp_fmt);
6499
6500                 if (!cb.cb_scripted && (first || cb.cb_verbose)) {
6501                         print_header(&cb);
6502                         first = B_FALSE;
6503                 }
6504                 ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
6505
6506                 if (interval == 0)
6507                         break;
6508
6509                 if (count != 0 && --count == 0)
6510                         break;
6511
6512                 pool_list_free(list);
6513
6514                 (void) fflush(stdout);
6515                 (void) fsleep(interval);
6516         }
6517
6518         if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
6519                 (void) printf(gettext("no pools available\n"));
6520                 ret = 0;
6521         }
6522
6523         pool_list_free(list);
6524         zprop_free_list(cb.cb_proplist);
6525         return (ret);
6526 }
6527
6528 static int
6529 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
6530 {
6531         boolean_t force = B_FALSE;
6532         boolean_t rebuild = B_FALSE;
6533         boolean_t wait = B_FALSE;
6534         int c;
6535         nvlist_t *nvroot;
6536         char *poolname, *old_disk, *new_disk;
6537         zpool_handle_t *zhp;
6538         nvlist_t *props = NULL;
6539         char *propval;
6540         int ret;
6541
6542         /* check options */
6543         while ((c = getopt(argc, argv, "fo:sw")) != -1) {
6544                 switch (c) {
6545                 case 'f':
6546                         force = B_TRUE;
6547                         break;
6548                 case 'o':
6549                         if ((propval = strchr(optarg, '=')) == NULL) {
6550                                 (void) fprintf(stderr, gettext("missing "
6551                                     "'=' for -o option\n"));
6552                                 usage(B_FALSE);
6553                         }
6554                         *propval = '\0';
6555                         propval++;
6556
6557                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
6558                             (add_prop_list(optarg, propval, &props, B_TRUE)))
6559                                 usage(B_FALSE);
6560                         break;
6561                 case 's':
6562                         rebuild = B_TRUE;
6563                         break;
6564                 case 'w':
6565                         wait = B_TRUE;
6566                         break;
6567                 case '?':
6568                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6569                             optopt);
6570                         usage(B_FALSE);
6571                 }
6572         }
6573
6574         argc -= optind;
6575         argv += optind;
6576
6577         /* get pool name and check number of arguments */
6578         if (argc < 1) {
6579                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6580                 usage(B_FALSE);
6581         }
6582
6583         poolname = argv[0];
6584
6585         if (argc < 2) {
6586                 (void) fprintf(stderr,
6587                     gettext("missing <device> specification\n"));
6588                 usage(B_FALSE);
6589         }
6590
6591         old_disk = argv[1];
6592
6593         if (argc < 3) {
6594                 if (!replacing) {
6595                         (void) fprintf(stderr,
6596                             gettext("missing <new_device> specification\n"));
6597                         usage(B_FALSE);
6598                 }
6599                 new_disk = old_disk;
6600                 argc -= 1;
6601                 argv += 1;
6602         } else {
6603                 new_disk = argv[2];
6604                 argc -= 2;
6605                 argv += 2;
6606         }
6607
6608         if (argc > 1) {
6609                 (void) fprintf(stderr, gettext("too many arguments\n"));
6610                 usage(B_FALSE);
6611         }
6612
6613         if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
6614                 nvlist_free(props);
6615                 return (1);
6616         }
6617
6618         if (zpool_get_config(zhp, NULL) == NULL) {
6619                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
6620                     poolname);
6621                 zpool_close(zhp);
6622                 nvlist_free(props);
6623                 return (1);
6624         }
6625
6626         /* unless manually specified use "ashift" pool property (if set) */
6627         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
6628                 int intval;
6629                 zprop_source_t src;
6630                 char strval[ZPOOL_MAXPROPLEN];
6631
6632                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
6633                 if (src != ZPROP_SRC_DEFAULT) {
6634                         (void) sprintf(strval, "%" PRId32, intval);
6635                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
6636                             &props, B_TRUE) == 0);
6637                 }
6638         }
6639
6640         nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
6641             argc, argv);
6642         if (nvroot == NULL) {
6643                 zpool_close(zhp);
6644                 nvlist_free(props);
6645                 return (1);
6646         }
6647
6648         ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing,
6649             rebuild);
6650
6651         if (ret == 0 && wait) {
6652                 zpool_wait_activity_t activity = ZPOOL_WAIT_RESILVER;
6653                 char raidz_prefix[] = "raidz";
6654                 if (replacing) {
6655                         activity = ZPOOL_WAIT_REPLACE;
6656                 } else if (strncmp(old_disk,
6657                     raidz_prefix, strlen(raidz_prefix)) == 0) {
6658                         activity = ZPOOL_WAIT_RAIDZ_EXPAND;
6659                 }
6660                 ret = zpool_wait(zhp, activity);
6661         }
6662
6663         nvlist_free(props);
6664         nvlist_free(nvroot);
6665         zpool_close(zhp);
6666
6667         return (ret);
6668 }
6669
6670 /*
6671  * zpool replace [-fsw] [-o property=value] <pool> <device> <new_device>
6672  *
6673  *      -f      Force attach, even if <new_device> appears to be in use.
6674  *      -s      Use sequential instead of healing reconstruction for resilver.
6675  *      -o      Set property=value.
6676  *      -w      Wait for replacing to complete before returning
6677  *
6678  * Replace <device> with <new_device>.
6679  */
6680 int
6681 zpool_do_replace(int argc, char **argv)
6682 {
6683         return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
6684 }
6685
6686 /*
6687  * zpool attach [-fsw] [-o property=value] <pool> <device>|<vdev> <new_device>
6688  *
6689  *      -f      Force attach, even if <new_device> appears to be in use.
6690  *      -s      Use sequential instead of healing reconstruction for resilver.
6691  *      -o      Set property=value.
6692  *      -w      Wait for resilvering (mirror) or expansion (raidz) to complete
6693  *              before returning.
6694  *
6695  * Attach <new_device> to a <device> or <vdev>, where the vdev can be of type
6696  * mirror or raidz. If <device> is not part of a mirror, then <device> will
6697  * be transformed into a mirror of <device> and <new_device>. When a mirror
6698  * is involved, <new_device> will begin life with a DTL of [0, now], and will
6699  * immediately begin to resilver itself. For the raidz case, a expansion will
6700  * commence and reflow the raidz data across all the disks including the
6701  * <new_device>.
6702  */
6703 int
6704 zpool_do_attach(int argc, char **argv)
6705 {
6706         return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
6707 }
6708
6709 /*
6710  * zpool detach [-f] <pool> <device>
6711  *
6712  *      -f      Force detach of <device>, even if DTLs argue against it
6713  *              (not supported yet)
6714  *
6715  * Detach a device from a mirror.  The operation will be refused if <device>
6716  * is the last device in the mirror, or if the DTLs indicate that this device
6717  * has the only valid copy of some data.
6718  */
6719 int
6720 zpool_do_detach(int argc, char **argv)
6721 {
6722         int c;
6723         char *poolname, *path;
6724         zpool_handle_t *zhp;
6725         int ret;
6726
6727         /* check options */
6728         while ((c = getopt(argc, argv, "")) != -1) {
6729                 switch (c) {
6730                 case '?':
6731                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6732                             optopt);
6733                         usage(B_FALSE);
6734                 }
6735         }
6736
6737         argc -= optind;
6738         argv += optind;
6739
6740         /* get pool name and check number of arguments */
6741         if (argc < 1) {
6742                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6743                 usage(B_FALSE);
6744         }
6745
6746         if (argc < 2) {
6747                 (void) fprintf(stderr,
6748                     gettext("missing <device> specification\n"));
6749                 usage(B_FALSE);
6750         }
6751
6752         poolname = argv[0];
6753         path = argv[1];
6754
6755         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6756                 return (1);
6757
6758         ret = zpool_vdev_detach(zhp, path);
6759
6760         zpool_close(zhp);
6761
6762         return (ret);
6763 }
6764
6765 /*
6766  * zpool split [-gLnP] [-o prop=val] ...
6767  *              [-o mntopt] ...
6768  *              [-R altroot] <pool> <newpool> [<device> ...]
6769  *
6770  *      -g      Display guid for individual vdev name.
6771  *      -L      Follow links when resolving vdev path name.
6772  *      -n      Do not split the pool, but display the resulting layout if
6773  *              it were to be split.
6774  *      -o      Set property=value, or set mount options.
6775  *      -P      Display full path for vdev name.
6776  *      -R      Mount the split-off pool under an alternate root.
6777  *      -l      Load encryption keys while importing.
6778  *
6779  * Splits the named pool and gives it the new pool name.  Devices to be split
6780  * off may be listed, provided that no more than one device is specified
6781  * per top-level vdev mirror.  The newly split pool is left in an exported
6782  * state unless -R is specified.
6783  *
6784  * Restrictions: the top-level of the pool pool must only be made up of
6785  * mirrors; all devices in the pool must be healthy; no device may be
6786  * undergoing a resilvering operation.
6787  */
6788 int
6789 zpool_do_split(int argc, char **argv)
6790 {
6791         char *srcpool, *newpool, *propval;
6792         char *mntopts = NULL;
6793         splitflags_t flags;
6794         int c, ret = 0;
6795         int ms_status = 0;
6796         boolean_t loadkeys = B_FALSE;
6797         zpool_handle_t *zhp;
6798         nvlist_t *config, *props = NULL;
6799
6800         flags.dryrun = B_FALSE;
6801         flags.import = B_FALSE;
6802         flags.name_flags = 0;
6803
6804         /* check options */
6805         while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
6806                 switch (c) {
6807                 case 'g':
6808                         flags.name_flags |= VDEV_NAME_GUID;
6809                         break;
6810                 case 'L':
6811                         flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
6812                         break;
6813                 case 'R':
6814                         flags.import = B_TRUE;
6815                         if (add_prop_list(
6816                             zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
6817                             &props, B_TRUE) != 0) {
6818                                 nvlist_free(props);
6819                                 usage(B_FALSE);
6820                         }
6821                         break;
6822                 case 'l':
6823                         loadkeys = B_TRUE;
6824                         break;
6825                 case 'n':
6826                         flags.dryrun = B_TRUE;
6827                         break;
6828                 case 'o':
6829                         if ((propval = strchr(optarg, '=')) != NULL) {
6830                                 *propval = '\0';
6831                                 propval++;
6832                                 if (add_prop_list(optarg, propval,
6833                                     &props, B_TRUE) != 0) {
6834                                         nvlist_free(props);
6835                                         usage(B_FALSE);
6836                                 }
6837                         } else {
6838                                 mntopts = optarg;
6839                         }
6840                         break;
6841                 case 'P':
6842                         flags.name_flags |= VDEV_NAME_PATH;
6843                         break;
6844                 case ':':
6845                         (void) fprintf(stderr, gettext("missing argument for "
6846                             "'%c' option\n"), optopt);
6847                         usage(B_FALSE);
6848                         break;
6849                 case '?':
6850                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6851                             optopt);
6852                         usage(B_FALSE);
6853                         break;
6854                 }
6855         }
6856
6857         if (!flags.import && mntopts != NULL) {
6858                 (void) fprintf(stderr, gettext("setting mntopts is only "
6859                     "valid when importing the pool\n"));
6860                 usage(B_FALSE);
6861         }
6862
6863         if (!flags.import && loadkeys) {
6864                 (void) fprintf(stderr, gettext("loading keys is only "
6865                     "valid when importing the pool\n"));
6866                 usage(B_FALSE);
6867         }
6868
6869         argc -= optind;
6870         argv += optind;
6871
6872         if (argc < 1) {
6873                 (void) fprintf(stderr, gettext("Missing pool name\n"));
6874                 usage(B_FALSE);
6875         }
6876         if (argc < 2) {
6877                 (void) fprintf(stderr, gettext("Missing new pool name\n"));
6878                 usage(B_FALSE);
6879         }
6880
6881         srcpool = argv[0];
6882         newpool = argv[1];
6883
6884         argc -= 2;
6885         argv += 2;
6886
6887         if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
6888                 nvlist_free(props);
6889                 return (1);
6890         }
6891
6892         config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
6893         if (config == NULL) {
6894                 ret = 1;
6895         } else {
6896                 if (flags.dryrun) {
6897                         (void) printf(gettext("would create '%s' with the "
6898                             "following layout:\n\n"), newpool);
6899                         print_vdev_tree(NULL, newpool, config, 0, "",
6900                             flags.name_flags);
6901                         print_vdev_tree(NULL, "dedup", config, 0,
6902                             VDEV_ALLOC_BIAS_DEDUP, 0);
6903                         print_vdev_tree(NULL, "special", config, 0,
6904                             VDEV_ALLOC_BIAS_SPECIAL, 0);
6905                 }
6906         }
6907
6908         zpool_close(zhp);
6909
6910         if (ret != 0 || flags.dryrun || !flags.import) {
6911                 nvlist_free(config);
6912                 nvlist_free(props);
6913                 return (ret);
6914         }
6915
6916         /*
6917          * The split was successful. Now we need to open the new
6918          * pool and import it.
6919          */
6920         if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
6921                 nvlist_free(config);
6922                 nvlist_free(props);
6923                 return (1);
6924         }
6925
6926         if (loadkeys) {
6927                 ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
6928                 if (ret != 0)
6929                         ret = 1;
6930         }
6931
6932         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL) {
6933                 ms_status = zpool_enable_datasets(zhp, mntopts, 0);
6934                 if (ms_status == EZFS_SHAREFAILED) {
6935                         (void) fprintf(stderr, gettext("Split was successful, "
6936                             "datasets are mounted but sharing of some datasets "
6937                             "has failed\n"));
6938                 } else if (ms_status == EZFS_MOUNTFAILED) {
6939                         (void) fprintf(stderr, gettext("Split was successful"
6940                             ", but some datasets could not be mounted\n"));
6941                         (void) fprintf(stderr, gettext("Try doing '%s' with a "
6942                             "different altroot\n"), "zpool import");
6943                 }
6944         }
6945         zpool_close(zhp);
6946         nvlist_free(config);
6947         nvlist_free(props);
6948
6949         return (ret);
6950 }
6951
6952
6953
6954 /*
6955  * zpool online <pool> <device> ...
6956  */
6957 int
6958 zpool_do_online(int argc, char **argv)
6959 {
6960         int c, i;
6961         char *poolname;
6962         zpool_handle_t *zhp;
6963         int ret = 0;
6964         vdev_state_t newstate;
6965         int flags = 0;
6966
6967         /* check options */
6968         while ((c = getopt(argc, argv, "e")) != -1) {
6969                 switch (c) {
6970                 case 'e':
6971                         flags |= ZFS_ONLINE_EXPAND;
6972                         break;
6973                 case '?':
6974                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6975                             optopt);
6976                         usage(B_FALSE);
6977                 }
6978         }
6979
6980         argc -= optind;
6981         argv += optind;
6982
6983         /* get pool name and check number of arguments */
6984         if (argc < 1) {
6985                 (void) fprintf(stderr, gettext("missing pool name\n"));
6986                 usage(B_FALSE);
6987         }
6988         if (argc < 2) {
6989                 (void) fprintf(stderr, gettext("missing device name\n"));
6990                 usage(B_FALSE);
6991         }
6992
6993         poolname = argv[0];
6994
6995         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
6996                 return (1);
6997
6998         for (i = 1; i < argc; i++) {
6999                 vdev_state_t oldstate;
7000                 boolean_t avail_spare, l2cache;
7001                 nvlist_t *tgt = zpool_find_vdev(zhp, argv[i], &avail_spare,
7002                     &l2cache, NULL);
7003                 if (tgt == NULL) {
7004                         ret = 1;
7005                         continue;
7006                 }
7007                 uint_t vsc;
7008                 oldstate = ((vdev_stat_t *)fnvlist_lookup_uint64_array(tgt,
7009                     ZPOOL_CONFIG_VDEV_STATS, &vsc))->vs_state;
7010                 if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
7011                         if (newstate != VDEV_STATE_HEALTHY) {
7012                                 (void) printf(gettext("warning: device '%s' "
7013                                     "onlined, but remains in faulted state\n"),
7014                                     argv[i]);
7015                                 if (newstate == VDEV_STATE_FAULTED)
7016                                         (void) printf(gettext("use 'zpool "
7017                                             "clear' to restore a faulted "
7018                                             "device\n"));
7019                                 else
7020                                         (void) printf(gettext("use 'zpool "
7021                                             "replace' to replace devices "
7022                                             "that are no longer present\n"));
7023                                 if ((flags & ZFS_ONLINE_EXPAND)) {
7024                                         (void) printf(gettext("%s: failed "
7025                                             "to expand usable space on "
7026                                             "unhealthy device '%s'\n"),
7027                                             (oldstate >= VDEV_STATE_DEGRADED ?
7028                                             "error" : "warning"), argv[i]);
7029                                         if (oldstate >= VDEV_STATE_DEGRADED) {
7030                                                 ret = 1;
7031                                                 break;
7032                                         }
7033                                 }
7034                         }
7035                 } else {
7036                         ret = 1;
7037                 }
7038         }
7039
7040         zpool_close(zhp);
7041
7042         return (ret);
7043 }
7044
7045 /*
7046  * zpool offline [-ft] <pool> <device> ...
7047  *
7048  *      -f      Force the device into a faulted state.
7049  *
7050  *      -t      Only take the device off-line temporarily.  The offline/faulted
7051  *              state will not be persistent across reboots.
7052  */
7053 int
7054 zpool_do_offline(int argc, char **argv)
7055 {
7056         int c, i;
7057         char *poolname;
7058         zpool_handle_t *zhp;
7059         int ret = 0;
7060         boolean_t istmp = B_FALSE;
7061         boolean_t fault = B_FALSE;
7062
7063         /* check options */
7064         while ((c = getopt(argc, argv, "ft")) != -1) {
7065                 switch (c) {
7066                 case 'f':
7067                         fault = B_TRUE;
7068                         break;
7069                 case 't':
7070                         istmp = B_TRUE;
7071                         break;
7072                 case '?':
7073                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7074                             optopt);
7075                         usage(B_FALSE);
7076                 }
7077         }
7078
7079         argc -= optind;
7080         argv += optind;
7081
7082         /* get pool name and check number of arguments */
7083         if (argc < 1) {
7084                 (void) fprintf(stderr, gettext("missing pool name\n"));
7085                 usage(B_FALSE);
7086         }
7087         if (argc < 2) {
7088                 (void) fprintf(stderr, gettext("missing device name\n"));
7089                 usage(B_FALSE);
7090         }
7091
7092         poolname = argv[0];
7093
7094         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
7095                 return (1);
7096
7097         for (i = 1; i < argc; i++) {
7098                 if (fault) {
7099                         uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
7100                         vdev_aux_t aux;
7101                         if (istmp == B_FALSE) {
7102                                 /* Force the fault to persist across imports */
7103                                 aux = VDEV_AUX_EXTERNAL_PERSIST;
7104                         } else {
7105                                 aux = VDEV_AUX_EXTERNAL;
7106                         }
7107
7108                         if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
7109                                 ret = 1;
7110                 } else {
7111                         if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
7112                                 ret = 1;
7113                 }
7114         }
7115
7116         zpool_close(zhp);
7117
7118         return (ret);
7119 }
7120
7121 /*
7122  * zpool clear <pool> [device]
7123  *
7124  * Clear all errors associated with a pool or a particular device.
7125  */
7126 int
7127 zpool_do_clear(int argc, char **argv)
7128 {
7129         int c;
7130         int ret = 0;
7131         boolean_t dryrun = B_FALSE;
7132         boolean_t do_rewind = B_FALSE;
7133         boolean_t xtreme_rewind = B_FALSE;
7134         uint32_t rewind_policy = ZPOOL_NO_REWIND;
7135         nvlist_t *policy = NULL;
7136         zpool_handle_t *zhp;
7137         char *pool, *device;
7138
7139         /* check options */
7140         while ((c = getopt(argc, argv, "FnX")) != -1) {
7141                 switch (c) {
7142                 case 'F':
7143                         do_rewind = B_TRUE;
7144                         break;
7145                 case 'n':
7146                         dryrun = B_TRUE;
7147                         break;
7148                 case 'X':
7149                         xtreme_rewind = B_TRUE;
7150                         break;
7151                 case '?':
7152                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7153                             optopt);
7154                         usage(B_FALSE);
7155                 }
7156         }
7157
7158         argc -= optind;
7159         argv += optind;
7160
7161         if (argc < 1) {
7162                 (void) fprintf(stderr, gettext("missing pool name\n"));
7163                 usage(B_FALSE);
7164         }
7165
7166         if (argc > 2) {
7167                 (void) fprintf(stderr, gettext("too many arguments\n"));
7168                 usage(B_FALSE);
7169         }
7170
7171         if ((dryrun || xtreme_rewind) && !do_rewind) {
7172                 (void) fprintf(stderr,
7173                     gettext("-n or -X only meaningful with -F\n"));
7174                 usage(B_FALSE);
7175         }
7176         if (dryrun)
7177                 rewind_policy = ZPOOL_TRY_REWIND;
7178         else if (do_rewind)
7179                 rewind_policy = ZPOOL_DO_REWIND;
7180         if (xtreme_rewind)
7181                 rewind_policy |= ZPOOL_EXTREME_REWIND;
7182
7183         /* In future, further rewind policy choices can be passed along here */
7184         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
7185             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
7186             rewind_policy) != 0) {
7187                 return (1);
7188         }
7189
7190         pool = argv[0];
7191         device = argc == 2 ? argv[1] : NULL;
7192
7193         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
7194                 nvlist_free(policy);
7195                 return (1);
7196         }
7197
7198         if (zpool_clear(zhp, device, policy) != 0)
7199                 ret = 1;
7200
7201         zpool_close(zhp);
7202
7203         nvlist_free(policy);
7204
7205         return (ret);
7206 }
7207
7208 /*
7209  * zpool reguid <pool>
7210  */
7211 int
7212 zpool_do_reguid(int argc, char **argv)
7213 {
7214         int c;
7215         char *poolname;
7216         zpool_handle_t *zhp;
7217         int ret = 0;
7218
7219         /* check options */
7220         while ((c = getopt(argc, argv, "")) != -1) {
7221                 switch (c) {
7222                 case '?':
7223                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7224                             optopt);
7225                         usage(B_FALSE);
7226                 }
7227         }
7228
7229         argc -= optind;
7230         argv += optind;
7231
7232         /* get pool name and check number of arguments */
7233         if (argc < 1) {
7234                 (void) fprintf(stderr, gettext("missing pool name\n"));
7235                 usage(B_FALSE);
7236         }
7237
7238         if (argc > 1) {
7239                 (void) fprintf(stderr, gettext("too many arguments\n"));
7240                 usage(B_FALSE);
7241         }
7242
7243         poolname = argv[0];
7244         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
7245                 return (1);
7246
7247         ret = zpool_reguid(zhp);
7248
7249         zpool_close(zhp);
7250         return (ret);
7251 }
7252
7253
7254 /*
7255  * zpool reopen <pool>
7256  *
7257  * Reopen the pool so that the kernel can update the sizes of all vdevs.
7258  */
7259 int
7260 zpool_do_reopen(int argc, char **argv)
7261 {
7262         int c;
7263         int ret = 0;
7264         boolean_t scrub_restart = B_TRUE;
7265
7266         /* check options */
7267         while ((c = getopt(argc, argv, "n")) != -1) {
7268                 switch (c) {
7269                 case 'n':
7270                         scrub_restart = B_FALSE;
7271                         break;
7272                 case '?':
7273                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7274                             optopt);
7275                         usage(B_FALSE);
7276                 }
7277         }
7278
7279         argc -= optind;
7280         argv += optind;
7281
7282         /* if argc == 0 we will execute zpool_reopen_one on all pools */
7283         ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7284             B_FALSE, zpool_reopen_one, &scrub_restart);
7285
7286         return (ret);
7287 }
7288
7289 typedef struct scrub_cbdata {
7290         int     cb_type;
7291         pool_scrub_cmd_t cb_scrub_cmd;
7292 } scrub_cbdata_t;
7293
7294 static boolean_t
7295 zpool_has_checkpoint(zpool_handle_t *zhp)
7296 {
7297         nvlist_t *config, *nvroot;
7298
7299         config = zpool_get_config(zhp, NULL);
7300
7301         if (config != NULL) {
7302                 pool_checkpoint_stat_t *pcs = NULL;
7303                 uint_t c;
7304
7305                 nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
7306                 (void) nvlist_lookup_uint64_array(nvroot,
7307                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
7308
7309                 if (pcs == NULL || pcs->pcs_state == CS_NONE)
7310                         return (B_FALSE);
7311
7312                 assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
7313                     pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
7314                 return (B_TRUE);
7315         }
7316
7317         return (B_FALSE);
7318 }
7319
7320 static int
7321 scrub_callback(zpool_handle_t *zhp, void *data)
7322 {
7323         scrub_cbdata_t *cb = data;
7324         int err;
7325
7326         /*
7327          * Ignore faulted pools.
7328          */
7329         if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
7330                 (void) fprintf(stderr, gettext("cannot scan '%s': pool is "
7331                     "currently unavailable\n"), zpool_get_name(zhp));
7332                 return (1);
7333         }
7334
7335         err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
7336
7337         if (err == 0 && zpool_has_checkpoint(zhp) &&
7338             cb->cb_type == POOL_SCAN_SCRUB) {
7339                 (void) printf(gettext("warning: will not scrub state that "
7340                     "belongs to the checkpoint of pool '%s'\n"),
7341                     zpool_get_name(zhp));
7342         }
7343
7344         return (err != 0);
7345 }
7346
7347 static int
7348 wait_callback(zpool_handle_t *zhp, void *data)
7349 {
7350         zpool_wait_activity_t *act = data;
7351         return (zpool_wait(zhp, *act));
7352 }
7353
7354 /*
7355  * zpool scrub [-s | -p] [-w] [-e] <pool> ...
7356  *
7357  *      -e      Only scrub blocks in the error log.
7358  *      -s      Stop.  Stops any in-progress scrub.
7359  *      -p      Pause. Pause in-progress scrub.
7360  *      -w      Wait.  Blocks until scrub has completed.
7361  */
7362 int
7363 zpool_do_scrub(int argc, char **argv)
7364 {
7365         int c;
7366         scrub_cbdata_t cb;
7367         boolean_t wait = B_FALSE;
7368         int error;
7369
7370         cb.cb_type = POOL_SCAN_SCRUB;
7371         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7372
7373         boolean_t is_error_scrub = B_FALSE;
7374         boolean_t is_pause = B_FALSE;
7375         boolean_t is_stop = B_FALSE;
7376
7377         /* check options */
7378         while ((c = getopt(argc, argv, "spwe")) != -1) {
7379                 switch (c) {
7380                 case 'e':
7381                         is_error_scrub = B_TRUE;
7382                         break;
7383                 case 's':
7384                         is_stop = B_TRUE;
7385                         break;
7386                 case 'p':
7387                         is_pause = B_TRUE;
7388                         break;
7389                 case 'w':
7390                         wait = B_TRUE;
7391                         break;
7392                 case '?':
7393                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7394                             optopt);
7395                         usage(B_FALSE);
7396                 }
7397         }
7398
7399         if (is_pause && is_stop) {
7400                 (void) fprintf(stderr, gettext("invalid option "
7401                     "combination :-s and -p are mutually exclusive\n"));
7402                 usage(B_FALSE);
7403         } else {
7404                 if (is_error_scrub)
7405                         cb.cb_type = POOL_SCAN_ERRORSCRUB;
7406
7407                 if (is_pause) {
7408                         cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
7409                 } else if (is_stop) {
7410                         cb.cb_type = POOL_SCAN_NONE;
7411                 } else {
7412                         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7413                 }
7414         }
7415
7416         if (wait && (cb.cb_type == POOL_SCAN_NONE ||
7417             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE)) {
7418                 (void) fprintf(stderr, gettext("invalid option combination: "
7419                     "-w cannot be used with -p or -s\n"));
7420                 usage(B_FALSE);
7421         }
7422
7423         argc -= optind;
7424         argv += optind;
7425
7426         if (argc < 1) {
7427                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7428                 usage(B_FALSE);
7429         }
7430
7431         error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7432             B_FALSE, scrub_callback, &cb);
7433
7434         if (wait && !error) {
7435                 zpool_wait_activity_t act = ZPOOL_WAIT_SCRUB;
7436                 error = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7437                     B_FALSE, wait_callback, &act);
7438         }
7439
7440         return (error);
7441 }
7442
7443 /*
7444  * zpool resilver <pool> ...
7445  *
7446  *      Restarts any in-progress resilver
7447  */
7448 int
7449 zpool_do_resilver(int argc, char **argv)
7450 {
7451         int c;
7452         scrub_cbdata_t cb;
7453
7454         cb.cb_type = POOL_SCAN_RESILVER;
7455         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
7456
7457         /* check options */
7458         while ((c = getopt(argc, argv, "")) != -1) {
7459                 switch (c) {
7460                 case '?':
7461                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7462                             optopt);
7463                         usage(B_FALSE);
7464                 }
7465         }
7466
7467         argc -= optind;
7468         argv += optind;
7469
7470         if (argc < 1) {
7471                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7472                 usage(B_FALSE);
7473         }
7474
7475         return (for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
7476             B_FALSE, scrub_callback, &cb));
7477 }
7478
7479 /*
7480  * zpool trim [-d] [-r <rate>] [-c | -s] <pool> [<device> ...]
7481  *
7482  *      -c              Cancel. Ends any in-progress trim.
7483  *      -d              Secure trim.  Requires kernel and device support.
7484  *      -r <rate>       Sets the TRIM rate in bytes (per second). Supports
7485  *                      adding a multiplier suffix such as 'k' or 'm'.
7486  *      -s              Suspend. TRIM can then be restarted with no flags.
7487  *      -w              Wait. Blocks until trimming has completed.
7488  */
7489 int
7490 zpool_do_trim(int argc, char **argv)
7491 {
7492         struct option long_options[] = {
7493                 {"cancel",      no_argument,            NULL,   'c'},
7494                 {"secure",      no_argument,            NULL,   'd'},
7495                 {"rate",        required_argument,      NULL,   'r'},
7496                 {"suspend",     no_argument,            NULL,   's'},
7497                 {"wait",        no_argument,            NULL,   'w'},
7498                 {0, 0, 0, 0}
7499         };
7500
7501         pool_trim_func_t cmd_type = POOL_TRIM_START;
7502         uint64_t rate = 0;
7503         boolean_t secure = B_FALSE;
7504         boolean_t wait = B_FALSE;
7505
7506         int c;
7507         while ((c = getopt_long(argc, argv, "cdr:sw", long_options, NULL))
7508             != -1) {
7509                 switch (c) {
7510                 case 'c':
7511                         if (cmd_type != POOL_TRIM_START &&
7512                             cmd_type != POOL_TRIM_CANCEL) {
7513                                 (void) fprintf(stderr, gettext("-c cannot be "
7514                                     "combined with other options\n"));
7515                                 usage(B_FALSE);
7516                         }
7517                         cmd_type = POOL_TRIM_CANCEL;
7518                         break;
7519                 case 'd':
7520                         if (cmd_type != POOL_TRIM_START) {
7521                                 (void) fprintf(stderr, gettext("-d cannot be "
7522                                     "combined with the -c or -s options\n"));
7523                                 usage(B_FALSE);
7524                         }
7525                         secure = B_TRUE;
7526                         break;
7527                 case 'r':
7528                         if (cmd_type != POOL_TRIM_START) {
7529                                 (void) fprintf(stderr, gettext("-r cannot be "
7530                                     "combined with the -c or -s options\n"));
7531                                 usage(B_FALSE);
7532                         }
7533                         if (zfs_nicestrtonum(g_zfs, optarg, &rate) == -1) {
7534                                 (void) fprintf(stderr, "%s: %s\n",
7535                                     gettext("invalid value for rate"),
7536                                     libzfs_error_description(g_zfs));
7537                                 usage(B_FALSE);
7538                         }
7539                         break;
7540                 case 's':
7541                         if (cmd_type != POOL_TRIM_START &&
7542                             cmd_type != POOL_TRIM_SUSPEND) {
7543                                 (void) fprintf(stderr, gettext("-s cannot be "
7544                                     "combined with other options\n"));
7545                                 usage(B_FALSE);
7546                         }
7547                         cmd_type = POOL_TRIM_SUSPEND;
7548                         break;
7549                 case 'w':
7550                         wait = B_TRUE;
7551                         break;
7552                 case '?':
7553                         if (optopt != 0) {
7554                                 (void) fprintf(stderr,
7555                                     gettext("invalid option '%c'\n"), optopt);
7556                         } else {
7557                                 (void) fprintf(stderr,
7558                                     gettext("invalid option '%s'\n"),
7559                                     argv[optind - 1]);
7560                         }
7561                         usage(B_FALSE);
7562                 }
7563         }
7564
7565         argc -= optind;
7566         argv += optind;
7567
7568         if (argc < 1) {
7569                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
7570                 usage(B_FALSE);
7571                 return (-1);
7572         }
7573
7574         if (wait && (cmd_type != POOL_TRIM_START)) {
7575                 (void) fprintf(stderr, gettext("-w cannot be used with -c or "
7576                     "-s\n"));
7577                 usage(B_FALSE);
7578         }
7579
7580         char *poolname = argv[0];
7581         zpool_handle_t *zhp = zpool_open(g_zfs, poolname);
7582         if (zhp == NULL)
7583                 return (-1);
7584
7585         trimflags_t trim_flags = {
7586                 .secure = secure,
7587                 .rate = rate,
7588                 .wait = wait,
7589         };
7590
7591         nvlist_t *vdevs = fnvlist_alloc();
7592         if (argc == 1) {
7593                 /* no individual leaf vdevs specified, so add them all */
7594                 nvlist_t *config = zpool_get_config(zhp, NULL);
7595                 nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
7596                     ZPOOL_CONFIG_VDEV_TREE);
7597                 zpool_collect_leaves(zhp, nvroot, vdevs);
7598                 trim_flags.fullpool = B_TRUE;
7599         } else {
7600                 trim_flags.fullpool = B_FALSE;
7601                 for (int i = 1; i < argc; i++) {
7602                         fnvlist_add_boolean(vdevs, argv[i]);
7603                 }
7604         }
7605
7606         int error = zpool_trim(zhp, cmd_type, vdevs, &trim_flags);
7607
7608         fnvlist_free(vdevs);
7609         zpool_close(zhp);
7610
7611         return (error);
7612 }
7613
7614 /*
7615  * Converts a total number of seconds to a human readable string broken
7616  * down in to days/hours/minutes/seconds.
7617  */
7618 static void
7619 secs_to_dhms(uint64_t total, char *buf)
7620 {
7621         uint64_t days = total / 60 / 60 / 24;
7622         uint64_t hours = (total / 60 / 60) % 24;
7623         uint64_t mins = (total / 60) % 60;
7624         uint64_t secs = (total % 60);
7625
7626         if (days > 0) {
7627                 (void) sprintf(buf, "%llu days %02llu:%02llu:%02llu",
7628                     (u_longlong_t)days, (u_longlong_t)hours,
7629                     (u_longlong_t)mins, (u_longlong_t)secs);
7630         } else {
7631                 (void) sprintf(buf, "%02llu:%02llu:%02llu",
7632                     (u_longlong_t)hours, (u_longlong_t)mins,
7633                     (u_longlong_t)secs);
7634         }
7635 }
7636
7637 /*
7638  * Print out detailed error scrub status.
7639  */
7640 static void
7641 print_err_scrub_status(pool_scan_stat_t *ps)
7642 {
7643         time_t start, end, pause;
7644         uint64_t total_secs_left;
7645         uint64_t secs_left, mins_left, hours_left, days_left;
7646         uint64_t examined, to_be_examined;
7647
7648         if (ps == NULL || ps->pss_error_scrub_func != POOL_SCAN_ERRORSCRUB) {
7649                 return;
7650         }
7651
7652         (void) printf(gettext(" scrub: "));
7653
7654         start = ps->pss_error_scrub_start;
7655         end = ps->pss_error_scrub_end;
7656         pause = ps->pss_pass_error_scrub_pause;
7657         examined = ps->pss_error_scrub_examined;
7658         to_be_examined = ps->pss_error_scrub_to_be_examined;
7659
7660         assert(ps->pss_error_scrub_func == POOL_SCAN_ERRORSCRUB);
7661
7662         if (ps->pss_error_scrub_state == DSS_FINISHED) {
7663                 total_secs_left = end - start;
7664                 days_left = total_secs_left / 60 / 60 / 24;
7665                 hours_left = (total_secs_left / 60 / 60) % 24;
7666                 mins_left = (total_secs_left / 60) % 60;
7667                 secs_left = (total_secs_left % 60);
7668
7669                 (void) printf(gettext("scrubbed %llu error blocks in %llu days "
7670                     "%02llu:%02llu:%02llu on %s"), (u_longlong_t)examined,
7671                     (u_longlong_t)days_left, (u_longlong_t)hours_left,
7672                     (u_longlong_t)mins_left, (u_longlong_t)secs_left,
7673                     ctime(&end));
7674
7675                 return;
7676         } else if (ps->pss_error_scrub_state == DSS_CANCELED) {
7677                 (void) printf(gettext("error scrub canceled on %s"),
7678                     ctime(&end));
7679                 return;
7680         }
7681         assert(ps->pss_error_scrub_state == DSS_ERRORSCRUBBING);
7682
7683         /* Error scrub is in progress. */
7684         if (pause == 0) {
7685                 (void) printf(gettext("error scrub in progress since %s"),
7686                     ctime(&start));
7687         } else {
7688                 (void) printf(gettext("error scrub paused since %s"),
7689                     ctime(&pause));
7690                 (void) printf(gettext("\terror scrub started on %s"),
7691                     ctime(&start));
7692         }
7693
7694         double fraction_done = (double)examined / (to_be_examined + examined);
7695         (void) printf(gettext("\t%.2f%% done, issued I/O for %llu error"
7696             " blocks"), 100 * fraction_done, (u_longlong_t)examined);
7697
7698         (void) printf("\n");
7699 }
7700
7701 /*
7702  * Print out detailed scrub status.
7703  */
7704 static void
7705 print_scan_scrub_resilver_status(pool_scan_stat_t *ps)
7706 {
7707         time_t start, end, pause;
7708         uint64_t pass_scanned, scanned, pass_issued, issued, total_s, total_i;
7709         uint64_t elapsed, scan_rate, issue_rate;
7710         double fraction_done;
7711         char processed_buf[7], scanned_buf[7], issued_buf[7], total_s_buf[7];
7712         char total_i_buf[7], srate_buf[7], irate_buf[7], time_buf[32];
7713
7714         printf("  ");
7715         printf_color(ANSI_BOLD, gettext("scan:"));
7716         printf(" ");
7717
7718         /* If there's never been a scan, there's not much to say. */
7719         if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
7720             ps->pss_func >= POOL_SCAN_FUNCS) {
7721                 (void) printf(gettext("none requested\n"));
7722                 return;
7723         }
7724
7725         start = ps->pss_start_time;
7726         end = ps->pss_end_time;
7727         pause = ps->pss_pass_scrub_pause;
7728
7729         zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
7730
7731         int is_resilver = ps->pss_func == POOL_SCAN_RESILVER;
7732         int is_scrub = ps->pss_func == POOL_SCAN_SCRUB;
7733         assert(is_resilver || is_scrub);
7734
7735         /* Scan is finished or canceled. */
7736         if (ps->pss_state == DSS_FINISHED) {
7737                 secs_to_dhms(end - start, time_buf);
7738
7739                 if (is_scrub) {
7740                         (void) printf(gettext("scrub repaired %s "
7741                             "in %s with %llu errors on %s"), processed_buf,
7742                             time_buf, (u_longlong_t)ps->pss_errors,
7743                             ctime(&end));
7744                 } else if (is_resilver) {
7745                         (void) printf(gettext("resilvered %s "
7746                             "in %s with %llu errors on %s"), processed_buf,
7747                             time_buf, (u_longlong_t)ps->pss_errors,
7748                             ctime(&end));
7749                 }
7750                 return;
7751         } else if (ps->pss_state == DSS_CANCELED) {
7752                 if (is_scrub) {
7753                         (void) printf(gettext("scrub canceled on %s"),
7754                             ctime(&end));
7755                 } else if (is_resilver) {
7756                         (void) printf(gettext("resilver canceled on %s"),
7757                             ctime(&end));
7758                 }
7759                 return;
7760         }
7761
7762         assert(ps->pss_state == DSS_SCANNING);
7763
7764         /* Scan is in progress. Resilvers can't be paused. */
7765         if (is_scrub) {
7766                 if (pause == 0) {
7767                         (void) printf(gettext("scrub in progress since %s"),
7768                             ctime(&start));
7769                 } else {
7770                         (void) printf(gettext("scrub paused since %s"),
7771                             ctime(&pause));
7772                         (void) printf(gettext("\tscrub started on %s"),
7773                             ctime(&start));
7774                 }
7775         } else if (is_resilver) {
7776                 (void) printf(gettext("resilver in progress since %s"),
7777                     ctime(&start));
7778         }
7779
7780         scanned = ps->pss_examined;
7781         pass_scanned = ps->pss_pass_exam;
7782         issued = ps->pss_issued;
7783         pass_issued = ps->pss_pass_issued;
7784         total_s = ps->pss_to_examine;
7785         total_i = ps->pss_to_examine - ps->pss_skipped;
7786
7787         /* we are only done with a block once we have issued the IO for it */
7788         fraction_done = (double)issued / total_i;
7789
7790         /* elapsed time for this pass, rounding up to 1 if it's 0 */
7791         elapsed = time(NULL) - ps->pss_pass_start;
7792         elapsed -= ps->pss_pass_scrub_spent_paused;
7793         elapsed = (elapsed != 0) ? elapsed : 1;
7794
7795         scan_rate = pass_scanned / elapsed;
7796         issue_rate = pass_issued / elapsed;
7797
7798         /* format all of the numbers we will be reporting */
7799         zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
7800         zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
7801         zfs_nicebytes(total_s, total_s_buf, sizeof (total_s_buf));
7802         zfs_nicebytes(total_i, total_i_buf, sizeof (total_i_buf));
7803
7804         /* do not print estimated time if we have a paused scrub */
7805         (void) printf(gettext("\t%s / %s scanned"), scanned_buf, total_s_buf);
7806         if (pause == 0 && scan_rate > 0) {
7807                 zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
7808                 (void) printf(gettext(" at %s/s"), srate_buf);
7809         }
7810         (void) printf(gettext(", %s / %s issued"), issued_buf, total_i_buf);
7811         if (pause == 0 && issue_rate > 0) {
7812                 zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
7813                 (void) printf(gettext(" at %s/s"), irate_buf);
7814         }
7815         (void) printf(gettext("\n"));
7816
7817         if (is_resilver) {
7818                 (void) printf(gettext("\t%s resilvered, %.2f%% done"),
7819                     processed_buf, 100 * fraction_done);
7820         } else if (is_scrub) {
7821                 (void) printf(gettext("\t%s repaired, %.2f%% done"),
7822                     processed_buf, 100 * fraction_done);
7823         }
7824
7825         if (pause == 0) {
7826                 /*
7827                  * Only provide an estimate iff:
7828                  * 1) we haven't yet issued all we expected, and
7829                  * 2) the issue rate exceeds 10 MB/s, and
7830                  * 3) it's either:
7831                  *    a) a resilver which has started repairs, or
7832                  *    b) a scrub which has entered the issue phase.
7833                  */
7834                 if (total_i >= issued && issue_rate >= 10 * 1024 * 1024 &&
7835                     ((is_resilver && ps->pss_processed > 0) ||
7836                     (is_scrub && issued > 0))) {
7837                         secs_to_dhms((total_i - issued) / issue_rate, time_buf);
7838                         (void) printf(gettext(", %s to go\n"), time_buf);
7839                 } else {
7840                         (void) printf(gettext(", no estimated "
7841                             "completion time\n"));
7842                 }
7843         } else {
7844                 (void) printf(gettext("\n"));
7845         }
7846 }
7847
7848 static void
7849 print_rebuild_status_impl(vdev_rebuild_stat_t *vrs, uint_t c, char *vdev_name)
7850 {
7851         if (vrs == NULL || vrs->vrs_state == VDEV_REBUILD_NONE)
7852                 return;
7853
7854         printf("  ");
7855         printf_color(ANSI_BOLD, gettext("scan:"));
7856         printf(" ");
7857
7858         uint64_t bytes_scanned = vrs->vrs_bytes_scanned;
7859         uint64_t bytes_issued = vrs->vrs_bytes_issued;
7860         uint64_t bytes_rebuilt = vrs->vrs_bytes_rebuilt;
7861         uint64_t bytes_est_s = vrs->vrs_bytes_est;
7862         uint64_t bytes_est_i = vrs->vrs_bytes_est;
7863         if (c > offsetof(vdev_rebuild_stat_t, vrs_pass_bytes_skipped) / 8)
7864                 bytes_est_i -= vrs->vrs_pass_bytes_skipped;
7865         uint64_t scan_rate = (vrs->vrs_pass_bytes_scanned /
7866             (vrs->vrs_pass_time_ms + 1)) * 1000;
7867         uint64_t issue_rate = (vrs->vrs_pass_bytes_issued /
7868             (vrs->vrs_pass_time_ms + 1)) * 1000;
7869         double scan_pct = MIN((double)bytes_scanned * 100 /
7870             (bytes_est_s + 1), 100);
7871
7872         /* Format all of the numbers we will be reporting */
7873         char bytes_scanned_buf[7], bytes_issued_buf[7];
7874         char bytes_rebuilt_buf[7], bytes_est_s_buf[7], bytes_est_i_buf[7];
7875         char scan_rate_buf[7], issue_rate_buf[7], time_buf[32];
7876         zfs_nicebytes(bytes_scanned, bytes_scanned_buf,
7877             sizeof (bytes_scanned_buf));
7878         zfs_nicebytes(bytes_issued, bytes_issued_buf,
7879             sizeof (bytes_issued_buf));
7880         zfs_nicebytes(bytes_rebuilt, bytes_rebuilt_buf,
7881             sizeof (bytes_rebuilt_buf));
7882         zfs_nicebytes(bytes_est_s, bytes_est_s_buf, sizeof (bytes_est_s_buf));
7883         zfs_nicebytes(bytes_est_i, bytes_est_i_buf, sizeof (bytes_est_i_buf));
7884
7885         time_t start = vrs->vrs_start_time;
7886         time_t end = vrs->vrs_end_time;
7887
7888         /* Rebuild is finished or canceled. */
7889         if (vrs->vrs_state == VDEV_REBUILD_COMPLETE) {
7890                 secs_to_dhms(vrs->vrs_scan_time_ms / 1000, time_buf);
7891                 (void) printf(gettext("resilvered (%s) %s in %s "
7892                     "with %llu errors on %s"), vdev_name, bytes_rebuilt_buf,
7893                     time_buf, (u_longlong_t)vrs->vrs_errors, ctime(&end));
7894                 return;
7895         } else if (vrs->vrs_state == VDEV_REBUILD_CANCELED) {
7896                 (void) printf(gettext("resilver (%s) canceled on %s"),
7897                     vdev_name, ctime(&end));
7898                 return;
7899         } else if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7900                 (void) printf(gettext("resilver (%s) in progress since %s"),
7901                     vdev_name, ctime(&start));
7902         }
7903
7904         assert(vrs->vrs_state == VDEV_REBUILD_ACTIVE);
7905
7906         (void) printf(gettext("\t%s / %s scanned"), bytes_scanned_buf,
7907             bytes_est_s_buf);
7908         if (scan_rate > 0) {
7909                 zfs_nicebytes(scan_rate, scan_rate_buf, sizeof (scan_rate_buf));
7910                 (void) printf(gettext(" at %s/s"), scan_rate_buf);
7911         }
7912         (void) printf(gettext(", %s / %s issued"), bytes_issued_buf,
7913             bytes_est_i_buf);
7914         if (issue_rate > 0) {
7915                 zfs_nicebytes(issue_rate, issue_rate_buf,
7916                     sizeof (issue_rate_buf));
7917                 (void) printf(gettext(" at %s/s"), issue_rate_buf);
7918         }
7919         (void) printf(gettext("\n"));
7920
7921         (void) printf(gettext("\t%s resilvered, %.2f%% done"),
7922             bytes_rebuilt_buf, scan_pct);
7923
7924         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
7925                 if (bytes_est_s >= bytes_scanned &&
7926                     scan_rate >= 10 * 1024 * 1024) {
7927                         secs_to_dhms((bytes_est_s - bytes_scanned) / scan_rate,
7928                             time_buf);
7929                         (void) printf(gettext(", %s to go\n"), time_buf);
7930                 } else {
7931                         (void) printf(gettext(", no estimated "
7932                             "completion time\n"));
7933                 }
7934         } else {
7935                 (void) printf(gettext("\n"));
7936         }
7937 }
7938
7939 /*
7940  * Print rebuild status for top-level vdevs.
7941  */
7942 static void
7943 print_rebuild_status(zpool_handle_t *zhp, nvlist_t *nvroot)
7944 {
7945         nvlist_t **child;
7946         uint_t children;
7947
7948         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
7949             &child, &children) != 0)
7950                 children = 0;
7951
7952         for (uint_t c = 0; c < children; c++) {
7953                 vdev_rebuild_stat_t *vrs;
7954                 uint_t i;
7955
7956                 if (nvlist_lookup_uint64_array(child[c],
7957                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
7958                         char *name = zpool_vdev_name(g_zfs, zhp,
7959                             child[c], VDEV_NAME_TYPE_ID);
7960                         print_rebuild_status_impl(vrs, i, name);
7961                         free(name);
7962                 }
7963         }
7964 }
7965
7966 /*
7967  * As we don't scrub checkpointed blocks, we want to warn the user that we
7968  * skipped scanning some blocks if a checkpoint exists or existed at any
7969  * time during the scan.  If a sequential instead of healing reconstruction
7970  * was performed then the blocks were reconstructed.  However, their checksums
7971  * have not been verified so we still print the warning.
7972  */
7973 static void
7974 print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
7975 {
7976         if (ps == NULL || pcs == NULL)
7977                 return;
7978
7979         if (pcs->pcs_state == CS_NONE ||
7980             pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
7981                 return;
7982
7983         assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
7984
7985         if (ps->pss_state == DSS_NONE)
7986                 return;
7987
7988         if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
7989             ps->pss_end_time < pcs->pcs_start_time)
7990                 return;
7991
7992         if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
7993                 (void) printf(gettext("    scan warning: skipped blocks "
7994                     "that are only referenced by the checkpoint.\n"));
7995         } else {
7996                 assert(ps->pss_state == DSS_SCANNING);
7997                 (void) printf(gettext("    scan warning: skipping blocks "
7998                     "that are only referenced by the checkpoint.\n"));
7999         }
8000 }
8001
8002 /*
8003  * Returns B_TRUE if there is an active rebuild in progress.  Otherwise,
8004  * B_FALSE is returned and 'rebuild_end_time' is set to the end time for
8005  * the last completed (or cancelled) rebuild.
8006  */
8007 static boolean_t
8008 check_rebuilding(nvlist_t *nvroot, uint64_t *rebuild_end_time)
8009 {
8010         nvlist_t **child;
8011         uint_t children;
8012         boolean_t rebuilding = B_FALSE;
8013         uint64_t end_time = 0;
8014
8015         if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8016             &child, &children) != 0)
8017                 children = 0;
8018
8019         for (uint_t c = 0; c < children; c++) {
8020                 vdev_rebuild_stat_t *vrs;
8021                 uint_t i;
8022
8023                 if (nvlist_lookup_uint64_array(child[c],
8024                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i) == 0) {
8025
8026                         if (vrs->vrs_end_time > end_time)
8027                                 end_time = vrs->vrs_end_time;
8028
8029                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
8030                                 rebuilding = B_TRUE;
8031                                 end_time = 0;
8032                                 break;
8033                         }
8034                 }
8035         }
8036
8037         if (rebuild_end_time != NULL)
8038                 *rebuild_end_time = end_time;
8039
8040         return (rebuilding);
8041 }
8042
8043 /*
8044  * Print the scan status.
8045  */
8046 static void
8047 print_scan_status(zpool_handle_t *zhp, nvlist_t *nvroot)
8048 {
8049         uint64_t rebuild_end_time = 0, resilver_end_time = 0;
8050         boolean_t have_resilver = B_FALSE, have_scrub = B_FALSE;
8051         boolean_t have_errorscrub = B_FALSE;
8052         boolean_t active_resilver = B_FALSE;
8053         pool_checkpoint_stat_t *pcs = NULL;
8054         pool_scan_stat_t *ps = NULL;
8055         uint_t c;
8056         time_t scrub_start = 0, errorscrub_start = 0;
8057
8058         if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_SCAN_STATS,
8059             (uint64_t **)&ps, &c) == 0) {
8060                 if (ps->pss_func == POOL_SCAN_RESILVER) {
8061                         resilver_end_time = ps->pss_end_time;
8062                         active_resilver = (ps->pss_state == DSS_SCANNING);
8063                 }
8064
8065                 have_resilver = (ps->pss_func == POOL_SCAN_RESILVER);
8066                 have_scrub = (ps->pss_func == POOL_SCAN_SCRUB);
8067                 scrub_start = ps->pss_start_time;
8068                 if (c > offsetof(pool_scan_stat_t,
8069                     pss_pass_error_scrub_pause) / 8) {
8070                         have_errorscrub = (ps->pss_error_scrub_func ==
8071                             POOL_SCAN_ERRORSCRUB);
8072                         errorscrub_start = ps->pss_error_scrub_start;
8073                 }
8074         }
8075
8076         boolean_t active_rebuild = check_rebuilding(nvroot, &rebuild_end_time);
8077         boolean_t have_rebuild = (active_rebuild || (rebuild_end_time > 0));
8078
8079         /* Always print the scrub status when available. */
8080         if (have_scrub && scrub_start > errorscrub_start)
8081                 print_scan_scrub_resilver_status(ps);
8082         else if (have_errorscrub && errorscrub_start >= scrub_start)
8083                 print_err_scrub_status(ps);
8084
8085         /*
8086          * When there is an active resilver or rebuild print its status.
8087          * Otherwise print the status of the last resilver or rebuild.
8088          */
8089         if (active_resilver || (!active_rebuild && have_resilver &&
8090             resilver_end_time && resilver_end_time > rebuild_end_time)) {
8091                 print_scan_scrub_resilver_status(ps);
8092         } else if (active_rebuild || (!active_resilver && have_rebuild &&
8093             rebuild_end_time && rebuild_end_time > resilver_end_time)) {
8094                 print_rebuild_status(zhp, nvroot);
8095         }
8096
8097         (void) nvlist_lookup_uint64_array(nvroot,
8098             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
8099         print_checkpoint_scan_warning(ps, pcs);
8100 }
8101
8102 /*
8103  * Print out detailed removal status.
8104  */
8105 static void
8106 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
8107 {
8108         char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
8109         time_t start, end;
8110         nvlist_t *config, *nvroot;
8111         nvlist_t **child;
8112         uint_t children;
8113         char *vdev_name;
8114
8115         if (prs == NULL || prs->prs_state == DSS_NONE)
8116                 return;
8117
8118         /*
8119          * Determine name of vdev.
8120          */
8121         config = zpool_get_config(zhp, NULL);
8122         nvroot = fnvlist_lookup_nvlist(config,
8123             ZPOOL_CONFIG_VDEV_TREE);
8124         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8125             &child, &children) == 0);
8126         assert(prs->prs_removing_vdev < children);
8127         vdev_name = zpool_vdev_name(g_zfs, zhp,
8128             child[prs->prs_removing_vdev], B_TRUE);
8129
8130         printf_color(ANSI_BOLD, gettext("remove: "));
8131
8132         start = prs->prs_start_time;
8133         end = prs->prs_end_time;
8134         zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
8135
8136         /*
8137          * Removal is finished or canceled.
8138          */
8139         if (prs->prs_state == DSS_FINISHED) {
8140                 uint64_t minutes_taken = (end - start) / 60;
8141
8142                 (void) printf(gettext("Removal of vdev %llu copied %s "
8143                     "in %lluh%um, completed on %s"),
8144                     (longlong_t)prs->prs_removing_vdev,
8145                     copied_buf,
8146                     (u_longlong_t)(minutes_taken / 60),
8147                     (uint_t)(minutes_taken % 60),
8148                     ctime((time_t *)&end));
8149         } else if (prs->prs_state == DSS_CANCELED) {
8150                 (void) printf(gettext("Removal of %s canceled on %s"),
8151                     vdev_name, ctime(&end));
8152         } else {
8153                 uint64_t copied, total, elapsed, mins_left, hours_left;
8154                 double fraction_done;
8155                 uint_t rate;
8156
8157                 assert(prs->prs_state == DSS_SCANNING);
8158
8159                 /*
8160                  * Removal is in progress.
8161                  */
8162                 (void) printf(gettext(
8163                     "Evacuation of %s in progress since %s"),
8164                     vdev_name, ctime(&start));
8165
8166                 copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
8167                 total = prs->prs_to_copy;
8168                 fraction_done = (double)copied / total;
8169
8170                 /* elapsed time for this pass */
8171                 elapsed = time(NULL) - prs->prs_start_time;
8172                 elapsed = elapsed > 0 ? elapsed : 1;
8173                 rate = copied / elapsed;
8174                 rate = rate > 0 ? rate : 1;
8175                 mins_left = ((total - copied) / rate) / 60;
8176                 hours_left = mins_left / 60;
8177
8178                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
8179                 zfs_nicenum(total, total_buf, sizeof (total_buf));
8180                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
8181
8182                 /*
8183                  * do not print estimated time if hours_left is more than
8184                  * 30 days
8185                  */
8186                 (void) printf(gettext(
8187                     "\t%s copied out of %s at %s/s, %.2f%% done"),
8188                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
8189                 if (hours_left < (30 * 24)) {
8190                         (void) printf(gettext(", %lluh%um to go\n"),
8191                             (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
8192                 } else {
8193                         (void) printf(gettext(
8194                             ", (copy is slow, no estimated time)\n"));
8195                 }
8196         }
8197         free(vdev_name);
8198
8199         if (prs->prs_mapping_memory > 0) {
8200                 char mem_buf[7];
8201                 zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
8202                 (void) printf(gettext(
8203                     "\t%s memory used for removed device mappings\n"),
8204                     mem_buf);
8205         }
8206 }
8207
8208 /*
8209  * Print out detailed raidz expansion status.
8210  */
8211 static void
8212 print_raidz_expand_status(zpool_handle_t *zhp, pool_raidz_expand_stat_t *pres)
8213 {
8214         char copied_buf[7];
8215
8216         if (pres == NULL || pres->pres_state == DSS_NONE)
8217                 return;
8218
8219         /*
8220          * Determine name of vdev.
8221          */
8222         nvlist_t *config = zpool_get_config(zhp, NULL);
8223         nvlist_t *nvroot = fnvlist_lookup_nvlist(config,
8224             ZPOOL_CONFIG_VDEV_TREE);
8225         nvlist_t **child;
8226         uint_t children;
8227         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
8228             &child, &children) == 0);
8229         assert(pres->pres_expanding_vdev < children);
8230
8231         printf_color(ANSI_BOLD, gettext("expand: "));
8232
8233         time_t start = pres->pres_start_time;
8234         time_t end = pres->pres_end_time;
8235         char *vname =
8236             zpool_vdev_name(g_zfs, zhp, child[pres->pres_expanding_vdev], 0);
8237         zfs_nicenum(pres->pres_reflowed, copied_buf, sizeof (copied_buf));
8238
8239         /*
8240          * Expansion is finished or canceled.
8241          */
8242         if (pres->pres_state == DSS_FINISHED) {
8243                 char time_buf[32];
8244                 secs_to_dhms(end - start, time_buf);
8245
8246                 (void) printf(gettext("expanded %s-%u copied %s in %s, "
8247                     "on %s"), vname, (int)pres->pres_expanding_vdev,
8248                     copied_buf, time_buf, ctime((time_t *)&end));
8249         } else {
8250                 char examined_buf[7], total_buf[7], rate_buf[7];
8251                 uint64_t copied, total, elapsed, secs_left;
8252                 double fraction_done;
8253                 uint_t rate;
8254
8255                 assert(pres->pres_state == DSS_SCANNING);
8256
8257                 /*
8258                  * Expansion is in progress.
8259                  */
8260                 (void) printf(gettext(
8261                     "expansion of %s-%u in progress since %s"),
8262                     vname, (int)pres->pres_expanding_vdev, ctime(&start));
8263
8264                 copied = pres->pres_reflowed > 0 ? pres->pres_reflowed : 1;
8265                 total = pres->pres_to_reflow;
8266                 fraction_done = (double)copied / total;
8267
8268                 /* elapsed time for this pass */
8269                 elapsed = time(NULL) - pres->pres_start_time;
8270                 elapsed = elapsed > 0 ? elapsed : 1;
8271                 rate = copied / elapsed;
8272                 rate = rate > 0 ? rate : 1;
8273                 secs_left = (total - copied) / rate;
8274
8275                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
8276                 zfs_nicenum(total, total_buf, sizeof (total_buf));
8277                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
8278
8279                 /*
8280                  * do not print estimated time if hours_left is more than
8281                  * 30 days
8282                  */
8283                 (void) printf(gettext("\t%s / %s copied at %s/s, %.2f%% done"),
8284                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
8285                 if (pres->pres_waiting_for_resilver) {
8286                         (void) printf(gettext(", paused for resilver or "
8287                             "clear\n"));
8288                 } else if (secs_left < (30 * 24 * 3600)) {
8289                         char time_buf[32];
8290                         secs_to_dhms(secs_left, time_buf);
8291                         (void) printf(gettext(", %s to go\n"), time_buf);
8292                 } else {
8293                         (void) printf(gettext(
8294                             ", (copy is slow, no estimated time)\n"));
8295                 }
8296         }
8297         free(vname);
8298 }
8299 static void
8300 print_checkpoint_status(pool_checkpoint_stat_t *pcs)
8301 {
8302         time_t start;
8303         char space_buf[7];
8304
8305         if (pcs == NULL || pcs->pcs_state == CS_NONE)
8306                 return;
8307
8308         (void) printf(gettext("checkpoint: "));
8309
8310         start = pcs->pcs_start_time;
8311         zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
8312
8313         if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
8314                 char *date = ctime(&start);
8315
8316                 /*
8317                  * ctime() adds a newline at the end of the generated
8318                  * string, thus the weird format specifier and the
8319                  * strlen() call used to chop it off from the output.
8320                  */
8321                 (void) printf(gettext("created %.*s, consumes %s\n"),
8322                     (int)(strlen(date) - 1), date, space_buf);
8323                 return;
8324         }
8325
8326         assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
8327
8328         (void) printf(gettext("discarding, %s remaining.\n"),
8329             space_buf);
8330 }
8331
8332 static void
8333 print_error_log(zpool_handle_t *zhp)
8334 {
8335         nvlist_t *nverrlist = NULL;
8336         nvpair_t *elem;
8337         char *pathname;
8338         size_t len = MAXPATHLEN * 2;
8339
8340         if (zpool_get_errlog(zhp, &nverrlist) != 0)
8341                 return;
8342
8343         (void) printf("errors: Permanent errors have been "
8344             "detected in the following files:\n\n");
8345
8346         pathname = safe_malloc(len);
8347         elem = NULL;
8348         while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
8349                 nvlist_t *nv;
8350                 uint64_t dsobj, obj;
8351
8352                 verify(nvpair_value_nvlist(elem, &nv) == 0);
8353                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
8354                     &dsobj) == 0);
8355                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
8356                     &obj) == 0);
8357                 zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
8358                 (void) printf("%7s %s\n", "", pathname);
8359         }
8360         free(pathname);
8361         nvlist_free(nverrlist);
8362 }
8363
8364 static void
8365 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
8366     uint_t nspares)
8367 {
8368         uint_t i;
8369         char *name;
8370
8371         if (nspares == 0)
8372                 return;
8373
8374         (void) printf(gettext("\tspares\n"));
8375
8376         for (i = 0; i < nspares; i++) {
8377                 name = zpool_vdev_name(g_zfs, zhp, spares[i],
8378                     cb->cb_name_flags);
8379                 print_status_config(zhp, cb, name, spares[i], 2, B_TRUE, NULL);
8380                 free(name);
8381         }
8382 }
8383
8384 static void
8385 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
8386     uint_t nl2cache)
8387 {
8388         uint_t i;
8389         char *name;
8390
8391         if (nl2cache == 0)
8392                 return;
8393
8394         (void) printf(gettext("\tcache\n"));
8395
8396         for (i = 0; i < nl2cache; i++) {
8397                 name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
8398                     cb->cb_name_flags);
8399                 print_status_config(zhp, cb, name, l2cache[i], 2,
8400                     B_FALSE, NULL);
8401                 free(name);
8402         }
8403 }
8404
8405 static void
8406 print_dedup_stats(nvlist_t *config)
8407 {
8408         ddt_histogram_t *ddh;
8409         ddt_stat_t *dds;
8410         ddt_object_t *ddo;
8411         uint_t c;
8412         char dspace[6], mspace[6];
8413
8414         /*
8415          * If the pool was faulted then we may not have been able to
8416          * obtain the config. Otherwise, if we have anything in the dedup
8417          * table continue processing the stats.
8418          */
8419         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
8420             (uint64_t **)&ddo, &c) != 0)
8421                 return;
8422
8423         (void) printf("\n");
8424         (void) printf(gettext(" dedup: "));
8425         if (ddo->ddo_count == 0) {
8426                 (void) printf(gettext("no DDT entries\n"));
8427                 return;
8428         }
8429
8430         zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
8431         zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
8432         (void) printf("DDT entries %llu, size %s on disk, %s in core\n",
8433             (u_longlong_t)ddo->ddo_count,
8434             dspace,
8435             mspace);
8436
8437         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
8438             (uint64_t **)&dds, &c) == 0);
8439         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
8440             (uint64_t **)&ddh, &c) == 0);
8441         zpool_dump_ddt(dds, ddh);
8442 }
8443
8444 /*
8445  * Display a summary of pool status.  Displays a summary such as:
8446  *
8447  *        pool: tank
8448  *      status: DEGRADED
8449  *      reason: One or more devices ...
8450  *         see: https://openzfs.github.io/openzfs-docs/msg/ZFS-xxxx-01
8451  *      config:
8452  *              mirror          DEGRADED
8453  *                c1t0d0        OK
8454  *                c2t0d0        UNAVAIL
8455  *
8456  * When given the '-v' option, we print out the complete config.  If the '-e'
8457  * option is specified, then we print out error rate information as well.
8458  */
8459 static int
8460 status_callback(zpool_handle_t *zhp, void *data)
8461 {
8462         status_cbdata_t *cbp = data;
8463         nvlist_t *config, *nvroot;
8464         const char *msgid;
8465         zpool_status_t reason;
8466         zpool_errata_t errata;
8467         const char *health;
8468         uint_t c;
8469         vdev_stat_t *vs;
8470
8471         config = zpool_get_config(zhp, NULL);
8472         reason = zpool_get_status(zhp, &msgid, &errata);
8473
8474         cbp->cb_count++;
8475
8476         /*
8477          * If we were given 'zpool status -x', only report those pools with
8478          * problems.
8479          */
8480         if (cbp->cb_explain &&
8481             (reason == ZPOOL_STATUS_OK ||
8482             reason == ZPOOL_STATUS_VERSION_OLDER ||
8483             reason == ZPOOL_STATUS_FEAT_DISABLED ||
8484             reason == ZPOOL_STATUS_COMPATIBILITY_ERR ||
8485             reason == ZPOOL_STATUS_INCOMPATIBLE_FEAT)) {
8486                 if (!cbp->cb_allpools) {
8487                         (void) printf(gettext("pool '%s' is healthy\n"),
8488                             zpool_get_name(zhp));
8489                         if (cbp->cb_first)
8490                                 cbp->cb_first = B_FALSE;
8491                 }
8492                 return (0);
8493         }
8494
8495         if (cbp->cb_first)
8496                 cbp->cb_first = B_FALSE;
8497         else
8498                 (void) printf("\n");
8499
8500         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
8501         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
8502             (uint64_t **)&vs, &c) == 0);
8503
8504         health = zpool_get_state_str(zhp);
8505
8506         printf("  ");
8507         printf_color(ANSI_BOLD, gettext("pool:"));
8508         printf(" %s\n", zpool_get_name(zhp));
8509         fputc(' ', stdout);
8510         printf_color(ANSI_BOLD, gettext("state: "));
8511
8512         printf_color(health_str_to_color(health), "%s", health);
8513
8514         fputc('\n', stdout);
8515
8516         switch (reason) {
8517         case ZPOOL_STATUS_MISSING_DEV_R:
8518                 printf_color(ANSI_BOLD, gettext("status: "));
8519                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8520                     "not be opened.  Sufficient replicas exist for\n\tthe pool "
8521                     "to continue functioning in a degraded state.\n"));
8522                 printf_color(ANSI_BOLD, gettext("action: "));
8523                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
8524                     "and online it using 'zpool online'.\n"));
8525                 break;
8526
8527         case ZPOOL_STATUS_MISSING_DEV_NR:
8528                 printf_color(ANSI_BOLD, gettext("status: "));
8529                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8530                     "not be opened.  There are insufficient\n\treplicas for the"
8531                     " pool to continue functioning.\n"));
8532                 printf_color(ANSI_BOLD, gettext("action: "));
8533                 printf_color(ANSI_YELLOW, gettext("Attach the missing device "
8534                     "and online it using 'zpool online'.\n"));
8535                 break;
8536
8537         case ZPOOL_STATUS_CORRUPT_LABEL_R:
8538                 printf_color(ANSI_BOLD, gettext("status: "));
8539                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8540                     "not be used because the label is missing or\n\tinvalid.  "
8541                     "Sufficient replicas exist for the pool to continue\n\t"
8542                     "functioning in a degraded state.\n"));
8543                 printf_color(ANSI_BOLD, gettext("action: "));
8544                 printf_color(ANSI_YELLOW, gettext("Replace the device using "
8545                     "'zpool replace'.\n"));
8546                 break;
8547
8548         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
8549                 printf_color(ANSI_BOLD, gettext("status: "));
8550                 printf_color(ANSI_YELLOW, gettext("One or more devices could "
8551                     "not be used because the label is missing \n\tor invalid.  "
8552                     "There are insufficient replicas for the pool to "
8553                     "continue\n\tfunctioning.\n"));
8554                 zpool_explain_recover(zpool_get_handle(zhp),
8555                     zpool_get_name(zhp), reason, config);
8556                 break;
8557
8558         case ZPOOL_STATUS_FAILING_DEV:
8559                 printf_color(ANSI_BOLD, gettext("status: "));
8560                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8561                     "experienced an unrecoverable error.  An\n\tattempt was "
8562                     "made to correct the error.  Applications are "
8563                     "unaffected.\n"));
8564                 printf_color(ANSI_BOLD, gettext("action: "));
8565                         printf_color(ANSI_YELLOW, gettext("Determine if the "
8566                     "device needs to be replaced, and clear the errors\n\tusing"
8567                     " 'zpool clear' or replace the device with 'zpool "
8568                     "replace'.\n"));
8569                 break;
8570
8571         case ZPOOL_STATUS_OFFLINE_DEV:
8572                 printf_color(ANSI_BOLD, gettext("status: "));
8573                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8574                     "been taken offline by the administrator.\n\tSufficient "
8575                     "replicas exist for the pool to continue functioning in "
8576                     "a\n\tdegraded state.\n"));
8577                 printf_color(ANSI_BOLD, gettext("action: "));
8578                 printf_color(ANSI_YELLOW, gettext("Online the device "
8579                     "using 'zpool online' or replace the device with\n\t'zpool "
8580                     "replace'.\n"));
8581                 break;
8582
8583         case ZPOOL_STATUS_REMOVED_DEV:
8584                 printf_color(ANSI_BOLD, gettext("status: "));
8585                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8586                     "been removed by the administrator.\n\tSufficient "
8587                     "replicas exist for the pool to continue functioning in "
8588                     "a\n\tdegraded state.\n"));
8589                 printf_color(ANSI_BOLD, gettext("action: "));
8590                 printf_color(ANSI_YELLOW, gettext("Online the device "
8591                     "using zpool online' or replace the device with\n\t'zpool "
8592                     "replace'.\n"));
8593                 break;
8594
8595         case ZPOOL_STATUS_RESILVERING:
8596         case ZPOOL_STATUS_REBUILDING:
8597                 printf_color(ANSI_BOLD, gettext("status: "));
8598                 printf_color(ANSI_YELLOW, gettext("One or more devices is "
8599                     "currently being resilvered.  The pool will\n\tcontinue "
8600                     "to function, possibly in a degraded state.\n"));
8601                 printf_color(ANSI_BOLD, gettext("action: "));
8602                 printf_color(ANSI_YELLOW, gettext("Wait for the resilver to "
8603                     "complete.\n"));
8604                 break;
8605
8606         case ZPOOL_STATUS_REBUILD_SCRUB:
8607                 printf_color(ANSI_BOLD, gettext("status: "));
8608                 printf_color(ANSI_YELLOW, gettext("One or more devices have "
8609                     "been sequentially resilvered, scrubbing\n\tthe pool "
8610                     "is recommended.\n"));
8611                 printf_color(ANSI_BOLD, gettext("action: "));
8612                 printf_color(ANSI_YELLOW, gettext("Use 'zpool scrub' to "
8613                     "verify all data checksums.\n"));
8614                 break;
8615
8616         case ZPOOL_STATUS_CORRUPT_DATA:
8617                 printf_color(ANSI_BOLD, gettext("status: "));
8618                 printf_color(ANSI_YELLOW, gettext("One or more devices has "
8619                     "experienced an error resulting in data\n\tcorruption.  "
8620                     "Applications may be affected.\n"));
8621                 printf_color(ANSI_BOLD, gettext("action: "));
8622                 printf_color(ANSI_YELLOW, gettext("Restore the file in question"
8623                     " if possible.  Otherwise restore the\n\tentire pool from "
8624                     "backup.\n"));
8625                 break;
8626
8627         case ZPOOL_STATUS_CORRUPT_POOL:
8628                 printf_color(ANSI_BOLD, gettext("status: "));
8629                 printf_color(ANSI_YELLOW, gettext("The pool metadata is "
8630                     "corrupted and the pool cannot be opened.\n"));
8631                 zpool_explain_recover(zpool_get_handle(zhp),
8632                     zpool_get_name(zhp), reason, config);
8633                 break;
8634
8635         case ZPOOL_STATUS_VERSION_OLDER:
8636                 printf_color(ANSI_BOLD, gettext("status: "));
8637                 printf_color(ANSI_YELLOW, gettext("The pool is formatted using "
8638                     "a legacy on-disk format.  The pool can\n\tstill be used, "
8639                     "but some features are unavailable.\n"));
8640                 printf_color(ANSI_BOLD, gettext("action: "));
8641                 printf_color(ANSI_YELLOW, gettext("Upgrade the pool using "
8642                     "'zpool upgrade'.  Once this is done, the\n\tpool will no "
8643                     "longer be accessible on software that does not support\n\t"
8644                     "feature flags.\n"));
8645                 break;
8646
8647         case ZPOOL_STATUS_VERSION_NEWER:
8648                 printf_color(ANSI_BOLD, gettext("status: "));
8649                 printf_color(ANSI_YELLOW, gettext("The pool has been upgraded "
8650                     "to a newer, incompatible on-disk version.\n\tThe pool "
8651                     "cannot be accessed on this system.\n"));
8652                 printf_color(ANSI_BOLD, gettext("action: "));
8653                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8654                     "system running more recent software, or\n\trestore the "
8655                     "pool from backup.\n"));
8656                 break;
8657
8658         case ZPOOL_STATUS_FEAT_DISABLED:
8659                 printf_color(ANSI_BOLD, gettext("status: "));
8660                 printf_color(ANSI_YELLOW, gettext("Some supported and "
8661                     "requested features are not enabled on the pool.\n\t"
8662                     "The pool can still be used, but some features are "
8663                     "unavailable.\n"));
8664                 printf_color(ANSI_BOLD, gettext("action: "));
8665                 printf_color(ANSI_YELLOW, gettext("Enable all features using "
8666                     "'zpool upgrade'. Once this is done,\n\tthe pool may no "
8667                     "longer be accessible by software that does not support\n\t"
8668                     "the features. See zpool-features(7) for details.\n"));
8669                 break;
8670
8671         case ZPOOL_STATUS_COMPATIBILITY_ERR:
8672                 printf_color(ANSI_BOLD, gettext("status: "));
8673                 printf_color(ANSI_YELLOW, gettext("This pool has a "
8674                     "compatibility list specified, but it could not be\n\t"
8675                     "read/parsed at this time. The pool can still be used, "
8676                     "but this\n\tshould be investigated.\n"));
8677                 printf_color(ANSI_BOLD, gettext("action: "));
8678                 printf_color(ANSI_YELLOW, gettext("Check the value of the "
8679                     "'compatibility' property against the\n\t"
8680                     "appropriate file in " ZPOOL_SYSCONF_COMPAT_D " or "
8681                     ZPOOL_DATA_COMPAT_D ".\n"));
8682                 break;
8683
8684         case ZPOOL_STATUS_INCOMPATIBLE_FEAT:
8685                 printf_color(ANSI_BOLD, gettext("status: "));
8686                 printf_color(ANSI_YELLOW, gettext("One or more features "
8687                     "are enabled on the pool despite not being\n\t"
8688                     "requested by the 'compatibility' property.\n"));
8689                 printf_color(ANSI_BOLD, gettext("action: "));
8690                 printf_color(ANSI_YELLOW, gettext("Consider setting "
8691                     "'compatibility' to an appropriate value, or\n\t"
8692                     "adding needed features to the relevant file in\n\t"
8693                     ZPOOL_SYSCONF_COMPAT_D " or " ZPOOL_DATA_COMPAT_D ".\n"));
8694                 break;
8695
8696         case ZPOOL_STATUS_UNSUP_FEAT_READ:
8697                 printf_color(ANSI_BOLD, gettext("status: "));
8698                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8699                     "on this system because it uses the\n\tfollowing feature(s)"
8700                     " not supported on this system:\n"));
8701                 zpool_print_unsup_feat(config);
8702                 (void) printf("\n");
8703                 printf_color(ANSI_BOLD, gettext("action: "));
8704                 printf_color(ANSI_YELLOW, gettext("Access the pool from a "
8705                     "system that supports the required feature(s),\n\tor "
8706                     "restore the pool from backup.\n"));
8707                 break;
8708
8709         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
8710                 printf_color(ANSI_BOLD, gettext("status: "));
8711                 printf_color(ANSI_YELLOW, gettext("The pool can only be "
8712                     "accessed in read-only mode on this system. It\n\tcannot be"
8713                     " accessed in read-write mode because it uses the "
8714                     "following\n\tfeature(s) not supported on this system:\n"));
8715                 zpool_print_unsup_feat(config);
8716                 (void) printf("\n");
8717                 printf_color(ANSI_BOLD, gettext("action: "));
8718                 printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
8719                     "in read-write mode. Import the pool with\n"
8720                     "\t\"-o readonly=on\", access the pool from a system that "
8721                     "supports the\n\trequired feature(s), or restore the "
8722                     "pool from backup.\n"));
8723                 break;
8724
8725         case ZPOOL_STATUS_FAULTED_DEV_R:
8726                 printf_color(ANSI_BOLD, gettext("status: "));
8727                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8728                     "faulted in response to persistent errors.\n\tSufficient "
8729                     "replicas exist for the pool to continue functioning "
8730                     "in a\n\tdegraded state.\n"));
8731                 printf_color(ANSI_BOLD, gettext("action: "));
8732                 printf_color(ANSI_YELLOW, gettext("Replace the faulted device, "
8733                     "or use 'zpool clear' to mark the device\n\trepaired.\n"));
8734                 break;
8735
8736         case ZPOOL_STATUS_FAULTED_DEV_NR:
8737                 printf_color(ANSI_BOLD, gettext("status: "));
8738                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8739                     "faulted in response to persistent errors.  There are "
8740                     "insufficient replicas for the pool to\n\tcontinue "
8741                     "functioning.\n"));
8742                 printf_color(ANSI_BOLD, gettext("action: "));
8743                 printf_color(ANSI_YELLOW, gettext("Destroy and re-create the "
8744                     "pool from a backup source.  Manually marking the device\n"
8745                     "\trepaired using 'zpool clear' may allow some data "
8746                     "to be recovered.\n"));
8747                 break;
8748
8749         case ZPOOL_STATUS_IO_FAILURE_MMP:
8750                 printf_color(ANSI_BOLD, gettext("status: "));
8751                 printf_color(ANSI_YELLOW, gettext("The pool is suspended "
8752                     "because multihost writes failed or were delayed;\n\t"
8753                     "another system could import the pool undetected.\n"));
8754                 printf_color(ANSI_BOLD, gettext("action: "));
8755                 printf_color(ANSI_YELLOW, gettext("Make sure the pool's devices"
8756                     " are connected, then reboot your system and\n\timport the "
8757                     "pool.\n"));
8758                 break;
8759
8760         case ZPOOL_STATUS_IO_FAILURE_WAIT:
8761         case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
8762                 printf_color(ANSI_BOLD, gettext("status: "));
8763                 printf_color(ANSI_YELLOW, gettext("One or more devices are "
8764                     "faulted in response to IO failures.\n"));
8765                 printf_color(ANSI_BOLD, gettext("action: "));
8766                 printf_color(ANSI_YELLOW, gettext("Make sure the affected "
8767                     "devices are connected, then run 'zpool clear'.\n"));
8768                 break;
8769
8770         case ZPOOL_STATUS_BAD_LOG:
8771                 printf_color(ANSI_BOLD, gettext("status: "));
8772                 printf_color(ANSI_YELLOW, gettext("An intent log record "
8773                     "could not be read.\n"
8774                     "\tWaiting for administrator intervention to fix the "
8775                     "faulted pool.\n"));
8776                 printf_color(ANSI_BOLD, gettext("action: "));
8777                 printf_color(ANSI_YELLOW, gettext("Either restore the affected "
8778                     "device(s) and run 'zpool online',\n"
8779                     "\tor ignore the intent log records by running "
8780                     "'zpool clear'.\n"));
8781                 break;
8782
8783         case ZPOOL_STATUS_NON_NATIVE_ASHIFT:
8784                 (void) printf(gettext("status: One or more devices are "
8785                     "configured to use a non-native block size.\n"
8786                     "\tExpect reduced performance.\n"));
8787                 (void) printf(gettext("action: Replace affected devices with "
8788                     "devices that support the\n\tconfigured block size, or "
8789                     "migrate data to a properly configured\n\tpool.\n"));
8790                 break;
8791
8792         case ZPOOL_STATUS_HOSTID_MISMATCH:
8793                 printf_color(ANSI_BOLD, gettext("status: "));
8794                 printf_color(ANSI_YELLOW, gettext("Mismatch between pool hostid"
8795                     " and system hostid on imported pool.\n\tThis pool was "
8796                     "previously imported into a system with a different "
8797                     "hostid,\n\tand then was verbatim imported into this "
8798                     "system.\n"));
8799                 printf_color(ANSI_BOLD, gettext("action: "));
8800                 printf_color(ANSI_YELLOW, gettext("Export this pool on all "
8801                     "systems on which it is imported.\n"
8802                     "\tThen import it to correct the mismatch.\n"));
8803                 break;
8804
8805         case ZPOOL_STATUS_ERRATA:
8806                 printf_color(ANSI_BOLD, gettext("status: "));
8807                 printf_color(ANSI_YELLOW, gettext("Errata #%d detected.\n"),
8808                     errata);
8809
8810                 switch (errata) {
8811                 case ZPOOL_ERRATA_NONE:
8812                         break;
8813
8814                 case ZPOOL_ERRATA_ZOL_2094_SCRUB:
8815                         printf_color(ANSI_BOLD, gettext("action: "));
8816                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
8817                             " run 'zpool scrub'.\n"));
8818                         break;
8819
8820                 case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
8821                         (void) printf(gettext("\tExisting encrypted datasets "
8822                             "contain an on-disk incompatibility\n\twhich "
8823                             "needs to be corrected.\n"));
8824                         printf_color(ANSI_BOLD, gettext("action: "));
8825                         printf_color(ANSI_YELLOW, gettext("To correct the issue"
8826                             " backup existing encrypted datasets to new\n\t"
8827                             "encrypted datasets and destroy the old ones. "
8828                             "'zfs mount -o ro' can\n\tbe used to temporarily "
8829                             "mount existing encrypted datasets readonly.\n"));
8830                         break;
8831
8832                 case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
8833                         (void) printf(gettext("\tExisting encrypted snapshots "
8834                             "and bookmarks contain an on-disk\n\tincompat"
8835                             "ibility. This may cause on-disk corruption if "
8836                             "they are used\n\twith 'zfs recv'.\n"));
8837                         printf_color(ANSI_BOLD, gettext("action: "));
8838                         printf_color(ANSI_YELLOW, gettext("To correct the"
8839                             "issue, enable the bookmark_v2 feature. No "
8840                             "additional\n\taction is needed if there are no "
8841                             "encrypted snapshots or bookmarks.\n\tIf preserving"
8842                             "the encrypted snapshots and bookmarks is required,"
8843                             " use\n\ta non-raw send to backup and restore them."
8844                             " Alternately, they may be\n\tremoved to resolve "
8845                             "the incompatibility.\n"));
8846                         break;
8847
8848                 default:
8849                         /*
8850                          * All errata which allow the pool to be imported
8851                          * must contain an action message.
8852                          */
8853                         assert(0);
8854                 }
8855                 break;
8856
8857         default:
8858                 /*
8859                  * The remaining errors can't actually be generated, yet.
8860                  */
8861                 assert(reason == ZPOOL_STATUS_OK);
8862         }
8863
8864         if (msgid != NULL) {
8865                 printf("   ");
8866                 printf_color(ANSI_BOLD, gettext("see:"));
8867                 printf(gettext(
8868                     " https://openzfs.github.io/openzfs-docs/msg/%s\n"),
8869                     msgid);
8870         }
8871
8872         if (config != NULL) {
8873                 uint64_t nerr;
8874                 nvlist_t **spares, **l2cache;
8875                 uint_t nspares, nl2cache;
8876
8877                 print_scan_status(zhp, nvroot);
8878
8879                 pool_removal_stat_t *prs = NULL;
8880                 (void) nvlist_lookup_uint64_array(nvroot,
8881                     ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
8882                 print_removal_status(zhp, prs);
8883
8884                 pool_checkpoint_stat_t *pcs = NULL;
8885                 (void) nvlist_lookup_uint64_array(nvroot,
8886                     ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
8887                 print_checkpoint_status(pcs);
8888
8889                 pool_raidz_expand_stat_t *pres = NULL;
8890                 (void) nvlist_lookup_uint64_array(nvroot,
8891                     ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t **)&pres, &c);
8892                 print_raidz_expand_status(zhp, pres);
8893
8894                 cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
8895                     cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
8896                 if (cbp->cb_namewidth < 10)
8897                         cbp->cb_namewidth = 10;
8898
8899                 color_start(ANSI_BOLD);
8900                 (void) printf(gettext("config:\n\n"));
8901                 (void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
8902                     cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
8903                     "CKSUM");
8904                 color_end();
8905
8906                 if (cbp->cb_print_slow_ios) {
8907                         printf_color(ANSI_BOLD, " %5s", gettext("SLOW"));
8908                 }
8909
8910                 if (cbp->vcdl != NULL)
8911                         print_cmd_columns(cbp->vcdl, 0);
8912
8913                 printf("\n");
8914
8915                 print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
8916                     B_FALSE, NULL);
8917
8918                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_DEDUP);
8919                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_BIAS_SPECIAL);
8920                 print_class_vdevs(zhp, cbp, nvroot, VDEV_ALLOC_CLASS_LOGS);
8921
8922                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
8923                     &l2cache, &nl2cache) == 0)
8924                         print_l2cache(zhp, cbp, l2cache, nl2cache);
8925
8926                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
8927                     &spares, &nspares) == 0)
8928                         print_spares(zhp, cbp, spares, nspares);
8929
8930                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
8931                     &nerr) == 0) {
8932                         (void) printf("\n");
8933                         if (nerr == 0) {
8934                                 (void) printf(gettext(
8935                                     "errors: No known data errors\n"));
8936                         } else if (!cbp->cb_verbose) {
8937                                 (void) printf(gettext("errors: %llu data "
8938                                     "errors, use '-v' for a list\n"),
8939                                     (u_longlong_t)nerr);
8940                         } else {
8941                                 print_error_log(zhp);
8942                         }
8943                 }
8944
8945                 if (cbp->cb_dedup_stats)
8946                         print_dedup_stats(config);
8947         } else {
8948                 (void) printf(gettext("config: The configuration cannot be "
8949                     "determined.\n"));
8950         }
8951
8952         return (0);
8953 }
8954
8955 /*
8956  * zpool status [-c [script1,script2,...]] [-igLpPstvx] [-T d|u] [pool] ...
8957  *              [interval [count]]
8958  *
8959  *      -c CMD  For each vdev, run command CMD
8960  *      -i      Display vdev initialization status.
8961  *      -g      Display guid for individual vdev name.
8962  *      -L      Follow links when resolving vdev path name.
8963  *      -p      Display values in parsable (exact) format.
8964  *      -P      Display full path for vdev name.
8965  *      -s      Display slow IOs column.
8966  *      -v      Display complete error logs
8967  *      -x      Display only pools with potential problems
8968  *      -D      Display dedup status (undocumented)
8969  *      -t      Display vdev TRIM status.
8970  *      -T      Display a timestamp in date(1) or Unix format
8971  *
8972  * Describes the health status of all pools or some subset.
8973  */
8974 int
8975 zpool_do_status(int argc, char **argv)
8976 {
8977         int c;
8978         int ret;
8979         float interval = 0;
8980         unsigned long count = 0;
8981         status_cbdata_t cb = { 0 };
8982         char *cmd = NULL;
8983
8984         /* check options */
8985         while ((c = getopt(argc, argv, "c:igLpPsvxDtT:")) != -1) {
8986                 switch (c) {
8987                 case 'c':
8988                         if (cmd != NULL) {
8989                                 fprintf(stderr,
8990                                     gettext("Can't set -c flag twice\n"));
8991                                 exit(1);
8992                         }
8993
8994                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
8995                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
8996                                 fprintf(stderr, gettext(
8997                                     "Can't run -c, disabled by "
8998                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
8999                                 exit(1);
9000                         }
9001
9002                         if ((getuid() <= 0 || geteuid() <= 0) &&
9003                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
9004                                 fprintf(stderr, gettext(
9005                                     "Can't run -c with root privileges "
9006                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
9007                                 exit(1);
9008                         }
9009                         cmd = optarg;
9010                         break;
9011                 case 'i':
9012                         cb.cb_print_vdev_init = B_TRUE;
9013                         break;
9014                 case 'g':
9015                         cb.cb_name_flags |= VDEV_NAME_GUID;
9016                         break;
9017                 case 'L':
9018                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
9019                         break;
9020                 case 'p':
9021                         cb.cb_literal = B_TRUE;
9022                         break;
9023                 case 'P':
9024                         cb.cb_name_flags |= VDEV_NAME_PATH;
9025                         break;
9026                 case 's':
9027                         cb.cb_print_slow_ios = B_TRUE;
9028                         break;
9029                 case 'v':
9030                         cb.cb_verbose = B_TRUE;
9031                         break;
9032                 case 'x':
9033                         cb.cb_explain = B_TRUE;
9034                         break;
9035                 case 'D':
9036                         cb.cb_dedup_stats = B_TRUE;
9037                         break;
9038                 case 't':
9039                         cb.cb_print_vdev_trim = B_TRUE;
9040                         break;
9041                 case 'T':
9042                         get_timestamp_arg(*optarg);
9043                         break;
9044                 case '?':
9045                         if (optopt == 'c') {
9046                                 print_zpool_script_list("status");
9047                                 exit(0);
9048                         } else {
9049                                 fprintf(stderr,
9050                                     gettext("invalid option '%c'\n"), optopt);
9051                         }
9052                         usage(B_FALSE);
9053                 }
9054         }
9055
9056         argc -= optind;
9057         argv += optind;
9058
9059         get_interval_count(&argc, argv, &interval, &count);
9060
9061         if (argc == 0)
9062                 cb.cb_allpools = B_TRUE;
9063
9064         cb.cb_first = B_TRUE;
9065         cb.cb_print_status = B_TRUE;
9066
9067         for (;;) {
9068                 if (timestamp_fmt != NODATE)
9069                         print_timestamp(timestamp_fmt);
9070
9071                 if (cmd != NULL)
9072                         cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
9073                             NULL, NULL, 0, 0);
9074
9075                 ret = for_each_pool(argc, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
9076                     cb.cb_literal, status_callback, &cb);
9077
9078                 if (cb.vcdl != NULL)
9079                         free_vdev_cmd_data_list(cb.vcdl);
9080
9081                 if (argc == 0 && cb.cb_count == 0)
9082                         (void) fprintf(stderr, gettext("no pools available\n"));
9083                 else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
9084                         (void) printf(gettext("all pools are healthy\n"));
9085
9086                 if (ret != 0)
9087                         return (ret);
9088
9089                 if (interval == 0)
9090                         break;
9091
9092                 if (count != 0 && --count == 0)
9093                         break;
9094
9095                 (void) fflush(stdout);
9096                 (void) fsleep(interval);
9097         }
9098
9099         return (0);
9100 }
9101
9102 typedef struct upgrade_cbdata {
9103         int     cb_first;
9104         int     cb_argc;
9105         uint64_t cb_version;
9106         char    **cb_argv;
9107 } upgrade_cbdata_t;
9108
9109 static int
9110 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
9111 {
9112         int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
9113         int *count = (int *)unsupp_fs;
9114
9115         if (zfs_version > ZPL_VERSION) {
9116                 (void) printf(gettext("%s (v%d) is not supported by this "
9117                     "implementation of ZFS.\n"),
9118                     zfs_get_name(zhp), zfs_version);
9119                 (*count)++;
9120         }
9121
9122         zfs_iter_filesystems_v2(zhp, 0, check_unsupp_fs, unsupp_fs);
9123
9124         zfs_close(zhp);
9125
9126         return (0);
9127 }
9128
9129 static int
9130 upgrade_version(zpool_handle_t *zhp, uint64_t version)
9131 {
9132         int ret;
9133         nvlist_t *config;
9134         uint64_t oldversion;
9135         int unsupp_fs = 0;
9136
9137         config = zpool_get_config(zhp, NULL);
9138         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9139             &oldversion) == 0);
9140
9141         char compat[ZFS_MAXPROPLEN];
9142         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
9143             ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
9144                 compat[0] = '\0';
9145
9146         assert(SPA_VERSION_IS_SUPPORTED(oldversion));
9147         assert(oldversion < version);
9148
9149         ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
9150         if (ret != 0)
9151                 return (ret);
9152
9153         if (unsupp_fs) {
9154                 (void) fprintf(stderr, gettext("Upgrade not performed due "
9155                     "to %d unsupported filesystems (max v%d).\n"),
9156                     unsupp_fs, (int)ZPL_VERSION);
9157                 return (1);
9158         }
9159
9160         if (strcmp(compat, ZPOOL_COMPAT_LEGACY) == 0) {
9161                 (void) fprintf(stderr, gettext("Upgrade not performed because "
9162                     "'compatibility' property set to '"
9163                     ZPOOL_COMPAT_LEGACY "'.\n"));
9164                 return (1);
9165         }
9166
9167         ret = zpool_upgrade(zhp, version);
9168         if (ret != 0)
9169                 return (ret);
9170
9171         if (version >= SPA_VERSION_FEATURES) {
9172                 (void) printf(gettext("Successfully upgraded "
9173                     "'%s' from version %llu to feature flags.\n"),
9174                     zpool_get_name(zhp), (u_longlong_t)oldversion);
9175         } else {
9176                 (void) printf(gettext("Successfully upgraded "
9177                     "'%s' from version %llu to version %llu.\n"),
9178                     zpool_get_name(zhp), (u_longlong_t)oldversion,
9179                     (u_longlong_t)version);
9180         }
9181
9182         return (0);
9183 }
9184
9185 static int
9186 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
9187 {
9188         int i, ret, count;
9189         boolean_t firstff = B_TRUE;
9190         nvlist_t *enabled = zpool_get_features(zhp);
9191
9192         char compat[ZFS_MAXPROPLEN];
9193         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY, compat,
9194             ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
9195                 compat[0] = '\0';
9196
9197         boolean_t requested_features[SPA_FEATURES];
9198         if (zpool_do_load_compat(compat, requested_features) !=
9199             ZPOOL_COMPATIBILITY_OK)
9200                 return (-1);
9201
9202         count = 0;
9203         for (i = 0; i < SPA_FEATURES; i++) {
9204                 const char *fname = spa_feature_table[i].fi_uname;
9205                 const char *fguid = spa_feature_table[i].fi_guid;
9206
9207                 if (!spa_feature_table[i].fi_zfs_mod_supported)
9208                         continue;
9209
9210                 if (!nvlist_exists(enabled, fguid) && requested_features[i]) {
9211                         char *propname;
9212                         verify(-1 != asprintf(&propname, "feature@%s", fname));
9213                         ret = zpool_set_prop(zhp, propname,
9214                             ZFS_FEATURE_ENABLED);
9215                         if (ret != 0) {
9216                                 free(propname);
9217                                 return (ret);
9218                         }
9219                         count++;
9220
9221                         if (firstff) {
9222                                 (void) printf(gettext("Enabled the "
9223                                     "following features on '%s':\n"),
9224                                     zpool_get_name(zhp));
9225                                 firstff = B_FALSE;
9226                         }
9227                         (void) printf(gettext("  %s\n"), fname);
9228                         free(propname);
9229                 }
9230         }
9231
9232         if (countp != NULL)
9233                 *countp = count;
9234         return (0);
9235 }
9236
9237 static int
9238 upgrade_cb(zpool_handle_t *zhp, void *arg)
9239 {
9240         upgrade_cbdata_t *cbp = arg;
9241         nvlist_t *config;
9242         uint64_t version;
9243         boolean_t modified_pool = B_FALSE;
9244         int ret;
9245
9246         config = zpool_get_config(zhp, NULL);
9247         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9248             &version) == 0);
9249
9250         assert(SPA_VERSION_IS_SUPPORTED(version));
9251
9252         if (version < cbp->cb_version) {
9253                 cbp->cb_first = B_FALSE;
9254                 ret = upgrade_version(zhp, cbp->cb_version);
9255                 if (ret != 0)
9256                         return (ret);
9257                 modified_pool = B_TRUE;
9258
9259                 /*
9260                  * If they did "zpool upgrade -a", then we could
9261                  * be doing ioctls to different pools.  We need
9262                  * to log this history once to each pool, and bypass
9263                  * the normal history logging that happens in main().
9264                  */
9265                 (void) zpool_log_history(g_zfs, history_str);
9266                 log_history = B_FALSE;
9267         }
9268
9269         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
9270                 int count;
9271                 ret = upgrade_enable_all(zhp, &count);
9272                 if (ret != 0)
9273                         return (ret);
9274
9275                 if (count > 0) {
9276                         cbp->cb_first = B_FALSE;
9277                         modified_pool = B_TRUE;
9278                 }
9279         }
9280
9281         if (modified_pool) {
9282                 (void) printf("\n");
9283                 (void) after_zpool_upgrade(zhp);
9284         }
9285
9286         return (0);
9287 }
9288
9289 static int
9290 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
9291 {
9292         upgrade_cbdata_t *cbp = arg;
9293         nvlist_t *config;
9294         uint64_t version;
9295
9296         config = zpool_get_config(zhp, NULL);
9297         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9298             &version) == 0);
9299
9300         assert(SPA_VERSION_IS_SUPPORTED(version));
9301
9302         if (version < SPA_VERSION_FEATURES) {
9303                 if (cbp->cb_first) {
9304                         (void) printf(gettext("The following pools are "
9305                             "formatted with legacy version numbers and can\n"
9306                             "be upgraded to use feature flags.  After "
9307                             "being upgraded, these pools\nwill no "
9308                             "longer be accessible by software that does not "
9309                             "support feature\nflags.\n\n"
9310                             "Note that setting a pool's 'compatibility' "
9311                             "feature to '" ZPOOL_COMPAT_LEGACY "' will\n"
9312                             "inhibit upgrades.\n\n"));
9313                         (void) printf(gettext("VER  POOL\n"));
9314                         (void) printf(gettext("---  ------------\n"));
9315                         cbp->cb_first = B_FALSE;
9316                 }
9317
9318                 (void) printf("%2llu   %s\n", (u_longlong_t)version,
9319                     zpool_get_name(zhp));
9320         }
9321
9322         return (0);
9323 }
9324
9325 static int
9326 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
9327 {
9328         upgrade_cbdata_t *cbp = arg;
9329         nvlist_t *config;
9330         uint64_t version;
9331
9332         config = zpool_get_config(zhp, NULL);
9333         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
9334             &version) == 0);
9335
9336         if (version >= SPA_VERSION_FEATURES) {
9337                 int i;
9338                 boolean_t poolfirst = B_TRUE;
9339                 nvlist_t *enabled = zpool_get_features(zhp);
9340
9341                 for (i = 0; i < SPA_FEATURES; i++) {
9342                         const char *fguid = spa_feature_table[i].fi_guid;
9343                         const char *fname = spa_feature_table[i].fi_uname;
9344
9345                         if (!spa_feature_table[i].fi_zfs_mod_supported)
9346                                 continue;
9347
9348                         if (!nvlist_exists(enabled, fguid)) {
9349                                 if (cbp->cb_first) {
9350                                         (void) printf(gettext("\nSome "
9351                                             "supported features are not "
9352                                             "enabled on the following pools. "
9353                                             "Once a\nfeature is enabled the "
9354                                             "pool may become incompatible with "
9355                                             "software\nthat does not support "
9356                                             "the feature. See "
9357                                             "zpool-features(7) for "
9358                                             "details.\n\n"
9359                                             "Note that the pool "
9360                                             "'compatibility' feature can be "
9361                                             "used to inhibit\nfeature "
9362                                             "upgrades.\n\n"));
9363                                         (void) printf(gettext("POOL  "
9364                                             "FEATURE\n"));
9365                                         (void) printf(gettext("------"
9366                                             "---------\n"));
9367                                         cbp->cb_first = B_FALSE;
9368                                 }
9369
9370                                 if (poolfirst) {
9371                                         (void) printf(gettext("%s\n"),
9372                                             zpool_get_name(zhp));
9373                                         poolfirst = B_FALSE;
9374                                 }
9375
9376                                 (void) printf(gettext("      %s\n"), fname);
9377                         }
9378                         /*
9379                          * If they did "zpool upgrade -a", then we could
9380                          * be doing ioctls to different pools.  We need
9381                          * to log this history once to each pool, and bypass
9382                          * the normal history logging that happens in main().
9383                          */
9384                         (void) zpool_log_history(g_zfs, history_str);
9385                         log_history = B_FALSE;
9386                 }
9387         }
9388
9389         return (0);
9390 }
9391
9392 static int
9393 upgrade_one(zpool_handle_t *zhp, void *data)
9394 {
9395         boolean_t modified_pool = B_FALSE;
9396         upgrade_cbdata_t *cbp = data;
9397         uint64_t cur_version;
9398         int ret;
9399
9400         if (strcmp("log", zpool_get_name(zhp)) == 0) {
9401                 (void) fprintf(stderr, gettext("'log' is now a reserved word\n"
9402                     "Pool 'log' must be renamed using export and import"
9403                     " to upgrade.\n"));
9404                 return (1);
9405         }
9406
9407         cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
9408         if (cur_version > cbp->cb_version) {
9409                 (void) printf(gettext("Pool '%s' is already formatted "
9410                     "using more current version '%llu'.\n\n"),
9411                     zpool_get_name(zhp), (u_longlong_t)cur_version);
9412                 return (0);
9413         }
9414
9415         if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
9416                 (void) printf(gettext("Pool '%s' is already formatted "
9417                     "using version %llu.\n\n"), zpool_get_name(zhp),
9418                     (u_longlong_t)cbp->cb_version);
9419                 return (0);
9420         }
9421
9422         if (cur_version != cbp->cb_version) {
9423                 modified_pool = B_TRUE;
9424                 ret = upgrade_version(zhp, cbp->cb_version);
9425                 if (ret != 0)
9426                         return (ret);
9427         }
9428
9429         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
9430                 int count = 0;
9431                 ret = upgrade_enable_all(zhp, &count);
9432                 if (ret != 0)
9433                         return (ret);
9434
9435                 if (count != 0) {
9436                         modified_pool = B_TRUE;
9437                 } else if (cur_version == SPA_VERSION) {
9438                         (void) printf(gettext("Pool '%s' already has all "
9439                             "supported and requested features enabled.\n"),
9440                             zpool_get_name(zhp));
9441                 }
9442         }
9443
9444         if (modified_pool) {
9445                 (void) printf("\n");
9446                 (void) after_zpool_upgrade(zhp);
9447         }
9448
9449         return (0);
9450 }
9451
9452 /*
9453  * zpool upgrade
9454  * zpool upgrade -v
9455  * zpool upgrade [-V version] <-a | pool ...>
9456  *
9457  * With no arguments, display downrev'd ZFS pool available for upgrade.
9458  * Individual pools can be upgraded by specifying the pool, and '-a' will
9459  * upgrade all pools.
9460  */
9461 int
9462 zpool_do_upgrade(int argc, char **argv)
9463 {
9464         int c;
9465         upgrade_cbdata_t cb = { 0 };
9466         int ret = 0;
9467         boolean_t showversions = B_FALSE;
9468         boolean_t upgradeall = B_FALSE;
9469         char *end;
9470
9471
9472         /* check options */
9473         while ((c = getopt(argc, argv, ":avV:")) != -1) {
9474                 switch (c) {
9475                 case 'a':
9476                         upgradeall = B_TRUE;
9477                         break;
9478                 case 'v':
9479                         showversions = B_TRUE;
9480                         break;
9481                 case 'V':
9482                         cb.cb_version = strtoll(optarg, &end, 10);
9483                         if (*end != '\0' ||
9484                             !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
9485                                 (void) fprintf(stderr,
9486                                     gettext("invalid version '%s'\n"), optarg);
9487                                 usage(B_FALSE);
9488                         }
9489                         break;
9490                 case ':':
9491                         (void) fprintf(stderr, gettext("missing argument for "
9492                             "'%c' option\n"), optopt);
9493                         usage(B_FALSE);
9494                         break;
9495                 case '?':
9496                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9497                             optopt);
9498                         usage(B_FALSE);
9499                 }
9500         }
9501
9502         cb.cb_argc = argc;
9503         cb.cb_argv = argv;
9504         argc -= optind;
9505         argv += optind;
9506
9507         if (cb.cb_version == 0) {
9508                 cb.cb_version = SPA_VERSION;
9509         } else if (!upgradeall && argc == 0) {
9510                 (void) fprintf(stderr, gettext("-V option is "
9511                     "incompatible with other arguments\n"));
9512                 usage(B_FALSE);
9513         }
9514
9515         if (showversions) {
9516                 if (upgradeall || argc != 0) {
9517                         (void) fprintf(stderr, gettext("-v option is "
9518                             "incompatible with other arguments\n"));
9519                         usage(B_FALSE);
9520                 }
9521         } else if (upgradeall) {
9522                 if (argc != 0) {
9523                         (void) fprintf(stderr, gettext("-a option should not "
9524                             "be used along with a pool name\n"));
9525                         usage(B_FALSE);
9526                 }
9527         }
9528
9529         (void) printf("%s", gettext("This system supports ZFS pool feature "
9530             "flags.\n\n"));
9531         if (showversions) {
9532                 int i;
9533
9534                 (void) printf(gettext("The following features are "
9535                     "supported:\n\n"));
9536                 (void) printf(gettext("FEAT DESCRIPTION\n"));
9537                 (void) printf("----------------------------------------------"
9538                     "---------------\n");
9539                 for (i = 0; i < SPA_FEATURES; i++) {
9540                         zfeature_info_t *fi = &spa_feature_table[i];
9541                         if (!fi->fi_zfs_mod_supported)
9542                                 continue;
9543                         const char *ro =
9544                             (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
9545                             " (read-only compatible)" : "";
9546
9547                         (void) printf("%-37s%s\n", fi->fi_uname, ro);
9548                         (void) printf("     %s\n", fi->fi_desc);
9549                 }
9550                 (void) printf("\n");
9551
9552                 (void) printf(gettext("The following legacy versions are also "
9553                     "supported:\n\n"));
9554                 (void) printf(gettext("VER  DESCRIPTION\n"));
9555                 (void) printf("---  -----------------------------------------"
9556                     "---------------\n");
9557                 (void) printf(gettext(" 1   Initial ZFS version\n"));
9558                 (void) printf(gettext(" 2   Ditto blocks "
9559                     "(replicated metadata)\n"));
9560                 (void) printf(gettext(" 3   Hot spares and double parity "
9561                     "RAID-Z\n"));
9562                 (void) printf(gettext(" 4   zpool history\n"));
9563                 (void) printf(gettext(" 5   Compression using the gzip "
9564                     "algorithm\n"));
9565                 (void) printf(gettext(" 6   bootfs pool property\n"));
9566                 (void) printf(gettext(" 7   Separate intent log devices\n"));
9567                 (void) printf(gettext(" 8   Delegated administration\n"));
9568                 (void) printf(gettext(" 9   refquota and refreservation "
9569                     "properties\n"));
9570                 (void) printf(gettext(" 10  Cache devices\n"));
9571                 (void) printf(gettext(" 11  Improved scrub performance\n"));
9572                 (void) printf(gettext(" 12  Snapshot properties\n"));
9573                 (void) printf(gettext(" 13  snapused property\n"));
9574                 (void) printf(gettext(" 14  passthrough-x aclinherit\n"));
9575                 (void) printf(gettext(" 15  user/group space accounting\n"));
9576                 (void) printf(gettext(" 16  stmf property support\n"));
9577                 (void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
9578                 (void) printf(gettext(" 18  Snapshot user holds\n"));
9579                 (void) printf(gettext(" 19  Log device removal\n"));
9580                 (void) printf(gettext(" 20  Compression using zle "
9581                     "(zero-length encoding)\n"));
9582                 (void) printf(gettext(" 21  Deduplication\n"));
9583                 (void) printf(gettext(" 22  Received properties\n"));
9584                 (void) printf(gettext(" 23  Slim ZIL\n"));
9585                 (void) printf(gettext(" 24  System attributes\n"));
9586                 (void) printf(gettext(" 25  Improved scrub stats\n"));
9587                 (void) printf(gettext(" 26  Improved snapshot deletion "
9588                     "performance\n"));
9589                 (void) printf(gettext(" 27  Improved snapshot creation "
9590                     "performance\n"));
9591                 (void) printf(gettext(" 28  Multiple vdev replacements\n"));
9592                 (void) printf(gettext("\nFor more information on a particular "
9593                     "version, including supported releases,\n"));
9594                 (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
9595         } else if (argc == 0 && upgradeall) {
9596                 cb.cb_first = B_TRUE;
9597                 ret = zpool_iter(g_zfs, upgrade_cb, &cb);
9598                 if (ret == 0 && cb.cb_first) {
9599                         if (cb.cb_version == SPA_VERSION) {
9600                                 (void) printf(gettext("All pools are already "
9601                                     "formatted using feature flags.\n\n"));
9602                                 (void) printf(gettext("Every feature flags "
9603                                     "pool already has all supported and "
9604                                     "requested features enabled.\n"));
9605                         } else {
9606                                 (void) printf(gettext("All pools are already "
9607                                     "formatted with version %llu or higher.\n"),
9608                                     (u_longlong_t)cb.cb_version);
9609                         }
9610                 }
9611         } else if (argc == 0) {
9612                 cb.cb_first = B_TRUE;
9613                 ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
9614                 assert(ret == 0);
9615
9616                 if (cb.cb_first) {
9617                         (void) printf(gettext("All pools are formatted "
9618                             "using feature flags.\n\n"));
9619                 } else {
9620                         (void) printf(gettext("\nUse 'zpool upgrade -v' "
9621                             "for a list of available legacy versions.\n"));
9622                 }
9623
9624                 cb.cb_first = B_TRUE;
9625                 ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
9626                 assert(ret == 0);
9627
9628                 if (cb.cb_first) {
9629                         (void) printf(gettext("Every feature flags pool has "
9630                             "all supported and requested features enabled.\n"));
9631                 } else {
9632                         (void) printf(gettext("\n"));
9633                 }
9634         } else {
9635                 ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
9636                     B_FALSE, upgrade_one, &cb);
9637         }
9638
9639         return (ret);
9640 }
9641
9642 typedef struct hist_cbdata {
9643         boolean_t first;
9644         boolean_t longfmt;
9645         boolean_t internal;
9646 } hist_cbdata_t;
9647
9648 static void
9649 print_history_records(nvlist_t *nvhis, hist_cbdata_t *cb)
9650 {
9651         nvlist_t **records;
9652         uint_t numrecords;
9653         int i;
9654
9655         verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
9656             &records, &numrecords) == 0);
9657         for (i = 0; i < numrecords; i++) {
9658                 nvlist_t *rec = records[i];
9659                 char tbuf[64] = "";
9660
9661                 if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
9662                         time_t tsec;
9663                         struct tm t;
9664
9665                         tsec = fnvlist_lookup_uint64(records[i],
9666                             ZPOOL_HIST_TIME);
9667                         (void) localtime_r(&tsec, &t);
9668                         (void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
9669                 }
9670
9671                 if (nvlist_exists(rec, ZPOOL_HIST_ELAPSED_NS)) {
9672                         uint64_t elapsed_ns = fnvlist_lookup_int64(records[i],
9673                             ZPOOL_HIST_ELAPSED_NS);
9674                         (void) snprintf(tbuf + strlen(tbuf),
9675                             sizeof (tbuf) - strlen(tbuf),
9676                             " (%lldms)", (long long)elapsed_ns / 1000 / 1000);
9677                 }
9678
9679                 if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
9680                         (void) printf("%s %s", tbuf,
9681                             fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
9682                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
9683                         int ievent =
9684                             fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
9685                         if (!cb->internal)
9686                                 continue;
9687                         if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
9688                                 (void) printf("%s unrecognized record:\n",
9689                                     tbuf);
9690                                 dump_nvlist(rec, 4);
9691                                 continue;
9692                         }
9693                         (void) printf("%s [internal %s txg:%lld] %s", tbuf,
9694                             zfs_history_event_names[ievent],
9695                             (longlong_t)fnvlist_lookup_uint64(
9696                             rec, ZPOOL_HIST_TXG),
9697                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
9698                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
9699                         if (!cb->internal)
9700                                 continue;
9701                         (void) printf("%s [txg:%lld] %s", tbuf,
9702                             (longlong_t)fnvlist_lookup_uint64(
9703                             rec, ZPOOL_HIST_TXG),
9704                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
9705                         if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
9706                                 (void) printf(" %s (%llu)",
9707                                     fnvlist_lookup_string(rec,
9708                                     ZPOOL_HIST_DSNAME),
9709                                     (u_longlong_t)fnvlist_lookup_uint64(rec,
9710                                     ZPOOL_HIST_DSID));
9711                         }
9712                         (void) printf(" %s", fnvlist_lookup_string(rec,
9713                             ZPOOL_HIST_INT_STR));
9714                 } else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
9715                         if (!cb->internal)
9716                                 continue;
9717                         (void) printf("%s ioctl %s\n", tbuf,
9718                             fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
9719                         if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
9720                                 (void) printf("    input:\n");
9721                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9722                                     ZPOOL_HIST_INPUT_NVL), 8);
9723                         }
9724                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
9725                                 (void) printf("    output:\n");
9726                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
9727                                     ZPOOL_HIST_OUTPUT_NVL), 8);
9728                         }
9729                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_SIZE)) {
9730                                 (void) printf("    output nvlist omitted; "
9731                                     "original size: %lldKB\n",
9732                                     (longlong_t)fnvlist_lookup_int64(rec,
9733                                     ZPOOL_HIST_OUTPUT_SIZE) / 1024);
9734                         }
9735                         if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
9736                                 (void) printf("    errno: %lld\n",
9737                                     (longlong_t)fnvlist_lookup_int64(rec,
9738                                     ZPOOL_HIST_ERRNO));
9739                         }
9740                 } else {
9741                         if (!cb->internal)
9742                                 continue;
9743                         (void) printf("%s unrecognized record:\n", tbuf);
9744                         dump_nvlist(rec, 4);
9745                 }
9746
9747                 if (!cb->longfmt) {
9748                         (void) printf("\n");
9749                         continue;
9750                 }
9751                 (void) printf(" [");
9752                 if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
9753                         uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
9754                         struct passwd *pwd = getpwuid(who);
9755                         (void) printf("user %d ", (int)who);
9756                         if (pwd != NULL)
9757                                 (void) printf("(%s) ", pwd->pw_name);
9758                 }
9759                 if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
9760                         (void) printf("on %s",
9761                             fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
9762                 }
9763                 if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
9764                         (void) printf(":%s",
9765                             fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
9766                 }
9767
9768                 (void) printf("]");
9769                 (void) printf("\n");
9770         }
9771 }
9772
9773 /*
9774  * Print out the command history for a specific pool.
9775  */
9776 static int
9777 get_history_one(zpool_handle_t *zhp, void *data)
9778 {
9779         nvlist_t *nvhis;
9780         int ret;
9781         hist_cbdata_t *cb = (hist_cbdata_t *)data;
9782         uint64_t off = 0;
9783         boolean_t eof = B_FALSE;
9784
9785         cb->first = B_FALSE;
9786
9787         (void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
9788
9789         while (!eof) {
9790                 if ((ret = zpool_get_history(zhp, &nvhis, &off, &eof)) != 0)
9791                         return (ret);
9792
9793                 print_history_records(nvhis, cb);
9794                 nvlist_free(nvhis);
9795         }
9796         (void) printf("\n");
9797
9798         return (ret);
9799 }
9800
9801 /*
9802  * zpool history <pool>
9803  *
9804  * Displays the history of commands that modified pools.
9805  */
9806 int
9807 zpool_do_history(int argc, char **argv)
9808 {
9809         hist_cbdata_t cbdata = { 0 };
9810         int ret;
9811         int c;
9812
9813         cbdata.first = B_TRUE;
9814         /* check options */
9815         while ((c = getopt(argc, argv, "li")) != -1) {
9816                 switch (c) {
9817                 case 'l':
9818                         cbdata.longfmt = B_TRUE;
9819                         break;
9820                 case 'i':
9821                         cbdata.internal = B_TRUE;
9822                         break;
9823                 case '?':
9824                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
9825                             optopt);
9826                         usage(B_FALSE);
9827                 }
9828         }
9829         argc -= optind;
9830         argv += optind;
9831
9832         ret = for_each_pool(argc, argv, B_FALSE, NULL, ZFS_TYPE_POOL,
9833             B_FALSE, get_history_one, &cbdata);
9834
9835         if (argc == 0 && cbdata.first == B_TRUE) {
9836                 (void) fprintf(stderr, gettext("no pools available\n"));
9837                 return (0);
9838         }
9839
9840         return (ret);
9841 }
9842
9843 typedef struct ev_opts {
9844         int verbose;
9845         int scripted;
9846         int follow;
9847         int clear;
9848         char poolname[ZFS_MAX_DATASET_NAME_LEN];
9849 } ev_opts_t;
9850
9851 static void
9852 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
9853 {
9854         char ctime_str[26], str[32];
9855         const char *ptr;
9856         int64_t *tv;
9857         uint_t n;
9858
9859         verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
9860         memset(str, ' ', 32);
9861         (void) ctime_r((const time_t *)&tv[0], ctime_str);
9862         (void) memcpy(str, ctime_str+4,  6);            /* 'Jun 30' */
9863         (void) memcpy(str+7, ctime_str+20, 4);          /* '1993' */
9864         (void) memcpy(str+12, ctime_str+11, 8);         /* '21:49:08' */
9865         (void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
9866         if (opts->scripted)
9867                 (void) printf(gettext("%s\t"), str);
9868         else
9869                 (void) printf(gettext("%s "), str);
9870
9871         verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
9872         (void) printf(gettext("%s\n"), ptr);
9873 }
9874
9875 static void
9876 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
9877 {
9878         nvpair_t *nvp;
9879
9880         for (nvp = nvlist_next_nvpair(nvl, NULL);
9881             nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
9882
9883                 data_type_t type = nvpair_type(nvp);
9884                 const char *name = nvpair_name(nvp);
9885
9886                 boolean_t b;
9887                 uint8_t i8;
9888                 uint16_t i16;
9889                 uint32_t i32;
9890                 uint64_t i64;
9891                 const char *str;
9892                 nvlist_t *cnv;
9893
9894                 printf(gettext("%*s%s = "), depth, "", name);
9895
9896                 switch (type) {
9897                 case DATA_TYPE_BOOLEAN:
9898                         printf(gettext("%s"), "1");
9899                         break;
9900
9901                 case DATA_TYPE_BOOLEAN_VALUE:
9902                         (void) nvpair_value_boolean_value(nvp, &b);
9903                         printf(gettext("%s"), b ? "1" : "0");
9904                         break;
9905
9906                 case DATA_TYPE_BYTE:
9907                         (void) nvpair_value_byte(nvp, &i8);
9908                         printf(gettext("0x%x"), i8);
9909                         break;
9910
9911                 case DATA_TYPE_INT8:
9912                         (void) nvpair_value_int8(nvp, (void *)&i8);
9913                         printf(gettext("0x%x"), i8);
9914                         break;
9915
9916                 case DATA_TYPE_UINT8:
9917                         (void) nvpair_value_uint8(nvp, &i8);
9918                         printf(gettext("0x%x"), i8);
9919                         break;
9920
9921                 case DATA_TYPE_INT16:
9922                         (void) nvpair_value_int16(nvp, (void *)&i16);
9923                         printf(gettext("0x%x"), i16);
9924                         break;
9925
9926                 case DATA_TYPE_UINT16:
9927                         (void) nvpair_value_uint16(nvp, &i16);
9928                         printf(gettext("0x%x"), i16);
9929                         break;
9930
9931                 case DATA_TYPE_INT32:
9932                         (void) nvpair_value_int32(nvp, (void *)&i32);
9933                         printf(gettext("0x%x"), i32);
9934                         break;
9935
9936                 case DATA_TYPE_UINT32:
9937                         (void) nvpair_value_uint32(nvp, &i32);
9938                         printf(gettext("0x%x"), i32);
9939                         break;
9940
9941                 case DATA_TYPE_INT64:
9942                         (void) nvpair_value_int64(nvp, (void *)&i64);
9943                         printf(gettext("0x%llx"), (u_longlong_t)i64);
9944                         break;
9945
9946                 case DATA_TYPE_UINT64:
9947                         (void) nvpair_value_uint64(nvp, &i64);
9948                         /*
9949                          * translate vdev state values to readable
9950                          * strings to aide zpool events consumers
9951                          */
9952                         if (strcmp(name,
9953                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
9954                             strcmp(name,
9955                             FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
9956                                 printf(gettext("\"%s\" (0x%llx)"),
9957                                     zpool_state_to_name(i64, VDEV_AUX_NONE),
9958                                     (u_longlong_t)i64);
9959                         } else {
9960                                 printf(gettext("0x%llx"), (u_longlong_t)i64);
9961                         }
9962                         break;
9963
9964                 case DATA_TYPE_HRTIME:
9965                         (void) nvpair_value_hrtime(nvp, (void *)&i64);
9966                         printf(gettext("0x%llx"), (u_longlong_t)i64);
9967                         break;
9968
9969                 case DATA_TYPE_STRING:
9970                         (void) nvpair_value_string(nvp, &str);
9971                         printf(gettext("\"%s\""), str ? str : "<NULL>");
9972                         break;
9973
9974                 case DATA_TYPE_NVLIST:
9975                         printf(gettext("(embedded nvlist)\n"));
9976                         (void) nvpair_value_nvlist(nvp, &cnv);
9977                         zpool_do_events_nvprint(cnv, depth + 8);
9978                         printf(gettext("%*s(end %s)"), depth, "", name);
9979                         break;
9980
9981                 case DATA_TYPE_NVLIST_ARRAY: {
9982                         nvlist_t **val;
9983                         uint_t i, nelem;
9984
9985                         (void) nvpair_value_nvlist_array(nvp, &val, &nelem);
9986                         printf(gettext("(%d embedded nvlists)\n"), nelem);
9987                         for (i = 0; i < nelem; i++) {
9988                                 printf(gettext("%*s%s[%d] = %s\n"),
9989                                     depth, "", name, i, "(embedded nvlist)");
9990                                 zpool_do_events_nvprint(val[i], depth + 8);
9991                                 printf(gettext("%*s(end %s[%i])\n"),
9992                                     depth, "", name, i);
9993                         }
9994                         printf(gettext("%*s(end %s)\n"), depth, "", name);
9995                         }
9996                         break;
9997
9998                 case DATA_TYPE_INT8_ARRAY: {
9999                         int8_t *val;
10000                         uint_t i, nelem;
10001
10002                         (void) nvpair_value_int8_array(nvp, &val, &nelem);
10003                         for (i = 0; i < nelem; i++)
10004                                 printf(gettext("0x%x "), val[i]);
10005
10006                         break;
10007                         }
10008
10009                 case DATA_TYPE_UINT8_ARRAY: {
10010                         uint8_t *val;
10011                         uint_t i, nelem;
10012
10013                         (void) nvpair_value_uint8_array(nvp, &val, &nelem);
10014                         for (i = 0; i < nelem; i++)
10015                                 printf(gettext("0x%x "), val[i]);
10016
10017                         break;
10018                         }
10019
10020                 case DATA_TYPE_INT16_ARRAY: {
10021                         int16_t *val;
10022                         uint_t i, nelem;
10023
10024                         (void) nvpair_value_int16_array(nvp, &val, &nelem);
10025                         for (i = 0; i < nelem; i++)
10026                                 printf(gettext("0x%x "), val[i]);
10027
10028                         break;
10029                         }
10030
10031                 case DATA_TYPE_UINT16_ARRAY: {
10032                         uint16_t *val;
10033                         uint_t i, nelem;
10034
10035                         (void) nvpair_value_uint16_array(nvp, &val, &nelem);
10036                         for (i = 0; i < nelem; i++)
10037                                 printf(gettext("0x%x "), val[i]);
10038
10039                         break;
10040                         }
10041
10042                 case DATA_TYPE_INT32_ARRAY: {
10043                         int32_t *val;
10044                         uint_t i, nelem;
10045
10046                         (void) nvpair_value_int32_array(nvp, &val, &nelem);
10047                         for (i = 0; i < nelem; i++)
10048                                 printf(gettext("0x%x "), val[i]);
10049
10050                         break;
10051                         }
10052
10053                 case DATA_TYPE_UINT32_ARRAY: {
10054                         uint32_t *val;
10055                         uint_t i, nelem;
10056
10057                         (void) nvpair_value_uint32_array(nvp, &val, &nelem);
10058                         for (i = 0; i < nelem; i++)
10059                                 printf(gettext("0x%x "), val[i]);
10060
10061                         break;
10062                         }
10063
10064                 case DATA_TYPE_INT64_ARRAY: {
10065                         int64_t *val;
10066                         uint_t i, nelem;
10067
10068                         (void) nvpair_value_int64_array(nvp, &val, &nelem);
10069                         for (i = 0; i < nelem; i++)
10070                                 printf(gettext("0x%llx "),
10071                                     (u_longlong_t)val[i]);
10072
10073                         break;
10074                         }
10075
10076                 case DATA_TYPE_UINT64_ARRAY: {
10077                         uint64_t *val;
10078                         uint_t i, nelem;
10079
10080                         (void) nvpair_value_uint64_array(nvp, &val, &nelem);
10081                         for (i = 0; i < nelem; i++)
10082                                 printf(gettext("0x%llx "),
10083                                     (u_longlong_t)val[i]);
10084
10085                         break;
10086                         }
10087
10088                 case DATA_TYPE_STRING_ARRAY: {
10089                         const char **str;
10090                         uint_t i, nelem;
10091
10092                         (void) nvpair_value_string_array(nvp, &str, &nelem);
10093                         for (i = 0; i < nelem; i++)
10094                                 printf(gettext("\"%s\" "),
10095                                     str[i] ? str[i] : "<NULL>");
10096
10097                         break;
10098                         }
10099
10100                 case DATA_TYPE_BOOLEAN_ARRAY:
10101                 case DATA_TYPE_BYTE_ARRAY:
10102                 case DATA_TYPE_DOUBLE:
10103                 case DATA_TYPE_DONTCARE:
10104                 case DATA_TYPE_UNKNOWN:
10105                         printf(gettext("<unknown>"));
10106                         break;
10107                 }
10108
10109                 printf(gettext("\n"));
10110         }
10111 }
10112
10113 static int
10114 zpool_do_events_next(ev_opts_t *opts)
10115 {
10116         nvlist_t *nvl;
10117         int zevent_fd, ret, dropped;
10118         const char *pool;
10119
10120         zevent_fd = open(ZFS_DEV, O_RDWR);
10121         VERIFY(zevent_fd >= 0);
10122
10123         if (!opts->scripted)
10124                 (void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
10125
10126         while (1) {
10127                 ret = zpool_events_next(g_zfs, &nvl, &dropped,
10128                     (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
10129                 if (ret || nvl == NULL)
10130                         break;
10131
10132                 if (dropped > 0)
10133                         (void) printf(gettext("dropped %d events\n"), dropped);
10134
10135                 if (strlen(opts->poolname) > 0 &&
10136                     nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
10137                     strcmp(opts->poolname, pool) != 0)
10138                         continue;
10139
10140                 zpool_do_events_short(nvl, opts);
10141
10142                 if (opts->verbose) {
10143                         zpool_do_events_nvprint(nvl, 8);
10144                         printf(gettext("\n"));
10145                 }
10146                 (void) fflush(stdout);
10147
10148                 nvlist_free(nvl);
10149         }
10150
10151         VERIFY(0 == close(zevent_fd));
10152
10153         return (ret);
10154 }
10155
10156 static int
10157 zpool_do_events_clear(void)
10158 {
10159         int count, ret;
10160
10161         ret = zpool_events_clear(g_zfs, &count);
10162         if (!ret)
10163                 (void) printf(gettext("cleared %d events\n"), count);
10164
10165         return (ret);
10166 }
10167
10168 /*
10169  * zpool events [-vHf [pool] | -c]
10170  *
10171  * Displays events logs by ZFS.
10172  */
10173 int
10174 zpool_do_events(int argc, char **argv)
10175 {
10176         ev_opts_t opts = { 0 };
10177         int ret;
10178         int c;
10179
10180         /* check options */
10181         while ((c = getopt(argc, argv, "vHfc")) != -1) {
10182                 switch (c) {
10183                 case 'v':
10184                         opts.verbose = 1;
10185                         break;
10186                 case 'H':
10187                         opts.scripted = 1;
10188                         break;
10189                 case 'f':
10190                         opts.follow = 1;
10191                         break;
10192                 case 'c':
10193                         opts.clear = 1;
10194                         break;
10195                 case '?':
10196                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10197                             optopt);
10198                         usage(B_FALSE);
10199                 }
10200         }
10201         argc -= optind;
10202         argv += optind;
10203
10204         if (argc > 1) {
10205                 (void) fprintf(stderr, gettext("too many arguments\n"));
10206                 usage(B_FALSE);
10207         } else if (argc == 1) {
10208                 (void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
10209                 if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
10210                         (void) fprintf(stderr,
10211                             gettext("invalid pool name '%s'\n"), opts.poolname);
10212                         usage(B_FALSE);
10213                 }
10214         }
10215
10216         if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
10217             opts.clear) {
10218                 (void) fprintf(stderr,
10219                     gettext("invalid options combined with -c\n"));
10220                 usage(B_FALSE);
10221         }
10222
10223         if (opts.clear)
10224                 ret = zpool_do_events_clear();
10225         else
10226                 ret = zpool_do_events_next(&opts);
10227
10228         return (ret);
10229 }
10230
10231 static int
10232 get_callback_vdev(zpool_handle_t *zhp, char *vdevname, void *data)
10233 {
10234         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
10235         char value[ZFS_MAXPROPLEN];
10236         zprop_source_t srctype;
10237
10238         for (zprop_list_t *pl = cbp->cb_proplist; pl != NULL;
10239             pl = pl->pl_next) {
10240                 char *prop_name;
10241                 /*
10242                  * If the first property is pool name, it is a special
10243                  * placeholder that we can skip. This will also skip
10244                  * over the name property when 'all' is specified.
10245                  */
10246                 if (pl->pl_prop == ZPOOL_PROP_NAME &&
10247                     pl == cbp->cb_proplist)
10248                         continue;
10249
10250                 if (pl->pl_prop == ZPROP_INVAL) {
10251                         prop_name = pl->pl_user_prop;
10252                 } else {
10253                         prop_name = (char *)vdev_prop_to_name(pl->pl_prop);
10254                 }
10255                 if (zpool_get_vdev_prop(zhp, vdevname, pl->pl_prop,
10256                     prop_name, value, sizeof (value), &srctype,
10257                     cbp->cb_literal) == 0) {
10258                         zprop_print_one_property(vdevname, cbp, prop_name,
10259                             value, srctype, NULL, NULL);
10260                 }
10261         }
10262
10263         return (0);
10264 }
10265
10266 static int
10267 get_callback_vdev_cb(void *zhp_data, nvlist_t *nv, void *data)
10268 {
10269         zpool_handle_t *zhp = zhp_data;
10270         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
10271         char *vdevname;
10272         const char *type;
10273         int ret;
10274
10275         /*
10276          * zpool_vdev_name() transforms the root vdev name (i.e., root-0) to the
10277          * pool name for display purposes, which is not desired. Fallback to
10278          * zpool_vdev_name() when not dealing with the root vdev.
10279          */
10280         type = fnvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE);
10281         if (zhp != NULL && strcmp(type, "root") == 0)
10282                 vdevname = strdup("root-0");
10283         else
10284                 vdevname = zpool_vdev_name(g_zfs, zhp, nv,
10285                     cbp->cb_vdevs.cb_name_flags);
10286
10287         (void) vdev_expand_proplist(zhp, vdevname, &cbp->cb_proplist);
10288
10289         ret = get_callback_vdev(zhp, vdevname, data);
10290
10291         free(vdevname);
10292
10293         return (ret);
10294 }
10295
10296 static int
10297 get_callback(zpool_handle_t *zhp, void *data)
10298 {
10299         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
10300         char value[ZFS_MAXPROPLEN];
10301         zprop_source_t srctype;
10302         zprop_list_t *pl;
10303         int vid;
10304
10305         if (cbp->cb_type == ZFS_TYPE_VDEV) {
10306                 if (strcmp(cbp->cb_vdevs.cb_names[0], "all-vdevs") == 0) {
10307                         for_each_vdev(zhp, get_callback_vdev_cb, data);
10308                 } else {
10309                         /* Adjust column widths for vdev properties */
10310                         for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
10311                             vid++) {
10312                                 vdev_expand_proplist(zhp,
10313                                     cbp->cb_vdevs.cb_names[vid],
10314                                     &cbp->cb_proplist);
10315                         }
10316                         /* Display the properties */
10317                         for (vid = 0; vid < cbp->cb_vdevs.cb_names_count;
10318                             vid++) {
10319                                 get_callback_vdev(zhp,
10320                                     cbp->cb_vdevs.cb_names[vid], data);
10321                         }
10322                 }
10323         } else {
10324                 assert(cbp->cb_type == ZFS_TYPE_POOL);
10325                 for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
10326                         /*
10327                          * Skip the special fake placeholder. This will also
10328                          * skip over the name property when 'all' is specified.
10329                          */
10330                         if (pl->pl_prop == ZPOOL_PROP_NAME &&
10331                             pl == cbp->cb_proplist)
10332                                 continue;
10333
10334                         if (pl->pl_prop == ZPROP_INVAL &&
10335                             zfs_prop_user(pl->pl_user_prop)) {
10336                                 srctype = ZPROP_SRC_LOCAL;
10337
10338                                 if (zpool_get_userprop(zhp, pl->pl_user_prop,
10339                                     value, sizeof (value), &srctype) != 0)
10340                                         continue;
10341
10342                                 zprop_print_one_property(zpool_get_name(zhp),
10343                                     cbp, pl->pl_user_prop, value, srctype,
10344                                     NULL, NULL);
10345                         } else if (pl->pl_prop == ZPROP_INVAL &&
10346                             (zpool_prop_feature(pl->pl_user_prop) ||
10347                             zpool_prop_unsupported(pl->pl_user_prop))) {
10348                                 srctype = ZPROP_SRC_LOCAL;
10349
10350                                 if (zpool_prop_get_feature(zhp,
10351                                     pl->pl_user_prop, value,
10352                                     sizeof (value)) == 0) {
10353                                         zprop_print_one_property(
10354                                             zpool_get_name(zhp), cbp,
10355                                             pl->pl_user_prop, value, srctype,
10356                                             NULL, NULL);
10357                                 }
10358                         } else {
10359                                 if (zpool_get_prop(zhp, pl->pl_prop, value,
10360                                     sizeof (value), &srctype,
10361                                     cbp->cb_literal) != 0)
10362                                         continue;
10363
10364                                 zprop_print_one_property(zpool_get_name(zhp),
10365                                     cbp, zpool_prop_to_name(pl->pl_prop),
10366                                     value, srctype, NULL, NULL);
10367                         }
10368                 }
10369         }
10370
10371         return (0);
10372 }
10373
10374 /*
10375  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
10376  *
10377  *      -H      Scripted mode.  Don't display headers, and separate properties
10378  *              by a single tab.
10379  *      -o      List of columns to display.  Defaults to
10380  *              "name,property,value,source".
10381  *      -p      Display values in parsable (exact) format.
10382  *
10383  * Get properties of pools in the system. Output space statistics
10384  * for each one as well as other attributes.
10385  */
10386 int
10387 zpool_do_get(int argc, char **argv)
10388 {
10389         zprop_get_cbdata_t cb = { 0 };
10390         zprop_list_t fake_name = { 0 };
10391         int ret;
10392         int c, i;
10393         char *propstr = NULL;
10394         char *vdev = NULL;
10395
10396         cb.cb_first = B_TRUE;
10397
10398         /*
10399          * Set up default columns and sources.
10400          */
10401         cb.cb_sources = ZPROP_SRC_ALL;
10402         cb.cb_columns[0] = GET_COL_NAME;
10403         cb.cb_columns[1] = GET_COL_PROPERTY;
10404         cb.cb_columns[2] = GET_COL_VALUE;
10405         cb.cb_columns[3] = GET_COL_SOURCE;
10406         cb.cb_type = ZFS_TYPE_POOL;
10407         cb.cb_vdevs.cb_name_flags |= VDEV_NAME_TYPE_ID;
10408         current_prop_type = cb.cb_type;
10409
10410         /* check options */
10411         while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
10412                 switch (c) {
10413                 case 'p':
10414                         cb.cb_literal = B_TRUE;
10415                         break;
10416                 case 'H':
10417                         cb.cb_scripted = B_TRUE;
10418                         break;
10419                 case 'o':
10420                         memset(&cb.cb_columns, 0, sizeof (cb.cb_columns));
10421                         i = 0;
10422
10423                         for (char *tok; (tok = strsep(&optarg, ",")); ) {
10424                                 static const char *const col_opts[] =
10425                                 { "name", "property", "value", "source",
10426                                     "all" };
10427                                 static const zfs_get_column_t col_cols[] =
10428                                 { GET_COL_NAME, GET_COL_PROPERTY, GET_COL_VALUE,
10429                                     GET_COL_SOURCE };
10430
10431                                 if (i == ZFS_GET_NCOLS - 1) {
10432                                         (void) fprintf(stderr, gettext("too "
10433                                         "many fields given to -o "
10434                                         "option\n"));
10435                                         usage(B_FALSE);
10436                                 }
10437
10438                                 for (c = 0; c < ARRAY_SIZE(col_opts); ++c)
10439                                         if (strcmp(tok, col_opts[c]) == 0)
10440                                                 goto found;
10441
10442                                 (void) fprintf(stderr,
10443                                     gettext("invalid column name '%s'\n"), tok);
10444                                 usage(B_FALSE);
10445
10446 found:
10447                                 if (c >= 4) {
10448                                         if (i > 0) {
10449                                                 (void) fprintf(stderr,
10450                                                     gettext("\"all\" conflicts "
10451                                                     "with specific fields "
10452                                                     "given to -o option\n"));
10453                                                 usage(B_FALSE);
10454                                         }
10455
10456                                         memcpy(cb.cb_columns, col_cols,
10457                                             sizeof (col_cols));
10458                                         i = ZFS_GET_NCOLS - 1;
10459                                 } else
10460                                         cb.cb_columns[i++] = col_cols[c];
10461                         }
10462                         break;
10463                 case '?':
10464                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10465                             optopt);
10466                         usage(B_FALSE);
10467                 }
10468         }
10469
10470         argc -= optind;
10471         argv += optind;
10472
10473         if (argc < 1) {
10474                 (void) fprintf(stderr, gettext("missing property "
10475                     "argument\n"));
10476                 usage(B_FALSE);
10477         }
10478
10479         /* Properties list is needed later by zprop_get_list() */
10480         propstr = argv[0];
10481
10482         argc--;
10483         argv++;
10484
10485         if (argc == 0) {
10486                 /* No args, so just print the defaults. */
10487         } else if (are_all_pools(argc, argv)) {
10488                 /* All the args are pool names */
10489         } else if (are_all_pools(1, argv)) {
10490                 /* The first arg is a pool name */
10491                 if ((argc == 2 && strcmp(argv[1], "all-vdevs") == 0) ||
10492                     (argc == 2 && strcmp(argv[1], "root") == 0) ||
10493                     are_vdevs_in_pool(argc - 1, argv + 1, argv[0],
10494                     &cb.cb_vdevs)) {
10495
10496                         if (strcmp(argv[1], "root") == 0)
10497                                 vdev = strdup("root-0");
10498                         else
10499                                 vdev = strdup(argv[1]);
10500
10501                         /* ... and the rest are vdev names */
10502                         cb.cb_vdevs.cb_names = &vdev;
10503                         cb.cb_vdevs.cb_names_count = argc - 1;
10504                         cb.cb_type = ZFS_TYPE_VDEV;
10505                         argc = 1; /* One pool to process */
10506                 } else {
10507                         fprintf(stderr, gettext("Expected a list of vdevs in"
10508                             " \"%s\", but got:\n"), argv[0]);
10509                         error_list_unresolved_vdevs(argc - 1, argv + 1,
10510                             argv[0], &cb.cb_vdevs);
10511                         fprintf(stderr, "\n");
10512                         usage(B_FALSE);
10513                         return (1);
10514                 }
10515         } else {
10516                 /*
10517                  * The first arg isn't a pool name,
10518                  */
10519                 fprintf(stderr, gettext("missing pool name.\n"));
10520                 fprintf(stderr, "\n");
10521                 usage(B_FALSE);
10522                 return (1);
10523         }
10524
10525         if (zprop_get_list(g_zfs, propstr, &cb.cb_proplist,
10526             cb.cb_type) != 0) {
10527                 /* Use correct list of valid properties (pool or vdev) */
10528                 current_prop_type = cb.cb_type;
10529                 usage(B_FALSE);
10530         }
10531
10532         if (cb.cb_proplist != NULL) {
10533                 fake_name.pl_prop = ZPOOL_PROP_NAME;
10534                 fake_name.pl_width = strlen(gettext("NAME"));
10535                 fake_name.pl_next = cb.cb_proplist;
10536                 cb.cb_proplist = &fake_name;
10537         }
10538
10539         ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist, cb.cb_type,
10540             cb.cb_literal, get_callback, &cb);
10541
10542         if (cb.cb_proplist == &fake_name)
10543                 zprop_free_list(fake_name.pl_next);
10544         else
10545                 zprop_free_list(cb.cb_proplist);
10546
10547         if (vdev != NULL)
10548                 free(vdev);
10549
10550         return (ret);
10551 }
10552
10553 typedef struct set_cbdata {
10554         char *cb_propname;
10555         char *cb_value;
10556         zfs_type_t cb_type;
10557         vdev_cbdata_t cb_vdevs;
10558         boolean_t cb_any_successful;
10559 } set_cbdata_t;
10560
10561 static int
10562 set_pool_callback(zpool_handle_t *zhp, set_cbdata_t *cb)
10563 {
10564         int error;
10565
10566         /* Check if we have out-of-bounds features */
10567         if (strcmp(cb->cb_propname, ZPOOL_CONFIG_COMPATIBILITY) == 0) {
10568                 boolean_t features[SPA_FEATURES];
10569                 if (zpool_do_load_compat(cb->cb_value, features) !=
10570                     ZPOOL_COMPATIBILITY_OK)
10571                         return (-1);
10572
10573                 nvlist_t *enabled = zpool_get_features(zhp);
10574                 spa_feature_t i;
10575                 for (i = 0; i < SPA_FEATURES; i++) {
10576                         const char *fguid = spa_feature_table[i].fi_guid;
10577                         if (nvlist_exists(enabled, fguid) && !features[i])
10578                                 break;
10579                 }
10580                 if (i < SPA_FEATURES)
10581                         (void) fprintf(stderr, gettext("Warning: one or "
10582                             "more features already enabled on pool '%s'\n"
10583                             "are not present in this compatibility set.\n"),
10584                             zpool_get_name(zhp));
10585         }
10586
10587         /* if we're setting a feature, check it's in compatibility set */
10588         if (zpool_prop_feature(cb->cb_propname) &&
10589             strcmp(cb->cb_value, ZFS_FEATURE_ENABLED) == 0) {
10590                 char *fname = strchr(cb->cb_propname, '@') + 1;
10591                 spa_feature_t f;
10592
10593                 if (zfeature_lookup_name(fname, &f) == 0) {
10594                         char compat[ZFS_MAXPROPLEN];
10595                         if (zpool_get_prop(zhp, ZPOOL_PROP_COMPATIBILITY,
10596                             compat, ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
10597                                 compat[0] = '\0';
10598
10599                         boolean_t features[SPA_FEATURES];
10600                         if (zpool_do_load_compat(compat, features) !=
10601                             ZPOOL_COMPATIBILITY_OK) {
10602                                 (void) fprintf(stderr, gettext("Error: "
10603                                     "cannot enable feature '%s' on pool '%s'\n"
10604                                     "because the pool's 'compatibility' "
10605                                     "property cannot be parsed.\n"),
10606                                     fname, zpool_get_name(zhp));
10607                                 return (-1);
10608                         }
10609
10610                         if (!features[f]) {
10611                                 (void) fprintf(stderr, gettext("Error: "
10612                                     "cannot enable feature '%s' on pool '%s'\n"
10613                                     "as it is not specified in this pool's "
10614                                     "current compatibility set.\n"
10615                                     "Consider setting 'compatibility' to a "
10616                                     "less restrictive set, or to 'off'.\n"),
10617                                     fname, zpool_get_name(zhp));
10618                                 return (-1);
10619                         }
10620                 }
10621         }
10622
10623         error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
10624
10625         return (error);
10626 }
10627
10628 static int
10629 set_callback(zpool_handle_t *zhp, void *data)
10630 {
10631         int error;
10632         set_cbdata_t *cb = (set_cbdata_t *)data;
10633
10634         if (cb->cb_type == ZFS_TYPE_VDEV) {
10635                 error = zpool_set_vdev_prop(zhp, *cb->cb_vdevs.cb_names,
10636                     cb->cb_propname, cb->cb_value);
10637         } else {
10638                 assert(cb->cb_type == ZFS_TYPE_POOL);
10639                 error = set_pool_callback(zhp, cb);
10640         }
10641
10642         cb->cb_any_successful = !error;
10643         return (error);
10644 }
10645
10646 int
10647 zpool_do_set(int argc, char **argv)
10648 {
10649         set_cbdata_t cb = { 0 };
10650         int error;
10651         char *vdev = NULL;
10652
10653         current_prop_type = ZFS_TYPE_POOL;
10654         if (argc > 1 && argv[1][0] == '-') {
10655                 (void) fprintf(stderr, gettext("invalid option '%c'\n"),
10656                     argv[1][1]);
10657                 usage(B_FALSE);
10658         }
10659
10660         if (argc < 2) {
10661                 (void) fprintf(stderr, gettext("missing property=value "
10662                     "argument\n"));
10663                 usage(B_FALSE);
10664         }
10665
10666         if (argc < 3) {
10667                 (void) fprintf(stderr, gettext("missing pool name\n"));
10668                 usage(B_FALSE);
10669         }
10670
10671         if (argc > 4) {
10672                 (void) fprintf(stderr, gettext("too many pool names\n"));
10673                 usage(B_FALSE);
10674         }
10675
10676         cb.cb_propname = argv[1];
10677         cb.cb_type = ZFS_TYPE_POOL;
10678         cb.cb_vdevs.cb_name_flags |= VDEV_NAME_TYPE_ID;
10679         cb.cb_value = strchr(cb.cb_propname, '=');
10680         if (cb.cb_value == NULL) {
10681                 (void) fprintf(stderr, gettext("missing value in "
10682                     "property=value argument\n"));
10683                 usage(B_FALSE);
10684         }
10685
10686         *(cb.cb_value) = '\0';
10687         cb.cb_value++;
10688         argc -= 2;
10689         argv += 2;
10690
10691         /* argv[0] is pool name */
10692         if (!is_pool(argv[0])) {
10693                 (void) fprintf(stderr,
10694                     gettext("cannot open '%s': is not a pool\n"), argv[0]);
10695                 return (EINVAL);
10696         }
10697
10698         /* argv[1], when supplied, is vdev name */
10699         if (argc == 2) {
10700
10701                 if (strcmp(argv[1], "root") == 0)
10702                         vdev = strdup("root-0");
10703                 else
10704                         vdev = strdup(argv[1]);
10705
10706                 if (!are_vdevs_in_pool(1, &vdev, argv[0], &cb.cb_vdevs)) {
10707                         (void) fprintf(stderr, gettext(
10708                             "cannot find '%s' in '%s': device not in pool\n"),
10709                             vdev, argv[0]);
10710                         free(vdev);
10711                         return (EINVAL);
10712                 }
10713                 cb.cb_vdevs.cb_names = &vdev;
10714                 cb.cb_vdevs.cb_names_count = 1;
10715                 cb.cb_type = ZFS_TYPE_VDEV;
10716         }
10717
10718         error = for_each_pool(1, argv, B_TRUE, NULL, ZFS_TYPE_POOL,
10719             B_FALSE, set_callback, &cb);
10720
10721         if (vdev != NULL)
10722                 free(vdev);
10723
10724         return (error);
10725 }
10726
10727 /* Add up the total number of bytes left to initialize/trim across all vdevs */
10728 static uint64_t
10729 vdev_activity_remaining(nvlist_t *nv, zpool_wait_activity_t activity)
10730 {
10731         uint64_t bytes_remaining;
10732         nvlist_t **child;
10733         uint_t c, children;
10734         vdev_stat_t *vs;
10735
10736         assert(activity == ZPOOL_WAIT_INITIALIZE ||
10737             activity == ZPOOL_WAIT_TRIM);
10738
10739         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
10740             (uint64_t **)&vs, &c) == 0);
10741
10742         if (activity == ZPOOL_WAIT_INITIALIZE &&
10743             vs->vs_initialize_state == VDEV_INITIALIZE_ACTIVE)
10744                 bytes_remaining = vs->vs_initialize_bytes_est -
10745                     vs->vs_initialize_bytes_done;
10746         else if (activity == ZPOOL_WAIT_TRIM &&
10747             vs->vs_trim_state == VDEV_TRIM_ACTIVE)
10748                 bytes_remaining = vs->vs_trim_bytes_est -
10749                     vs->vs_trim_bytes_done;
10750         else
10751                 bytes_remaining = 0;
10752
10753         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10754             &child, &children) != 0)
10755                 children = 0;
10756
10757         for (c = 0; c < children; c++)
10758                 bytes_remaining += vdev_activity_remaining(child[c], activity);
10759
10760         return (bytes_remaining);
10761 }
10762
10763 /* Add up the total number of bytes left to rebuild across top-level vdevs */
10764 static uint64_t
10765 vdev_activity_top_remaining(nvlist_t *nv)
10766 {
10767         uint64_t bytes_remaining = 0;
10768         nvlist_t **child;
10769         uint_t children;
10770         int error;
10771
10772         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10773             &child, &children) != 0)
10774                 children = 0;
10775
10776         for (uint_t c = 0; c < children; c++) {
10777                 vdev_rebuild_stat_t *vrs;
10778                 uint_t i;
10779
10780                 error = nvlist_lookup_uint64_array(child[c],
10781                     ZPOOL_CONFIG_REBUILD_STATS, (uint64_t **)&vrs, &i);
10782                 if (error == 0) {
10783                         if (vrs->vrs_state == VDEV_REBUILD_ACTIVE) {
10784                                 bytes_remaining += (vrs->vrs_bytes_est -
10785                                     vrs->vrs_bytes_rebuilt);
10786                         }
10787                 }
10788         }
10789
10790         return (bytes_remaining);
10791 }
10792
10793 /* Whether any vdevs are 'spare' or 'replacing' vdevs */
10794 static boolean_t
10795 vdev_any_spare_replacing(nvlist_t *nv)
10796 {
10797         nvlist_t **child;
10798         uint_t c, children;
10799         const char *vdev_type;
10800
10801         (void) nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &vdev_type);
10802
10803         if (strcmp(vdev_type, VDEV_TYPE_REPLACING) == 0 ||
10804             strcmp(vdev_type, VDEV_TYPE_SPARE) == 0 ||
10805             strcmp(vdev_type, VDEV_TYPE_DRAID_SPARE) == 0) {
10806                 return (B_TRUE);
10807         }
10808
10809         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
10810             &child, &children) != 0)
10811                 children = 0;
10812
10813         for (c = 0; c < children; c++) {
10814                 if (vdev_any_spare_replacing(child[c]))
10815                         return (B_TRUE);
10816         }
10817
10818         return (B_FALSE);
10819 }
10820
10821 typedef struct wait_data {
10822         char *wd_poolname;
10823         boolean_t wd_scripted;
10824         boolean_t wd_exact;
10825         boolean_t wd_headers_once;
10826         boolean_t wd_should_exit;
10827         /* Which activities to wait for */
10828         boolean_t wd_enabled[ZPOOL_WAIT_NUM_ACTIVITIES];
10829         float wd_interval;
10830         pthread_cond_t wd_cv;
10831         pthread_mutex_t wd_mutex;
10832 } wait_data_t;
10833
10834 /*
10835  * Print to stdout a single line, containing one column for each activity that
10836  * we are waiting for specifying how many bytes of work are left for that
10837  * activity.
10838  */
10839 static void
10840 print_wait_status_row(wait_data_t *wd, zpool_handle_t *zhp, int row)
10841 {
10842         nvlist_t *config, *nvroot;
10843         uint_t c;
10844         int i;
10845         pool_checkpoint_stat_t *pcs = NULL;
10846         pool_scan_stat_t *pss = NULL;
10847         pool_removal_stat_t *prs = NULL;
10848         pool_raidz_expand_stat_t *pres = NULL;
10849         const char *const headers[] = {"DISCARD", "FREE", "INITIALIZE",
10850             "REPLACE", "REMOVE", "RESILVER", "SCRUB", "TRIM", "RAIDZ_EXPAND"};
10851         int col_widths[ZPOOL_WAIT_NUM_ACTIVITIES];
10852
10853         /* Calculate the width of each column */
10854         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10855                 /*
10856                  * Make sure we have enough space in the col for pretty-printed
10857                  * numbers and for the column header, and then leave a couple
10858                  * spaces between cols for readability.
10859                  */
10860                 col_widths[i] = MAX(strlen(headers[i]), 6) + 2;
10861         }
10862
10863         /* Print header if appropriate */
10864         int term_height = terminal_height();
10865         boolean_t reprint_header = (!wd->wd_headers_once && term_height > 0 &&
10866             row % (term_height-1) == 0);
10867         if (!wd->wd_scripted && (row == 0 || reprint_header)) {
10868                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10869                         if (wd->wd_enabled[i])
10870                                 (void) printf("%*s", col_widths[i], headers[i]);
10871                 }
10872                 (void) fputc('\n', stdout);
10873         }
10874
10875         /* Bytes of work remaining in each activity */
10876         int64_t bytes_rem[ZPOOL_WAIT_NUM_ACTIVITIES] = {0};
10877
10878         bytes_rem[ZPOOL_WAIT_FREE] =
10879             zpool_get_prop_int(zhp, ZPOOL_PROP_FREEING, NULL);
10880
10881         config = zpool_get_config(zhp, NULL);
10882         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
10883
10884         (void) nvlist_lookup_uint64_array(nvroot,
10885             ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
10886         if (pcs != NULL && pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
10887                 bytes_rem[ZPOOL_WAIT_CKPT_DISCARD] = pcs->pcs_space;
10888
10889         (void) nvlist_lookup_uint64_array(nvroot,
10890             ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
10891         if (prs != NULL && prs->prs_state == DSS_SCANNING)
10892                 bytes_rem[ZPOOL_WAIT_REMOVE] = prs->prs_to_copy -
10893                     prs->prs_copied;
10894
10895         (void) nvlist_lookup_uint64_array(nvroot,
10896             ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&pss, &c);
10897         if (pss != NULL && pss->pss_state == DSS_SCANNING &&
10898             pss->pss_pass_scrub_pause == 0) {
10899                 int64_t rem = pss->pss_to_examine - pss->pss_issued;
10900                 if (pss->pss_func == POOL_SCAN_SCRUB)
10901                         bytes_rem[ZPOOL_WAIT_SCRUB] = rem;
10902                 else
10903                         bytes_rem[ZPOOL_WAIT_RESILVER] = rem;
10904         } else if (check_rebuilding(nvroot, NULL)) {
10905                 bytes_rem[ZPOOL_WAIT_RESILVER] =
10906                     vdev_activity_top_remaining(nvroot);
10907         }
10908
10909         (void) nvlist_lookup_uint64_array(nvroot,
10910             ZPOOL_CONFIG_RAIDZ_EXPAND_STATS, (uint64_t **)&pres, &c);
10911         if (pres != NULL && pres->pres_state == DSS_SCANNING) {
10912                 int64_t rem = pres->pres_to_reflow - pres->pres_reflowed;
10913                 bytes_rem[ZPOOL_WAIT_RAIDZ_EXPAND] = rem;
10914         }
10915
10916         bytes_rem[ZPOOL_WAIT_INITIALIZE] =
10917             vdev_activity_remaining(nvroot, ZPOOL_WAIT_INITIALIZE);
10918         bytes_rem[ZPOOL_WAIT_TRIM] =
10919             vdev_activity_remaining(nvroot, ZPOOL_WAIT_TRIM);
10920
10921         /*
10922          * A replace finishes after resilvering finishes, so the amount of work
10923          * left for a replace is the same as for resilvering.
10924          *
10925          * It isn't quite correct to say that if we have any 'spare' or
10926          * 'replacing' vdevs and a resilver is happening, then a replace is in
10927          * progress, like we do here. When a hot spare is used, the faulted vdev
10928          * is not removed after the hot spare is resilvered, so parent 'spare'
10929          * vdev is not removed either. So we could have a 'spare' vdev, but be
10930          * resilvering for a different reason. However, we use it as a heuristic
10931          * because we don't have access to the DTLs, which could tell us whether
10932          * or not we have really finished resilvering a hot spare.
10933          */
10934         if (vdev_any_spare_replacing(nvroot))
10935                 bytes_rem[ZPOOL_WAIT_REPLACE] =  bytes_rem[ZPOOL_WAIT_RESILVER];
10936
10937         if (timestamp_fmt != NODATE)
10938                 print_timestamp(timestamp_fmt);
10939
10940         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
10941                 char buf[64];
10942                 if (!wd->wd_enabled[i])
10943                         continue;
10944
10945                 if (wd->wd_exact) {
10946                         (void) snprintf(buf, sizeof (buf), "%" PRIi64,
10947                             bytes_rem[i]);
10948                 } else {
10949                         zfs_nicenum(bytes_rem[i], buf, sizeof (buf));
10950                 }
10951
10952                 if (wd->wd_scripted)
10953                         (void) printf(i == 0 ? "%s" : "\t%s", buf);
10954                 else
10955                         (void) printf(" %*s", col_widths[i] - 1, buf);
10956         }
10957         (void) printf("\n");
10958         (void) fflush(stdout);
10959 }
10960
10961 static void *
10962 wait_status_thread(void *arg)
10963 {
10964         wait_data_t *wd = (wait_data_t *)arg;
10965         zpool_handle_t *zhp;
10966
10967         if ((zhp = zpool_open(g_zfs, wd->wd_poolname)) == NULL)
10968                 return (void *)(1);
10969
10970         for (int row = 0; ; row++) {
10971                 boolean_t missing;
10972                 struct timespec timeout;
10973                 int ret = 0;
10974                 (void) clock_gettime(CLOCK_REALTIME, &timeout);
10975
10976                 if (zpool_refresh_stats(zhp, &missing) != 0 || missing ||
10977                     zpool_props_refresh(zhp) != 0) {
10978                         zpool_close(zhp);
10979                         return (void *)(uintptr_t)(missing ? 0 : 1);
10980                 }
10981
10982                 print_wait_status_row(wd, zhp, row);
10983
10984                 timeout.tv_sec += floor(wd->wd_interval);
10985                 long nanos = timeout.tv_nsec +
10986                     (wd->wd_interval - floor(wd->wd_interval)) * NANOSEC;
10987                 if (nanos >= NANOSEC) {
10988                         timeout.tv_sec++;
10989                         timeout.tv_nsec = nanos - NANOSEC;
10990                 } else {
10991                         timeout.tv_nsec = nanos;
10992                 }
10993                 pthread_mutex_lock(&wd->wd_mutex);
10994                 if (!wd->wd_should_exit)
10995                         ret = pthread_cond_timedwait(&wd->wd_cv, &wd->wd_mutex,
10996                             &timeout);
10997                 pthread_mutex_unlock(&wd->wd_mutex);
10998                 if (ret == 0) {
10999                         break; /* signaled by main thread */
11000                 } else if (ret != ETIMEDOUT) {
11001                         (void) fprintf(stderr, gettext("pthread_cond_timedwait "
11002                             "failed: %s\n"), strerror(ret));
11003                         zpool_close(zhp);
11004                         return (void *)(uintptr_t)(1);
11005                 }
11006         }
11007
11008         zpool_close(zhp);
11009         return (void *)(0);
11010 }
11011
11012 int
11013 zpool_do_wait(int argc, char **argv)
11014 {
11015         boolean_t verbose = B_FALSE;
11016         int c, i;
11017         unsigned long count;
11018         pthread_t status_thr;
11019         int error = 0;
11020         zpool_handle_t *zhp;
11021
11022         wait_data_t wd;
11023         wd.wd_scripted = B_FALSE;
11024         wd.wd_exact = B_FALSE;
11025         wd.wd_headers_once = B_FALSE;
11026         wd.wd_should_exit = B_FALSE;
11027
11028         pthread_mutex_init(&wd.wd_mutex, NULL);
11029         pthread_cond_init(&wd.wd_cv, NULL);
11030
11031         /* By default, wait for all types of activity. */
11032         for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++)
11033                 wd.wd_enabled[i] = B_TRUE;
11034
11035         while ((c = getopt(argc, argv, "HpT:t:")) != -1) {
11036                 switch (c) {
11037                 case 'H':
11038                         wd.wd_scripted = B_TRUE;
11039                         break;
11040                 case 'n':
11041                         wd.wd_headers_once = B_TRUE;
11042                         break;
11043                 case 'p':
11044                         wd.wd_exact = B_TRUE;
11045                         break;
11046                 case 'T':
11047                         get_timestamp_arg(*optarg);
11048                         break;
11049                 case 't':
11050                         /* Reset activities array */
11051                         memset(&wd.wd_enabled, 0, sizeof (wd.wd_enabled));
11052
11053                         for (char *tok; (tok = strsep(&optarg, ",")); ) {
11054                                 static const char *const col_opts[] = {
11055                                     "discard", "free", "initialize", "replace",
11056                                     "remove", "resilver", "scrub", "trim",
11057                                     "raidz_expand" };
11058
11059                                 for (i = 0; i < ARRAY_SIZE(col_opts); ++i)
11060                                         if (strcmp(tok, col_opts[i]) == 0) {
11061                                                 wd.wd_enabled[i] = B_TRUE;
11062                                                 goto found;
11063                                         }
11064
11065                                 (void) fprintf(stderr,
11066                                     gettext("invalid activity '%s'\n"), tok);
11067                                 usage(B_FALSE);
11068 found:;
11069                         }
11070                         break;
11071                 case '?':
11072                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
11073                             optopt);
11074                         usage(B_FALSE);
11075                 }
11076         }
11077
11078         argc -= optind;
11079         argv += optind;
11080
11081         get_interval_count(&argc, argv, &wd.wd_interval, &count);
11082         if (count != 0) {
11083                 /* This subcmd only accepts an interval, not a count */
11084                 (void) fprintf(stderr, gettext("too many arguments\n"));
11085                 usage(B_FALSE);
11086         }
11087
11088         if (wd.wd_interval != 0)
11089                 verbose = B_TRUE;
11090
11091         if (argc < 1) {
11092                 (void) fprintf(stderr, gettext("missing 'pool' argument\n"));
11093                 usage(B_FALSE);
11094         }
11095         if (argc > 1) {
11096                 (void) fprintf(stderr, gettext("too many arguments\n"));
11097                 usage(B_FALSE);
11098         }
11099
11100         wd.wd_poolname = argv[0];
11101
11102         if ((zhp = zpool_open(g_zfs, wd.wd_poolname)) == NULL)
11103                 return (1);
11104
11105         if (verbose) {
11106                 /*
11107                  * We use a separate thread for printing status updates because
11108                  * the main thread will call lzc_wait(), which blocks as long
11109                  * as an activity is in progress, which can be a long time.
11110                  */
11111                 if (pthread_create(&status_thr, NULL, wait_status_thread, &wd)
11112                     != 0) {
11113                         (void) fprintf(stderr, gettext("failed to create status"
11114                             "thread: %s\n"), strerror(errno));
11115                         zpool_close(zhp);
11116                         return (1);
11117                 }
11118         }
11119
11120         /*
11121          * Loop over all activities that we are supposed to wait for until none
11122          * of them are in progress. Note that this means we can end up waiting
11123          * for more activities to complete than just those that were in progress
11124          * when we began waiting; if an activity we are interested in begins
11125          * while we are waiting for another activity, we will wait for both to
11126          * complete before exiting.
11127          */
11128         for (;;) {
11129                 boolean_t missing = B_FALSE;
11130                 boolean_t any_waited = B_FALSE;
11131
11132                 for (i = 0; i < ZPOOL_WAIT_NUM_ACTIVITIES; i++) {
11133                         boolean_t waited;
11134
11135                         if (!wd.wd_enabled[i])
11136                                 continue;
11137
11138                         error = zpool_wait_status(zhp, i, &missing, &waited);
11139                         if (error != 0 || missing)
11140                                 break;
11141
11142                         any_waited = (any_waited || waited);
11143                 }
11144
11145                 if (error != 0 || missing || !any_waited)
11146                         break;
11147         }
11148
11149         zpool_close(zhp);
11150
11151         if (verbose) {
11152                 uintptr_t status;
11153                 pthread_mutex_lock(&wd.wd_mutex);
11154                 wd.wd_should_exit = B_TRUE;
11155                 pthread_cond_signal(&wd.wd_cv);
11156                 pthread_mutex_unlock(&wd.wd_mutex);
11157                 (void) pthread_join(status_thr, (void *)&status);
11158                 if (status != 0)
11159                         error = status;
11160         }
11161
11162         pthread_mutex_destroy(&wd.wd_mutex);
11163         pthread_cond_destroy(&wd.wd_cv);
11164         return (error);
11165 }
11166
11167 static int
11168 find_command_idx(const char *command, int *idx)
11169 {
11170         for (int i = 0; i < NCOMMAND; ++i) {
11171                 if (command_table[i].name == NULL)
11172                         continue;
11173
11174                 if (strcmp(command, command_table[i].name) == 0) {
11175                         *idx = i;
11176                         return (0);
11177                 }
11178         }
11179         return (1);
11180 }
11181
11182 /*
11183  * Display version message
11184  */
11185 static int
11186 zpool_do_version(int argc, char **argv)
11187 {
11188         (void) argc, (void) argv;
11189         return (zfs_version_print() != 0);
11190 }
11191
11192 /* Display documentation */
11193 static int
11194 zpool_do_help(int argc, char **argv)
11195 {
11196         char page[MAXNAMELEN];
11197         if (argc < 3 || strcmp(argv[2], "zpool") == 0)
11198                 strcpy(page, "zpool");
11199         else if (strcmp(argv[2], "concepts") == 0 ||
11200             strcmp(argv[2], "props") == 0)
11201                 snprintf(page, sizeof (page), "zpool%s", argv[2]);
11202         else
11203                 snprintf(page, sizeof (page), "zpool-%s", argv[2]);
11204
11205         execlp("man", "man", page, NULL);
11206
11207         fprintf(stderr, "couldn't run man program: %s", strerror(errno));
11208         return (-1);
11209 }
11210
11211 /*
11212  * Do zpool_load_compat() and print error message on failure
11213  */
11214 static zpool_compat_status_t
11215 zpool_do_load_compat(const char *compat, boolean_t *list)
11216 {
11217         char report[1024];
11218
11219         zpool_compat_status_t ret;
11220
11221         ret = zpool_load_compat(compat, list, report, 1024);
11222         switch (ret) {
11223
11224         case ZPOOL_COMPATIBILITY_OK:
11225                 break;
11226
11227         case ZPOOL_COMPATIBILITY_NOFILES:
11228         case ZPOOL_COMPATIBILITY_BADFILE:
11229         case ZPOOL_COMPATIBILITY_BADTOKEN:
11230                 (void) fprintf(stderr, "Error: %s\n", report);
11231                 break;
11232
11233         case ZPOOL_COMPATIBILITY_WARNTOKEN:
11234                 (void) fprintf(stderr, "Warning: %s\n", report);
11235                 ret = ZPOOL_COMPATIBILITY_OK;
11236                 break;
11237         }
11238         return (ret);
11239 }
11240
11241 int
11242 main(int argc, char **argv)
11243 {
11244         int ret = 0;
11245         int i = 0;
11246         char *cmdname;
11247         char **newargv;
11248
11249         (void) setlocale(LC_ALL, "");
11250         (void) setlocale(LC_NUMERIC, "C");
11251         (void) textdomain(TEXT_DOMAIN);
11252         srand(time(NULL));
11253
11254         opterr = 0;
11255
11256         /*
11257          * Make sure the user has specified some command.
11258          */
11259         if (argc < 2) {
11260                 (void) fprintf(stderr, gettext("missing command\n"));
11261                 usage(B_FALSE);
11262         }
11263
11264         cmdname = argv[1];
11265
11266         /*
11267          * Special case '-?'
11268          */
11269         if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
11270                 usage(B_TRUE);
11271
11272         /*
11273          * Special case '-V|--version'
11274          */
11275         if ((strcmp(cmdname, "-V") == 0) || (strcmp(cmdname, "--version") == 0))
11276                 return (zpool_do_version(argc, argv));
11277
11278         /*
11279          * Special case 'help'
11280          */
11281         if (strcmp(cmdname, "help") == 0)
11282                 return (zpool_do_help(argc, argv));
11283
11284         if ((g_zfs = libzfs_init()) == NULL) {
11285                 (void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
11286                 return (1);
11287         }
11288
11289         libzfs_print_on_error(g_zfs, B_TRUE);
11290
11291         zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
11292
11293         /*
11294          * Many commands modify input strings for string parsing reasons.
11295          * We create a copy to protect the original argv.
11296          */
11297         newargv = safe_malloc((argc + 1) * sizeof (newargv[0]));
11298         for (i = 0; i < argc; i++)
11299                 newargv[i] = strdup(argv[i]);
11300         newargv[argc] = NULL;
11301
11302         /*
11303          * Run the appropriate command.
11304          */
11305         if (find_command_idx(cmdname, &i) == 0) {
11306                 current_command = &command_table[i];
11307                 ret = command_table[i].func(argc - 1, newargv + 1);
11308         } else if (strchr(cmdname, '=')) {
11309                 verify(find_command_idx("set", &i) == 0);
11310                 current_command = &command_table[i];
11311                 ret = command_table[i].func(argc, newargv);
11312         } else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
11313                 /*
11314                  * 'freeze' is a vile debugging abomination, so we treat
11315                  * it as such.
11316                  */
11317                 zfs_cmd_t zc = {"\0"};
11318
11319                 (void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
11320                 ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
11321                 if (ret != 0) {
11322                         (void) fprintf(stderr,
11323                         gettext("failed to freeze pool: %d\n"), errno);
11324                         ret = 1;
11325                 }
11326
11327                 log_history = 0;
11328         } else {
11329                 (void) fprintf(stderr, gettext("unrecognized "
11330                     "command '%s'\n"), cmdname);
11331                 usage(B_FALSE);
11332                 ret = 1;
11333         }
11334
11335         for (i = 0; i < argc; i++)
11336                 free(newargv[i]);
11337         free(newargv);
11338
11339         if (ret == 0 && log_history)
11340                 (void) zpool_log_history(g_zfs, history_str);
11341
11342         libzfs_fini(g_zfs);
11343
11344         /*
11345          * The 'ZFS_ABORT' environment variable causes us to dump core on exit
11346          * for the purposes of running ::findleaks.
11347          */
11348         if (getenv("ZFS_ABORT") != NULL) {
11349                 (void) printf("dumping core by request\n");
11350                 abort();
11351         }
11352
11353         return (ret);
11354 }