4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
28 * This userland component takes a set of options and uses libzpool to translate
29 * from a user-visible object type and name to an internal representation.
30 * There are two basic types of faults: device faults and data faults.
35 * Errors can be injected into a particular vdev using the '-d' option. This
36 * option takes a path or vdev GUID to uniquely identify the device within a
37 * pool. There are two types of errors that can be injected, EIO and ENXIO,
38 * that can be controlled through the '-e' option. The default is ENXIO. For
39 * EIO failures, any attempt to read data from the device will return EIO, but
40 * subsequent attempt to reopen the device will succeed. For ENXIO failures,
41 * any attempt to read from the device will return EIO, but any attempt to
42 * reopen the device will also return ENXIO.
43 * For label faults, the -L option must be specified. This allows faults
44 * to be injected into either the nvlist, uberblock, pad1, or pad2 region
45 * of all the labels for the specified device.
47 * This form of the command looks like:
49 * zinject -d device [-e errno] [-L <uber | nvlist | pad1 | pad2>] pool
54 * We begin with a tuple of the form:
56 * <type,level,range,object>
58 * type A string describing the type of data to target. Each type
59 * implicitly describes how to interpret 'object'. Currently,
60 * the following values are supported:
62 * data User data for a file
63 * dnode Dnode for a file or directory
65 * The following MOS objects are special. Instead of injecting
66 * errors on a particular object or blkid, we inject errors across
67 * all objects of the given type.
69 * mos Any data in the MOS
70 * mosdir object directory
71 * config pool configuration
75 * errlog persistent error log
77 * level Object level. Defaults to '0', not applicable to all types. If
78 * a range is given, this corresponds to the indirect block
79 * corresponding to the specific range.
81 * range A numerical range [start,end) within the object. Defaults to
82 * the full size of the file.
84 * object A string describing the logical location of the object. For
85 * files and directories (currently the only supported types),
86 * this is the path of the object on disk.
88 * This is translated, via libzpool, into the following internal representation:
90 * <type,objset,object,level,range>
92 * These types should be self-explanatory. This tuple is then passed to the
93 * kernel via a special ioctl() to initiate fault injection for the given
94 * object. Note that 'type' is not strictly necessary for fault injection, but
95 * is used when translating existing faults into a human-readable string.
98 * The command itself takes one of the forms:
101 * zinject <-a | -u pool>
102 * zinject -c <id|all>
103 * zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
104 * [-r range] <object>
105 * zinject [-f freq] [-a] [-m] [-u] -b objset:object:level:start:end pool
107 * With no arguments, the command prints all currently registered injection
108 * handlers, with their numeric identifiers.
110 * The '-c' option will clear the given handler, or all handlers if 'all' is
113 * The '-e' option takes a string describing the errno to simulate. This must
114 * be either 'io' or 'checksum'. In most cases this will result in the same
115 * behavior, but RAID-Z will produce a different set of ereports for this
118 * The '-a', '-u', and '-m' flags toggle internal flush behavior. If '-a' is
119 * specified, then the ARC cache is flushed appropriately. If '-u' is
120 * specified, then the underlying SPA is unloaded. Either of these flags can be
121 * specified independently of any other handlers. The '-m' flag automatically
122 * does an unmount and remount of the underlying dataset to aid in flushing the
125 * The '-f' flag controls the frequency of errors injected, expressed as a
126 * integer percentage between 1 and 100. The default is 100.
128 * The this form is responsible for actually injecting the handler into the
129 * framework. It takes the arguments described above, translates them to the
130 * internal tuple using libzpool, and then issues an ioctl() to register the
133 * The final form can target a specific bookmark, regardless of whether a
134 * human-readable interface has been designed. It allows developers to specify
135 * a particular block by number.
145 #include <sys/fs/zfs.h>
146 #include <sys/param.h>
147 #include <sys/mount.h>
151 #undef verify /* both libzfs.h and zfs_context.h want to define this */
155 libzfs_handle_t *g_zfs;
162 static const char *errtable[TYPE_INVAL] = {
179 name_to_type(const char *arg)
182 for (i = 0; i < TYPE_INVAL; i++)
183 if (strcmp(errtable[i], arg) == 0)
190 type_to_name(uint64_t type)
193 case DMU_OT_OBJECT_DIRECTORY:
195 case DMU_OT_OBJECT_ARRAY:
197 case DMU_OT_PACKED_NVLIST:
201 case DMU_OT_SPACE_MAP:
203 case DMU_OT_ERROR_LOG:
212 * Print usage message.
222 "\t\tList all active injection records.\n"
224 "\tzinject -c <id|all>\n"
226 "\t\tClear the particular record (if given a numeric ID), or\n"
227 "\t\tall records if 'all' is specificed.\n"
229 "\tzinject -p <function name> pool\n"
230 "\t\tInject a panic fault at the specified function. Only \n"
231 "\t\tfunctions which call spa_vdev_config_exit(), or \n"
232 "\t\tspa_vdev_exit() will trigger a panic.\n"
234 "\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n"
235 "\t [-T <read|write|free|claim|all> pool\n"
236 "\t\tInject a fault into a particular device or the device's\n"
237 "\t\tlabel. Label injection can either be 'nvlist', 'uber',\n "
238 "\t\t'pad1', or 'pad2'.\n"
239 "\t\t'errno' can be 'nxio' (the default), 'io', or 'dtl'.\n"
241 "\tzinject -d device -A <degrade|fault> pool\n"
242 "\t\tPerform a specific action on a particular device\n"
244 "\tzinject -I [-s <seconds> | -g <txgs>] pool\n"
245 "\t\tCause the pool to stop writing blocks yet not\n"
246 "\t\treport errors for a duration. Simulates buggy hardware\n"
247 "\t\tthat fails to honor cache flush requests.\n"
248 "\t\tDefault duration is 30 seconds. The machine is panicked\n"
249 "\t\tat the end of the duration.\n"
251 "\tzinject -b objset:object:level:blkid pool\n"
253 "\t\tInject an error into pool 'pool' with the numeric bookmark\n"
254 "\t\tspecified by the remaining tuple. Each number is in\n"
255 "\t\thexidecimal, and only one block can be specified.\n"
257 "\tzinject [-q] <-t type> [-e errno] [-l level] [-r range]\n"
258 "\t [-a] [-m] [-u] [-f freq] <object>\n"
260 "\t\tInject an error into the object specified by the '-t' option\n"
261 "\t\tand the object descriptor. The 'object' parameter is\n"
262 "\t\tinterperted depending on the '-t' option.\n"
264 "\t\t-q\tQuiet mode. Only print out the handler number added.\n"
265 "\t\t-e\tInject a specific error. Must be either 'io' or\n"
266 "\t\t\t'checksum'. Default is 'io'.\n"
267 "\t\t-l\tInject error at a particular block level. Default is "
269 "\t\t-m\tAutomatically remount underlying filesystem.\n"
270 "\t\t-r\tInject error over a particular logical range of an\n"
271 "\t\t\tobject. Will be translated to the appropriate blkid\n"
272 "\t\t\trange according to the object's properties.\n"
273 "\t\t-a\tFlush the ARC cache. Can be specified without any\n"
274 "\t\t\tassociated object.\n"
275 "\t\t-u\tUnload the associated pool. Can be specified with only\n"
276 "\t\t\ta pool object.\n"
277 "\t\t-f\tOnly inject errors a fraction of the time. Expressed as\n"
278 "\t\t\ta percentage between 1 and 100.\n"
280 "\t-t data\t\tInject an error into the plain file contents of a\n"
281 "\t\t\tfile. The object must be specified as a complete path\n"
282 "\t\t\tto a file on a ZFS filesystem.\n"
284 "\t-t dnode\tInject an error into the metadnode in the block\n"
285 "\t\t\tcorresponding to the dnode for a file or directory. The\n"
286 "\t\t\t'-r' option is incompatible with this mode. The object\n"
287 "\t\t\tis specified as a complete path to a file or directory\n"
288 "\t\t\ton a ZFS filesystem.\n"
290 "\t-t <mos>\tInject errors into the MOS for objects of the given\n"
291 "\t\t\ttype. Valid types are: mos, mosdir, config, bpobj,\n"
292 "\t\t\tspacemap, metaslab, errlog. The only valid <object> is\n"
293 "\t\t\tthe poolname.\n");
297 iter_handlers(int (*func)(int, const char *, zinject_record_t *, void *),
300 zfs_cmd_t zc = { 0 };
303 while (ioctl(zfs_fd, ZFS_IOC_INJECT_LIST_NEXT, &zc) == 0)
304 if ((ret = func((int)zc.zc_guid, zc.zc_name,
305 &zc.zc_inject_record, data)) != 0)
308 if (errno != ENOENT) {
309 (void) fprintf(stderr, "Unable to list handlers: %s\n",
318 print_data_handler(int id, const char *pool, zinject_record_t *record,
323 if (record->zi_guid != 0 || record->zi_func[0] != '\0')
327 (void) printf("%3s %-15s %-6s %-6s %-8s %3s %-15s\n",
328 "ID", "POOL", "OBJSET", "OBJECT", "TYPE", "LVL", "RANGE");
329 (void) printf("--- --------------- ------ "
330 "------ -------- --- ---------------\n");
335 (void) printf("%3d %-15s %-6llu %-6llu %-8s %3d ", id, pool,
336 (u_longlong_t)record->zi_objset, (u_longlong_t)record->zi_object,
337 type_to_name(record->zi_type), record->zi_level);
339 if (record->zi_start == 0 &&
340 record->zi_end == -1ULL)
341 (void) printf("all\n");
343 (void) printf("[%llu, %llu]\n", (u_longlong_t)record->zi_start,
344 (u_longlong_t)record->zi_end);
350 print_device_handler(int id, const char *pool, zinject_record_t *record,
355 if (record->zi_guid == 0 || record->zi_func[0] != '\0')
359 (void) printf("%3s %-15s %s\n", "ID", "POOL", "GUID");
360 (void) printf("--- --------------- ----------------\n");
365 (void) printf("%3d %-15s %llx\n", id, pool,
366 (u_longlong_t)record->zi_guid);
372 print_panic_handler(int id, const char *pool, zinject_record_t *record,
377 if (record->zi_func[0] == '\0')
381 (void) printf("%3s %-15s %s\n", "ID", "POOL", "FUNCTION");
382 (void) printf("--- --------------- ----------------\n");
387 (void) printf("%3d %-15s %s\n", id, pool, record->zi_func);
393 * Print all registered error handlers. Returns the number of handlers
397 print_all_handlers(void)
399 int count = 0, total = 0;
401 (void) iter_handlers(print_device_handler, &count);
408 (void) iter_handlers(print_data_handler, &count);
415 (void) iter_handlers(print_panic_handler, &count);
417 return (count + total);
422 cancel_one_handler(int id, const char *pool, zinject_record_t *record,
425 zfs_cmd_t zc = { 0 };
427 zc.zc_guid = (uint64_t)id;
429 if (ioctl(zfs_fd, ZFS_IOC_CLEAR_FAULT, &zc) != 0) {
430 (void) fprintf(stderr, "failed to remove handler %d: %s\n",
431 id, strerror(errno));
439 * Remove all fault injection handlers.
442 cancel_all_handlers(void)
444 int ret = iter_handlers(cancel_one_handler, NULL);
447 (void) printf("removed all registered handlers\n");
453 * Remove a specific fault injection handler.
456 cancel_handler(int id)
458 zfs_cmd_t zc = { 0 };
460 zc.zc_guid = (uint64_t)id;
462 if (ioctl(zfs_fd, ZFS_IOC_CLEAR_FAULT, &zc) != 0) {
463 (void) fprintf(stderr, "failed to remove handler %d: %s\n",
464 id, strerror(errno));
468 (void) printf("removed handler %d\n", id);
474 * Register a new fault injection handler.
477 register_handler(const char *pool, int flags, zinject_record_t *record,
480 zfs_cmd_t zc = { 0 };
482 (void) strcpy(zc.zc_name, pool);
483 zc.zc_inject_record = *record;
486 if (ioctl(zfs_fd, ZFS_IOC_INJECT_FAULT, &zc) != 0) {
487 (void) fprintf(stderr, "failed to add handler: %s\n",
492 if (flags & ZINJECT_NULL)
496 (void) printf("%llu\n", (u_longlong_t)zc.zc_guid);
498 (void) printf("Added handler %llu with the following "
499 "properties:\n", (u_longlong_t)zc.zc_guid);
500 (void) printf(" pool: %s\n", pool);
501 if (record->zi_guid) {
502 (void) printf(" vdev: %llx\n",
503 (u_longlong_t)record->zi_guid);
504 } else if (record->zi_func[0] != '\0') {
505 (void) printf(" panic function: %s\n",
507 } else if (record->zi_duration > 0) {
508 (void) printf(" time: %lld seconds\n",
509 (u_longlong_t)record->zi_duration);
510 } else if (record->zi_duration < 0) {
511 (void) printf(" txgs: %lld \n",
512 (u_longlong_t)-record->zi_duration);
514 (void) printf("objset: %llu\n",
515 (u_longlong_t)record->zi_objset);
516 (void) printf("object: %llu\n",
517 (u_longlong_t)record->zi_object);
518 (void) printf(" type: %llu\n",
519 (u_longlong_t)record->zi_type);
520 (void) printf(" level: %d\n", record->zi_level);
521 if (record->zi_start == 0 &&
522 record->zi_end == -1ULL)
523 (void) printf(" range: all\n");
525 (void) printf(" range: [%llu, %llu)\n",
526 (u_longlong_t)record->zi_start,
527 (u_longlong_t)record->zi_end);
535 perform_action(const char *pool, zinject_record_t *record, int cmd)
537 zfs_cmd_t zc = { 0 };
539 ASSERT(cmd == VDEV_STATE_DEGRADED || cmd == VDEV_STATE_FAULTED);
540 (void) strlcpy(zc.zc_name, pool, sizeof (zc.zc_name));
541 zc.zc_guid = record->zi_guid;
544 if (ioctl(zfs_fd, ZFS_IOC_VDEV_SET_STATE, &zc) == 0)
551 main(int argc, char **argv)
563 int io_type = ZIO_TYPES;
564 int action = VDEV_STATE_UNKNOWN;
565 err_type_t type = TYPE_INVAL;
566 err_type_t label = TYPE_INVAL;
567 zinject_record_t record = { 0 };
568 char pool[MAXNAMELEN];
569 char dataset[MAXNAMELEN];
577 if ((g_zfs = libzfs_init()) == NULL) {
578 (void) fprintf(stderr, "internal error: failed to "
579 "initialize ZFS library\n");
583 libzfs_print_on_error(g_zfs, B_TRUE);
585 if ((zfs_fd = open(ZFS_DEV, O_RDWR)) < 0) {
586 (void) fprintf(stderr, "failed to open ZFS device\n");
592 * No arguments. Print the available handlers. If there are no
593 * available handlers, direct the user to '-h' for help
596 if (print_all_handlers() == 0) {
597 (void) printf("No handlers registered.\n");
598 (void) printf("Run 'zinject -h' for usage "
605 while ((c = getopt(argc, argv,
606 ":aA:b:d:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:")) != -1) {
609 flags |= ZINJECT_FLUSH_ARC;
612 if (strcasecmp(optarg, "degrade") == 0) {
613 action = VDEV_STATE_DEGRADED;
614 } else if (strcasecmp(optarg, "fault") == 0) {
615 action = VDEV_STATE_FAULTED;
617 (void) fprintf(stderr, "invalid action '%s': "
618 "must be 'degrade' or 'fault'\n", optarg);
633 if (strcasecmp(optarg, "io") == 0) {
635 } else if (strcasecmp(optarg, "checksum") == 0) {
637 } else if (strcasecmp(optarg, "nxio") == 0) {
639 } else if (strcasecmp(optarg, "dtl") == 0) {
642 (void) fprintf(stderr, "invalid error type "
643 "'%s': must be 'io', 'checksum' or "
650 record.zi_freq = atoi(optarg);
651 if (record.zi_freq < 1 || record.zi_freq > 100) {
652 (void) fprintf(stderr, "frequency range must "
653 "be in the range (0, 100]\n");
658 record.zi_failfast = B_TRUE;
662 record.zi_duration = (int)strtol(optarg, &end, 10);
663 if (record.zi_duration <= 0 || *end != '\0') {
664 (void) fprintf(stderr, "invalid duration '%s': "
665 "must be a positive integer\n", optarg);
669 /* store duration of txgs as its negative */
670 record.zi_duration *= -1;
676 /* default duration, if one hasn't yet been defined */
678 if (dur_secs == 0 && dur_txg == 0)
679 record.zi_duration = 30;
682 level = (int)strtol(optarg, &end, 10);
684 (void) fprintf(stderr, "invalid level '%s': "
685 "must be an integer\n", optarg);
694 (void) strlcpy(record.zi_func, optarg,
695 sizeof (record.zi_func));
705 record.zi_duration = (int)strtol(optarg, &end, 10);
706 if (record.zi_duration <= 0 || *end != '\0') {
707 (void) fprintf(stderr, "invalid duration '%s': "
708 "must be a positive integer\n", optarg);
714 if (strcasecmp(optarg, "read") == 0) {
715 io_type = ZIO_TYPE_READ;
716 } else if (strcasecmp(optarg, "write") == 0) {
717 io_type = ZIO_TYPE_WRITE;
718 } else if (strcasecmp(optarg, "free") == 0) {
719 io_type = ZIO_TYPE_FREE;
720 } else if (strcasecmp(optarg, "claim") == 0) {
721 io_type = ZIO_TYPE_CLAIM;
722 } else if (strcasecmp(optarg, "all") == 0) {
725 (void) fprintf(stderr, "invalid I/O type "
726 "'%s': must be 'read', 'write', 'free', "
727 "'claim' or 'all'\n", optarg);
733 if ((type = name_to_type(optarg)) == TYPE_INVAL &&
735 (void) fprintf(stderr, "invalid type '%s'\n",
742 flags |= ZINJECT_UNLOAD_SPA;
745 if ((label = name_to_type(optarg)) == TYPE_INVAL &&
747 (void) fprintf(stderr, "invalid label type "
754 (void) fprintf(stderr, "option -%c requires an "
755 "operand\n", optopt);
759 (void) fprintf(stderr, "invalid option '%c'\n",
769 if (cancel != NULL) {
771 * '-c' is invalid with any other options.
773 if (raw != NULL || range != NULL || type != TYPE_INVAL ||
774 level != 0 || record.zi_func[0] != '\0' ||
775 record.zi_duration != 0) {
776 (void) fprintf(stderr, "cancel (-c) incompatible with "
777 "any other options\n");
782 (void) fprintf(stderr, "extraneous argument to '-c'\n");
787 if (strcmp(cancel, "all") == 0) {
788 return (cancel_all_handlers());
790 int id = (int)strtol(cancel, &end, 10);
792 (void) fprintf(stderr, "invalid handle id '%s':"
793 " must be an integer or 'all'\n", cancel);
797 return (cancel_handler(id));
801 if (device != NULL) {
803 * Device (-d) injection uses a completely different mechanism
804 * for doing injection, so handle it separately here.
806 if (raw != NULL || range != NULL || type != TYPE_INVAL ||
807 level != 0 || record.zi_func[0] != '\0' ||
808 record.zi_duration != 0) {
809 (void) fprintf(stderr, "device (-d) incompatible with "
810 "data error injection\n");
816 (void) fprintf(stderr, "device (-d) injection requires "
817 "a single pool name\n");
822 (void) strcpy(pool, argv[0]);
825 if (error == ECKSUM) {
826 (void) fprintf(stderr, "device error type must be "
831 record.zi_iotype = io_type;
832 if (translate_device(pool, device, label, &record) != 0)
837 if (action != VDEV_STATE_UNKNOWN)
838 return (perform_action(pool, &record, action));
840 } else if (raw != NULL) {
841 if (range != NULL || type != TYPE_INVAL || level != 0 ||
842 record.zi_func[0] != '\0' || record.zi_duration != 0) {
843 (void) fprintf(stderr, "raw (-b) format with "
844 "any other options\n");
850 (void) fprintf(stderr, "raw (-b) format expects a "
851 "single pool name\n");
856 (void) strcpy(pool, argv[0]);
859 if (error == ENXIO) {
860 (void) fprintf(stderr, "data error type must be "
861 "'checksum' or 'io'\n");
865 if (translate_raw(raw, &record) != 0)
869 } else if (record.zi_func[0] != '\0') {
870 if (raw != NULL || range != NULL || type != TYPE_INVAL ||
871 level != 0 || device != NULL || record.zi_duration != 0) {
872 (void) fprintf(stderr, "panic (-p) incompatible with "
878 if (argc < 1 || argc > 2) {
879 (void) fprintf(stderr, "panic (-p) injection requires "
880 "a single pool name and an optional id\n");
885 (void) strcpy(pool, argv[0]);
887 record.zi_type = atoi(argv[1]);
889 } else if (record.zi_duration != 0) {
891 (void) fprintf(stderr, "-s or -g meaningless "
892 "without -I (ignore writes)\n");
895 } else if (dur_secs && dur_txg) {
896 (void) fprintf(stderr, "choose a duration either "
897 "in seconds (-s) or a number of txgs (-g) "
901 } else if (argc != 1) {
902 (void) fprintf(stderr, "ignore writes (-I) "
903 "injection requires a single pool name\n");
908 (void) strcpy(pool, argv[0]);
910 } else if (type == TYPE_INVAL) {
912 (void) fprintf(stderr, "at least one of '-b', '-d', "
913 "'-t', '-a', '-p', '-I' or '-u' "
914 "must be specified\n");
919 if (argc == 1 && (flags & ZINJECT_UNLOAD_SPA)) {
920 (void) strcpy(pool, argv[0]);
922 } else if (argc != 0) {
923 (void) fprintf(stderr, "extraneous argument for "
929 flags |= ZINJECT_NULL;
932 (void) fprintf(stderr, "missing object\n");
937 if (error == ENXIO) {
938 (void) fprintf(stderr, "data error type must be "
939 "'checksum' or 'io'\n");
943 if (translate_record(type, argv[0], range, level, &record, pool,
951 * If this is pool-wide metadata, unmount everything. The ioctl() will
952 * unload the pool, so that we trigger spa-wide reopen of metadata next
953 * time we access the pool.
955 if (dataset[0] != '\0' && domount) {
956 if ((zhp = zfs_open(g_zfs, dataset, ZFS_TYPE_DATASET)) == NULL)
959 if (zfs_unmount(zhp, NULL, 0) != 0)
963 record.zi_error = error;
965 ret = register_handler(pool, flags, &record, quiet);
967 if (dataset[0] != '\0' && domount)
968 ret = (zfs_mount(zhp, NULL, 0) != 0);