4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
27 * Copyright (c) 2012,2020 by Delphix. All rights reserved.
31 #include <sys/spa_impl.h>
33 #include <sys/vdev_impl.h>
35 #include <sys/zio_checksum.h>
37 #include <sys/fm/fs/zfs.h>
38 #include <sys/fm/protocol.h>
39 #include <sys/fm/util.h>
40 #include <sys/sysevent.h>
43 * This general routine is responsible for generating all the different ZFS
44 * ereports. The payload is dependent on the class, and which arguments are
45 * supplied to the function:
47 * EREPORT POOL VDEV IO
53 * If we are in a loading state, all errors are chained together by the same
54 * SPA-wide ENA (Error Numeric Association).
56 * For isolated I/O requests, we get the ENA from the zio_t. The propagation
57 * gets very complicated due to RAID-Z, gang blocks, and vdev caching. We want
58 * to chain together all ereports associated with a logical piece of data. For
59 * read I/Os, there are basically three 'types' of I/O, which form a roughly
63 * | Aggregate I/O | No associated logical data or device
67 * +---------------+ Reads associated with a piece of logical data.
68 * | Read I/O | This includes reads on behalf of RAID-Z,
69 * +---------------+ mirrors, gang blocks, retries, etc.
72 * +---------------+ Reads associated with a particular device, but
73 * | Physical I/O | no logical data. Issued as part of vdev caching
74 * +---------------+ and I/O aggregation.
76 * Note that 'physical I/O' here is not the same terminology as used in the rest
77 * of ZIO. Typically, 'physical I/O' simply means that there is no attached
78 * blockpointer. But I/O with no associated block pointer can still be related
79 * to a logical piece of data (i.e. RAID-Z requests).
81 * Purely physical I/O always have unique ENAs. They are not related to a
82 * particular piece of logical data, and therefore cannot be chained together.
83 * We still generate an ereport, but the DE doesn't correlate it with any
84 * logical piece of data. When such an I/O fails, the delegated I/O requests
85 * will issue a retry, which will trigger the 'real' ereport with the correct
88 * We keep track of the ENA for a ZIO chain through the 'io_logical' member.
89 * When a new logical I/O is issued, we set this to point to itself. Child I/Os
90 * then inherit this pointer, so that when it is first set subsequent failures
91 * will use the same ENA. For vdev cache fill and queue aggregation I/O,
92 * this pointer is set to NULL, and no ereport will be generated (since it
93 * doesn't actually correspond to any particular device or piece of data,
94 * and the caller will always retry without caching or queueing anyway).
96 * For checksum errors, we want to include more information about the actual
97 * error which occurs. Accordingly, we build an ereport when the error is
98 * noticed, but instead of sending it in immediately, we hang it off of the
99 * io_cksum_report field of the logical IO. When the logical IO completes
100 * (successfully or not), zfs_ereport_finish_checksum() is called with the
101 * good and bad versions of the buffer (if available), and we annotate the
102 * ereport with information about the differences.
107 * Duplicate ereport Detection
109 * Some ereports are retained momentarily for detecting duplicates. These
110 * are kept in a recent_events_node_t in both a time-ordered list and an AVL
111 * tree of recent unique ereports.
113 * The lifespan of these recent ereports is bounded (15 mins) and a cleaner
114 * task is used to purge stale entries.
116 static list_t recent_events_list;
117 static avl_tree_t recent_events_tree;
118 static kmutex_t recent_events_lock;
119 static taskqid_t recent_events_cleaner_tqid;
122 * Each node is about 128 bytes so 2,000 would consume 1/4 MiB.
124 * This setting can be changed dynamically and setting it to zero
125 * disables duplicate detection.
127 unsigned int zfs_zevent_retain_max = 2000;
130 * The lifespan for a recent ereport entry. The default of 15 minutes is
131 * intended to outlive the zfs diagnosis engine's threshold of 10 errors
132 * over a period of 10 minutes.
134 unsigned int zfs_zevent_retain_expire_secs = 900;
136 typedef enum zfs_subclass {
143 /* common criteria */
144 uint64_t re_pool_guid;
145 uint64_t re_vdev_guid;
148 uint64_t re_io_offset;
149 zfs_subclass_t re_subclass;
150 zio_priority_t re_io_priority;
152 /* logical zio criteria (optional) */
153 zbookmark_phys_t re_io_bookmark;
156 avl_node_t re_tree_link;
157 list_node_t re_list_link;
158 uint64_t re_timestamp;
159 } recent_events_node_t;
162 recent_events_compare(const void *a, const void *b)
164 const recent_events_node_t *node1 = a;
165 const recent_events_node_t *node2 = b;
169 * The comparison order here is somewhat arbitrary.
170 * What's important is that if every criteria matches, then it
171 * is a duplicate (i.e. compare returns 0)
173 if ((cmp = TREE_CMP(node1->re_subclass, node2->re_subclass)) != 0)
175 if ((cmp = TREE_CMP(node1->re_pool_guid, node2->re_pool_guid)) != 0)
177 if ((cmp = TREE_CMP(node1->re_vdev_guid, node2->re_vdev_guid)) != 0)
179 if ((cmp = TREE_CMP(node1->re_io_error, node2->re_io_error)) != 0)
181 if ((cmp = TREE_CMP(node1->re_io_priority, node2->re_io_priority)) != 0)
183 if ((cmp = TREE_CMP(node1->re_io_size, node2->re_io_size)) != 0)
185 if ((cmp = TREE_CMP(node1->re_io_offset, node2->re_io_offset)) != 0)
188 const zbookmark_phys_t *zb1 = &node1->re_io_bookmark;
189 const zbookmark_phys_t *zb2 = &node2->re_io_bookmark;
191 if ((cmp = TREE_CMP(zb1->zb_objset, zb2->zb_objset)) != 0)
193 if ((cmp = TREE_CMP(zb1->zb_object, zb2->zb_object)) != 0)
195 if ((cmp = TREE_CMP(zb1->zb_level, zb2->zb_level)) != 0)
197 if ((cmp = TREE_CMP(zb1->zb_blkid, zb2->zb_blkid)) != 0)
203 static void zfs_ereport_schedule_cleaner(void);
206 * background task to clean stale recent event nodes.
210 zfs_ereport_cleaner(void *arg)
212 recent_events_node_t *entry;
213 uint64_t now = gethrtime();
216 * purge expired entries
218 mutex_enter(&recent_events_lock);
219 while ((entry = list_tail(&recent_events_list)) != NULL) {
220 uint64_t age = NSEC2SEC(now - entry->re_timestamp);
221 if (age <= zfs_zevent_retain_expire_secs)
224 /* remove expired node */
225 avl_remove(&recent_events_tree, entry);
226 list_remove(&recent_events_list, entry);
227 kmem_free(entry, sizeof (*entry));
230 /* Restart the cleaner if more entries remain */
231 recent_events_cleaner_tqid = 0;
232 if (!list_is_empty(&recent_events_list))
233 zfs_ereport_schedule_cleaner();
235 mutex_exit(&recent_events_lock);
239 zfs_ereport_schedule_cleaner(void)
241 ASSERT(MUTEX_HELD(&recent_events_lock));
243 uint64_t timeout = SEC2NSEC(zfs_zevent_retain_expire_secs + 1);
245 recent_events_cleaner_tqid = taskq_dispatch_delay(
246 system_delay_taskq, zfs_ereport_cleaner, NULL, TQ_SLEEP,
247 ddi_get_lbolt() + NSEC_TO_TICK(timeout));
251 * Check if an ereport would be a duplicate of one recently posted.
253 * An ereport is considered a duplicate if the set of criteria in
254 * recent_events_node_t all match.
256 * Only FM_EREPORT_ZFS_IO, FM_EREPORT_ZFS_DATA, and FM_EREPORT_ZFS_CHECKSUM
257 * are candidates for duplicate checking.
260 zfs_ereport_is_duplicate(const char *subclass, spa_t *spa, vdev_t *vd,
261 const zbookmark_phys_t *zb, zio_t *zio, uint64_t offset, uint64_t size)
263 recent_events_node_t search = {0}, *entry;
265 if (vd == NULL || zio == NULL)
268 if (zfs_zevent_retain_max == 0)
271 if (strcmp(subclass, FM_EREPORT_ZFS_IO) == 0)
272 search.re_subclass = ZSC_IO;
273 else if (strcmp(subclass, FM_EREPORT_ZFS_DATA) == 0)
274 search.re_subclass = ZSC_DATA;
275 else if (strcmp(subclass, FM_EREPORT_ZFS_CHECKSUM) == 0)
276 search.re_subclass = ZSC_CHECKSUM;
280 search.re_pool_guid = spa_guid(spa);
281 search.re_vdev_guid = vd->vdev_guid;
282 search.re_io_error = zio->io_error;
283 search.re_io_priority = zio->io_priority;
284 /* if size is supplied use it over what's in zio */
286 search.re_io_size = size;
287 search.re_io_offset = offset;
289 search.re_io_size = zio->io_size;
290 search.re_io_offset = zio->io_offset;
293 /* grab optional logical zio criteria */
295 search.re_io_bookmark.zb_objset = zb->zb_objset;
296 search.re_io_bookmark.zb_object = zb->zb_object;
297 search.re_io_bookmark.zb_level = zb->zb_level;
298 search.re_io_bookmark.zb_blkid = zb->zb_blkid;
301 uint64_t now = gethrtime();
303 mutex_enter(&recent_events_lock);
305 /* check if we have seen this one recently */
306 entry = avl_find(&recent_events_tree, &search, NULL);
308 uint64_t age = NSEC2SEC(now - entry->re_timestamp);
311 * There is still an active cleaner (since we're here).
312 * Reset the last seen time for this duplicate entry
313 * so that its lifespand gets extended.
315 list_remove(&recent_events_list, entry);
316 list_insert_head(&recent_events_list, entry);
317 entry->re_timestamp = now;
319 zfs_zevent_track_duplicate();
320 mutex_exit(&recent_events_lock);
322 return (age <= zfs_zevent_retain_expire_secs);
325 if (avl_numnodes(&recent_events_tree) >= zfs_zevent_retain_max) {
326 /* recycle oldest node */
327 entry = list_tail(&recent_events_list);
328 ASSERT(entry != NULL);
329 list_remove(&recent_events_list, entry);
330 avl_remove(&recent_events_tree, entry);
332 entry = kmem_alloc(sizeof (recent_events_node_t), KM_SLEEP);
335 /* record this as a recent ereport */
337 avl_add(&recent_events_tree, entry);
338 list_insert_head(&recent_events_list, entry);
339 entry->re_timestamp = now;
341 /* Start a cleaner if not already scheduled */
342 if (recent_events_cleaner_tqid == 0)
343 zfs_ereport_schedule_cleaner();
345 mutex_exit(&recent_events_lock);
350 zfs_zevent_post_cb(nvlist_t *nvl, nvlist_t *detector)
353 fm_nvlist_destroy(nvl, FM_NVA_FREE);
356 fm_nvlist_destroy(detector, FM_NVA_FREE);
360 * We want to rate limit ZIO delay and checksum events so as to not
361 * flood ZED when a disk is acting up.
363 * Returns 1 if we're ratelimiting, 0 if not.
366 zfs_is_ratelimiting_event(const char *subclass, vdev_t *vd)
370 * __ratelimit() returns 1 if we're *not* ratelimiting and 0 if we
371 * are. Invert it to get our return value.
373 if (strcmp(subclass, FM_EREPORT_ZFS_DELAY) == 0) {
374 rc = !zfs_ratelimit(&vd->vdev_delay_rl);
375 } else if (strcmp(subclass, FM_EREPORT_ZFS_CHECKSUM) == 0) {
376 rc = !zfs_ratelimit(&vd->vdev_checksum_rl);
380 /* We're rate limiting */
381 fm_erpt_dropped_increment();
388 * Return B_TRUE if the event actually posted, B_FALSE if not.
391 zfs_ereport_start(nvlist_t **ereport_out, nvlist_t **detector_out,
392 const char *subclass, spa_t *spa, vdev_t *vd, const zbookmark_phys_t *zb,
393 zio_t *zio, uint64_t stateoroffset, uint64_t size)
395 nvlist_t *ereport, *detector;
400 if ((ereport = fm_nvlist_create(NULL)) == NULL)
403 if ((detector = fm_nvlist_create(NULL)) == NULL) {
404 fm_nvlist_destroy(ereport, FM_NVA_FREE);
409 * Serialize ereport generation
411 mutex_enter(&spa->spa_errlist_lock);
414 * Determine the ENA to use for this event. If we are in a loading
415 * state, use a SPA-wide ENA. Otherwise, if we are in an I/O state, use
416 * a root zio-wide ENA. Otherwise, simply use a unique ENA.
418 if (spa_load_state(spa) != SPA_LOAD_NONE) {
419 if (spa->spa_ena == 0)
420 spa->spa_ena = fm_ena_generate(0, FM_ENA_FMT1);
422 } else if (zio != NULL && zio->io_logical != NULL) {
423 if (zio->io_logical->io_ena == 0)
424 zio->io_logical->io_ena =
425 fm_ena_generate(0, FM_ENA_FMT1);
426 ena = zio->io_logical->io_ena;
428 ena = fm_ena_generate(0, FM_ENA_FMT1);
432 * Construct the full class, detector, and other standard FMA fields.
434 (void) snprintf(class, sizeof (class), "%s.%s",
435 ZFS_ERROR_CLASS, subclass);
437 fm_fmri_zfs_set(detector, FM_ZFS_SCHEME_VERSION, spa_guid(spa),
438 vd != NULL ? vd->vdev_guid : 0);
440 fm_ereport_set(ereport, FM_EREPORT_VERSION, class, ena, detector, NULL);
443 * Construct the per-ereport payload, depending on which parameters are
448 * Generic payload members common to all ereports.
450 fm_payload_set(ereport,
451 FM_EREPORT_PAYLOAD_ZFS_POOL, DATA_TYPE_STRING, spa_name(spa),
452 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, DATA_TYPE_UINT64, spa_guid(spa),
453 FM_EREPORT_PAYLOAD_ZFS_POOL_STATE, DATA_TYPE_UINT64,
454 (uint64_t)spa_state(spa),
455 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, DATA_TYPE_INT32,
456 (int32_t)spa_load_state(spa), NULL);
458 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE,
460 spa_get_failmode(spa) == ZIO_FAILURE_MODE_WAIT ?
461 FM_EREPORT_FAILMODE_WAIT :
462 spa_get_failmode(spa) == ZIO_FAILURE_MODE_CONTINUE ?
463 FM_EREPORT_FAILMODE_CONTINUE : FM_EREPORT_FAILMODE_PANIC,
467 vdev_t *pvd = vd->vdev_parent;
468 vdev_queue_t *vq = &vd->vdev_queue;
469 vdev_stat_t *vs = &vd->vdev_stat;
471 uint64_t *spare_guids;
475 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
476 DATA_TYPE_UINT64, vd->vdev_guid,
477 FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
478 DATA_TYPE_STRING, vd->vdev_ops->vdev_op_type, NULL);
479 if (vd->vdev_path != NULL)
480 fm_payload_set(ereport,
481 FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH,
482 DATA_TYPE_STRING, vd->vdev_path, NULL);
483 if (vd->vdev_devid != NULL)
484 fm_payload_set(ereport,
485 FM_EREPORT_PAYLOAD_ZFS_VDEV_DEVID,
486 DATA_TYPE_STRING, vd->vdev_devid, NULL);
487 if (vd->vdev_fru != NULL)
488 fm_payload_set(ereport,
489 FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU,
490 DATA_TYPE_STRING, vd->vdev_fru, NULL);
491 if (vd->vdev_enc_sysfs_path != NULL)
492 fm_payload_set(ereport,
493 FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH,
494 DATA_TYPE_STRING, vd->vdev_enc_sysfs_path, NULL);
496 fm_payload_set(ereport,
497 FM_EREPORT_PAYLOAD_ZFS_VDEV_ASHIFT,
498 DATA_TYPE_UINT64, vd->vdev_ashift, NULL);
501 fm_payload_set(ereport,
502 FM_EREPORT_PAYLOAD_ZFS_VDEV_COMP_TS,
503 DATA_TYPE_UINT64, vq->vq_io_complete_ts, NULL);
504 fm_payload_set(ereport,
505 FM_EREPORT_PAYLOAD_ZFS_VDEV_DELTA_TS,
506 DATA_TYPE_UINT64, vq->vq_io_delta_ts, NULL);
510 fm_payload_set(ereport,
511 FM_EREPORT_PAYLOAD_ZFS_VDEV_READ_ERRORS,
512 DATA_TYPE_UINT64, vs->vs_read_errors,
513 FM_EREPORT_PAYLOAD_ZFS_VDEV_WRITE_ERRORS,
514 DATA_TYPE_UINT64, vs->vs_write_errors,
515 FM_EREPORT_PAYLOAD_ZFS_VDEV_CKSUM_ERRORS,
516 DATA_TYPE_UINT64, vs->vs_checksum_errors,
517 FM_EREPORT_PAYLOAD_ZFS_VDEV_DELAYS,
518 DATA_TYPE_UINT64, vs->vs_slow_ios,
523 fm_payload_set(ereport,
524 FM_EREPORT_PAYLOAD_ZFS_PARENT_GUID,
525 DATA_TYPE_UINT64, pvd->vdev_guid,
526 FM_EREPORT_PAYLOAD_ZFS_PARENT_TYPE,
527 DATA_TYPE_STRING, pvd->vdev_ops->vdev_op_type,
530 fm_payload_set(ereport,
531 FM_EREPORT_PAYLOAD_ZFS_PARENT_PATH,
532 DATA_TYPE_STRING, pvd->vdev_path, NULL);
534 fm_payload_set(ereport,
535 FM_EREPORT_PAYLOAD_ZFS_PARENT_DEVID,
536 DATA_TYPE_STRING, pvd->vdev_devid, NULL);
539 spare_count = spa->spa_spares.sav_count;
540 spare_paths = kmem_zalloc(sizeof (char *) * spare_count,
542 spare_guids = kmem_zalloc(sizeof (uint64_t) * spare_count,
545 for (i = 0; i < spare_count; i++) {
546 spare_vd = spa->spa_spares.sav_vdevs[i];
548 spare_paths[i] = spare_vd->vdev_path;
549 spare_guids[i] = spare_vd->vdev_guid;
553 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_VDEV_SPARE_PATHS,
554 DATA_TYPE_STRING_ARRAY, spare_count, spare_paths,
555 FM_EREPORT_PAYLOAD_ZFS_VDEV_SPARE_GUIDS,
556 DATA_TYPE_UINT64_ARRAY, spare_count, spare_guids, NULL);
558 kmem_free(spare_guids, sizeof (uint64_t) * spare_count);
559 kmem_free(spare_paths, sizeof (char *) * spare_count);
564 * Payload common to all I/Os.
566 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_ERR,
567 DATA_TYPE_INT32, zio->io_error, NULL);
568 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_FLAGS,
569 DATA_TYPE_INT32, zio->io_flags, NULL);
570 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_STAGE,
571 DATA_TYPE_UINT32, zio->io_stage, NULL);
572 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_PIPELINE,
573 DATA_TYPE_UINT32, zio->io_pipeline, NULL);
574 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_DELAY,
575 DATA_TYPE_UINT64, zio->io_delay, NULL);
576 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_TIMESTAMP,
577 DATA_TYPE_UINT64, zio->io_timestamp, NULL);
578 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_DELTA,
579 DATA_TYPE_UINT64, zio->io_delta, NULL);
580 fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_ZIO_PRIORITY,
581 DATA_TYPE_UINT32, zio->io_priority, NULL);
584 * If the 'size' parameter is non-zero, it indicates this is a
585 * RAID-Z or other I/O where the physical offset and length are
586 * provided for us, instead of within the zio_t.
590 fm_payload_set(ereport,
591 FM_EREPORT_PAYLOAD_ZFS_ZIO_OFFSET,
592 DATA_TYPE_UINT64, stateoroffset,
593 FM_EREPORT_PAYLOAD_ZFS_ZIO_SIZE,
594 DATA_TYPE_UINT64, size, NULL);
596 fm_payload_set(ereport,
597 FM_EREPORT_PAYLOAD_ZFS_ZIO_OFFSET,
598 DATA_TYPE_UINT64, zio->io_offset,
599 FM_EREPORT_PAYLOAD_ZFS_ZIO_SIZE,
600 DATA_TYPE_UINT64, zio->io_size, NULL);
602 } else if (vd != NULL) {
604 * If we have a vdev but no zio, this is a device fault, and the
605 * 'stateoroffset' parameter indicates the previous state of the
608 fm_payload_set(ereport,
609 FM_EREPORT_PAYLOAD_ZFS_PREV_STATE,
610 DATA_TYPE_UINT64, stateoroffset, NULL);
614 * Payload for I/Os with corresponding logical information.
616 if (zb != NULL && (zio == NULL || zio->io_logical != NULL)) {
617 fm_payload_set(ereport,
618 FM_EREPORT_PAYLOAD_ZFS_ZIO_OBJSET,
619 DATA_TYPE_UINT64, zb->zb_objset,
620 FM_EREPORT_PAYLOAD_ZFS_ZIO_OBJECT,
621 DATA_TYPE_UINT64, zb->zb_object,
622 FM_EREPORT_PAYLOAD_ZFS_ZIO_LEVEL,
623 DATA_TYPE_INT64, zb->zb_level,
624 FM_EREPORT_PAYLOAD_ZFS_ZIO_BLKID,
625 DATA_TYPE_UINT64, zb->zb_blkid, NULL);
628 mutex_exit(&spa->spa_errlist_lock);
630 *ereport_out = ereport;
631 *detector_out = detector;
635 /* if it's <= 128 bytes, save the corruption directly */
636 #define ZFM_MAX_INLINE (128 / sizeof (uint64_t))
638 #define MAX_RANGES 16
640 typedef struct zfs_ecksum_info {
641 /* histograms of set and cleared bits by bit number in a 64-bit word */
642 uint32_t zei_histogram_set[sizeof (uint64_t) * NBBY];
643 uint32_t zei_histogram_cleared[sizeof (uint64_t) * NBBY];
645 /* inline arrays of bits set and cleared. */
646 uint64_t zei_bits_set[ZFM_MAX_INLINE];
647 uint64_t zei_bits_cleared[ZFM_MAX_INLINE];
650 * for each range, the number of bits set and cleared. The Hamming
651 * distance between the good and bad buffers is the sum of them all.
653 uint32_t zei_range_sets[MAX_RANGES];
654 uint32_t zei_range_clears[MAX_RANGES];
659 } zei_ranges[MAX_RANGES];
661 size_t zei_range_count;
663 uint32_t zei_allowed_mingap;
668 update_histogram(uint64_t value_arg, uint32_t *hist, uint32_t *count)
672 uint64_t value = BE_64(value_arg);
674 /* We store the bits in big-endian (largest-first) order */
675 for (i = 0; i < 64; i++) {
676 if (value & (1ull << i)) {
681 /* update the count of bits changed */
686 * We've now filled up the range array, and need to increase "mingap" and
687 * shrink the range list accordingly. zei_mingap is always the smallest
688 * distance between array entries, so we set the new_allowed_gap to be
689 * one greater than that. We then go through the list, joining together
690 * any ranges which are closer than the new_allowed_gap.
692 * By construction, there will be at least one. We also update zei_mingap
693 * to the new smallest gap, to prepare for our next invocation.
696 zei_shrink_ranges(zfs_ecksum_info_t *eip)
698 uint32_t mingap = UINT32_MAX;
699 uint32_t new_allowed_gap = eip->zei_mingap + 1;
702 size_t max = eip->zei_range_count;
704 struct zei_ranges *r = eip->zei_ranges;
706 ASSERT3U(eip->zei_range_count, >, 0);
707 ASSERT3U(eip->zei_range_count, <=, MAX_RANGES);
710 while (idx < max - 1) {
711 uint32_t start = r[idx].zr_start;
712 uint32_t end = r[idx].zr_end;
714 while (idx < max - 1) {
717 uint32_t nstart = r[idx].zr_start;
718 uint32_t nend = r[idx].zr_end;
720 uint32_t gap = nstart - end;
721 if (gap < new_allowed_gap) {
729 r[output].zr_start = start;
730 r[output].zr_end = end;
733 ASSERT3U(output, <, eip->zei_range_count);
734 eip->zei_range_count = output;
735 eip->zei_mingap = mingap;
736 eip->zei_allowed_mingap = new_allowed_gap;
740 zei_add_range(zfs_ecksum_info_t *eip, int start, int end)
742 struct zei_ranges *r = eip->zei_ranges;
743 size_t count = eip->zei_range_count;
745 if (count >= MAX_RANGES) {
746 zei_shrink_ranges(eip);
747 count = eip->zei_range_count;
750 eip->zei_mingap = UINT32_MAX;
751 eip->zei_allowed_mingap = 1;
753 int gap = start - r[count - 1].zr_end;
755 if (gap < eip->zei_allowed_mingap) {
756 r[count - 1].zr_end = end;
759 if (gap < eip->zei_mingap)
760 eip->zei_mingap = gap;
762 r[count].zr_start = start;
763 r[count].zr_end = end;
764 eip->zei_range_count++;
768 zei_range_total_size(zfs_ecksum_info_t *eip)
770 struct zei_ranges *r = eip->zei_ranges;
771 size_t count = eip->zei_range_count;
775 for (idx = 0; idx < count; idx++)
776 result += (r[idx].zr_end - r[idx].zr_start);
781 static zfs_ecksum_info_t *
782 annotate_ecksum(nvlist_t *ereport, zio_bad_cksum_t *info,
783 const abd_t *goodabd, const abd_t *badabd, size_t size,
784 boolean_t drop_if_identical)
786 const uint64_t *good;
790 uint64_t allcleared = 0;
792 size_t nui64s = size / sizeof (uint64_t);
802 zfs_ecksum_info_t *eip = kmem_zalloc(sizeof (*eip), KM_SLEEP);
804 /* don't do any annotation for injected checksum errors */
805 if (info != NULL && info->zbc_injected)
808 if (info != NULL && info->zbc_has_cksum) {
809 fm_payload_set(ereport,
810 FM_EREPORT_PAYLOAD_ZFS_CKSUM_EXPECTED,
811 DATA_TYPE_UINT64_ARRAY,
812 sizeof (info->zbc_expected) / sizeof (uint64_t),
813 (uint64_t *)&info->zbc_expected,
814 FM_EREPORT_PAYLOAD_ZFS_CKSUM_ACTUAL,
815 DATA_TYPE_UINT64_ARRAY,
816 sizeof (info->zbc_actual) / sizeof (uint64_t),
817 (uint64_t *)&info->zbc_actual,
818 FM_EREPORT_PAYLOAD_ZFS_CKSUM_ALGO,
820 info->zbc_checksum_name,
823 if (info->zbc_byteswapped) {
824 fm_payload_set(ereport,
825 FM_EREPORT_PAYLOAD_ZFS_CKSUM_BYTESWAP,
826 DATA_TYPE_BOOLEAN, 1,
831 if (badabd == NULL || goodabd == NULL)
834 ASSERT3U(nui64s, <=, UINT32_MAX);
835 ASSERT3U(size, ==, nui64s * sizeof (uint64_t));
836 ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
837 ASSERT3U(size, <=, UINT32_MAX);
839 good = (const uint64_t *) abd_borrow_buf_copy((abd_t *)goodabd, size);
840 bad = (const uint64_t *) abd_borrow_buf_copy((abd_t *)badabd, size);
842 /* build up the range list by comparing the two buffers. */
843 for (idx = 0; idx < nui64s; idx++) {
844 if (good[idx] == bad[idx]) {
848 zei_add_range(eip, start, idx);
858 zei_add_range(eip, start, idx);
860 /* See if it will fit in our inline buffers */
861 inline_size = zei_range_total_size(eip);
862 if (inline_size > ZFM_MAX_INLINE)
866 * If there is no change and we want to drop if the buffers are
869 if (inline_size == 0 && drop_if_identical) {
870 kmem_free(eip, sizeof (*eip));
871 abd_return_buf((abd_t *)goodabd, (void *)good, size);
872 abd_return_buf((abd_t *)badabd, (void *)bad, size);
877 * Now walk through the ranges, filling in the details of the
878 * differences. Also convert our uint64_t-array offsets to byte
881 for (range = 0; range < eip->zei_range_count; range++) {
882 size_t start = eip->zei_ranges[range].zr_start;
883 size_t end = eip->zei_ranges[range].zr_end;
885 for (idx = start; idx < end; idx++) {
886 uint64_t set, cleared;
888 // bits set in bad, but not in good
889 set = ((~good[idx]) & bad[idx]);
890 // bits set in good, but not in bad
891 cleared = (good[idx] & (~bad[idx]));
894 allcleared |= cleared;
897 ASSERT3U(offset, <, inline_size);
898 eip->zei_bits_set[offset] = set;
899 eip->zei_bits_cleared[offset] = cleared;
903 update_histogram(set, eip->zei_histogram_set,
904 &eip->zei_range_sets[range]);
905 update_histogram(cleared, eip->zei_histogram_cleared,
906 &eip->zei_range_clears[range]);
909 /* convert to byte offsets */
910 eip->zei_ranges[range].zr_start *= sizeof (uint64_t);
911 eip->zei_ranges[range].zr_end *= sizeof (uint64_t);
914 abd_return_buf((abd_t *)goodabd, (void *)good, size);
915 abd_return_buf((abd_t *)badabd, (void *)bad, size);
917 eip->zei_allowed_mingap *= sizeof (uint64_t);
918 inline_size *= sizeof (uint64_t);
920 /* fill in ereport */
921 fm_payload_set(ereport,
922 FM_EREPORT_PAYLOAD_ZFS_BAD_OFFSET_RANGES,
923 DATA_TYPE_UINT32_ARRAY, 2 * eip->zei_range_count,
924 (uint32_t *)eip->zei_ranges,
925 FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_MIN_GAP,
926 DATA_TYPE_UINT32, eip->zei_allowed_mingap,
927 FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_SETS,
928 DATA_TYPE_UINT32_ARRAY, eip->zei_range_count, eip->zei_range_sets,
929 FM_EREPORT_PAYLOAD_ZFS_BAD_RANGE_CLEARS,
930 DATA_TYPE_UINT32_ARRAY, eip->zei_range_count, eip->zei_range_clears,
934 fm_payload_set(ereport,
935 FM_EREPORT_PAYLOAD_ZFS_BAD_SET_BITS,
936 DATA_TYPE_UINT8_ARRAY,
937 inline_size, (uint8_t *)eip->zei_bits_set,
938 FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_BITS,
939 DATA_TYPE_UINT8_ARRAY,
940 inline_size, (uint8_t *)eip->zei_bits_cleared,
943 fm_payload_set(ereport,
944 FM_EREPORT_PAYLOAD_ZFS_BAD_SET_HISTOGRAM,
945 DATA_TYPE_UINT32_ARRAY,
946 NBBY * sizeof (uint64_t), eip->zei_histogram_set,
947 FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_HISTOGRAM,
948 DATA_TYPE_UINT32_ARRAY,
949 NBBY * sizeof (uint64_t), eip->zei_histogram_cleared,
957 * Make sure our event is still valid for the given zio/vdev/pool. For example,
958 * we don't want to keep logging events for a faulted or missing vdev.
961 zfs_ereport_is_valid(const char *subclass, spa_t *spa, vdev_t *vd, zio_t *zio)
965 * If we are doing a spa_tryimport() or in recovery mode,
968 if (spa_load_state(spa) == SPA_LOAD_TRYIMPORT ||
969 spa_load_state(spa) == SPA_LOAD_RECOVER)
973 * If we are in the middle of opening a pool, and the previous attempt
974 * failed, don't bother logging any new ereports - we're just going to
975 * get the same diagnosis anyway.
977 if (spa_load_state(spa) != SPA_LOAD_NONE &&
978 spa->spa_last_open_failed)
983 * If this is not a read or write zio, ignore the error. This
984 * can occur if the DKIOCFLUSHWRITECACHE ioctl fails.
986 if (zio->io_type != ZIO_TYPE_READ &&
987 zio->io_type != ZIO_TYPE_WRITE)
992 * If the vdev has already been marked as failing due
993 * to a failed probe, then ignore any subsequent I/O
994 * errors, as the DE will automatically fault the vdev
995 * on the first such failure. This also catches cases
996 * where vdev_remove_wanted is set and the device has
997 * not yet been asynchronously placed into the REMOVED
1000 if (zio->io_vd == vd && !vdev_accessible(vd, zio))
1004 * Ignore checksum errors for reads from DTL regions of
1007 if (zio->io_type == ZIO_TYPE_READ &&
1008 zio->io_error == ECKSUM &&
1009 vd->vdev_ops->vdev_op_leaf &&
1010 vdev_dtl_contains(vd, DTL_MISSING, zio->io_txg, 1))
1016 * For probe failure, we want to avoid posting ereports if we've
1017 * already removed the device in the meantime.
1020 strcmp(subclass, FM_EREPORT_ZFS_PROBE_FAILURE) == 0 &&
1021 (vd->vdev_remove_wanted || vd->vdev_state == VDEV_STATE_REMOVED))
1024 /* Ignore bogus delay events (like from ioctls or unqueued IOs) */
1025 if ((strcmp(subclass, FM_EREPORT_ZFS_DELAY) == 0) &&
1026 (zio != NULL) && (!zio->io_timestamp)) {
1034 * Post an ereport for the given subclass
1037 * - 0 if an event was posted
1038 * - EINVAL if there was a problem posting event
1039 * - EBUSY if the event was rate limited
1040 * - EALREADY if the event was already posted (duplicate)
1043 zfs_ereport_post(const char *subclass, spa_t *spa, vdev_t *vd,
1044 const zbookmark_phys_t *zb, zio_t *zio, uint64_t state)
1048 nvlist_t *ereport = NULL;
1049 nvlist_t *detector = NULL;
1051 if (!zfs_ereport_is_valid(subclass, spa, vd, zio))
1054 if (zfs_ereport_is_duplicate(subclass, spa, vd, zb, zio, 0, 0))
1055 return (SET_ERROR(EALREADY));
1057 if (zfs_is_ratelimiting_event(subclass, vd))
1058 return (SET_ERROR(EBUSY));
1060 if (!zfs_ereport_start(&ereport, &detector, subclass, spa, vd,
1062 return (SET_ERROR(EINVAL)); /* couldn't post event */
1064 if (ereport == NULL)
1065 return (SET_ERROR(EINVAL));
1067 /* Cleanup is handled by the callback function */
1068 rc = zfs_zevent_post(ereport, detector, zfs_zevent_post_cb);
1074 * Prepare a checksum ereport
1077 * - 0 if an event was posted
1078 * - EINVAL if there was a problem posting event
1079 * - EBUSY if the event was rate limited
1080 * - EALREADY if the event was already posted (duplicate)
1083 zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, const zbookmark_phys_t *zb,
1084 struct zio *zio, uint64_t offset, uint64_t length, void *arg,
1085 zio_bad_cksum_t *info)
1087 zio_cksum_report_t *report;
1090 if (!zfs_ereport_is_valid(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zio))
1091 return (SET_ERROR(EINVAL));
1093 if (zfs_ereport_is_duplicate(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zb, zio,
1095 return (SET_ERROR(EALREADY));
1097 if (zfs_is_ratelimiting_event(FM_EREPORT_ZFS_CHECKSUM, vd))
1098 return (SET_ERROR(EBUSY));
1101 report = kmem_zalloc(sizeof (*report), KM_SLEEP);
1103 if (zio->io_vsd != NULL)
1104 zio->io_vsd_ops->vsd_cksum_report(zio, report, arg);
1106 zio_vsd_default_cksum_report(zio, report, arg);
1108 /* copy the checksum failure information if it was provided */
1110 report->zcr_ckinfo = kmem_zalloc(sizeof (*info), KM_SLEEP);
1111 bcopy(info, report->zcr_ckinfo, sizeof (*info));
1114 report->zcr_align = 1ULL << vd->vdev_top->vdev_ashift;
1115 report->zcr_length = length;
1118 (void) zfs_ereport_start(&report->zcr_ereport, &report->zcr_detector,
1119 FM_EREPORT_ZFS_CHECKSUM, spa, vd, zb, zio, offset, length);
1121 if (report->zcr_ereport == NULL) {
1122 zfs_ereport_free_checksum(report);
1127 mutex_enter(&spa->spa_errlist_lock);
1128 report->zcr_next = zio->io_logical->io_cksum_report;
1129 zio->io_logical->io_cksum_report = report;
1130 mutex_exit(&spa->spa_errlist_lock);
1135 zfs_ereport_finish_checksum(zio_cksum_report_t *report, const abd_t *good_data,
1136 const abd_t *bad_data, boolean_t drop_if_identical)
1139 zfs_ecksum_info_t *info;
1141 info = annotate_ecksum(report->zcr_ereport, report->zcr_ckinfo,
1142 good_data, bad_data, report->zcr_length, drop_if_identical);
1144 zfs_zevent_post(report->zcr_ereport,
1145 report->zcr_detector, zfs_zevent_post_cb);
1147 zfs_zevent_post_cb(report->zcr_ereport, report->zcr_detector);
1149 report->zcr_ereport = report->zcr_detector = NULL;
1151 kmem_free(info, sizeof (*info));
1156 zfs_ereport_free_checksum(zio_cksum_report_t *rpt)
1159 if (rpt->zcr_ereport != NULL) {
1160 fm_nvlist_destroy(rpt->zcr_ereport,
1162 fm_nvlist_destroy(rpt->zcr_detector,
1166 rpt->zcr_free(rpt->zcr_cbdata, rpt->zcr_cbinfo);
1168 if (rpt->zcr_ckinfo != NULL)
1169 kmem_free(rpt->zcr_ckinfo, sizeof (*rpt->zcr_ckinfo));
1171 kmem_free(rpt, sizeof (*rpt));
1175 * Post a checksum ereport
1178 * - 0 if an event was posted
1179 * - EINVAL if there was a problem posting event
1180 * - EBUSY if the event was rate limited
1181 * - EALREADY if the event was already posted (duplicate)
1184 zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd, const zbookmark_phys_t *zb,
1185 struct zio *zio, uint64_t offset, uint64_t length,
1186 const abd_t *good_data, const abd_t *bad_data, zio_bad_cksum_t *zbc)
1190 nvlist_t *ereport = NULL;
1191 nvlist_t *detector = NULL;
1192 zfs_ecksum_info_t *info;
1194 if (!zfs_ereport_is_valid(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zio))
1195 return (SET_ERROR(EINVAL));
1197 if (zfs_ereport_is_duplicate(FM_EREPORT_ZFS_CHECKSUM, spa, vd, zb, zio,
1199 return (SET_ERROR(EALREADY));
1201 if (zfs_is_ratelimiting_event(FM_EREPORT_ZFS_CHECKSUM, vd))
1202 return (SET_ERROR(EBUSY));
1204 if (!zfs_ereport_start(&ereport, &detector, FM_EREPORT_ZFS_CHECKSUM,
1205 spa, vd, zb, zio, offset, length) || (ereport == NULL)) {
1206 return (SET_ERROR(EINVAL));
1209 info = annotate_ecksum(ereport, zbc, good_data, bad_data, length,
1213 rc = zfs_zevent_post(ereport, detector, zfs_zevent_post_cb);
1214 kmem_free(info, sizeof (*info));
1221 * The 'sysevent.fs.zfs.*' events are signals posted to notify user space of
1222 * change in the pool. All sysevents are listed in sys/sysevent/eventdefs.h
1223 * and are designed to be consumed by the ZFS Event Daemon (ZED). For
1224 * additional details refer to the zed(8) man page.
1227 zfs_event_create(spa_t *spa, vdev_t *vd, const char *type, const char *name,
1230 nvlist_t *resource = NULL;
1234 if (spa_load_state(spa) == SPA_LOAD_TRYIMPORT)
1237 if ((resource = fm_nvlist_create(NULL)) == NULL)
1240 (void) snprintf(class, sizeof (class), "%s.%s.%s", type,
1241 ZFS_ERROR_CLASS, name);
1242 VERIFY0(nvlist_add_uint8(resource, FM_VERSION, FM_RSRC_VERSION));
1243 VERIFY0(nvlist_add_string(resource, FM_CLASS, class));
1244 VERIFY0(nvlist_add_string(resource,
1245 FM_EREPORT_PAYLOAD_ZFS_POOL, spa_name(spa)));
1246 VERIFY0(nvlist_add_uint64(resource,
1247 FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, spa_guid(spa)));
1248 VERIFY0(nvlist_add_uint64(resource,
1249 FM_EREPORT_PAYLOAD_ZFS_POOL_STATE, spa_state(spa)));
1250 VERIFY0(nvlist_add_int32(resource,
1251 FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, spa_load_state(spa)));
1254 VERIFY0(nvlist_add_uint64(resource,
1255 FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, vd->vdev_guid));
1256 VERIFY0(nvlist_add_uint64(resource,
1257 FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, vd->vdev_state));
1258 if (vd->vdev_path != NULL)
1259 VERIFY0(nvlist_add_string(resource,
1260 FM_EREPORT_PAYLOAD_ZFS_VDEV_PATH, vd->vdev_path));
1261 if (vd->vdev_devid != NULL)
1262 VERIFY0(nvlist_add_string(resource,
1263 FM_EREPORT_PAYLOAD_ZFS_VDEV_DEVID, vd->vdev_devid));
1264 if (vd->vdev_fru != NULL)
1265 VERIFY0(nvlist_add_string(resource,
1266 FM_EREPORT_PAYLOAD_ZFS_VDEV_FRU, vd->vdev_fru));
1267 if (vd->vdev_enc_sysfs_path != NULL)
1268 VERIFY0(nvlist_add_string(resource,
1269 FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH,
1270 vd->vdev_enc_sysfs_path));
1273 /* also copy any optional payload data */
1275 nvpair_t *elem = NULL;
1277 while ((elem = nvlist_next_nvpair(aux, elem)) != NULL)
1278 (void) nvlist_add_nvpair(resource, elem);
1286 zfs_post_common(spa_t *spa, vdev_t *vd, const char *type, const char *name,
1292 resource = zfs_event_create(spa, vd, type, name, aux);
1294 zfs_zevent_post(resource, NULL, zfs_zevent_post_cb);
1299 * The 'resource.fs.zfs.removed' event is an internal signal that the given vdev
1300 * has been removed from the system. This will cause the DE to ignore any
1301 * recent I/O errors, inferring that they are due to the asynchronous device
1305 zfs_post_remove(spa_t *spa, vdev_t *vd)
1307 zfs_post_common(spa, vd, FM_RSRC_CLASS, FM_RESOURCE_REMOVED, NULL);
1311 * The 'resource.fs.zfs.autoreplace' event is an internal signal that the pool
1312 * has the 'autoreplace' property set, and therefore any broken vdevs will be
1313 * handled by higher level logic, and no vdev fault should be generated.
1316 zfs_post_autoreplace(spa_t *spa, vdev_t *vd)
1318 zfs_post_common(spa, vd, FM_RSRC_CLASS, FM_RESOURCE_AUTOREPLACE, NULL);
1322 * The 'resource.fs.zfs.statechange' event is an internal signal that the
1323 * given vdev has transitioned its state to DEGRADED or HEALTHY. This will
1324 * cause the retire agent to repair any outstanding fault management cases
1325 * open because the device was not found (fault.fs.zfs.device).
1328 zfs_post_state_change(spa_t *spa, vdev_t *vd, uint64_t laststate)
1334 * Add optional supplemental keys to payload
1336 aux = fm_nvlist_create(NULL);
1338 if (vd->vdev_physpath) {
1339 (void) nvlist_add_string(aux,
1340 FM_EREPORT_PAYLOAD_ZFS_VDEV_PHYSPATH,
1343 if (vd->vdev_enc_sysfs_path) {
1344 (void) nvlist_add_string(aux,
1345 FM_EREPORT_PAYLOAD_ZFS_VDEV_ENC_SYSFS_PATH,
1346 vd->vdev_enc_sysfs_path);
1349 (void) nvlist_add_uint64(aux,
1350 FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE, laststate);
1353 zfs_post_common(spa, vd, FM_RSRC_CLASS, FM_RESOURCE_STATECHANGE,
1357 fm_nvlist_destroy(aux, FM_NVA_FREE);
1363 zfs_ereport_init(void)
1365 mutex_init(&recent_events_lock, NULL, MUTEX_DEFAULT, NULL);
1366 list_create(&recent_events_list, sizeof (recent_events_node_t),
1367 offsetof(recent_events_node_t, re_list_link));
1368 avl_create(&recent_events_tree, recent_events_compare,
1369 sizeof (recent_events_node_t), offsetof(recent_events_node_t,
1374 * This 'early' fini needs to run before zfs_fini() which on Linux waits
1375 * for the system_delay_taskq to drain.
1378 zfs_ereport_taskq_fini(void)
1380 mutex_enter(&recent_events_lock);
1381 if (recent_events_cleaner_tqid != 0) {
1382 taskq_cancel_id(system_delay_taskq, recent_events_cleaner_tqid);
1383 recent_events_cleaner_tqid = 0;
1385 mutex_exit(&recent_events_lock);
1389 zfs_ereport_fini(void)
1391 recent_events_node_t *entry;
1393 while ((entry = list_head(&recent_events_list)) != NULL) {
1394 avl_remove(&recent_events_tree, entry);
1395 list_remove(&recent_events_list, entry);
1396 kmem_free(entry, sizeof (*entry));
1398 avl_destroy(&recent_events_tree);
1399 list_destroy(&recent_events_list);
1400 mutex_destroy(&recent_events_lock);
1403 EXPORT_SYMBOL(zfs_ereport_post);
1404 EXPORT_SYMBOL(zfs_ereport_is_valid);
1405 EXPORT_SYMBOL(zfs_ereport_post_checksum);
1406 EXPORT_SYMBOL(zfs_post_remove);
1407 EXPORT_SYMBOL(zfs_post_autoreplace);
1408 EXPORT_SYMBOL(zfs_post_state_change);
1410 ZFS_MODULE_PARAM(zfs_zevent, zfs_zevent_, retain_max, UINT, ZMOD_RW,
1411 "Maximum recent zevents records to retain for duplicate checking");
1412 ZFS_MODULE_PARAM(zfs_zevent, zfs_zevent_, retain_expire_secs, UINT, ZMOD_RW,
1413 "Expiration time for recent zevents records");
1414 #endif /* _KERNEL */