cmd/zed/agents/zfs_diagnosis.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  25  * Copyright (c) 2016, Intel Corporation.
  26  */
  27
  28 #include <stddef.h>
  29 #include <string.h>
  30 #include <strings.h>
  31 #include <libuutil.h>
  32 #include <libzfs.h>
  33 #include <sys/types.h>
  34 #include <sys/time.h>
  35 #include <sys/fs/zfs.h>
  36 #include <sys/fm/protocol.h>
  37 #include <sys/fm/fs/zfs.h>
  38
  39 #include "zfs_agents.h"
  40 #include "fmd_api.h"
  41
  42 /*
  43  * Our serd engines are named 'zfs_<pool_guid>_<vdev_guid>_{checksum,io}'.  This
  44  * #define reserves enough space for two 64-bit hex values plus the length of
  45  * the longest string.
  46  */
  47 #define MAX_SERDLEN     (16 * 2 + sizeof ("zfs___checksum"))
  48
  49 /*
  50  * On-disk case structure.  This must maintain backwards compatibility with
  51  * previous versions of the DE.  By default, any members appended to the end
  52  * will be filled with zeros if they don't exist in a previous version.
  53  */
  54 typedef struct zfs_case_data {
  55         uint64_t        zc_version;
  56         uint64_t        zc_ena;
  57         uint64_t        zc_pool_guid;
  58         uint64_t        zc_vdev_guid;
  59         int             zc_pool_state;
  60         char            zc_serd_checksum[MAX_SERDLEN];
  61         char            zc_serd_io[MAX_SERDLEN];
  62         int             zc_has_remove_timer;
  63 } zfs_case_data_t;
  64
  65 /*
  66  * Time-of-day
  67  */
  68 typedef struct er_timeval {
  69         uint64_t        ertv_sec;
  70         uint64_t        ertv_nsec;
  71 } er_timeval_t;
  72
  73 /*
  74  * In-core case structure.
  75  */
  76 typedef struct zfs_case {
  77         boolean_t       zc_present;
  78         uint32_t        zc_version;
  79         zfs_case_data_t zc_data;
  80         fmd_case_t      *zc_case;
  81         uu_list_node_t  zc_node;
  82         id_t            zc_remove_timer;
  83         char            *zc_fru;
  84         er_timeval_t    zc_when;
  85 } zfs_case_t;
  86
  87 #define CASE_DATA                       "data"
  88 #define CASE_FRU                        "fru"
  89 #define CASE_DATA_VERSION_INITIAL       1
  90 #define CASE_DATA_VERSION_SERD          2
  91
  92 typedef struct zfs_de_stats {
  93         fmd_stat_t      old_drops;
  94         fmd_stat_t      dev_drops;
  95         fmd_stat_t      vdev_drops;
  96         fmd_stat_t      import_drops;
  97         fmd_stat_t      resource_drops;
  98 } zfs_de_stats_t;
  99
 100 zfs_de_stats_t zfs_stats = {
 101         { "old_drops", FMD_TYPE_UINT64, "ereports dropped (from before load)" },
 102         { "dev_drops", FMD_TYPE_UINT64, "ereports dropped (dev during open)"},
 103         { "vdev_drops", FMD_TYPE_UINT64, "ereports dropped (weird vdev types)"},
 104         { "import_drops", FMD_TYPE_UINT64, "ereports dropped (during import)" },
 105         { "resource_drops", FMD_TYPE_UINT64, "resource related ereports" }
 106 };
 107
 108 static hrtime_t zfs_remove_timeout;
 109
 110 uu_list_pool_t *zfs_case_pool;
 111 uu_list_t *zfs_cases;
 112
 113 #define ZFS_MAKE_RSRC(type)     \
 114     FM_RSRC_CLASS "." ZFS_ERROR_CLASS "." type
 115 #define ZFS_MAKE_EREPORT(type)  \
 116     FM_EREPORT_CLASS "." ZFS_ERROR_CLASS "." type
 117
 118 /*
 119  * Write out the persistent representation of an active case.
 120  */
 121 static void
 122 zfs_case_serialize(fmd_hdl_t *hdl, zfs_case_t *zcp)
 123 {
 124         zcp->zc_data.zc_version = CASE_DATA_VERSION_SERD;
 125 }
 126
 127 /*
 128  * Read back the persistent representation of an active case.
 129  */
 130 static zfs_case_t *
 131 zfs_case_unserialize(fmd_hdl_t *hdl, fmd_case_t *cp)
 132 {
 133         zfs_case_t *zcp;
 134
 135         zcp = fmd_hdl_zalloc(hdl, sizeof (zfs_case_t), FMD_SLEEP);
 136         zcp->zc_case = cp;
 137
 138         fmd_buf_read(hdl, cp, CASE_DATA, &zcp->zc_data,
 139             sizeof (zcp->zc_data));
 140
 141         if (zcp->zc_data.zc_version > CASE_DATA_VERSION_SERD) {
 142                 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
 143                 return (NULL);
 144         }
 145
 146         /*
 147          * fmd_buf_read() will have already zeroed out the remainder of the
 148          * buffer, so we don't have to do anything special if the version
 149          * doesn't include the SERD engine name.
 150          */
 151
 152         if (zcp->zc_data.zc_has_remove_timer)
 153                 zcp->zc_remove_timer = fmd_timer_install(hdl, zcp,
 154                     NULL, zfs_remove_timeout);
 155
 156         uu_list_node_init(zcp, &zcp->zc_node, zfs_case_pool);
 157         (void) uu_list_insert_before(zfs_cases, NULL, zcp);
 158
 159         fmd_case_setspecific(hdl, cp, zcp);
 160
 161         return (zcp);
 162 }
 163
 164 /*
 165  * Iterate over any active cases.  If any cases are associated with a pool or
 166  * vdev which is no longer present on the system, close the associated case.
 167  */
 168 static void
 169 zfs_mark_vdev(uint64_t pool_guid, nvlist_t *vd, er_timeval_t *loaded)
 170 {
 171         uint64_t vdev_guid = 0;
 172         uint_t c, children;
 173         nvlist_t **child;
 174         zfs_case_t *zcp;
 175
 176         (void) nvlist_lookup_uint64(vd, ZPOOL_CONFIG_GUID, &vdev_guid);
 177
 178         /*
 179          * Mark any cases associated with this (pool, vdev) pair.
 180          */
 181         for (zcp = uu_list_first(zfs_cases); zcp != NULL;
 182             zcp = uu_list_next(zfs_cases, zcp)) {
 183                 if (zcp->zc_data.zc_pool_guid == pool_guid &&
 184                     zcp->zc_data.zc_vdev_guid == vdev_guid) {
 185                         zcp->zc_present = B_TRUE;
 186                         zcp->zc_when = *loaded;
 187                 }
 188         }
 189
 190         /*
 191          * Iterate over all children.
 192          */
 193         if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_CHILDREN, &child,
 194             &children) == 0) {
 195                 for (c = 0; c < children; c++)
 196                         zfs_mark_vdev(pool_guid, child[c], loaded);
 197         }
 198
 199         if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_L2CACHE, &child,
 200             &children) == 0) {
 201                 for (c = 0; c < children; c++)
 202                         zfs_mark_vdev(pool_guid, child[c], loaded);
 203         }
 204
 205         if (nvlist_lookup_nvlist_array(vd, ZPOOL_CONFIG_SPARES, &child,
 206             &children) == 0) {
 207                 for (c = 0; c < children; c++)
 208                         zfs_mark_vdev(pool_guid, child[c], loaded);
 209         }
 210 }
 211
 212 /*ARGSUSED*/
 213 static int
 214 zfs_mark_pool(zpool_handle_t *zhp, void *unused)
 215 {
 216         zfs_case_t *zcp;
 217         uint64_t pool_guid;
 218         uint64_t *tod;
 219         er_timeval_t loaded = { 0 };
 220         nvlist_t *config, *vd;
 221         uint_t nelem = 0;
 222         int ret;
 223
 224         pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
 225         /*
 226          * Mark any cases associated with just this pool.
 227          */
 228         for (zcp = uu_list_first(zfs_cases); zcp != NULL;
 229             zcp = uu_list_next(zfs_cases, zcp)) {
 230                 if (zcp->zc_data.zc_pool_guid == pool_guid &&
 231                     zcp->zc_data.zc_vdev_guid == 0)
 232                         zcp->zc_present = B_TRUE;
 233         }
 234
 235         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
 236                 zpool_close(zhp);
 237                 return (-1);
 238         }
 239
 240         (void) nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
 241             &tod, &nelem);
 242         if (nelem == 2) {
 243                 loaded.ertv_sec = tod[0];
 244                 loaded.ertv_nsec = tod[1];
 245                 for (zcp = uu_list_first(zfs_cases); zcp != NULL;
 246                     zcp = uu_list_next(zfs_cases, zcp)) {
 247                         if (zcp->zc_data.zc_pool_guid == pool_guid &&
 248                             zcp->zc_data.zc_vdev_guid == 0) {
 249                                 zcp->zc_when = loaded;
 250                         }
 251                 }
 252         }
 253
 254         ret = nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &vd);
 255         if (ret) {
 256                 zpool_close(zhp);
 257                 return (-1);
 258         }
 259
 260         zfs_mark_vdev(pool_guid, vd, &loaded);
 261
 262         zpool_close(zhp);
 263
 264         return (0);
 265 }
 266
 267 struct load_time_arg {
 268         uint64_t lt_guid;
 269         er_timeval_t *lt_time;
 270         boolean_t lt_found;
 271 };
 272
 273 static int
 274 zpool_find_load_time(zpool_handle_t *zhp, void *arg)
 275 {
 276         struct load_time_arg *lta = arg;
 277         uint64_t pool_guid;
 278         uint64_t *tod;
 279         nvlist_t *config;
 280         uint_t nelem;
 281
 282         if (lta->lt_found) {
 283                 zpool_close(zhp);
 284                 return (0);
 285         }
 286
 287         pool_guid = zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL);
 288         if (pool_guid != lta->lt_guid) {
 289                 zpool_close(zhp);
 290                 return (0);
 291         }
 292
 293         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
 294                 zpool_close(zhp);
 295                 return (-1);
 296         }
 297
 298         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_LOADED_TIME,
 299             &tod, &nelem) == 0 && nelem == 2) {
 300                 lta->lt_found = B_TRUE;
 301                 lta->lt_time->ertv_sec = tod[0];
 302                 lta->lt_time->ertv_nsec = tod[1];
 303         }
 304
 305         zpool_close(zhp);
 306
 307         return (0);
 308 }
 309
 310 static void
 311 zfs_purge_cases(fmd_hdl_t *hdl)
 312 {
 313         zfs_case_t *zcp;
 314         uu_list_walk_t *walk;
 315         libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
 316
 317         /*
 318          * There is no way to open a pool by GUID, or lookup a vdev by GUID.  No
 319          * matter what we do, we're going to have to stomach an O(vdevs * cases)
 320          * algorithm.  In reality, both quantities are likely so small that
 321          * neither will matter. Given that iterating over pools is more
 322          * expensive than iterating over the in-memory case list, we opt for a
 323          * 'present' flag in each case that starts off cleared.  We then iterate
 324          * over all pools, marking those that are still present, and removing
 325          * those that aren't found.
 326          *
 327          * Note that we could also construct an FMRI and rely on
 328          * fmd_nvl_fmri_present(), but this would end up doing the same search.
 329          */
 330
 331         /*
 332          * Mark the cases as not present.
 333          */
 334         for (zcp = uu_list_first(zfs_cases); zcp != NULL;
 335             zcp = uu_list_next(zfs_cases, zcp))
 336                 zcp->zc_present = B_FALSE;
 337
 338         /*
 339          * Iterate over all pools and mark the pools and vdevs found.  If this
 340          * fails (most probably because we're out of memory), then don't close
 341          * any of the cases and we cannot be sure they are accurate.
 342          */
 343         if (zpool_iter(zhdl, zfs_mark_pool, NULL) != 0)
 344                 return;
 345
 346         /*
 347          * Remove those cases which were not found.
 348          */
 349         walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
 350         while ((zcp = uu_list_walk_next(walk)) != NULL) {
 351                 if (!zcp->zc_present)
 352                         fmd_case_close(hdl, zcp->zc_case);
 353         }
 354         uu_list_walk_end(walk);
 355 }
 356
 357 /*
 358  * Construct the name of a serd engine given the pool/vdev GUID and type (io or
 359  * checksum).
 360  */
 361 static void
 362 zfs_serd_name(char *buf, uint64_t pool_guid, uint64_t vdev_guid,
 363     const char *type)
 364 {
 365         (void) snprintf(buf, MAX_SERDLEN, "zfs_%llx_%llx_%s",
 366             (long long unsigned int)pool_guid,
 367             (long long unsigned int)vdev_guid, type);
 368 }
 369
 370 /*
 371  * Solve a given ZFS case.  This first checks to make sure the diagnosis is
 372  * still valid, as well as cleaning up any pending timer associated with the
 373  * case.
 374  */
 375 static void
 376 zfs_case_solve(fmd_hdl_t *hdl, zfs_case_t *zcp, const char *faultname,
 377     boolean_t checkunusable)
 378 {
 379         nvlist_t *detector, *fault;
 380         boolean_t serialize;
 381         nvlist_t *fru = NULL;
 382         fmd_hdl_debug(hdl, "solving fault '%s'", faultname);
 383
 384         /*
 385          * Construct the detector from the case data.  The detector is in the
 386          * ZFS scheme, and is either the pool or the vdev, depending on whether
 387          * this is a vdev or pool fault.
 388          */
 389         detector = fmd_nvl_alloc(hdl, FMD_SLEEP);
 390
 391         (void) nvlist_add_uint8(detector, FM_VERSION, ZFS_SCHEME_VERSION0);
 392         (void) nvlist_add_string(detector, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
 393         (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_POOL,
 394             zcp->zc_data.zc_pool_guid);
 395         if (zcp->zc_data.zc_vdev_guid != 0) {
 396                 (void) nvlist_add_uint64(detector, FM_FMRI_ZFS_VDEV,
 397                     zcp->zc_data.zc_vdev_guid);
 398         }
 399
 400         fault = fmd_nvl_create_fault(hdl, faultname, 100, detector,
 401             fru, detector);
 402         fmd_case_add_suspect(hdl, zcp->zc_case, fault);
 403
 404         nvlist_free(fru);
 405
 406         fmd_case_solve(hdl, zcp->zc_case);
 407
 408         serialize = B_FALSE;
 409         if (zcp->zc_data.zc_has_remove_timer) {
 410                 fmd_timer_remove(hdl, zcp->zc_remove_timer);
 411                 zcp->zc_data.zc_has_remove_timer = 0;
 412                 serialize = B_TRUE;
 413         }
 414         if (serialize)
 415                 zfs_case_serialize(hdl, zcp);
 416
 417         nvlist_free(detector);
 418 }
 419
 420 static boolean_t
 421 timeval_earlier(er_timeval_t *a, er_timeval_t *b)
 422 {
 423         return (a->ertv_sec < b->ertv_sec ||
 424             (a->ertv_sec == b->ertv_sec && a->ertv_nsec < b->ertv_nsec));
 425 }
 426
 427 /*ARGSUSED*/
 428 static void
 429 zfs_ereport_when(fmd_hdl_t *hdl, nvlist_t *nvl, er_timeval_t *when)
 430 {
 431         int64_t *tod;
 432         uint_t  nelem;
 433
 434         if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tod,
 435             &nelem) == 0 && nelem == 2) {
 436                 when->ertv_sec = tod[0];
 437                 when->ertv_nsec = tod[1];
 438         } else {
 439                 when->ertv_sec = when->ertv_nsec = UINT64_MAX;
 440         }
 441 }
 442
 443 /*
 444  * Main fmd entry point.
 445  */
 446 /*ARGSUSED*/
 447 static void
 448 zfs_fm_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
 449 {
 450         zfs_case_t *zcp, *dcp;
 451         int32_t pool_state;
 452         uint64_t ena, pool_guid, vdev_guid;
 453         er_timeval_t pool_load;
 454         er_timeval_t er_when;
 455         nvlist_t *detector;
 456         boolean_t pool_found = B_FALSE;
 457         boolean_t isresource;
 458         char *type;
 459
 460         /*
 461          * We subscribe to notifications for vdev or pool removal.  In these
 462          * cases, there may be cases that no longer apply.  Purge any cases
 463          * that no longer apply.
 464          */
 465         if (fmd_nvl_class_match(hdl, nvl, "sysevent.fs.zfs.*")) {
 466                 fmd_hdl_debug(hdl, "purging orphaned cases from %s",
 467                     strrchr(class, '.') + 1);
 468                 zfs_purge_cases(hdl);
 469                 zfs_stats.resource_drops.fmds_value.ui64++;
 470                 return;
 471         }
 472
 473         isresource = fmd_nvl_class_match(hdl, nvl, "resource.fs.zfs.*");
 474
 475         if (isresource) {
 476                 /*
 477                  * For resources, we don't have a normal payload.
 478                  */
 479                 if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
 480                     &vdev_guid) != 0)
 481                         pool_state = SPA_LOAD_OPEN;
 482                 else
 483                         pool_state = SPA_LOAD_NONE;
 484                 detector = NULL;
 485         } else {
 486                 (void) nvlist_lookup_nvlist(nvl,
 487                     FM_EREPORT_DETECTOR, &detector);
 488                 (void) nvlist_lookup_int32(nvl,
 489                     FM_EREPORT_PAYLOAD_ZFS_POOL_CONTEXT, &pool_state);
 490         }
 491
 492         /*
 493          * We also ignore all ereports generated during an import of a pool,
 494          * since the only possible fault (.pool) would result in import failure,
 495          * and hence no persistent fault.  Some day we may want to do something
 496          * with these ereports, so we continue generating them internally.
 497          */
 498         if (pool_state == SPA_LOAD_IMPORT) {
 499                 zfs_stats.import_drops.fmds_value.ui64++;
 500                 fmd_hdl_debug(hdl, "ignoring '%s' during import", class);
 501                 return;
 502         }
 503
 504         /*
 505          * Device I/O errors are ignored during pool open.
 506          */
 507         if (pool_state == SPA_LOAD_OPEN &&
 508             (fmd_nvl_class_match(hdl, nvl,
 509             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
 510             fmd_nvl_class_match(hdl, nvl,
 511             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
 512             fmd_nvl_class_match(hdl, nvl,
 513             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE)))) {
 514                 fmd_hdl_debug(hdl, "ignoring '%s' during pool open", class);
 515                 zfs_stats.dev_drops.fmds_value.ui64++;
 516                 return;
 517         }
 518
 519         /*
 520          * We ignore ereports for anything except disks and files.
 521          */
 522         if (nvlist_lookup_string(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_TYPE,
 523             &type) == 0) {
 524                 if (strcmp(type, VDEV_TYPE_DISK) != 0 &&
 525                     strcmp(type, VDEV_TYPE_FILE) != 0) {
 526                         zfs_stats.vdev_drops.fmds_value.ui64++;
 527                         return;
 528                 }
 529         }
 530
 531         /*
 532          * Determine if this ereport corresponds to an open case.
 533          * Each vdev or pool can have a single case.
 534          */
 535         (void) nvlist_lookup_uint64(nvl,
 536             FM_EREPORT_PAYLOAD_ZFS_POOL_GUID, &pool_guid);
 537         if (nvlist_lookup_uint64(nvl,
 538             FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
 539                 vdev_guid = 0;
 540         if (nvlist_lookup_uint64(nvl, FM_EREPORT_ENA, &ena) != 0)
 541                 ena = 0;
 542
 543         zfs_ereport_when(hdl, nvl, &er_when);
 544
 545         for (zcp = uu_list_first(zfs_cases); zcp != NULL;
 546             zcp = uu_list_next(zfs_cases, zcp)) {
 547                 if (zcp->zc_data.zc_pool_guid == pool_guid) {
 548                         pool_found = B_TRUE;
 549                         pool_load = zcp->zc_when;
 550                 }
 551                 if (zcp->zc_data.zc_vdev_guid == vdev_guid)
 552                         break;
 553         }
 554
 555         /*
 556          * Avoid falsely accusing a pool of being faulty.  Do so by
 557          * not replaying ereports that were generated prior to the
 558          * current import.  If the failure that generated them was
 559          * transient because the device was actually removed but we
 560          * didn't receive the normal asynchronous notification, we
 561          * don't want to mark it as faulted and potentially panic. If
 562          * there is still a problem we'd expect not to be able to
 563          * import the pool, or that new ereports will be generated
 564          * once the pool is used.
 565          */
 566         if (pool_found && timeval_earlier(&er_when, &pool_load)) {
 567                 fmd_hdl_debug(hdl, "ignoring pool %llx, "
 568                     "ereport time %lld.%lld, pool load time = %lld.%lld",
 569                     pool_guid, er_when.ertv_sec, er_when.ertv_nsec,
 570                     pool_load.ertv_sec, pool_load.ertv_nsec);
 571                 zfs_stats.old_drops.fmds_value.ui64++;
 572                 return;
 573         }
 574
 575         if (!pool_found) {
 576                 /*
 577                  * Haven't yet seen this pool, but same situation
 578                  * may apply.
 579                  */
 580                 libzfs_handle_t *zhdl = fmd_hdl_getspecific(hdl);
 581                 struct load_time_arg la;
 582
 583                 la.lt_guid = pool_guid;
 584                 la.lt_time = &pool_load;
 585                 la.lt_found = B_FALSE;
 586
 587                 if (zhdl != NULL &&
 588                     zpool_iter(zhdl, zpool_find_load_time, &la) == 0 &&
 589                     la.lt_found == B_TRUE) {
 590                         pool_found = B_TRUE;
 591
 592                         if (timeval_earlier(&er_when, &pool_load)) {
 593                                 fmd_hdl_debug(hdl, "ignoring pool %llx, "
 594                                     "ereport time %lld.%lld, "
 595                                     "pool load time = %lld.%lld",
 596                                     pool_guid, er_when.ertv_sec,
 597                                     er_when.ertv_nsec, pool_load.ertv_sec,
 598                                     pool_load.ertv_nsec);
 599                                 zfs_stats.old_drops.fmds_value.ui64++;
 600                                 return;
 601                         }
 602                 }
 603         }
 604
 605         if (zcp == NULL) {
 606                 fmd_case_t *cs;
 607                 zfs_case_data_t data = { 0 };
 608
 609                 /*
 610                  * If this is one of our 'fake' resource ereports, and there is
 611                  * no case open, simply discard it.
 612                  */
 613                 if (isresource) {
 614                         zfs_stats.resource_drops.fmds_value.ui64++;
 615                         fmd_hdl_debug(hdl, "discarding '%s for vdev %llu",
 616                             class, vdev_guid);
 617                         return;
 618                 }
 619
 620                 /*
 621                  * Skip tracking some ereports
 622                  */
 623                 if (strcmp(class,
 624                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DATA)) == 0 ||
 625                     strcmp(class,
 626                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CONFIG_CACHE_WRITE)) == 0 ||
 627                     strcmp(class,
 628                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_DELAY)) == 0) {
 629                         zfs_stats.resource_drops.fmds_value.ui64++;
 630                         return;
 631                 }
 632
 633                 /*
 634                  * Open a new case.
 635                  */
 636                 cs = fmd_case_open(hdl, NULL);
 637
 638                 fmd_hdl_debug(hdl, "opening case for vdev %llu due to '%s'",
 639                     vdev_guid, class);
 640
 641                 /*
 642                  * Initialize the case buffer.  To commonize code, we actually
 643                  * create the buffer with existing data, and then call
 644                  * zfs_case_unserialize() to instantiate the in-core structure.
 645                  */
 646                 fmd_buf_create(hdl, cs, CASE_DATA, sizeof (zfs_case_data_t));
 647
 648                 data.zc_version = CASE_DATA_VERSION_SERD;
 649                 data.zc_ena = ena;
 650                 data.zc_pool_guid = pool_guid;
 651                 data.zc_vdev_guid = vdev_guid;
 652                 data.zc_pool_state = (int)pool_state;
 653
 654                 fmd_buf_write(hdl, cs, CASE_DATA, &data, sizeof (data));
 655
 656                 zcp = zfs_case_unserialize(hdl, cs);
 657                 assert(zcp != NULL);
 658                 if (pool_found)
 659                         zcp->zc_when = pool_load;
 660         }
 661
 662         if (isresource) {
 663                 fmd_hdl_debug(hdl, "resource event '%s'", class);
 664
 665                 if (fmd_nvl_class_match(hdl, nvl,
 666                     ZFS_MAKE_RSRC(FM_RESOURCE_AUTOREPLACE))) {
 667                         /*
 668                          * The 'resource.fs.zfs.autoreplace' event indicates
 669                          * that the pool was loaded with the 'autoreplace'
 670                          * property set.  In this case, any pending device
 671                          * failures should be ignored, as the asynchronous
 672                          * autoreplace handling will take care of them.
 673                          */
 674                         fmd_case_close(hdl, zcp->zc_case);
 675                 } else if (fmd_nvl_class_match(hdl, nvl,
 676                     ZFS_MAKE_RSRC(FM_RESOURCE_REMOVED))) {
 677                         /*
 678                          * The 'resource.fs.zfs.removed' event indicates that
 679                          * device removal was detected, and the device was
 680                          * closed asynchronously.  If this is the case, we
 681                          * assume that any recent I/O errors were due to the
 682                          * device removal, not any fault of the device itself.
 683                          * We reset the SERD engine, and cancel any pending
 684                          * timers.
 685                          */
 686                         if (zcp->zc_data.zc_has_remove_timer) {
 687                                 fmd_timer_remove(hdl, zcp->zc_remove_timer);
 688                                 zcp->zc_data.zc_has_remove_timer = 0;
 689                                 zfs_case_serialize(hdl, zcp);
 690                         }
 691                         if (zcp->zc_data.zc_serd_io[0] != '\0')
 692                                 fmd_serd_reset(hdl, zcp->zc_data.zc_serd_io);
 693                         if (zcp->zc_data.zc_serd_checksum[0] != '\0')
 694                                 fmd_serd_reset(hdl,
 695                                     zcp->zc_data.zc_serd_checksum);
 696                 } else if (fmd_nvl_class_match(hdl, nvl,
 697                     ZFS_MAKE_RSRC(FM_RESOURCE_STATECHANGE))) {
 698                         uint64_t state = 0;
 699
 700                         if (zcp != NULL &&
 701                             nvlist_lookup_uint64(nvl,
 702                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE, &state) == 0 &&
 703                             state == VDEV_STATE_HEALTHY) {
 704                                 fmd_hdl_debug(hdl, "closing case after a "
 705                                     "device statechange to healthy");
 706                                 fmd_case_close(hdl, zcp->zc_case);
 707                         }
 708                 }
 709                 zfs_stats.resource_drops.fmds_value.ui64++;
 710                 return;
 711         }
 712
 713         /*
 714          * Associate the ereport with this case.
 715          */
 716         fmd_case_add_ereport(hdl, zcp->zc_case, ep);
 717
 718         /*
 719          * Don't do anything else if this case is already solved.
 720          */
 721         if (fmd_case_solved(hdl, zcp->zc_case))
 722                 return;
 723
 724         fmd_hdl_debug(hdl, "error event '%s'", class);
 725
 726         /*
 727          * Determine if we should solve the case and generate a fault.  We solve
 728          * a case if:
 729          *
 730          *      a. A pool failed to open (ereport.fs.zfs.pool)
 731          *      b. A device failed to open (ereport.fs.zfs.pool) while a pool
 732          *         was up and running.
 733          *
 734          * We may see a series of ereports associated with a pool open, all
 735          * chained together by the same ENA.  If the pool open succeeds, then
 736          * we'll see no further ereports.  To detect when a pool open has
 737          * succeeded, we associate a timer with the event.  When it expires, we
 738          * close the case.
 739          */
 740         if (fmd_nvl_class_match(hdl, nvl,
 741             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_POOL))) {
 742                 /*
 743                  * Pool level fault.  Before solving the case, go through and
 744                  * close any open device cases that may be pending.
 745                  */
 746                 for (dcp = uu_list_first(zfs_cases); dcp != NULL;
 747                     dcp = uu_list_next(zfs_cases, dcp)) {
 748                         if (dcp->zc_data.zc_pool_guid ==
 749                             zcp->zc_data.zc_pool_guid &&
 750                             dcp->zc_data.zc_vdev_guid != 0)
 751                                 fmd_case_close(hdl, dcp->zc_case);
 752                 }
 753
 754                 zfs_case_solve(hdl, zcp, "fault.fs.zfs.pool", B_TRUE);
 755         } else if (fmd_nvl_class_match(hdl, nvl,
 756             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_LOG_REPLAY))) {
 757                 /*
 758                  * Pool level fault for reading the intent logs.
 759                  */
 760                 zfs_case_solve(hdl, zcp, "fault.fs.zfs.log_replay", B_TRUE);
 761         } else if (fmd_nvl_class_match(hdl, nvl, "ereport.fs.zfs.vdev.*")) {
 762                 /*
 763                  * Device fault.
 764                  */
 765                 zfs_case_solve(hdl, zcp, "fault.fs.zfs.device",  B_TRUE);
 766         } else if (fmd_nvl_class_match(hdl, nvl,
 767             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO)) ||
 768             fmd_nvl_class_match(hdl, nvl,
 769             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM)) ||
 770             fmd_nvl_class_match(hdl, nvl,
 771             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) ||
 772             fmd_nvl_class_match(hdl, nvl,
 773             ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
 774                 char *failmode = NULL;
 775                 boolean_t checkremove = B_FALSE;
 776
 777                 /*
 778                  * If this is a checksum or I/O error, then toss it into the
 779                  * appropriate SERD engine and check to see if it has fired.
 780                  * Ideally, we want to do something more sophisticated,
 781                  * (persistent errors for a single data block, etc).  For now,
 782                  * a single SERD engine is sufficient.
 783                  */
 784                 if (fmd_nvl_class_match(hdl, nvl,
 785                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO))) {
 786                         if (zcp->zc_data.zc_serd_io[0] == '\0') {
 787                                 zfs_serd_name(zcp->zc_data.zc_serd_io,
 788                                     pool_guid, vdev_guid, "io");
 789                                 fmd_serd_create(hdl, zcp->zc_data.zc_serd_io,
 790                                     fmd_prop_get_int32(hdl, "io_N"),
 791                                     fmd_prop_get_int64(hdl, "io_T"));
 792                                 zfs_case_serialize(hdl, zcp);
 793                         }
 794                         if (fmd_serd_record(hdl, zcp->zc_data.zc_serd_io, ep))
 795                                 checkremove = B_TRUE;
 796                 } else if (fmd_nvl_class_match(hdl, nvl,
 797                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_CHECKSUM))) {
 798                         if (zcp->zc_data.zc_serd_checksum[0] == '\0') {
 799                                 zfs_serd_name(zcp->zc_data.zc_serd_checksum,
 800                                     pool_guid, vdev_guid, "checksum");
 801                                 fmd_serd_create(hdl,
 802                                     zcp->zc_data.zc_serd_checksum,
 803                                     fmd_prop_get_int32(hdl, "checksum_N"),
 804                                     fmd_prop_get_int64(hdl, "checksum_T"));
 805                                 zfs_case_serialize(hdl, zcp);
 806                         }
 807                         if (fmd_serd_record(hdl,
 808                             zcp->zc_data.zc_serd_checksum, ep)) {
 809                                 zfs_case_solve(hdl, zcp,
 810                                     "fault.fs.zfs.vdev.checksum", B_FALSE);
 811                         }
 812                 } else if (fmd_nvl_class_match(hdl, nvl,
 813                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_IO_FAILURE)) &&
 814                     (nvlist_lookup_string(nvl,
 815                     FM_EREPORT_PAYLOAD_ZFS_POOL_FAILMODE, &failmode) == 0) &&
 816                     failmode != NULL) {
 817                         if (strncmp(failmode, FM_EREPORT_FAILMODE_CONTINUE,
 818                             strlen(FM_EREPORT_FAILMODE_CONTINUE)) == 0) {
 819                                 zfs_case_solve(hdl, zcp,
 820                                     "fault.fs.zfs.io_failure_continue",
 821                                     B_FALSE);
 822                         } else if (strncmp(failmode, FM_EREPORT_FAILMODE_WAIT,
 823                             strlen(FM_EREPORT_FAILMODE_WAIT)) == 0) {
 824                                 zfs_case_solve(hdl, zcp,
 825                                     "fault.fs.zfs.io_failure_wait", B_FALSE);
 826                         }
 827                 } else if (fmd_nvl_class_match(hdl, nvl,
 828                     ZFS_MAKE_EREPORT(FM_EREPORT_ZFS_PROBE_FAILURE))) {
 829 #ifndef __linux__
 830                         /* This causes an unexpected fault diagnosis on linux */
 831                         checkremove = B_TRUE;
 832 #endif
 833                 }
 834
 835                 /*
 836                  * Because I/O errors may be due to device removal, we postpone
 837                  * any diagnosis until we're sure that we aren't about to
 838                  * receive a 'resource.fs.zfs.removed' event.
 839                  */
 840                 if (checkremove) {
 841                         if (zcp->zc_data.zc_has_remove_timer)
 842                                 fmd_timer_remove(hdl, zcp->zc_remove_timer);
 843                         zcp->zc_remove_timer = fmd_timer_install(hdl, zcp, NULL,
 844                             zfs_remove_timeout);
 845                         if (!zcp->zc_data.zc_has_remove_timer) {
 846                                 zcp->zc_data.zc_has_remove_timer = 1;
 847                                 zfs_case_serialize(hdl, zcp);
 848                         }
 849                 }
 850         }
 851 }
 852
 853 /*
 854  * The timeout is fired when we diagnosed an I/O error, and it was not due to
 855  * device removal (which would cause the timeout to be cancelled).
 856  */
 857 /* ARGSUSED */
 858 static void
 859 zfs_fm_timeout(fmd_hdl_t *hdl, id_t id, void *data)
 860 {
 861         zfs_case_t *zcp = data;
 862
 863         if (id == zcp->zc_remove_timer)
 864                 zfs_case_solve(hdl, zcp, "fault.fs.zfs.vdev.io", B_FALSE);
 865 }
 866
 867 /*
 868  * The specified case has been closed and any case-specific
 869  * data structures should be deallocated.
 870  */
 871 static void
 872 zfs_fm_close(fmd_hdl_t *hdl, fmd_case_t *cs)
 873 {
 874         zfs_case_t *zcp = fmd_case_getspecific(hdl, cs);
 875
 876         if (zcp->zc_data.zc_serd_checksum[0] != '\0')
 877                 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_checksum);
 878         if (zcp->zc_data.zc_serd_io[0] != '\0')
 879                 fmd_serd_destroy(hdl, zcp->zc_data.zc_serd_io);
 880         if (zcp->zc_data.zc_has_remove_timer)
 881                 fmd_timer_remove(hdl, zcp->zc_remove_timer);
 882
 883         uu_list_remove(zfs_cases, zcp);
 884         uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
 885         fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
 886 }
 887
 888 /*
 889  * We use the fmd gc entry point to look for old cases that no longer apply.
 890  * This allows us to keep our set of case data small in a long running system.
 891  */
 892 static void
 893 zfs_fm_gc(fmd_hdl_t *hdl)
 894 {
 895         zfs_purge_cases(hdl);
 896 }
 897
 898 static const fmd_hdl_ops_t fmd_ops = {
 899         zfs_fm_recv,    /* fmdo_recv */
 900         zfs_fm_timeout, /* fmdo_timeout */
 901         zfs_fm_close,   /* fmdo_close */
 902         NULL,           /* fmdo_stats */
 903         zfs_fm_gc,      /* fmdo_gc */
 904 };
 905
 906 static const fmd_prop_t fmd_props[] = {
 907         { "checksum_N", FMD_TYPE_UINT32, "10" },
 908         { "checksum_T", FMD_TYPE_TIME, "10min" },
 909         { "io_N", FMD_TYPE_UINT32, "10" },
 910         { "io_T", FMD_TYPE_TIME, "10min" },
 911         { "remove_timeout", FMD_TYPE_TIME, "15sec" },
 912         { NULL, 0, NULL }
 913 };
 914
 915 static const fmd_hdl_info_t fmd_info = {
 916         "ZFS Diagnosis Engine", "1.0", &fmd_ops, fmd_props
 917 };
 918
 919 void
 920 _zfs_diagnosis_init(fmd_hdl_t *hdl)
 921 {
 922         libzfs_handle_t *zhdl;
 923
 924         if ((zhdl = libzfs_init()) == NULL)
 925                 return;
 926
 927         if ((zfs_case_pool = uu_list_pool_create("zfs_case_pool",
 928             sizeof (zfs_case_t), offsetof(zfs_case_t, zc_node),
 929             NULL, UU_LIST_POOL_DEBUG)) == NULL) {
 930                 libzfs_fini(zhdl);
 931                 return;
 932         }
 933
 934         if ((zfs_cases = uu_list_create(zfs_case_pool, NULL,
 935             UU_LIST_DEBUG)) == NULL) {
 936                 uu_list_pool_destroy(zfs_case_pool);
 937                 libzfs_fini(zhdl);
 938                 return;
 939         }
 940
 941         if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
 942                 uu_list_destroy(zfs_cases);
 943                 uu_list_pool_destroy(zfs_case_pool);
 944                 libzfs_fini(zhdl);
 945                 return;
 946         }
 947
 948         fmd_hdl_setspecific(hdl, zhdl);
 949
 950         (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (zfs_stats) /
 951             sizeof (fmd_stat_t), (fmd_stat_t *)&zfs_stats);
 952
 953         zfs_remove_timeout = fmd_prop_get_int64(hdl, "remove_timeout");
 954 }
 955
 956 void
 957 _zfs_diagnosis_fini(fmd_hdl_t *hdl)
 958 {
 959         zfs_case_t *zcp;
 960         uu_list_walk_t *walk;
 961         libzfs_handle_t *zhdl;
 962
 963         /*
 964          * Remove all active cases.
 965          */
 966         walk = uu_list_walk_start(zfs_cases, UU_WALK_ROBUST);
 967         while ((zcp = uu_list_walk_next(walk)) != NULL) {
 968                 fmd_hdl_debug(hdl, "removing case ena %llu",
 969                     (long long unsigned)zcp->zc_data.zc_ena);
 970                 uu_list_remove(zfs_cases, zcp);
 971                 uu_list_node_fini(zcp, &zcp->zc_node, zfs_case_pool);
 972                 fmd_hdl_free(hdl, zcp, sizeof (zfs_case_t));
 973         }
 974         uu_list_walk_end(walk);
 975
 976         uu_list_destroy(zfs_cases);
 977         uu_list_pool_destroy(zfs_case_pool);
 978
 979         zhdl = fmd_hdl_getspecific(hdl);
 980         libzfs_fini(zhdl);
 981 }