2 * Copyright (c) 2012, 2013, 2014 Spectra Logic Corporation
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions, and the following disclaimer,
10 * without modification.
11 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
12 * substantially similar to the "NO WARRANTY" disclaimer below
13 * ("Disclaimer") and any redistribution must be conditioned upon
14 * including a substantially similar Disclaimer requirement for further
15 * binary redistribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 * POSSIBILITY OF SUCH DAMAGES.
30 * Authors: Alan Somers (Spectra Logic Corporation)
32 #include <sys/cdefs.h>
37 #include <libnvpair.h>
45 #include <gmock/gmock.h>
46 #include <gtest/gtest.h>
48 #include <devdctl/guid.h>
49 #include <devdctl/event.h>
50 #include <devdctl/event_factory.h>
51 #include <devdctl/exception.h>
52 #include <devdctl/consumer.h>
54 #include <zfsd/callout.h>
55 #include <zfsd/vdev_iterator.h>
56 #include <zfsd/zfsd_event.h>
57 #include <zfsd/case_file.h>
58 #include <zfsd/vdev.h>
59 #include <zfsd/zfsd.h>
60 #include <zfsd/zfsd_exception.h>
61 #include <zfsd/zpool_list.h>
65 __FBSDID("$FreeBSD$");
67 /*================================== Macros ==================================*/
68 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
70 /*============================ Namespace Control =============================*/
72 using std::stringstream;
75 using DevdCtl::EventFactory;
76 using DevdCtl::EventList;
78 using DevdCtl::NVPairMap;
80 /* redefine zpool_handle here because libzfs_impl.h is not includable */
83 libzfs_handle_t *zpool_hdl;
84 zpool_handle_t *zpool_next;
85 char zpool_name[ZFS_MAX_DATASET_NAME_LEN];
87 size_t zpool_config_size;
88 nvlist_t *zpool_config;
89 nvlist_t *zpool_old_config;
90 nvlist_t *zpool_props;
91 diskaddr_t zpool_start_block;
94 class MockZfsEvent : public ZfsEvent
97 MockZfsEvent(Event::Type, NVPairMap&, const string&);
98 virtual ~MockZfsEvent() {}
100 static BuildMethod MockZfsEventBuilder;
102 MOCK_CONST_METHOD0(ProcessPoolEvent, void());
104 static EventFactory::Record s_buildRecords[];
107 EventFactory::Record MockZfsEvent::s_buildRecords[] =
109 { Event::NOTIFY, "ZFS", &MockZfsEvent::MockZfsEventBuilder }
112 MockZfsEvent::MockZfsEvent(Event::Type type, NVPairMap& map,
114 : ZfsEvent(type, map, str)
119 MockZfsEvent::MockZfsEventBuilder(Event::Type type,
121 const string &eventString)
123 return (new MockZfsEvent(type, nvpairs, eventString));
127 * A dummy Vdev class used for testing other classes
129 class MockVdev : public Vdev
132 MockVdev(nvlist_t *vdevConfig);
133 virtual ~MockVdev() {}
135 MOCK_CONST_METHOD0(GUID, Guid());
136 MOCK_CONST_METHOD0(PoolGUID, Guid());
137 MOCK_CONST_METHOD0(State, vdev_state());
138 MOCK_CONST_METHOD0(PhysicalPath, string());
141 MockVdev::MockVdev(nvlist_t *vdevConfig)
147 * A CaseFile class with side effects removed, for testing
149 class TestableCaseFile : public CaseFile
152 static TestableCaseFile &Create(Vdev &vdev);
153 TestableCaseFile(Vdev &vdev);
154 virtual ~TestableCaseFile() {}
156 MOCK_METHOD0(Close, void());
157 MOCK_METHOD1(RegisterCallout, void(const Event &event));
158 MOCK_METHOD0(RefreshVdevState, bool());
159 MOCK_METHOD1(ReEvaluate, bool(const ZfsEvent &event));
161 bool RealReEvaluate(const ZfsEvent &event)
163 return (CaseFile::ReEvaluate(event));
167 * This splices the event lists, a procedure that would normally be done
168 * by OnGracePeriodEnded, but we don't necessarily call that in the
174 * Used by some of our expectations. CaseFile does not publicize this
176 static int getActiveCases()
178 return (s_activeCases.size());
182 TestableCaseFile::TestableCaseFile(Vdev &vdev)
188 TestableCaseFile::Create(Vdev &vdev)
190 TestableCaseFile *newCase;
191 newCase = new TestableCaseFile(vdev);
196 TestableCaseFile::SpliceEvents()
198 m_events.splice(m_events.begin(), m_tentativeEvents);
203 * Test class ZfsdException
205 class ZfsdExceptionTest : public ::testing::Test
210 ASSERT_EQ(0, nvlist_alloc(&poolConfig, NV_UNIQUE_NAME, 0));
211 ASSERT_EQ(0, nvlist_add_string(poolConfig,
212 ZPOOL_CONFIG_POOL_NAME, "unit_test_pool"));
213 ASSERT_EQ(0, nvlist_add_uint64(poolConfig,
214 ZPOOL_CONFIG_POOL_GUID, 0x1234));
216 ASSERT_EQ(0, nvlist_alloc(&vdevConfig, NV_UNIQUE_NAME, 0));
217 ASSERT_EQ(0, nvlist_add_uint64(vdevConfig,
218 ZPOOL_CONFIG_GUID, 0x5678));
219 bzero(&poolHandle, sizeof(poolHandle));
220 poolHandle.zpool_config = poolConfig;
223 virtual void TearDown()
225 nvlist_free(poolConfig);
226 nvlist_free(vdevConfig);
229 nvlist_t *poolConfig;
230 nvlist_t *vdevConfig;
231 zpool_handle_t poolHandle;
234 TEST_F(ZfsdExceptionTest, StringConstructorNull)
236 ZfsdException ze("");
237 EXPECT_STREQ("", ze.GetString().c_str());
240 TEST_F(ZfsdExceptionTest, StringConstructorFormatted)
242 ZfsdException ze(" %d %s", 55, "hello world");
243 EXPECT_STREQ(" 55 hello world", ze.GetString().c_str());
246 TEST_F(ZfsdExceptionTest, LogSimple)
248 ZfsdException ze("unit test w/o vdev or pool");
250 EXPECT_EQ(LOG_ERR, syslog_last_priority);
251 EXPECT_STREQ("unit test w/o vdev or pool\n", syslog_last_message);
254 TEST_F(ZfsdExceptionTest, Pool)
256 const char msg[] = "Exception with pool name";
258 sprintf(expected, "Pool unit_test_pool: %s\n", msg);
259 ZfsdException ze(poolConfig, msg);
261 EXPECT_STREQ(expected, syslog_last_message);
264 TEST_F(ZfsdExceptionTest, PoolHandle)
266 const char msg[] = "Exception with pool handle";
268 sprintf(expected, "Pool unit_test_pool: %s\n", msg);
269 ZfsdException ze(&poolHandle, msg);
271 EXPECT_STREQ(expected, syslog_last_message);
277 class VdevTest : public ::testing::Test
282 ASSERT_EQ(0, nvlist_alloc(&m_poolConfig, NV_UNIQUE_NAME, 0));
283 ASSERT_EQ(0, nvlist_add_uint64(m_poolConfig,
284 ZPOOL_CONFIG_POOL_GUID,
287 ASSERT_EQ(0, nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0));
288 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_GUID,
292 virtual void TearDown()
294 nvlist_free(m_poolConfig);
295 nvlist_free(m_vdevConfig);
298 nvlist_t *m_poolConfig;
299 nvlist_t *m_vdevConfig;
303 TEST_F(VdevTest, StateFromConfig)
307 vs.vs_state = VDEV_STATE_OFFLINE;
309 ASSERT_EQ(0, nvlist_add_uint64_array(m_vdevConfig,
310 ZPOOL_CONFIG_VDEV_STATS,
312 sizeof(vs) / sizeof(uint64_t)));
314 Vdev vdev(m_poolConfig, m_vdevConfig);
316 EXPECT_EQ(VDEV_STATE_OFFLINE, vdev.State());
319 TEST_F(VdevTest, StateFaulted)
321 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_FAULTED, 1));
323 Vdev vdev(m_poolConfig, m_vdevConfig);
325 EXPECT_EQ(VDEV_STATE_FAULTED, vdev.State());
329 * Test that we can construct a Vdev from the label information that is stored
330 * on an available spare drive
332 TEST_F(VdevTest, ConstructAvailSpare)
334 nvlist_t *labelConfig;
336 ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
337 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
338 1948339428197961030));
339 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
342 EXPECT_NO_THROW(Vdev vdev(labelConfig));
344 nvlist_free(labelConfig);
347 /* Available spares will always show the HEALTHY state */
348 TEST_F(VdevTest, AvailSpareState) {
349 nvlist_t *labelConfig;
351 ASSERT_EQ(0, nvlist_alloc(&labelConfig, NV_UNIQUE_NAME, 0));
352 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_GUID,
353 1948339428197961030));
354 ASSERT_EQ(0, nvlist_add_uint64(labelConfig, ZPOOL_CONFIG_POOL_STATE,
357 Vdev vdev(labelConfig);
358 EXPECT_EQ(VDEV_STATE_HEALTHY, vdev.State());
360 nvlist_free(labelConfig);
363 /* Test the Vdev::IsSpare method */
364 TEST_F(VdevTest, IsSpare) {
365 Vdev notSpare(m_poolConfig, m_vdevConfig);
366 EXPECT_EQ(false, notSpare.IsSpare());
368 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig, ZPOOL_CONFIG_IS_SPARE, 1));
369 Vdev isSpare(m_poolConfig, m_vdevConfig);
370 EXPECT_EQ(true, isSpare.IsSpare());
374 * Test class ZFSEvent
376 class ZfsEventTest : public ::testing::Test
381 m_eventFactory = new EventFactory();
382 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
383 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
388 virtual void TearDown()
390 delete m_eventFactory;
394 EventFactory *m_eventFactory;
398 TEST_F(ZfsEventTest, ProcessPoolEventGetsCalled)
400 string evString("!system=ZFS "
402 "type=misc.fs.zfs.vdev_remove "
404 "pool_guid=9756779504028057996 "
405 "vdev_guid=1631193447431603339 "
406 "vdev_path=/dev/da1 "
407 "timestamp=1348871594");
408 m_event = Event::CreateEvent(*m_eventFactory, evString);
409 MockZfsEvent *mock_event = static_cast<MockZfsEvent*>(m_event);
411 EXPECT_CALL(*mock_event, ProcessPoolEvent()).Times(1);
412 mock_event->Process();
416 * Test class CaseFile
419 class CaseFileTest : public ::testing::Test
424 m_eventFactory = new EventFactory();
425 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
426 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
430 nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
431 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
432 ZPOOL_CONFIG_GUID, 0xbeef));
433 m_vdev = new MockVdev(m_vdevConfig);
434 ON_CALL(*m_vdev, GUID())
435 .WillByDefault(::testing::Return(Guid(123)));
436 ON_CALL(*m_vdev, PoolGUID())
437 .WillByDefault(::testing::Return(Guid(456)));
438 ON_CALL(*m_vdev, State())
439 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
440 m_caseFile = &TestableCaseFile::Create(*m_vdev);
441 ON_CALL(*m_caseFile, ReEvaluate(::testing::_))
442 .WillByDefault(::testing::Invoke(m_caseFile, &TestableCaseFile::RealReEvaluate));
446 virtual void TearDown()
449 nvlist_free(m_vdevConfig);
452 delete m_eventFactory;
455 nvlist_t *m_vdevConfig;
457 TestableCaseFile *m_caseFile;
459 EventFactory *m_eventFactory;
463 * A Vdev with no events should not be degraded or faulted
465 TEST_F(CaseFileTest, HealthyVdev)
467 EXPECT_FALSE(m_caseFile->ShouldDegrade());
468 EXPECT_FALSE(m_caseFile->ShouldFault());
472 * A Vdev with only one event should not be degraded or faulted
473 * For performance reasons, RefreshVdevState should not be called.
475 TEST_F(CaseFileTest, HealthyishVdev)
477 string evString("!system=ZFS "
478 "class=ereport.fs.zfs.io "
479 "ena=12091638756982918145 "
480 "parent_guid=13237004955564865395 "
482 "pool=testpool.4415 "
484 "pool_failmode=wait "
487 "timestamp=1348867914 "
488 "type=ereport.fs.zfs.io "
490 "vdev_path=/dev/da400 "
497 "zio_offset=25598976 "
499 m_event = Event::CreateEvent(*m_eventFactory, evString);
500 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
502 EXPECT_CALL(*m_caseFile, RefreshVdevState())
503 .Times(::testing::Exactly(0));
504 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
505 EXPECT_FALSE(m_caseFile->ShouldDegrade());
506 EXPECT_FALSE(m_caseFile->ShouldFault());
509 /* The case file should be closed when its pool is destroyed */
510 TEST_F(CaseFileTest, PoolDestroy)
512 string evString("!system=ZFS "
513 "pool_name=testpool.4415 "
516 "timestamp=1348867914 "
517 "type=misc.fs.zfs.pool_destroy ");
518 m_event = Event::CreateEvent(*m_eventFactory, evString);
519 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
520 EXPECT_CALL(*m_caseFile, Close());
521 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
525 * A Vdev with a very large number of IO errors should fault
526 * For performance reasons, RefreshVdevState should be called at most once
528 TEST_F(CaseFileTest, VeryManyIOErrors)
530 EXPECT_CALL(*m_caseFile, RefreshVdevState())
531 .Times(::testing::AtMost(1))
532 .WillRepeatedly(::testing::Return(true));
534 for(int i=0; i<100; i++) {
535 stringstream evStringStream;
538 "class=ereport.fs.zfs.io "
539 "ena=12091638756982918145 "
540 "parent_guid=13237004955564865395 "
542 "pool=testpool.4415 "
544 "pool_failmode=wait "
548 evStringStream << i << " ";
550 "type=ereport.fs.zfs.io "
552 "vdev_path=/dev/da400 "
559 "zio_offset=25598976 "
561 Event *event(Event::CreateEvent(*m_eventFactory,
562 evStringStream.str()));
563 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
564 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
568 m_caseFile->SpliceEvents();
569 EXPECT_FALSE(m_caseFile->ShouldDegrade());
570 EXPECT_TRUE(m_caseFile->ShouldFault());
574 * A Vdev with a very large number of checksum errors should degrade
575 * For performance reasons, RefreshVdevState should be called at most once
577 TEST_F(CaseFileTest, VeryManyChecksumErrors)
579 EXPECT_CALL(*m_caseFile, RefreshVdevState())
580 .Times(::testing::AtMost(1))
581 .WillRepeatedly(::testing::Return(true));
583 for(int i=0; i<100; i++) {
584 stringstream evStringStream;
587 "bad_cleared_bits=03000000000000803f50b00000000000 "
588 "bad_range_clears=0000000e "
589 "bad_range_sets=00000000 "
590 "bad_ranges=0000000000000010 "
591 "bad_ranges_min_gap=8 "
592 "bad_set_bits=00000000000000000000000000000000 "
593 "class=ereport.fs.zfs.checksum "
594 "ena=12272856582652437505 "
595 "parent_guid=5838204195352909894 "
596 "parent_type=raidz pool=testpool.7640 "
598 "pool_failmode=wait "
600 "subsystem=ZFS timestamp=";
601 evStringStream << i << " ";
603 "type=ereport.fs.zfs.checksum "
605 "vdev_path=/mnt/tmp/file1.7702 "
614 Event *event(Event::CreateEvent(*m_eventFactory,
615 evStringStream.str()));
616 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(event);
617 EXPECT_TRUE(m_caseFile->ReEvaluate(*zfs_event));
621 m_caseFile->SpliceEvents();
622 EXPECT_TRUE(m_caseFile->ShouldDegrade());
623 EXPECT_FALSE(m_caseFile->ShouldFault());
627 * Test CaseFile::ReEvaluateByGuid
629 class ReEvaluateByGuidTest : public ::testing::Test
634 m_eventFactory = new EventFactory();
635 m_eventFactory->UpdateRegistry(MockZfsEvent::s_buildRecords,
636 NUM_ELEMENTS(MockZfsEvent::s_buildRecords));
637 m_event = Event::CreateEvent(*m_eventFactory, s_evString);
638 nvlist_alloc(&m_vdevConfig, NV_UNIQUE_NAME, 0);
639 ASSERT_EQ(0, nvlist_add_uint64(m_vdevConfig,
640 ZPOOL_CONFIG_GUID, 0xbeef));
641 m_vdev456 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
642 m_vdev789 = new ::testing::NiceMock<MockVdev>(m_vdevConfig);
643 ON_CALL(*m_vdev456, GUID())
644 .WillByDefault(::testing::Return(Guid(123)));
645 ON_CALL(*m_vdev456, PoolGUID())
646 .WillByDefault(::testing::Return(Guid(456)));
647 ON_CALL(*m_vdev456, State())
648 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
649 ON_CALL(*m_vdev789, GUID())
650 .WillByDefault(::testing::Return(Guid(123)));
651 ON_CALL(*m_vdev789, PoolGUID())
652 .WillByDefault(::testing::Return(Guid(789)));
653 ON_CALL(*m_vdev789, State())
654 .WillByDefault(::testing::Return(VDEV_STATE_HEALTHY));
655 m_caseFile456 = NULL;
656 m_caseFile789 = NULL;
660 virtual void TearDown()
662 delete m_caseFile456;
663 delete m_caseFile789;
664 nvlist_free(m_vdevConfig);
668 delete m_eventFactory;
671 static string s_evString;
672 nvlist_t *m_vdevConfig;
673 ::testing::NiceMock<MockVdev> *m_vdev456;
674 ::testing::NiceMock<MockVdev> *m_vdev789;
675 TestableCaseFile *m_caseFile456;
676 TestableCaseFile *m_caseFile789;
678 EventFactory *m_eventFactory;
681 string ReEvaluateByGuidTest::s_evString(
683 "pool_guid=16271873792808333580 "
686 "timestamp=1360620391 "
687 "type=misc.fs.zfs.config_sync");
691 * Test the ReEvaluateByGuid method on an empty list of casefiles.
692 * We must create one event, even though it never gets used, because it will
693 * be passed by reference to ReEvaluateByGuid
695 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_empty)
697 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
699 EXPECT_EQ(0, TestableCaseFile::getActiveCases());
700 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
701 EXPECT_EQ(0, TestableCaseFile::getActiveCases());
705 * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
706 * one CaseFile, which doesn't match the criteria
708 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneFalse)
710 m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
711 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
713 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
714 EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
715 .Times(::testing::Exactly(0));
716 CaseFile::ReEvaluateByGuid(Guid(789), *zfs_event);
717 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
721 * Test the ReEvaluateByGuid method on a list of CaseFiles that contains only
722 * one CaseFile, which does match the criteria
724 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_oneTrue)
726 m_caseFile456 = &TestableCaseFile::Create(*m_vdev456);
727 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
729 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
730 EXPECT_CALL(*m_caseFile456, ReEvaluate(::testing::_))
731 .Times(::testing::Exactly(1))
732 .WillRepeatedly(::testing::Return(false));
733 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
734 EXPECT_EQ(1, TestableCaseFile::getActiveCases());
738 * Test the ReEvaluateByGuid method on a long list of CaseFiles that contains a
739 * few cases which meet the criteria
741 TEST_F(ReEvaluateByGuidTest, ReEvaluateByGuid_five)
743 TestableCaseFile *CaseFile1 = &TestableCaseFile::Create(*m_vdev456);
744 TestableCaseFile *CaseFile2 = &TestableCaseFile::Create(*m_vdev789);
745 TestableCaseFile *CaseFile3 = &TestableCaseFile::Create(*m_vdev456);
746 TestableCaseFile *CaseFile4 = &TestableCaseFile::Create(*m_vdev789);
747 TestableCaseFile *CaseFile5 = &TestableCaseFile::Create(*m_vdev789);
748 ZfsEvent *zfs_event = static_cast<ZfsEvent*>(m_event);
750 EXPECT_EQ(5, TestableCaseFile::getActiveCases());
751 EXPECT_CALL(*CaseFile1, ReEvaluate(::testing::_))
752 .Times(::testing::Exactly(1))
753 .WillRepeatedly(::testing::Return(false));
754 EXPECT_CALL(*CaseFile3, ReEvaluate(::testing::_))
755 .Times(::testing::Exactly(1))
756 .WillRepeatedly(::testing::Return(false));
757 EXPECT_CALL(*CaseFile2, ReEvaluate(::testing::_))
758 .Times(::testing::Exactly(0));
759 EXPECT_CALL(*CaseFile4, ReEvaluate(::testing::_))
760 .Times(::testing::Exactly(0));
761 EXPECT_CALL(*CaseFile5, ReEvaluate(::testing::_))
762 .Times(::testing::Exactly(0));
763 CaseFile::ReEvaluateByGuid(Guid(456), *zfs_event);
764 EXPECT_EQ(5, TestableCaseFile::getActiveCases());