contrib/googletest/googletest/test/googletest-json-output-unittest.py

   1 #!/usr/bin/env python
   2 # Copyright 2018, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit test for the gtest_json_output module."""
  32
  33 import datetime
  34 import errno
  35 import json
  36 import os
  37 import re
  38 import sys
  39
  40 import gtest_json_test_utils
  41 import gtest_test_utils
  42
  43 GTEST_FILTER_FLAG = '--gtest_filter'
  44 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
  45 GTEST_OUTPUT_FLAG = '--gtest_output'
  46 GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json'
  47 GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_'
  48
  49 # The flag indicating stacktraces are not supported
  50 NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'
  51
  52 SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
  53
  54 if SUPPORTS_STACK_TRACES:
  55   STACK_TRACE_TEMPLATE = '\nStack trace:\n*'
  56 else:
  57   STACK_TRACE_TEMPLATE = ''
  58
  59 EXPECTED_NON_EMPTY = {
  60     u'tests': 23,
  61     u'failures': 4,
  62     u'disabled': 2,
  63     u'errors': 0,
  64     u'timestamp': u'*',
  65     u'time': u'*',
  66     u'ad_hoc_property': u'42',
  67     u'name': u'AllTests',
  68     u'testsuites': [
  69         {
  70             u'name': u'SuccessfulTest',
  71             u'tests': 1,
  72             u'failures': 0,
  73             u'disabled': 0,
  74             u'errors': 0,
  75             u'time': u'*',
  76             u'testsuite': [
  77                 {
  78                     u'name': u'Succeeds',
  79                     u'status': u'RUN',
  80                     u'time': u'*',
  81                     u'classname': u'SuccessfulTest'
  82                 }
  83             ]
  84         },
  85         {
  86             u'name': u'FailedTest',
  87             u'tests': 1,
  88             u'failures': 1,
  89             u'disabled': 0,
  90             u'errors': 0,
  91             u'time': u'*',
  92             u'testsuite': [
  93                 {
  94                     u'name': u'Fails',
  95                     u'status': u'RUN',
  96                     u'time': u'*',
  97                     u'classname': u'FailedTest',
  98                     u'failures': [
  99                         {
 100                             u'failure':
 101                                 u'gtest_xml_output_unittest_.cc:*\n'
 102                                 u'Expected equality of these values:\n'
 103                                 u'  1\n  2' + STACK_TRACE_TEMPLATE,
 104                             u'type': u''
 105                         }
 106                     ]
 107                 }
 108             ]
 109         },
 110         {
 111             u'name': u'DisabledTest',
 112             u'tests': 1,
 113             u'failures': 0,
 114             u'disabled': 1,
 115             u'errors': 0,
 116             u'time': u'*',
 117             u'testsuite': [
 118                 {
 119                     u'name': u'DISABLED_test_not_run',
 120                     u'status': u'NOTRUN',
 121                     u'time': u'*',
 122                     u'classname': u'DisabledTest'
 123                 }
 124             ]
 125         },
 126         {
 127             u'name': u'MixedResultTest',
 128             u'tests': 3,
 129             u'failures': 1,
 130             u'disabled': 1,
 131             u'errors': 0,
 132             u'time': u'*',
 133             u'testsuite': [
 134                 {
 135                     u'name': u'Succeeds',
 136                     u'status': u'RUN',
 137                     u'time': u'*',
 138                     u'classname': u'MixedResultTest'
 139                 },
 140                 {
 141                     u'name': u'Fails',
 142                     u'status': u'RUN',
 143                     u'time': u'*',
 144                     u'classname': u'MixedResultTest',
 145                     u'failures': [
 146                         {
 147                             u'failure':
 148                                 u'gtest_xml_output_unittest_.cc:*\n'
 149                                 u'Expected equality of these values:\n'
 150                                 u'  1\n  2' + STACK_TRACE_TEMPLATE,
 151                             u'type': u''
 152                         },
 153                         {
 154                             u'failure':
 155                                 u'gtest_xml_output_unittest_.cc:*\n'
 156                                 u'Expected equality of these values:\n'
 157                                 u'  2\n  3' + STACK_TRACE_TEMPLATE,
 158                             u'type': u''
 159                         }
 160                     ]
 161                 },
 162                 {
 163                     u'name': u'DISABLED_test',
 164                     u'status': u'NOTRUN',
 165                     u'time': u'*',
 166                     u'classname': u'MixedResultTest'
 167                 }
 168             ]
 169         },
 170         {
 171             u'name': u'XmlQuotingTest',
 172             u'tests': 1,
 173             u'failures': 1,
 174             u'disabled': 0,
 175             u'errors': 0,
 176             u'time': u'*',
 177             u'testsuite': [
 178                 {
 179                     u'name': u'OutputsCData',
 180                     u'status': u'RUN',
 181                     u'time': u'*',
 182                     u'classname': u'XmlQuotingTest',
 183                     u'failures': [
 184                         {
 185                             u'failure':
 186                                 u'gtest_xml_output_unittest_.cc:*\n'
 187                                 u'Failed\nXML output: <?xml encoding="utf-8">'
 188                                 u'<top><![CDATA[cdata text]]></top>' +
 189                                 STACK_TRACE_TEMPLATE,
 190                             u'type': u''
 191                         }
 192                     ]
 193                 }
 194             ]
 195         },
 196         {
 197             u'name': u'InvalidCharactersTest',
 198             u'tests': 1,
 199             u'failures': 1,
 200             u'disabled': 0,
 201             u'errors': 0,
 202             u'time': u'*',
 203             u'testsuite': [
 204                 {
 205                     u'name': u'InvalidCharactersInMessage',
 206                     u'status': u'RUN',
 207                     u'time': u'*',
 208                     u'classname': u'InvalidCharactersTest',
 209                     u'failures': [
 210                         {
 211                             u'failure':
 212                                 u'gtest_xml_output_unittest_.cc:*\n'
 213                                 u'Failed\nInvalid characters in brackets'
 214                                 u' [\x01\x02]' + STACK_TRACE_TEMPLATE,
 215                             u'type': u''
 216                         }
 217                     ]
 218                 }
 219             ]
 220         },
 221         {
 222             u'name': u'PropertyRecordingTest',
 223             u'tests': 4,
 224             u'failures': 0,
 225             u'disabled': 0,
 226             u'errors': 0,
 227             u'time': u'*',
 228             u'SetUpTestCase': u'yes',
 229             u'TearDownTestCase': u'aye',
 230             u'testsuite': [
 231                 {
 232                     u'name': u'OneProperty',
 233                     u'status': u'RUN',
 234                     u'time': u'*',
 235                     u'classname': u'PropertyRecordingTest',
 236                     u'key_1': u'1'
 237                 },
 238                 {
 239                     u'name': u'IntValuedProperty',
 240                     u'status': u'RUN',
 241                     u'time': u'*',
 242                     u'classname': u'PropertyRecordingTest',
 243                     u'key_int': u'1'
 244                 },
 245                 {
 246                     u'name': u'ThreeProperties',
 247                     u'status': u'RUN',
 248                     u'time': u'*',
 249                     u'classname': u'PropertyRecordingTest',
 250                     u'key_1': u'1',
 251                     u'key_2': u'2',
 252                     u'key_3': u'3'
 253                 },
 254                 {
 255                     u'name': u'TwoValuesForOneKeyUsesLastValue',
 256                     u'status': u'RUN',
 257                     u'time': u'*',
 258                     u'classname': u'PropertyRecordingTest',
 259                     u'key_1': u'2'
 260                 }
 261             ]
 262         },
 263         {
 264             u'name': u'NoFixtureTest',
 265             u'tests': 3,
 266             u'failures': 0,
 267             u'disabled': 0,
 268             u'errors': 0,
 269             u'time': u'*',
 270             u'testsuite': [
 271                 {
 272                     u'name': u'RecordProperty',
 273                     u'status': u'RUN',
 274                     u'time': u'*',
 275                     u'classname': u'NoFixtureTest',
 276                     u'key': u'1'
 277                 },
 278                 {
 279                     u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty',
 280                     u'status': u'RUN',
 281                     u'time': u'*',
 282                     u'classname': u'NoFixtureTest',
 283                     u'key_for_utility_int': u'1'
 284                 },
 285                 {
 286                     u'name':
 287                         u'ExternalUtilityThatCallsRecordStringValuedProperty',
 288                     u'status': u'RUN',
 289                     u'time': u'*',
 290                     u'classname': u'NoFixtureTest',
 291                     u'key_for_utility_string': u'1'
 292                 }
 293             ]
 294         },
 295         {
 296             u'name': u'TypedTest/0',
 297             u'tests': 1,
 298             u'failures': 0,
 299             u'disabled': 0,
 300             u'errors': 0,
 301             u'time': u'*',
 302             u'testsuite': [
 303                 {
 304                     u'name': u'HasTypeParamAttribute',
 305                     u'type_param': u'int',
 306                     u'status': u'RUN',
 307                     u'time': u'*',
 308                     u'classname': u'TypedTest/0'
 309                 }
 310             ]
 311         },
 312         {
 313             u'name': u'TypedTest/1',
 314             u'tests': 1,
 315             u'failures': 0,
 316             u'disabled': 0,
 317             u'errors': 0,
 318             u'time': u'*',
 319             u'testsuite': [
 320                 {
 321                     u'name': u'HasTypeParamAttribute',
 322                     u'type_param': u'long',
 323                     u'status': u'RUN',
 324                     u'time': u'*',
 325                     u'classname': u'TypedTest/1'
 326                 }
 327             ]
 328         },
 329         {
 330             u'name': u'Single/TypeParameterizedTestCase/0',
 331             u'tests': 1,
 332             u'failures': 0,
 333             u'disabled': 0,
 334             u'errors': 0,
 335             u'time': u'*',
 336             u'testsuite': [
 337                 {
 338                     u'name': u'HasTypeParamAttribute',
 339                     u'type_param': u'int',
 340                     u'status': u'RUN',
 341                     u'time': u'*',
 342                     u'classname': u'Single/TypeParameterizedTestCase/0'
 343                 }
 344             ]
 345         },
 346         {
 347             u'name': u'Single/TypeParameterizedTestCase/1',
 348             u'tests': 1,
 349             u'failures': 0,
 350             u'disabled': 0,
 351             u'errors': 0,
 352             u'time': u'*',
 353             u'testsuite': [
 354                 {
 355                     u'name': u'HasTypeParamAttribute',
 356                     u'type_param': u'long',
 357                     u'status': u'RUN',
 358                     u'time': u'*',
 359                     u'classname': u'Single/TypeParameterizedTestCase/1'
 360                 }
 361             ]
 362         },
 363         {
 364             u'name': u'Single/ValueParamTest',
 365             u'tests': 4,
 366             u'failures': 0,
 367             u'disabled': 0,
 368             u'errors': 0,
 369             u'time': u'*',
 370             u'testsuite': [
 371                 {
 372                     u'name': u'HasValueParamAttribute/0',
 373                     u'value_param': u'33',
 374                     u'status': u'RUN',
 375                     u'time': u'*',
 376                     u'classname': u'Single/ValueParamTest'
 377                 },
 378                 {
 379                     u'name': u'HasValueParamAttribute/1',
 380                     u'value_param': u'42',
 381                     u'status': u'RUN',
 382                     u'time': u'*',
 383                     u'classname': u'Single/ValueParamTest'
 384                 },
 385                 {
 386                     u'name': u'AnotherTestThatHasValueParamAttribute/0',
 387                     u'value_param': u'33',
 388                     u'status': u'RUN',
 389                     u'time': u'*',
 390                     u'classname': u'Single/ValueParamTest'
 391                 },
 392                 {
 393                     u'name': u'AnotherTestThatHasValueParamAttribute/1',
 394                     u'value_param': u'42',
 395                     u'status': u'RUN',
 396                     u'time': u'*',
 397                     u'classname': u'Single/ValueParamTest'
 398                 }
 399             ]
 400         }
 401     ]
 402 }
 403
 404 EXPECTED_FILTERED = {
 405     u'tests': 1,
 406     u'failures': 0,
 407     u'disabled': 0,
 408     u'errors': 0,
 409     u'time': u'*',
 410     u'timestamp': u'*',
 411     u'name': u'AllTests',
 412     u'ad_hoc_property': u'42',
 413     u'testsuites': [{
 414         u'name': u'SuccessfulTest',
 415         u'tests': 1,
 416         u'failures': 0,
 417         u'disabled': 0,
 418         u'errors': 0,
 419         u'time': u'*',
 420         u'testsuite': [{
 421             u'name': u'Succeeds',
 422             u'status': u'RUN',
 423             u'time': u'*',
 424             u'classname': u'SuccessfulTest',
 425         }]
 426     }],
 427 }
 428
 429 EXPECTED_EMPTY = {
 430     u'tests': 0,
 431     u'failures': 0,
 432     u'disabled': 0,
 433     u'errors': 0,
 434     u'time': u'*',
 435     u'timestamp': u'*',
 436     u'name': u'AllTests',
 437     u'testsuites': [],
 438 }
 439
 440 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
 441
 442 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess(
 443     [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output
 444
 445
 446 class GTestJsonOutputUnitTest(gtest_test_utils.TestCase):
 447   """Unit test for Google Test's JSON output functionality.
 448   """
 449
 450   # This test currently breaks on platforms that do not support typed and
 451   # type-parameterized tests, so we don't run it under them.
 452   if SUPPORTS_TYPED_TESTS:
 453
 454     def testNonEmptyJsonOutput(self):
 455       """Verifies JSON output for a Google Test binary with non-empty output.
 456
 457       Runs a test program that generates a non-empty JSON output, and
 458       tests that the JSON output is expected.
 459       """
 460       self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1)
 461
 462   def testEmptyJsonOutput(self):
 463     """Verifies JSON output for a Google Test binary without actual tests.
 464
 465     Runs a test program that generates an empty JSON output, and
 466     tests that the JSON output is expected.
 467     """
 468
 469     self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0)
 470
 471   def testTimestampValue(self):
 472     """Checks whether the timestamp attribute in the JSON output is valid.
 473
 474     Runs a test program that generates an empty JSON output, and checks if
 475     the timestamp attribute in the testsuites tag is valid.
 476     """
 477     actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0)
 478     date_time_str = actual['timestamp']
 479     # datetime.strptime() is only available in Python 2.5+ so we have to
 480     # parse the expected datetime manually.
 481     match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str)
 482     self.assertTrue(
 483         re.match,
 484         'JSON datettime string %s has incorrect format' % date_time_str)
 485     date_time_from_json = datetime.datetime(
 486         year=int(match.group(1)), month=int(match.group(2)),
 487         day=int(match.group(3)), hour=int(match.group(4)),
 488         minute=int(match.group(5)), second=int(match.group(6)))
 489
 490     time_delta = abs(datetime.datetime.now() - date_time_from_json)
 491     # timestamp value should be near the current local time
 492     self.assertTrue(time_delta < datetime.timedelta(seconds=600),
 493                     'time_delta is %s' % time_delta)
 494
 495   def testDefaultOutputFile(self):
 496     """Verifies the default output file name.
 497
 498     Confirms that Google Test produces an JSON output file with the expected
 499     default name if no name is explicitly specified.
 500     """
 501     output_file = os.path.join(gtest_test_utils.GetTempDir(),
 502                                GTEST_DEFAULT_OUTPUT_FILE)
 503     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(
 504         'gtest_no_test_unittest')
 505     try:
 506       os.remove(output_file)
 507     except OSError:
 508       e = sys.exc_info()[1]
 509       if e.errno != errno.ENOENT:
 510         raise
 511
 512     p = gtest_test_utils.Subprocess(
 513         [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG],
 514         working_dir=gtest_test_utils.GetTempDir())
 515     self.assert_(p.exited)
 516     self.assertEquals(0, p.exit_code)
 517     self.assert_(os.path.isfile(output_file))
 518
 519   def testSuppressedJsonOutput(self):
 520     """Verifies that no JSON output is generated.
 521
 522     Tests that no JSON file is generated if the default JSON listener is
 523     shut down before RUN_ALL_TESTS is invoked.
 524     """
 525
 526     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 527                              GTEST_PROGRAM_NAME + 'out.json')
 528     if os.path.isfile(json_path):
 529       os.remove(json_path)
 530
 531     command = [GTEST_PROGRAM_PATH,
 532                '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path),
 533                '--shut_down_xml']
 534     p = gtest_test_utils.Subprocess(command)
 535     if p.terminated_by_signal:
 536       # p.signal is available only if p.terminated_by_signal is True.
 537       self.assertFalse(
 538           p.terminated_by_signal,
 539           '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal))
 540     else:
 541       self.assert_(p.exited)
 542       self.assertEquals(1, p.exit_code,
 543                         "'%s' exited with code %s, which doesn't match "
 544                         'the expected exit code %s.'
 545                         % (command, p.exit_code, 1))
 546
 547     self.assert_(not os.path.isfile(json_path))
 548
 549   def testFilteredTestJsonOutput(self):
 550     """Verifies JSON output when a filter is applied.
 551
 552     Runs a test program that executes only some tests and verifies that
 553     non-selected tests do not show up in the JSON output.
 554     """
 555
 556     self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0,
 557                          extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
 558
 559   def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code):
 560     """Returns the JSON output generated by running the program gtest_prog_name.
 561
 562     Furthermore, the program's exit code must be expected_exit_code.
 563
 564     Args:
 565       gtest_prog_name: Google Test binary name.
 566       extra_args: extra arguments to binary invocation.
 567       expected_exit_code: program's exit code.
 568     """
 569     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 570                              gtest_prog_name + 'out.json')
 571     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
 572
 573     command = (
 574         [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] +
 575         extra_args
 576     )
 577     p = gtest_test_utils.Subprocess(command)
 578     if p.terminated_by_signal:
 579       self.assert_(False,
 580                    '%s was killed by signal %d' % (gtest_prog_name, p.signal))
 581     else:
 582       self.assert_(p.exited)
 583       self.assertEquals(expected_exit_code, p.exit_code,
 584                         "'%s' exited with code %s, which doesn't match "
 585                         'the expected exit code %s.'
 586                         % (command, p.exit_code, expected_exit_code))
 587     with open(json_path) as f:
 588       actual = json.load(f)
 589     return actual
 590
 591   def _TestJsonOutput(self, gtest_prog_name, expected,
 592                       expected_exit_code, extra_args=None):
 593     """Checks the JSON output generated by the Google Test binary.
 594
 595     Asserts that the JSON document generated by running the program
 596     gtest_prog_name matches expected_json, a string containing another
 597     JSON document.  Furthermore, the program's exit code must be
 598     expected_exit_code.
 599
 600     Args:
 601       gtest_prog_name: Google Test binary name.
 602       expected: expected output.
 603       expected_exit_code: program's exit code.
 604       extra_args: extra arguments to binary invocation.
 605     """
 606
 607     actual = self._GetJsonOutput(gtest_prog_name, extra_args or [],
 608                                  expected_exit_code)
 609     self.assertEqual(expected, gtest_json_test_utils.normalize(actual))
 610
 611
 612 if __name__ == '__main__':
 613   if NO_STACKTRACE_SUPPORT_FLAG in sys.argv:
 614     # unittest.main() can't handle unknown flags
 615     sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
 616
 617   os.environ['GTEST_STACK_TRACE_DEPTH'] = '1'
 618   gtest_test_utils.Main()