test/libcxx/test/format.py

   1 #===----------------------------------------------------------------------===##
   2 #
   3 #                     The LLVM Compiler Infrastructure
   4 #
   5 # This file is dual licensed under the MIT and the University of Illinois Open
   6 # Source Licenses. See LICENSE.TXT for details.
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 import errno
  11 import os
  12 import time
  13 import random
  14
  15 import lit.Test        # pylint: disable=import-error
  16 import lit.TestRunner  # pylint: disable=import-error
  17 from lit.TestRunner import ParserKind, IntegratedTestKeywordParser  \
  18     # pylint: disable=import-error
  19 import lit.util        # pylint: disable=import-error
  20
  21
  22 from libcxx.test.executor import LocalExecutor as LocalExecutor
  23 import libcxx.util
  24
  25
  26 class LibcxxTestFormat(object):
  27     """
  28     Custom test format handler for use with the test format use by libc++.
  29
  30     Tests fall into two categories:
  31       FOO.pass.cpp - Executable test which should compile, run, and exit with
  32                      code 0.
  33       FOO.fail.cpp - Negative test case which is expected to fail compilation.
  34       FOO.sh.cpp   - A test that uses LIT's ShTest format.
  35     """
  36
  37     def __init__(self, cxx, use_verify_for_fail, execute_external,
  38                  executor, exec_env):
  39         self.cxx = cxx.copy()
  40         self.use_verify_for_fail = use_verify_for_fail
  41         self.execute_external = execute_external
  42         self.executor = executor
  43         self.exec_env = dict(exec_env)
  44
  45     @staticmethod
  46     def _make_custom_parsers():
  47         return [
  48             IntegratedTestKeywordParser('FLAKY_TEST.', ParserKind.TAG,
  49                                         initial_value=False),
  50             IntegratedTestKeywordParser('MODULES_DEFINES:', ParserKind.LIST,
  51                                         initial_value=[])
  52         ]
  53
  54     @staticmethod
  55     def _get_parser(key, parsers):
  56         for p in parsers:
  57             if p.keyword == key:
  58                 return p
  59         assert False and "parser not found"
  60
  61     # TODO: Move this into lit's FileBasedTest
  62     def getTestsInDirectory(self, testSuite, path_in_suite,
  63                             litConfig, localConfig):
  64         source_path = testSuite.getSourcePath(path_in_suite)
  65         for filename in os.listdir(source_path):
  66             # Ignore dot files and excluded tests.
  67             if filename.startswith('.') or filename in localConfig.excludes:
  68                 continue
  69
  70             filepath = os.path.join(source_path, filename)
  71             if not os.path.isdir(filepath):
  72                 if any([filename.endswith(ext)
  73                         for ext in localConfig.suffixes]):
  74                     yield lit.Test.Test(testSuite, path_in_suite + (filename,),
  75                                         localConfig)
  76
  77     def execute(self, test, lit_config):
  78         while True:
  79             try:
  80                 return self._execute(test, lit_config)
  81             except OSError as oe:
  82                 if oe.errno != errno.ETXTBSY:
  83                     raise
  84                 time.sleep(0.1)
  85
  86     def _execute(self, test, lit_config):
  87         name = test.path_in_suite[-1]
  88         name_root, name_ext = os.path.splitext(name)
  89         is_libcxx_test = test.path_in_suite[0] == 'libcxx'
  90         is_sh_test = name_root.endswith('.sh')
  91         is_pass_test = name.endswith('.pass.cpp')
  92         is_fail_test = name.endswith('.fail.cpp')
  93         assert is_sh_test or name_ext == '.cpp', 'non-cpp file must be sh test'
  94
  95         if test.config.unsupported:
  96             return (lit.Test.UNSUPPORTED,
  97                     "A lit.local.cfg marked this unsupported")
  98
  99         parsers = self._make_custom_parsers()
 100         script = lit.TestRunner.parseIntegratedTestScript(
 101             test, additional_parsers=parsers, require_script=is_sh_test)
 102         # Check if a result for the test was returned. If so return that
 103         # result.
 104         if isinstance(script, lit.Test.Result):
 105             return script
 106         if lit_config.noExecute:
 107             return lit.Test.Result(lit.Test.PASS)
 108
 109         # Check that we don't have run lines on tests that don't support them.
 110         if not is_sh_test and len(script) != 0:
 111             lit_config.fatal('Unsupported RUN line found in test %s' % name)
 112
 113         tmpDir, tmpBase = lit.TestRunner.getTempPaths(test)
 114         substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir,
 115                                                                tmpBase)
 116         script = lit.TestRunner.applySubstitutions(script, substitutions)
 117
 118         test_cxx = self.cxx.copy()
 119         if is_fail_test:
 120             test_cxx.useCCache(False)
 121             test_cxx.useWarnings(False)
 122         extra_modules_defines = self._get_parser('MODULES_DEFINES:',
 123                                                  parsers).getValue()
 124         if '-fmodules' in test.config.available_features:
 125             test_cxx.compile_flags += [('-D%s' % mdef.strip()) for
 126                                        mdef in extra_modules_defines]
 127             test_cxx.addWarningFlagIfSupported('-Wno-macro-redefined')
 128             # FIXME: libc++ debug tests #define _LIBCPP_ASSERT to override it
 129             # If we see this we need to build the test against uniquely built
 130             # modules.
 131             if is_libcxx_test:
 132                 with open(test.getSourcePath(), 'r') as f:
 133                     contents = f.read()
 134                 if '#define _LIBCPP_ASSERT' in contents:
 135                     test_cxx.useModules(False)
 136
 137         # Dispatch the test based on its suffix.
 138         if is_sh_test:
 139             if not isinstance(self.executor, LocalExecutor):
 140                 # We can't run ShTest tests with a executor yet.
 141                 # For now, bail on trying to run them
 142                 return lit.Test.UNSUPPORTED, 'ShTest format not yet supported'
 143             return lit.TestRunner._runShTest(test, lit_config,
 144                                              self.execute_external, script,
 145                                              tmpBase)
 146         elif is_fail_test:
 147             return self._evaluate_fail_test(test, test_cxx, parsers)
 148         elif is_pass_test:
 149             return self._evaluate_pass_test(test, tmpBase, lit_config,
 150                                             test_cxx, parsers)
 151         else:
 152             # No other test type is supported
 153             assert False
 154
 155     def _clean(self, exec_path):  # pylint: disable=no-self-use
 156         libcxx.util.cleanFile(exec_path)
 157
 158     def _evaluate_pass_test(self, test, tmpBase, lit_config,
 159                             test_cxx, parsers):
 160         execDir = os.path.dirname(test.getExecPath())
 161         source_path = test.getSourcePath()
 162         exec_path = tmpBase + '.exe'
 163         object_path = tmpBase + '.o'
 164         # Create the output directory if it does not already exist.
 165         lit.util.mkdir_p(os.path.dirname(tmpBase))
 166         try:
 167             # Compile the test
 168             cmd, out, err, rc = test_cxx.compileLinkTwoSteps(
 169                 source_path, out=exec_path, object_file=object_path,
 170                 cwd=execDir)
 171             compile_cmd = cmd
 172             if rc != 0:
 173                 report = libcxx.util.makeReport(cmd, out, err, rc)
 174                 report += "Compilation failed unexpectedly!"
 175                 return lit.Test.FAIL, report
 176             # Run the test
 177             local_cwd = os.path.dirname(source_path)
 178             env = None
 179             if self.exec_env:
 180                 env = self.exec_env
 181             # TODO: Only list actually needed files in file_deps.
 182             # Right now we just mark all of the .dat files in the same
 183             # directory as dependencies, but it's likely less than that. We
 184             # should add a `// FILE-DEP: foo.dat` to each test to track this.
 185             data_files = [os.path.join(local_cwd, f)
 186                           for f in os.listdir(local_cwd) if f.endswith('.dat')]
 187             is_flaky = self._get_parser('FLAKY_TEST.', parsers).getValue()
 188             max_retry = 3 if is_flaky else 1
 189             for retry_count in range(max_retry):
 190                 cmd, out, err, rc = self.executor.run(exec_path, [exec_path],
 191                                                       local_cwd, data_files,
 192                                                       env)
 193                 if rc == 0:
 194                     res = lit.Test.PASS if retry_count == 0 else lit.Test.FLAKYPASS
 195                     return res, ''
 196                 elif rc != 0 and retry_count + 1 == max_retry:
 197                     report = libcxx.util.makeReport(cmd, out, err, rc)
 198                     report = "Compiled With: %s\n%s" % (compile_cmd, report)
 199                     report += "Compiled test failed unexpectedly!"
 200                     return lit.Test.FAIL, report
 201
 202             assert False # Unreachable
 203         finally:
 204             # Note that cleanup of exec_file happens in `_clean()`. If you
 205             # override this, cleanup is your reponsibility.
 206             libcxx.util.cleanFile(object_path)
 207             self._clean(exec_path)
 208
 209     def _evaluate_fail_test(self, test, test_cxx, parsers):
 210         source_path = test.getSourcePath()
 211         # FIXME: lift this detection into LLVM/LIT.
 212         with open(source_path, 'r') as f:
 213             contents = f.read()
 214         verify_tags = ['expected-note', 'expected-remark', 'expected-warning',
 215                        'expected-error', 'expected-no-diagnostics']
 216         use_verify = self.use_verify_for_fail and \
 217                      any([tag in contents for tag in verify_tags])
 218         # FIXME(EricWF): GCC 5 does not evaluate static assertions that
 219         # are dependant on a template parameter when '-fsyntax-only' is passed.
 220         # This is fixed in GCC 6. However for now we only pass "-fsyntax-only"
 221         # when using Clang.
 222         if test_cxx.type != 'gcc':
 223             test_cxx.flags += ['-fsyntax-only']
 224         if use_verify:
 225             test_cxx.useVerify()
 226         cmd, out, err, rc = test_cxx.compile(source_path, out=os.devnull)
 227         expected_rc = 0 if use_verify else 1
 228         if rc == expected_rc:
 229             return lit.Test.PASS, ''
 230         else:
 231             report = libcxx.util.makeReport(cmd, out, err, rc)
 232             report_msg = ('Expected compilation to fail!' if not use_verify else
 233                           'Expected compilation using verify to pass!')
 234             return lit.Test.FAIL, report + report_msg + '\n'