packages/Python/lldbsuite/test/test_result.py

   1 """
   2                      The LLVM Compiler Infrastructure
   3
   4 This file is distributed under the University of Illinois Open Source
   5 License. See LICENSE.TXT for details.
   6
   7 Provides the LLDBTestResult class, which holds information about progress
   8 and results of a single test run.
   9 """
  10
  11 from __future__ import absolute_import
  12 from __future__ import print_function
  13
  14 # System modules
  15 import inspect
  16 import os
  17
  18 # Third-party modules
  19 import unittest2
  20
  21 # LLDB Modules
  22 from . import configuration
  23 from lldbsuite.test_event.event_builder import EventBuilder
  24 from lldbsuite.test_event import build_exception
  25
  26
  27 class LLDBTestResult(unittest2.TextTestResult):
  28     """
  29     Enforce a singleton pattern to allow introspection of test progress.
  30
  31     Overwrite addError(), addFailure(), and addExpectedFailure() methods
  32     to enable each test instance to track its failure/error status.  It
  33     is used in the LLDB test framework to emit detailed trace messages
  34     to a log file for easier human inspection of test failures/errors.
  35     """
  36     __singleton__ = None
  37     __ignore_singleton__ = False
  38
  39     @staticmethod
  40     def getTerminalSize():
  41         import os
  42         env = os.environ
  43
  44         def ioctl_GWINSZ(fd):
  45             try:
  46                 import fcntl
  47                 import termios
  48                 import struct
  49                 import os
  50                 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
  51                                                      '1234'))
  52             except:
  53                 return
  54             return cr
  55         cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
  56         if not cr:
  57             try:
  58                 fd = os.open(os.ctermid(), os.O_RDONLY)
  59                 cr = ioctl_GWINSZ(fd)
  60                 os.close(fd)
  61             except:
  62                 pass
  63         if not cr:
  64             cr = (env.get('LINES', 25), env.get('COLUMNS', 80))
  65         return int(cr[1]), int(cr[0])
  66
  67     def __init__(self, *args):
  68         if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
  69             raise Exception("LLDBTestResult instantiated more than once")
  70         super(LLDBTestResult, self).__init__(*args)
  71         LLDBTestResult.__singleton__ = self
  72         # Now put this singleton into the lldb module namespace.
  73         configuration.test_result = self
  74         # Computes the format string for displaying the counter.
  75         counterWidth = len(str(configuration.suite.countTestCases()))
  76         self.fmt = "%" + str(counterWidth) + "d: "
  77         self.indentation = ' ' * (counterWidth + 2)
  78         # This counts from 1 .. suite.countTestCases().
  79         self.counter = 0
  80         (width, height) = LLDBTestResult.getTerminalSize()
  81         self.results_formatter = configuration.results_formatter_object
  82
  83     def _config_string(self, test):
  84         compiler = getattr(test, "getCompiler", None)
  85         arch = getattr(test, "getArchitecture", None)
  86         return "%s-%s" % (compiler() if compiler else "",
  87                           arch() if arch else "")
  88
  89     def _exc_info_to_string(self, err, test):
  90         """Overrides superclass TestResult's method in order to append
  91         our test config info string to the exception info string."""
  92         if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
  93             return '%sConfig=%s-%s' % (super(LLDBTestResult,
  94                                              self)._exc_info_to_string(err,
  95                                                                        test),
  96                                        test.getArchitecture(),
  97                                        test.getCompiler())
  98         else:
  99             return super(LLDBTestResult, self)._exc_info_to_string(err, test)
 100
 101     def getDescription(self, test):
 102         doc_first_line = test.shortDescription()
 103         if self.descriptions and doc_first_line:
 104             return '\n'.join((str(test), self.indentation + doc_first_line))
 105         else:
 106             return str(test)
 107
 108     def getCategoriesForTest(self, test):
 109         """
 110         Gets all the categories for the currently running test method in test case
 111         """
 112         test_categories = []
 113         test_method = getattr(test, test._testMethodName)
 114         if test_method is not None and hasattr(test_method, "categories"):
 115             test_categories.extend(test_method.categories)
 116
 117         test_categories.extend(test.getCategories())
 118
 119         return test_categories
 120
 121     def hardMarkAsSkipped(self, test):
 122         getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
 123         getattr(
 124             test,
 125             test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
 126
 127     def checkExclusion(self, exclusion_list, name):
 128         if exclusion_list:
 129             import re
 130             for item in exclusion_list:
 131                 if re.search(item, name):
 132                     return True
 133         return False
 134
 135     def startTest(self, test):
 136         if configuration.shouldSkipBecauseOfCategories(
 137                 self.getCategoriesForTest(test)):
 138             self.hardMarkAsSkipped(test)
 139         if self.checkExclusion(
 140                 configuration.skip_tests, test.id()):
 141             self.hardMarkAsSkipped(test)
 142
 143         configuration.setCrashInfoHook(
 144             "%s at %s" %
 145             (str(test), inspect.getfile(
 146                 test.__class__)))
 147         self.counter += 1
 148         # if self.counter == 4:
 149         #    import crashinfo
 150         #    crashinfo.testCrashReporterDescription(None)
 151         test.test_number = self.counter
 152         if self.showAll:
 153             self.stream.write(self.fmt % self.counter)
 154         super(LLDBTestResult, self).startTest(test)
 155         if self.results_formatter:
 156             self.results_formatter.handle_event(
 157                 EventBuilder.event_for_start(test))
 158
 159     def addSuccess(self, test):
 160         if self.checkExclusion(
 161                 configuration.xfail_tests, test.id()):
 162             self.addUnexpectedSuccess(test, None)
 163             return
 164
 165         super(LLDBTestResult, self).addSuccess(test)
 166         if configuration.parsable:
 167             self.stream.write(
 168                 "PASS: LLDB (%s) :: %s\n" %
 169                 (self._config_string(test), str(test)))
 170         if self.results_formatter:
 171             self.results_formatter.handle_event(
 172                 EventBuilder.event_for_success(test))
 173
 174     def _isBuildError(self, err_tuple):
 175         exception = err_tuple[1]
 176         return isinstance(exception, build_exception.BuildError)
 177
 178     def _getTestPath(self, test):
 179         if test is None:
 180             return ""
 181         elif hasattr(test, "test_filename"):
 182             return test.test_filename
 183         else:
 184             return inspect.getsourcefile(test.__class__)
 185
 186     def _saveBuildErrorTuple(self, test, err):
 187         # Adjust the error description so it prints the build command and build error
 188         # rather than an uninformative Python backtrace.
 189         build_error = err[1]
 190         error_description = "{}\nTest Directory:\n{}".format(
 191             str(build_error),
 192             os.path.dirname(self._getTestPath(test)))
 193         self.errors.append((test, error_description))
 194         self._mirrorOutput = True
 195
 196     def addError(self, test, err):
 197         configuration.sdir_has_content = True
 198         if self._isBuildError(err):
 199             self._saveBuildErrorTuple(test, err)
 200         else:
 201             super(LLDBTestResult, self).addError(test, err)
 202
 203         method = getattr(test, "markError", None)
 204         if method:
 205             method()
 206         if configuration.parsable:
 207             self.stream.write(
 208                 "FAIL: LLDB (%s) :: %s\n" %
 209                 (self._config_string(test), str(test)))
 210         if self.results_formatter:
 211             # Handle build errors as a separate event type
 212             if self._isBuildError(err):
 213                 error_event = EventBuilder.event_for_build_error(test, err)
 214             else:
 215                 error_event = EventBuilder.event_for_error(test, err)
 216             self.results_formatter.handle_event(error_event)
 217
 218     def addCleanupError(self, test, err):
 219         configuration.sdir_has_content = True
 220         super(LLDBTestResult, self).addCleanupError(test, err)
 221         method = getattr(test, "markCleanupError", None)
 222         if method:
 223             method()
 224         if configuration.parsable:
 225             self.stream.write(
 226                 "CLEANUP ERROR: LLDB (%s) :: %s\n" %
 227                 (self._config_string(test), str(test)))
 228         if self.results_formatter:
 229             self.results_formatter.handle_event(
 230                 EventBuilder.event_for_cleanup_error(
 231                     test, err))
 232
 233     def addFailure(self, test, err):
 234         if self.checkExclusion(
 235                 configuration.xfail_tests, test.id()):
 236             self.addExpectedFailure(test, err, None)
 237             return
 238
 239         configuration.sdir_has_content = True
 240         super(LLDBTestResult, self).addFailure(test, err)
 241         method = getattr(test, "markFailure", None)
 242         if method:
 243             method()
 244         if configuration.parsable:
 245             self.stream.write(
 246                 "FAIL: LLDB (%s) :: %s\n" %
 247                 (self._config_string(test), str(test)))
 248         if configuration.useCategories:
 249             test_categories = self.getCategoriesForTest(test)
 250             for category in test_categories:
 251                 if category in configuration.failuresPerCategory:
 252                     configuration.failuresPerCategory[
 253                         category] = configuration.failuresPerCategory[category] + 1
 254                 else:
 255                     configuration.failuresPerCategory[category] = 1
 256         if self.results_formatter:
 257             self.results_formatter.handle_event(
 258                 EventBuilder.event_for_failure(test, err))
 259
 260     def addExpectedFailure(self, test, err, bugnumber):
 261         configuration.sdir_has_content = True
 262         super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
 263         method = getattr(test, "markExpectedFailure", None)
 264         if method:
 265             method(err, bugnumber)
 266         if configuration.parsable:
 267             self.stream.write(
 268                 "XFAIL: LLDB (%s) :: %s\n" %
 269                 (self._config_string(test), str(test)))
 270         if self.results_formatter:
 271             self.results_formatter.handle_event(
 272                 EventBuilder.event_for_expected_failure(
 273                     test, err, bugnumber))
 274
 275     def addSkip(self, test, reason):
 276         configuration.sdir_has_content = True
 277         super(LLDBTestResult, self).addSkip(test, reason)
 278         method = getattr(test, "markSkippedTest", None)
 279         if method:
 280             method()
 281         if configuration.parsable:
 282             self.stream.write(
 283                 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" %
 284                 (self._config_string(test), str(test), reason))
 285         if self.results_formatter:
 286             self.results_formatter.handle_event(
 287                 EventBuilder.event_for_skip(test, reason))
 288
 289     def addUnexpectedSuccess(self, test, bugnumber):
 290         configuration.sdir_has_content = True
 291         super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
 292         method = getattr(test, "markUnexpectedSuccess", None)
 293         if method:
 294             method(bugnumber)
 295         if configuration.parsable:
 296             self.stream.write(
 297                 "XPASS: LLDB (%s) :: %s\n" %
 298                 (self._config_string(test), str(test)))
 299         if self.results_formatter:
 300             self.results_formatter.handle_event(
 301                 EventBuilder.event_for_unexpected_success(
 302                     test, bugnumber))