utils/test/MultiTestRunner.py

   1 #!/usr/bin/python
   2
   3 """
   4 MultiTestRunner - Harness for running multiple tests in the simple clang style.
   5
   6 TODO
   7 --
   8  - Fix Ctrl-c issues
   9  - Use a timeout
  10  - Detect signalled failures (abort)
  11  - Better support for finding tests
  12 """
  13
  14 # TOD
  15 import os, sys, re, random, time
  16 import threading
  17 import ProgressBar
  18 import TestRunner
  19 from TestRunner import TestStatus
  20 from Queue import Queue
  21
  22 kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll'])
  23
  24 def getTests(inputs):
  25     for path in inputs:
  26         if not os.path.exists(path):
  27             print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,)
  28             continue
  29
  30         if os.path.isdir(path):
  31             for dirpath,dirnames,filenames in os.walk(path):
  32                 dotTests = os.path.join(dirpath,'.tests')
  33                 if os.path.exists(dotTests):
  34                     for ln in open(dotTests):
  35                         if ln.strip():
  36                             yield os.path.join(dirpath,ln.strip())
  37                 else:
  38                     # FIXME: This doesn't belong here
  39                     if 'Output' in dirnames:
  40                         dirnames.remove('Output')
  41                     for f in filenames:
  42                         base,ext = os.path.splitext(f)
  43                         if ext in kTestFileExtensions:
  44                             yield os.path.join(dirpath,f)
  45         else:
  46             yield path
  47
  48 class TestingProgressDisplay:
  49     def __init__(self, opts, numTests, progressBar=None):
  50         self.opts = opts
  51         self.numTests = numTests
  52         self.digits = len(str(self.numTests))
  53         self.current = None
  54         self.lock = threading.Lock()
  55         self.progressBar = progressBar
  56         self.progress = 0.
  57
  58     def update(self, index, tr):
  59         # Avoid locking overhead in quiet mode
  60         if self.opts.quiet and not tr.failed():
  61             return
  62
  63         # Output lock
  64         self.lock.acquire()
  65         try:
  66             self.handleUpdate(index, tr)
  67         finally:
  68             self.lock.release()
  69
  70     def finish(self):
  71         if self.progressBar:
  72             self.progressBar.clear()
  73         elif self.opts.succinct:
  74             sys.stdout.write('\n')
  75
  76     def handleUpdate(self, index, tr):
  77         if self.progressBar:
  78             if tr.failed():
  79                 self.progressBar.clear()
  80             else:
  81                 # Force monotonicity
  82                 self.progress = max(self.progress, float(index)/self.numTests)
  83                 self.progressBar.update(self.progress, tr.path)
  84                 return
  85         elif self.opts.succinct:
  86             if not tr.failed():
  87                 sys.stdout.write('.')
  88                 sys.stdout.flush()
  89                 return
  90             else:
  91                 sys.stdout.write('\n')
  92
  93         extra = ''
  94         if tr.code==TestStatus.Invalid:
  95             extra = ' - (Invalid test)'
  96         elif tr.code==TestStatus.NoRunLine:
  97             extra = ' - (No RUN line)'
  98         elif tr.failed():
  99             extra = ' - %s'%(TestStatus.getName(tr.code).upper(),)
 100         print '%*d/%*d - %s%s'%(self.digits, index+1, self.digits,
 101                               self.numTests, tr.path, extra)
 102
 103         if tr.failed() and self.opts.showOutput:
 104             TestRunner.cat(tr.testResults, sys.stdout)
 105
 106 class TestResult:
 107     def __init__(self, path, code, testResults):
 108         self.path = path
 109         self.code = code
 110         self.testResults = testResults
 111
 112     def failed(self):
 113         return self.code in (TestStatus.Fail,TestStatus.XPass)
 114
 115 class TestProvider:
 116     def __init__(self, opts, tests, display):
 117         self.opts = opts
 118         self.tests = tests
 119         self.index = 0
 120         self.lock = threading.Lock()
 121         self.results = [None]*len(self.tests)
 122         self.startTime = time.time()
 123         self.progress = display
 124
 125     def get(self):
 126         self.lock.acquire()
 127         try:
 128             if self.opts.maxTime is not None:
 129                 if time.time() - self.startTime > self.opts.maxTime:
 130                     return None
 131             if self.index >= len(self.tests):
 132                 return None
 133             item = self.tests[self.index],self.index
 134             self.index += 1
 135             return item
 136         finally:
 137             self.lock.release()
 138
 139     def setResult(self, index, result):
 140         self.results[index] = result
 141         self.progress.update(index, result)
 142
 143 class Tester(threading.Thread):
 144     def __init__(self, provider):
 145         threading.Thread.__init__(self)
 146         self.provider = provider
 147
 148     def run(self):
 149         while 1:
 150             item = self.provider.get()
 151             if item is None:
 152                 break
 153             self.runTest(item)
 154
 155     def runTest(self, (path,index)):
 156         command = path
 157         # Use hand concatentation here because we want to override
 158         # absolute paths.
 159         output = 'Output/' + path + '.out'
 160         testname = path
 161         testresults = 'Output/' + path + '.testresults'
 162         TestRunner.mkdir_p(os.path.dirname(testresults))
 163         numTests = len(self.provider.tests)
 164         digits = len(str(numTests))
 165         code = None
 166         try:
 167             opts = self.provider.opts
 168             if opts.debugDoNotTest:
 169                 code = None
 170             else:
 171                 code = TestRunner.runOneTest(path, command, output, testname,
 172                                              opts.clang, opts.clangcc,
 173                                              useValgrind=opts.useValgrind,
 174                                              useDGCompat=opts.useDGCompat,
 175                                              useScript=opts.testScript,
 176                                              output=open(testresults,'w'))
 177         except KeyboardInterrupt:
 178             # This is a sad hack. Unfortunately subprocess goes
 179             # bonkers with ctrl-c and we start forking merrily.
 180             print 'Ctrl-C detected, goodbye.'
 181             os.kill(0,9)
 182
 183         self.provider.setResult(index, TestResult(path, code, testresults))
 184
 185 def detectCPUs():
 186     """
 187     Detects the number of CPUs on a system. Cribbed from pp.
 188     """
 189     # Linux, Unix and MacOS:
 190     if hasattr(os, "sysconf"):
 191         if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
 192             # Linux & Unix:
 193             ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
 194             if isinstance(ncpus, int) and ncpus > 0:
 195                 return ncpus
 196         else: # OSX:
 197             return int(os.popen2("sysctl -n hw.ncpu")[1].read())
 198     # Windows:
 199     if os.environ.has_key("NUMBER_OF_PROCESSORS"):
 200         ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]);
 201         if ncpus > 0:
 202             return ncpus
 203     return 1 # Default
 204
 205 def main():
 206     global options
 207     from optparse import OptionParser
 208     parser = OptionParser("usage: %prog [options] {inputs}")
 209     parser.add_option("-j", "--threads", dest="numThreads",
 210                       help="Number of testing threads",
 211                       type=int, action="store",
 212                       default=detectCPUs())
 213     parser.add_option("", "--clang", dest="clang",
 214                       help="Program to use as \"clang\"",
 215                       action="store", default=None)
 216     parser.add_option("", "--clang-cc", dest="clangcc",
 217                       help="Program to use as \"clang-cc\"",
 218                       action="store", default=None)
 219     parser.add_option("", "--vg", dest="useValgrind",
 220                       help="Run tests under valgrind",
 221                       action="store_true", default=False)
 222     parser.add_option("", "--dg", dest="useDGCompat",
 223                       help="Use llvm dejagnu compatibility mode",
 224                       action="store_true", default=False)
 225     parser.add_option("", "--script", dest="testScript",
 226                       help="Default script to use",
 227                       action="store", default=None)
 228     parser.add_option("-v", "--verbose", dest="showOutput",
 229                       help="Show all test output",
 230                       action="store_true", default=False)
 231     parser.add_option("-q", "--quiet", dest="quiet",
 232                       help="Suppress no error output",
 233                       action="store_true", default=False)
 234     parser.add_option("-s", "--succinct", dest="succinct",
 235                       help="Reduce amount of output",
 236                       action="store_true", default=False)
 237     parser.add_option("", "--max-tests", dest="maxTests",
 238                       help="Maximum number of tests to run",
 239                       action="store", type=int, default=None)
 240     parser.add_option("", "--max-time", dest="maxTime",
 241                       help="Maximum time to spend testing (in seconds)",
 242                       action="store", type=float, default=None)
 243     parser.add_option("", "--shuffle", dest="shuffle",
 244                       help="Run tests in random order",
 245                       action="store_true", default=False)
 246     parser.add_option("", "--seed", dest="seed",
 247                       help="Seed for random number generator (default: random)",
 248                       action="store", default=None)
 249     parser.add_option("", "--no-progress-bar", dest="useProgressBar",
 250                       help="Do not use curses based progress bar",
 251                       action="store_false", default=True)
 252     parser.add_option("", "--debug-do-not-test", dest="debugDoNotTest",
 253                       help="DEBUG: Skip running actual test script",
 254                       action="store_true", default=False)
 255     parser.add_option("", "--path", dest="path",
 256                       help="Additional paths to add to testing environment",
 257                       action="store", type=str, default=None)
 258
 259     (opts, args) = parser.parse_args()
 260
 261     if not args:
 262         parser.error('No inputs specified')
 263
 264     if opts.clang is None:
 265         opts.clang = TestRunner.inferClang()
 266     if opts.clangcc is None:
 267         opts.clangcc = TestRunner.inferClangCC(opts.clang)
 268
 269     # FIXME: It could be worth loading these in parallel with testing.
 270     allTests = list(getTests(args))
 271     allTests.sort()
 272
 273     tests = allTests
 274     if opts.seed is not None:
 275         try:
 276             seed = int(opts.seed)
 277         except:
 278             parser.error('--seed argument should be an integer')
 279         random.seed(seed)
 280     if opts.shuffle:
 281         random.shuffle(tests)
 282     if opts.maxTests is not None:
 283         tests = tests[:opts.maxTests]
 284     if opts.path is not None:
 285         os.environ["PATH"] = opts.path + ":" + os.environ["PATH"];
 286
 287     extra = ''
 288     if len(tests) != len(allTests):
 289         extra = ' of %d'%(len(allTests),)
 290     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 291                                                       opts.numThreads)
 292
 293     progressBar = None
 294     if not opts.quiet:
 295         if opts.useProgressBar:
 296             try:
 297                 tc = ProgressBar.TerminalController()
 298                 progressBar = ProgressBar.ProgressBar(tc, header)
 299             except ValueError:
 300                 pass
 301
 302         if not progressBar:
 303             print header
 304
 305     display = TestingProgressDisplay(opts, len(tests), progressBar)
 306     provider = TestProvider(opts, tests, display)
 307
 308     testers = [Tester(provider) for i in range(opts.numThreads)]
 309     startTime = time.time()
 310     for t in testers:
 311         t.start()
 312     try:
 313         for t in testers:
 314             t.join()
 315     except KeyboardInterrupt:
 316         sys.exit(1)
 317
 318     display.finish()
 319
 320     if not opts.quiet:
 321         print 'Testing Time: %.2fs'%(time.time() - startTime)
 322
 323     # List test results organized organized by kind.
 324     byCode = {}
 325     for t in provider.results:
 326         if t:
 327             if t.code not in byCode:
 328                 byCode[t.code] = []
 329             byCode[t.code].append(t)
 330     for title,code in (('Expected Failures', TestStatus.XFail),
 331                        ('Unexpected Passing Tests', TestStatus.XPass),
 332                        ('Failing Tests', TestStatus.Fail)):
 333         elts = byCode.get(code)
 334         if not elts:
 335             continue
 336         print '*'*20
 337         print '%s (%d):' % (title, len(elts))
 338         for tr in elts:
 339             print '\t%s'%(tr.path,)
 340
 341     numFailures = len(byCode.get(TestStatus.Fail,[]))
 342     if numFailures:
 343         print '\nFailures: %d' % (numFailures,)
 344         sys.exit(1)
 345
 346 if __name__=='__main__':
 347     main()