4 Static Analyzer qualification infrastructure.
6 The goal is to test the analyzer against different projects, check for failures,
7 compare results, and measure performance.
9 Repository Directory will contain sources of the projects as well as the
10 information on how to build them and the expected output.
11 Repository Directory structure:
13 - Historical Performance Data
20 To test the build of the analyzer one would:
21 - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
22 the build directory does not pollute the repository to min network traffic).
23 - Build all projects, until error. Produce logs to report errors.
26 The files which should be kept around for failure investigations:
27 RepositoryCopy/Project DirI/ScanBuildResults
28 RepositoryCopy/Project DirI/run_static_analyzer.log
30 Assumptions (TODO: shouldn't need to assume these.):
31 The script is being run from the Repository Directory.
32 The compiler for scan-build and scan-build are in the PATH.
33 export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
35 For more logging, set the env variables:
36 zaks:TI zaks$ export CCC_ANALYZER_LOG=1
37 zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
49 from subprocess import check_call, CalledProcessError
51 #------------------------------------------------------------------------------
53 #------------------------------------------------------------------------------
57 Detects the number of CPUs on a system. Cribbed from pp.
59 # Linux, Unix and MacOS:
60 if hasattr(os, "sysconf"):
61 if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
63 ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
64 if isinstance(ncpus, int) and ncpus > 0:
67 return int(capture(['sysctl', '-n', 'hw.ncpu']))
69 if os.environ.has_key("NUMBER_OF_PROCESSORS"):
70 ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
75 def which(command, paths = None):
76 """which(command, [paths]) - Look up the given command in the paths string
77 (or the PATH environment variable, if unspecified)."""
80 paths = os.environ.get('PATH','')
82 # Check for absolute match first.
83 if os.path.exists(command):
86 # Would be nice if Python had a lib function for this.
90 # Get suffixes to search.
91 # On Cygwin, 'PATHEXT' may exist but it should not be used.
93 pathext = os.environ.get('PATHEXT', '').split(';')
98 for path in paths.split(os.pathsep):
100 p = os.path.join(path, command + ext)
101 if os.path.exists(p):
106 # Make sure we flush the output after every print statement.
107 class flushfile(object):
108 def __init__(self, f):
114 sys.stdout = flushfile(sys.stdout)
116 def getProjectMapPath():
117 ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
119 if not os.path.exists(ProjectMapPath):
120 print "Error: Cannot find the Project Map file " + ProjectMapPath +\
121 "\nRunning script for the wrong directory?"
123 return ProjectMapPath
125 def getProjectDir(ID):
126 return os.path.join(os.path.abspath(os.curdir), ID)
128 def getSBOutputDirName(IsReferenceBuild) :
129 if IsReferenceBuild == True :
130 return SBOutputDirReferencePrefix + SBOutputDirName
132 return SBOutputDirName
134 #------------------------------------------------------------------------------
135 # Configuration setup.
136 #------------------------------------------------------------------------------
138 # Find Clang for static analysis.
139 Clang = which("clang", os.environ['PATH'])
141 print "Error: cannot find 'clang' in PATH"
145 Jobs = int(math.ceil(detectCPUs() * 0.75))
147 # Project map stores info about all the "registered" projects.
148 ProjectMapFile = "projectMap.csv"
150 # Names of the project specific scripts.
151 # The script that needs to be executed before the build can start.
152 CleanupScript = "cleanup_run_static_analyzer.sh"
153 # This is a file containing commands for scan-build.
154 BuildScript = "run_static_analyzer.cmd"
157 LogFolderName = "Logs"
158 BuildLogName = "run_static_analyzer.log"
159 # Summary file - contains the summary of the failures. Ex: This info can be be
160 # displayed when buildbot detects a build failure.
161 NumOfFailuresInSummary = 10
162 FailuresSummaryFileName = "failures.txt"
163 # Summary of the result diffs.
164 DiffsSummaryFileName = "diffs.txt"
166 # The scan-build result directory.
167 SBOutputDirName = "ScanBuildResults"
168 SBOutputDirReferencePrefix = "Ref"
170 # The list of checkers used during analyzes.
171 # Currently, consists of all the non-experimental checkers, plus a few alpha
172 # checkers we don't want to regress on.
173 Checkers="alpha.unix.SimpleStream,alpha.security.taint,alpha.cplusplus.NewDeleteLeaks,core,cplusplus,deadcode,security,unix,osx"
177 #------------------------------------------------------------------------------
178 # Test harness logic.
179 #------------------------------------------------------------------------------
181 # Run pre-processing script if any.
182 def runCleanupScript(Dir, PBuildLogFile):
183 ScriptPath = os.path.join(Dir, CleanupScript)
184 if os.path.exists(ScriptPath):
187 print " Executing: %s" % (ScriptPath,)
188 check_call("chmod +x %s" % ScriptPath, cwd = Dir,
189 stderr=PBuildLogFile,
190 stdout=PBuildLogFile,
192 check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile,
193 stdout=PBuildLogFile,
196 print "Error: The pre-processing step failed. See ", \
197 PBuildLogFile.name, " for details."
200 # Build the project with scan-build by reading in the commands and
201 # prefixing them with the scan-build options.
202 def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
203 BuildScriptPath = os.path.join(Dir, BuildScript)
204 if not os.path.exists(BuildScriptPath):
205 print "Error: build script is not defined: %s" % BuildScriptPath
207 SBOptions = "--use-analyzer " + Clang + " "
208 SBOptions += "-plist-html -o " + SBOutputDir + " "
209 SBOptions += "-enable-checker " + Checkers + " "
210 SBOptions += "--keep-empty "
211 # Always use ccc-analyze to ensure that we can locate the failures
213 SBOptions += "--override-compiler "
215 SBCommandFile = open(BuildScriptPath, "r")
216 SBPrefix = "scan-build " + SBOptions + " "
217 for Command in SBCommandFile:
218 Command = Command.strip()
219 # If using 'make', auto imply a -jX argument
220 # to speed up analysis. xcodebuild will
221 # automatically use the maximum number of cores.
222 if (Command.startswith("make ") or Command == "make") and \
224 Command += " -j%d" % Jobs
225 SBCommand = SBPrefix + Command
227 print " Executing: %s" % (SBCommand,)
228 check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile,
229 stdout=PBuildLogFile,
232 print "Error: scan-build failed. See ",PBuildLogFile.name,\
236 def hasNoExtension(FileName):
237 (Root, Ext) = os.path.splitext(FileName)
242 def isValidSingleInputFile(FileName):
243 (Root, Ext) = os.path.splitext(FileName)
244 if ((Ext == ".i") | (Ext == ".ii") |
245 (Ext == ".c") | (Ext == ".cpp") |
246 (Ext == ".m") | (Ext == "")) :
250 # Run analysis on a set of preprocessed files.
251 def runAnalyzePreprocessed(Dir, SBOutputDir, Mode):
252 if os.path.exists(os.path.join(Dir, BuildScript)):
253 print "Error: The preprocessed files project should not contain %s" % \
257 CmdPrefix = Clang + " -cc1 -analyze -analyzer-output=plist -w "
258 CmdPrefix += "-analyzer-checker=" + Checkers +" -fcxx-exceptions -fblocks "
261 CmdPrefix += "-std=c++11 "
263 PlistPath = os.path.join(Dir, SBOutputDir, "date")
264 FailPath = os.path.join(PlistPath, "failures");
265 os.makedirs(FailPath);
267 for FullFileName in glob.glob(Dir + "/*"):
268 FileName = os.path.basename(FullFileName)
271 # Only run the analyzes on supported files.
272 if (hasNoExtension(FileName)):
274 if (isValidSingleInputFile(FileName) == False):
275 print "Error: Invalid single input file %s." % (FullFileName,)
278 # Build and call the analyzer command.
279 OutputOption = "-o " + os.path.join(PlistPath, FileName) + ".plist "
280 Command = CmdPrefix + OutputOption + os.path.join(Dir, FileName)
281 LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
284 print " Executing: %s" % (Command,)
285 check_call(Command, cwd = Dir, stderr=LogFile,
288 except CalledProcessError, e:
289 print "Error: Analyzes of %s failed. See %s for details." \
291 (FullFileName, LogFile.name, e.returncode)
296 # If command did not fail, erase the log file.
298 os.remove(LogFile.name);
300 def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
303 BuildLogPath = os.path.join(SBOutputDir, LogFolderName, BuildLogName)
304 print "Log file: %s" % (BuildLogPath,)
305 print "Output directory: %s" %(SBOutputDir, )
307 # Clean up the log file.
308 if (os.path.exists(BuildLogPath)) :
309 RmCommand = "rm " + BuildLogPath
311 print " Executing: %s" % (RmCommand,)
312 check_call(RmCommand, shell=True)
314 # Clean up scan build results.
315 if (os.path.exists(SBOutputDir)) :
316 RmCommand = "rm -r " + SBOutputDir
318 print " Executing: %s" % (RmCommand,)
319 check_call(RmCommand, shell=True)
320 assert(not os.path.exists(SBOutputDir))
321 os.makedirs(os.path.join(SBOutputDir, LogFolderName))
324 PBuildLogFile = open(BuildLogPath, "wb+")
326 # Build and analyze the project.
328 runCleanupScript(Dir, PBuildLogFile)
330 if (ProjectBuildMode == 1):
331 runScanBuild(Dir, SBOutputDir, PBuildLogFile)
333 runAnalyzePreprocessed(Dir, SBOutputDir, ProjectBuildMode)
335 if IsReferenceBuild :
336 runCleanupScript(Dir, PBuildLogFile)
339 PBuildLogFile.close()
341 print "Build complete (time: %.2f). See the log for more details: %s" % \
342 ((time.time()-TBegin), BuildLogPath)
344 # A plist file is created for each call to the analyzer(each source file).
345 # We are only interested on the once that have bug reports, so delete the rest.
346 def CleanUpEmptyPlists(SBOutputDir):
347 for F in glob.glob(SBOutputDir + "/*/*.plist"):
348 P = os.path.join(SBOutputDir, F)
350 Data = plistlib.readPlist(P)
351 # Delete empty reports.
352 if not Data['files']:
356 # Given the scan-build output directory, checks if the build failed
357 # (by searching for the failures directories). If there are failures, it
358 # creates a summary file in the output directory.
359 def checkBuild(SBOutputDir):
360 # Check if there are failures.
361 Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
362 TotalFailed = len(Failures);
364 CleanUpEmptyPlists(SBOutputDir)
365 Plists = glob.glob(SBOutputDir + "/*/*.plist")
366 print "Number of bug reports (non-empty plist files) produced: %d" %\
370 # Create summary file to display when the build fails.
371 SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName)
373 print " Creating the failures summary file %s" % (SummaryPath,)
375 SummaryLog = open(SummaryPath, "w+")
377 SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
378 if TotalFailed > NumOfFailuresInSummary:
379 SummaryLog.write("See the first %d below.\n"
380 % (NumOfFailuresInSummary,))
381 # TODO: Add a line "See the results folder for more."
383 FailuresCopied = NumOfFailuresInSummary
385 for FailLogPathI in Failures:
386 if Idx >= NumOfFailuresInSummary:
389 SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
390 FailLogI = open(FailLogPathI, "r");
392 shutil.copyfileobj(FailLogI, SummaryLog);
398 print "Error: analysis failed. See ", SummaryPath
401 # Auxiliary object to discard stdout.
402 class Discarder(object):
403 def write(self, text):
406 # Compare the warnings produced by scan-build.
407 def runCmpResults(Dir):
410 RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
411 NewDir = os.path.join(Dir, SBOutputDirName)
413 # We have to go one level down the directory tree.
414 RefList = glob.glob(RefDir + "/*")
415 NewList = glob.glob(NewDir + "/*")
417 # Log folders are also located in the results dir, so ignore them.
418 RefLogDir = os.path.join(RefDir, LogFolderName)
419 if RefLogDir in RefList:
420 RefList.remove(RefLogDir)
421 NewList.remove(os.path.join(NewDir, LogFolderName))
423 if len(RefList) == 0 or len(NewList) == 0:
425 assert(len(RefList) == len(NewList))
427 # There might be more then one folder underneath - one per each scan-build
428 # command (Ex: one for configure and one for make).
429 if (len(RefList) > 1):
430 # Assume that the corresponding folders have the same names.
434 # Iterate and find the differences.
436 PairList = zip(RefList, NewList)
441 assert(RefDir != NewDir)
443 print " Comparing Results: %s %s" % (RefDir, NewDir)
445 DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
446 Opts = CmpRuns.CmpOptions(DiffsPath)
447 # Discard everything coming out of stdout (CmpRun produces a lot of them).
448 OLD_STDOUT = sys.stdout
449 sys.stdout = Discarder()
450 # Scan the results, delete empty plist files.
451 NumDiffs = CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
452 sys.stdout = OLD_STDOUT
454 print "Warning: %r differences in diagnostics. See %s" % \
455 (NumDiffs, DiffsPath,)
457 print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
458 return (NumDiffs > 0)
460 def updateSVN(Mode, ProjectsMap):
463 for I in csv.reader(ProjectsMap):
465 Path = os.path.join(ProjName, getSBOutputDirName(True))
468 Command = "svn delete %s" % (Path,)
470 Command = "svn add %s" % (Path,)
473 print " Executing: %s" % (Command,)
474 check_call(Command, shell=True)
477 CommitCommand = "svn commit -m \"[analyzer tests] Remove " \
478 "reference results.\""
480 CommitCommand = "svn commit -m \"[analyzer tests] Add new " \
481 "reference results.\""
483 print " Executing: %s" % (CommitCommand,)
484 check_call(CommitCommand, shell=True)
486 print "Error: SVN update failed."
489 def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None):
490 print " \n\n--- Building project %s" % (ID,)
495 Dir = getProjectDir(ID)
497 print " Build directory: %s." % (Dir,)
499 # Set the build results directory.
500 RelOutputDir = getSBOutputDirName(IsReferenceBuild)
501 SBOutputDir = os.path.join(Dir, RelOutputDir)
503 buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
505 checkBuild(SBOutputDir)
507 if IsReferenceBuild == False:
510 print "Completed tests for project %s (time: %.2f)." % \
511 (ID, (time.time()-TBegin))
513 def testAll(IsReferenceBuild = False, UpdateSVN = False):
514 PMapFile = open(getProjectMapPath(), "rb")
516 # Validate the input.
517 for I in csv.reader(PMapFile):
519 print "Error: Rows in the ProjectMapFile should have 3 entries."
521 if (not ((I[1] == "0") | (I[1] == "1") | (I[1] == "2"))):
522 print "Error: Second entry in the ProjectMapFile should be 0" \
523 " (single file), 1 (project), or 2(single file c++11)."
526 # When we are regenerating the reference results, we might need to
527 # update svn. Remove reference results from SVN.
528 if UpdateSVN == True:
529 assert(IsReferenceBuild == True);
530 updateSVN("delete", PMapFile);
534 for I in csv.reader(PMapFile):
535 testProject(I[0], int(I[1]), IsReferenceBuild)
537 # Add reference results to SVN.
538 if UpdateSVN == True:
539 updateSVN("add", PMapFile);
542 print "Error occurred. Premature termination."
547 if __name__ == '__main__':
550 if len(sys.argv) >= 2:
551 if sys.argv[1] == "-r":
553 elif sys.argv[1] == "-rs":
557 print >> sys.stderr, 'Usage: ', sys.argv[0],\
559 'Use -r to regenerate reference output' \
560 'Use -rs to regenerate reference output and update svn'
562 testAll(IsReference, UpdateSVN)