4 Static Analyzer qualification infrastructure.
6 The goal is to test the analyzer against different projects, check for failures,
7 compare results, and measure performance.
9 Repository Directory will contain sources of the projects as well as the
10 information on how to build them and the expected output.
11 Repository Directory structure:
13 - Historical Performance Data
19 Note that the build tree must be inside the project dir.
21 To test the build of the analyzer one would:
22 - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
23 the build directory does not pollute the repository to min network traffic).
24 - Build all projects, until error. Produce logs to report errors.
27 The files which should be kept around for failure investigations:
28 RepositoryCopy/Project DirI/ScanBuildResults
29 RepositoryCopy/Project DirI/run_static_analyzer.log
31 Assumptions (TODO: shouldn't need to assume these.):
32 The script is being run from the Repository Directory.
33 The compiler for scan-build and scan-build are in the PATH.
34 export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
36 For more logging, set the env variables:
37 zaks:TI zaks$ export CCC_ANALYZER_LOG=1
38 zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
40 The list of checkers tested are hardcoded in the Checkers variable.
41 For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment
42 variable. It should contain a comma separated list.
55 from subprocess import check_call, check_output, CalledProcessError
57 #------------------------------------------------------------------------------
59 #------------------------------------------------------------------------------
63 Detects the number of CPUs on a system. Cribbed from pp.
65 # Linux, Unix and MacOS:
66 if hasattr(os, "sysconf"):
67 if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
69 ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
70 if isinstance(ncpus, int) and ncpus > 0:
73 return int(capture(['sysctl', '-n', 'hw.ncpu']))
75 if os.environ.has_key("NUMBER_OF_PROCESSORS"):
76 ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
81 def which(command, paths = None):
82 """which(command, [paths]) - Look up the given command in the paths string
83 (or the PATH environment variable, if unspecified)."""
86 paths = os.environ.get('PATH','')
88 # Check for absolute match first.
89 if os.path.exists(command):
92 # Would be nice if Python had a lib function for this.
96 # Get suffixes to search.
97 # On Cygwin, 'PATHEXT' may exist but it should not be used.
99 pathext = os.environ.get('PATHEXT', '').split(';')
103 # Search the paths...
104 for path in paths.split(os.pathsep):
106 p = os.path.join(path, command + ext)
107 if os.path.exists(p):
112 # Make sure we flush the output after every print statement.
113 class flushfile(object):
114 def __init__(self, f):
120 sys.stdout = flushfile(sys.stdout)
122 def getProjectMapPath():
123 ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
125 if not os.path.exists(ProjectMapPath):
126 print "Error: Cannot find the Project Map file " + ProjectMapPath +\
127 "\nRunning script for the wrong directory?"
129 return ProjectMapPath
131 def getProjectDir(ID):
132 return os.path.join(os.path.abspath(os.curdir), ID)
134 def getSBOutputDirName(IsReferenceBuild) :
135 if IsReferenceBuild == True :
136 return SBOutputDirReferencePrefix + SBOutputDirName
138 return SBOutputDirName
140 #------------------------------------------------------------------------------
141 # Configuration setup.
142 #------------------------------------------------------------------------------
144 # Find Clang for static analysis.
145 Clang = which("clang", os.environ['PATH'])
147 print "Error: cannot find 'clang' in PATH"
151 Jobs = int(math.ceil(detectCPUs() * 0.75))
153 # Project map stores info about all the "registered" projects.
154 ProjectMapFile = "projectMap.csv"
156 # Names of the project specific scripts.
157 # The script that downloads the project.
158 DownloadScript = "download_project.sh"
159 # The script that needs to be executed before the build can start.
160 CleanupScript = "cleanup_run_static_analyzer.sh"
161 # This is a file containing commands for scan-build.
162 BuildScript = "run_static_analyzer.cmd"
165 LogFolderName = "Logs"
166 BuildLogName = "run_static_analyzer.log"
167 # Summary file - contains the summary of the failures. Ex: This info can be be
168 # displayed when buildbot detects a build failure.
169 NumOfFailuresInSummary = 10
170 FailuresSummaryFileName = "failures.txt"
171 # Summary of the result diffs.
172 DiffsSummaryFileName = "diffs.txt"
174 # The scan-build result directory.
175 SBOutputDirName = "ScanBuildResults"
176 SBOutputDirReferencePrefix = "Ref"
178 # The name of the directory storing the cached project source. If this directory
179 # does not exist, the download script will be executed. That script should
180 # create the "CachedSource" directory and download the project source into it.
181 CachedSourceDirName = "CachedSource"
183 # The name of the directory containing the source code that will be analyzed.
184 # Each time a project is analyzed, a fresh copy of its CachedSource directory
185 # will be copied to the PatchedSource directory and then the local patches
186 # in PatchfileName will be applied (if PatchfileName exists).
187 PatchedSourceDirName = "PatchedSource"
189 # The name of the patchfile specifying any changes that should be applied
190 # to the CachedSource before analyzing.
191 PatchfileName = "changes_for_analyzer.patch"
193 # The list of checkers used during analyzes.
194 # Currently, consists of all the non-experimental checkers, plus a few alpha
195 # checkers we don't want to regress on.
196 Checkers="alpha.unix.SimpleStream,alpha.security.taint,cplusplus.NewDeleteLeaks,core,cplusplus,deadcode,security,unix,osx"
200 #------------------------------------------------------------------------------
201 # Test harness logic.
202 #------------------------------------------------------------------------------
204 # Run pre-processing script if any.
205 def runCleanupScript(Dir, PBuildLogFile):
206 Cwd = os.path.join(Dir, PatchedSourceDirName)
207 ScriptPath = os.path.join(Dir, CleanupScript)
208 runScript(ScriptPath, PBuildLogFile, Cwd)
210 # Run the script to download the project, if it exists.
211 def runDownloadScript(Dir, PBuildLogFile):
212 ScriptPath = os.path.join(Dir, DownloadScript)
213 runScript(ScriptPath, PBuildLogFile, Dir)
215 # Run the provided script if it exists.
216 def runScript(ScriptPath, PBuildLogFile, Cwd):
217 if os.path.exists(ScriptPath):
220 print " Executing: %s" % (ScriptPath,)
221 check_call("chmod +x '%s'" % ScriptPath, cwd = Cwd,
222 stderr=PBuildLogFile,
223 stdout=PBuildLogFile,
225 check_call("'%s'" % ScriptPath, cwd = Cwd, stderr=PBuildLogFile,
226 stdout=PBuildLogFile,
229 print "Error: Running %s failed. See %s for details." % (ScriptPath,
233 # Download the project and apply the local patchfile if it exists.
234 def downloadAndPatch(Dir, PBuildLogFile):
235 CachedSourceDirPath = os.path.join(Dir, CachedSourceDirName)
237 # If the we don't already have the cached source, run the project's
238 # download script to download it.
239 if not os.path.exists(CachedSourceDirPath):
240 runDownloadScript(Dir, PBuildLogFile)
241 if not os.path.exists(CachedSourceDirPath):
242 print "Error: '%s' not found after download." % (CachedSourceDirPath)
245 PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
247 # Remove potentially stale patched source.
248 if os.path.exists(PatchedSourceDirPath):
249 shutil.rmtree(PatchedSourceDirPath)
251 # Copy the cached source and apply any patches to the copy.
252 shutil.copytree(CachedSourceDirPath, PatchedSourceDirPath, symlinks=True)
253 applyPatch(Dir, PBuildLogFile)
255 def applyPatch(Dir, PBuildLogFile):
256 PatchfilePath = os.path.join(Dir, PatchfileName)
257 PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
258 if not os.path.exists(PatchfilePath):
259 print " No local patches."
262 print " Applying patch."
264 check_call("patch -p1 < '%s'" % (PatchfilePath),
265 cwd = PatchedSourceDirPath,
266 stderr=PBuildLogFile,
267 stdout=PBuildLogFile,
270 print "Error: Patch failed. See %s for details." % (PBuildLogFile.name)
273 # Build the project with scan-build by reading in the commands and
274 # prefixing them with the scan-build options.
275 def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
276 BuildScriptPath = os.path.join(Dir, BuildScript)
277 if not os.path.exists(BuildScriptPath):
278 print "Error: build script is not defined: %s" % BuildScriptPath
281 AllCheckers = Checkers
282 if os.environ.has_key('SA_ADDITIONAL_CHECKERS'):
283 AllCheckers = AllCheckers + ',' + os.environ['SA_ADDITIONAL_CHECKERS']
285 # Run scan-build from within the patched source directory.
286 SBCwd = os.path.join(Dir, PatchedSourceDirName)
288 SBOptions = "--use-analyzer '%s' " % Clang
289 SBOptions += "-plist-html -o '%s' " % SBOutputDir
290 SBOptions += "-enable-checker " + AllCheckers + " "
291 SBOptions += "--keep-empty "
292 # Always use ccc-analyze to ensure that we can locate the failures
294 SBOptions += "--override-compiler "
296 SBCommandFile = open(BuildScriptPath, "r")
297 SBPrefix = "scan-build " + SBOptions + " "
298 for Command in SBCommandFile:
299 Command = Command.strip()
300 if len(Command) == 0:
302 # If using 'make', auto imply a -jX argument
303 # to speed up analysis. xcodebuild will
304 # automatically use the maximum number of cores.
305 if (Command.startswith("make ") or Command == "make") and \
307 Command += " -j%d" % Jobs
308 SBCommand = SBPrefix + Command
310 print " Executing: %s" % (SBCommand,)
311 check_call(SBCommand, cwd = SBCwd, stderr=PBuildLogFile,
312 stdout=PBuildLogFile,
315 print "Error: scan-build failed. See ",PBuildLogFile.name,\
319 def hasNoExtension(FileName):
320 (Root, Ext) = os.path.splitext(FileName)
325 def isValidSingleInputFile(FileName):
326 (Root, Ext) = os.path.splitext(FileName)
327 if ((Ext == ".i") | (Ext == ".ii") |
328 (Ext == ".c") | (Ext == ".cpp") |
329 (Ext == ".m") | (Ext == "")) :
333 # Get the path to the SDK for the given SDK name. Returns None if
334 # the path cannot be determined.
335 def getSDKPath(SDKName):
336 if which("xcrun") is None:
339 Cmd = "xcrun --sdk " + SDKName + " --show-sdk-path"
340 return check_output(Cmd, shell=True).rstrip()
342 # Run analysis on a set of preprocessed files.
343 def runAnalyzePreprocessed(Dir, SBOutputDir, Mode):
344 if os.path.exists(os.path.join(Dir, BuildScript)):
345 print "Error: The preprocessed files project should not contain %s" % \
349 CmdPrefix = Clang + " -cc1 "
351 # For now, we assume the preprocessed files should be analyzed
353 SDKPath = getSDKPath("macosx")
354 if SDKPath is not None:
355 CmdPrefix += "-isysroot " + SDKPath + " "
357 CmdPrefix += "-analyze -analyzer-output=plist -w "
358 CmdPrefix += "-analyzer-checker=" + Checkers +" -fcxx-exceptions -fblocks "
361 CmdPrefix += "-std=c++11 "
363 PlistPath = os.path.join(Dir, SBOutputDir, "date")
364 FailPath = os.path.join(PlistPath, "failures");
365 os.makedirs(FailPath);
367 for FullFileName in glob.glob(Dir + "/*"):
368 FileName = os.path.basename(FullFileName)
371 # Only run the analyzes on supported files.
372 if (hasNoExtension(FileName)):
374 if (isValidSingleInputFile(FileName) == False):
375 print "Error: Invalid single input file %s." % (FullFileName,)
378 # Build and call the analyzer command.
379 OutputOption = "-o '%s.plist' " % os.path.join(PlistPath, FileName)
380 Command = CmdPrefix + OutputOption + ("'%s'" % FileName)
381 LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
384 print " Executing: %s" % (Command,)
385 check_call(Command, cwd = Dir, stderr=LogFile,
388 except CalledProcessError, e:
389 print "Error: Analyzes of %s failed. See %s for details." \
391 (FullFileName, LogFile.name, e.returncode)
396 # If command did not fail, erase the log file.
398 os.remove(LogFile.name);
400 def getBuildLogPath(SBOutputDir):
401 return os.path.join(SBOutputDir, LogFolderName, BuildLogName)
403 def removeLogFile(SBOutputDir):
404 BuildLogPath = getBuildLogPath(SBOutputDir)
405 # Clean up the log file.
406 if (os.path.exists(BuildLogPath)) :
407 RmCommand = "rm '%s'" % BuildLogPath
409 print " Executing: %s" % (RmCommand,)
410 check_call(RmCommand, shell=True)
412 def buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
415 BuildLogPath = getBuildLogPath(SBOutputDir)
416 print "Log file: %s" % (BuildLogPath,)
417 print "Output directory: %s" %(SBOutputDir, )
419 removeLogFile(SBOutputDir)
421 # Clean up scan build results.
422 if (os.path.exists(SBOutputDir)) :
423 RmCommand = "rm -r '%s'" % SBOutputDir
425 print " Executing: %s" % (RmCommand,)
426 check_call(RmCommand, shell=True)
427 assert(not os.path.exists(SBOutputDir))
428 os.makedirs(os.path.join(SBOutputDir, LogFolderName))
431 PBuildLogFile = open(BuildLogPath, "wb+")
433 # Build and analyze the project.
435 if (ProjectBuildMode == 1):
436 downloadAndPatch(Dir, PBuildLogFile)
437 runCleanupScript(Dir, PBuildLogFile)
438 runScanBuild(Dir, SBOutputDir, PBuildLogFile)
440 runAnalyzePreprocessed(Dir, SBOutputDir, ProjectBuildMode)
442 if IsReferenceBuild :
443 runCleanupScript(Dir, PBuildLogFile)
445 # Make the absolute paths relative in the reference results.
446 for (DirPath, Dirnames, Filenames) in os.walk(SBOutputDir):
448 if (not F.endswith('plist')):
450 Plist = os.path.join(DirPath, F)
451 Data = plistlib.readPlist(Plist)
453 if (ProjectBuildMode == 1):
454 PathPrefix = os.path.join(Dir, PatchedSourceDirName)
455 Paths = [SourceFile[len(PathPrefix)+1:]\
456 if SourceFile.startswith(PathPrefix)\
457 else SourceFile for SourceFile in Data['files']]
458 Data['files'] = Paths
459 plistlib.writePlist(Data, Plist)
462 PBuildLogFile.close()
464 print "Build complete (time: %.2f). See the log for more details: %s" % \
465 ((time.time()-TBegin), BuildLogPath)
467 # A plist file is created for each call to the analyzer(each source file).
468 # We are only interested on the once that have bug reports, so delete the rest.
469 def CleanUpEmptyPlists(SBOutputDir):
470 for F in glob.glob(SBOutputDir + "/*/*.plist"):
471 P = os.path.join(SBOutputDir, F)
473 Data = plistlib.readPlist(P)
474 # Delete empty reports.
475 if not Data['files']:
479 # Given the scan-build output directory, checks if the build failed
480 # (by searching for the failures directories). If there are failures, it
481 # creates a summary file in the output directory.
482 def checkBuild(SBOutputDir):
483 # Check if there are failures.
484 Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
485 TotalFailed = len(Failures);
487 CleanUpEmptyPlists(SBOutputDir)
488 Plists = glob.glob(SBOutputDir + "/*/*.plist")
489 print "Number of bug reports (non-empty plist files) produced: %d" %\
493 # Create summary file to display when the build fails.
494 SummaryPath = os.path.join(SBOutputDir, LogFolderName, FailuresSummaryFileName)
496 print " Creating the failures summary file %s" % (SummaryPath,)
498 SummaryLog = open(SummaryPath, "w+")
500 SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
501 if TotalFailed > NumOfFailuresInSummary:
502 SummaryLog.write("See the first %d below.\n"
503 % (NumOfFailuresInSummary,))
504 # TODO: Add a line "See the results folder for more."
506 FailuresCopied = NumOfFailuresInSummary
508 for FailLogPathI in Failures:
509 if Idx >= NumOfFailuresInSummary:
512 SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
513 FailLogI = open(FailLogPathI, "r");
515 shutil.copyfileobj(FailLogI, SummaryLog);
521 print "Error: analysis failed. See ", SummaryPath
524 # Auxiliary object to discard stdout.
525 class Discarder(object):
526 def write(self, text):
529 # Compare the warnings produced by scan-build.
530 # Strictness defines the success criteria for the test:
531 # 0 - success if there are no crashes or analyzer failure.
532 # 1 - success if there are no difference in the number of reported bugs.
533 # 2 - success if all the bug reports are identical.
534 def runCmpResults(Dir, Strictness = 0):
537 RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
538 NewDir = os.path.join(Dir, SBOutputDirName)
540 # We have to go one level down the directory tree.
541 RefList = glob.glob(RefDir + "/*")
542 NewList = glob.glob(NewDir + "/*")
544 # Log folders are also located in the results dir, so ignore them.
545 RefLogDir = os.path.join(RefDir, LogFolderName)
546 if RefLogDir in RefList:
547 RefList.remove(RefLogDir)
548 NewList.remove(os.path.join(NewDir, LogFolderName))
550 if len(RefList) == 0 or len(NewList) == 0:
552 assert(len(RefList) == len(NewList))
554 # There might be more then one folder underneath - one per each scan-build
555 # command (Ex: one for configure and one for make).
556 if (len(RefList) > 1):
557 # Assume that the corresponding folders have the same names.
561 # Iterate and find the differences.
563 PairList = zip(RefList, NewList)
568 assert(RefDir != NewDir)
570 print " Comparing Results: %s %s" % (RefDir, NewDir)
572 DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
573 PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
574 Opts = CmpRuns.CmpOptions(DiffsPath, "", PatchedSourceDirPath)
575 # Discard everything coming out of stdout (CmpRun produces a lot of them).
576 OLD_STDOUT = sys.stdout
577 sys.stdout = Discarder()
578 # Scan the results, delete empty plist files.
579 NumDiffs, ReportsInRef, ReportsInNew = \
580 CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
581 sys.stdout = OLD_STDOUT
583 print "Warning: %r differences in diagnostics. See %s" % \
584 (NumDiffs, DiffsPath,)
585 if Strictness >= 2 and NumDiffs > 0:
586 print "Error: Diffs found in strict mode (2)."
588 elif Strictness >= 1 and ReportsInRef != ReportsInNew:
589 print "Error: The number of results are different in strict mode (1)."
592 print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
593 return (NumDiffs > 0)
595 def cleanupReferenceResults(SBOutputDir):
596 # Delete html, css, and js files from reference results. These can
597 # include multiple copies of the benchmark source and so get very large.
598 Extensions = ["html", "css", "js"]
600 for F in glob.glob("%s/*/*.%s" % (SBOutputDir, E)):
601 P = os.path.join(SBOutputDir, F)
602 RmCommand = "rm '%s'" % P
603 check_call(RmCommand, shell=True)
605 # Remove the log file. It leaks absolute path names.
606 removeLogFile(SBOutputDir)
608 def updateSVN(Mode, ProjectsMap):
611 for I in csv.reader(ProjectsMap):
613 Path = os.path.join(ProjName, getSBOutputDirName(True))
616 Command = "svn delete '%s'" % (Path,)
618 Command = "svn add '%s'" % (Path,)
621 print " Executing: %s" % (Command,)
622 check_call(Command, shell=True)
625 CommitCommand = "svn commit -m \"[analyzer tests] Remove " \
626 "reference results.\""
628 CommitCommand = "svn commit -m \"[analyzer tests] Add new " \
629 "reference results.\""
631 print " Executing: %s" % (CommitCommand,)
632 check_call(CommitCommand, shell=True)
634 print "Error: SVN update failed."
637 def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None, Strictness = 0):
638 print " \n\n--- Building project %s" % (ID,)
643 Dir = getProjectDir(ID)
645 print " Build directory: %s." % (Dir,)
647 # Set the build results directory.
648 RelOutputDir = getSBOutputDirName(IsReferenceBuild)
649 SBOutputDir = os.path.join(Dir, RelOutputDir)
651 buildProject(Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild)
653 checkBuild(SBOutputDir)
655 if IsReferenceBuild == False:
656 runCmpResults(Dir, Strictness)
658 cleanupReferenceResults(SBOutputDir)
660 print "Completed tests for project %s (time: %.2f)." % \
661 (ID, (time.time()-TBegin))
663 def isCommentCSVLine(Entries):
664 # Treat CSV lines starting with a '#' as a comment.
665 return len(Entries) > 0 and Entries[0].startswith("#")
667 def testAll(IsReferenceBuild = False, UpdateSVN = False, Strictness = 0):
668 PMapFile = open(getProjectMapPath(), "rb")
670 # Validate the input.
671 for I in csv.reader(PMapFile):
672 if (isCommentCSVLine(I)):
675 print "Error: Rows in the ProjectMapFile should have 3 entries."
677 if (not ((I[1] == "0") | (I[1] == "1") | (I[1] == "2"))):
678 print "Error: Second entry in the ProjectMapFile should be 0" \
679 " (single file), 1 (project), or 2(single file c++11)."
682 # When we are regenerating the reference results, we might need to
683 # update svn. Remove reference results from SVN.
684 if UpdateSVN == True:
685 assert(IsReferenceBuild == True);
686 updateSVN("delete", PMapFile);
690 for I in csv.reader(PMapFile):
691 if isCommentCSVLine(I):
693 testProject(I[0], int(I[1]), IsReferenceBuild, None, Strictness)
695 # Add reference results to SVN.
696 if UpdateSVN == True:
697 updateSVN("add", PMapFile);
700 print "Error occurred. Premature termination."
705 if __name__ == '__main__':
706 # Parse command line arguments.
707 Parser = argparse.ArgumentParser(description='Test the Clang Static Analyzer.')
708 Parser.add_argument('--strictness', dest='strictness', type=int, default=0,
709 help='0 to fail on runtime errors, 1 to fail when the number\
710 of found bugs are different from the reference, 2 to \
711 fail on any difference from the reference. Default is 0.')
712 Parser.add_argument('-r', dest='regenerate', action='store_true', default=False,
713 help='Regenerate reference output.')
714 Parser.add_argument('-rs', dest='update_reference', action='store_true',
715 default=False, help='Regenerate reference output and update svn.')
716 Args = Parser.parse_args()
720 Strictness = Args.strictness
723 elif Args.update_reference:
727 testAll(IsReference, UpdateSVN, Strictness)