2 The LLVM Compiler Infrastructure
4 This file is distributed under the University of Illinois Open Source
5 License. See LICENSE.TXT for details.
7 Provides an xUnit ResultsFormatter for integrating the LLDB
8 test suite with the Jenkins xUnit aggregator and other xUnit-compliant
9 test output processors.
11 from __future__ import absolute_import
12 from __future__ import print_function
17 import xml.sax.saxutils
23 from ..event_builder import EventBuilder
24 from ..build_exception import BuildError
25 from .results_formatter import ResultsFormatter
28 class XunitFormatter(ResultsFormatter):
29 """Provides xUnit-style formatted output.
32 # Result mapping arguments
34 RM_SUCCESS = 'success'
35 RM_FAILURE = 'failure'
36 RM_PASSTHRU = 'passthru'
39 def _build_illegal_xml_regex():
40 """Constructs a regex to match all illegal xml characters.
42 Expects to be used against a unicode string."""
43 # Construct the range pairs of invalid unicode characters.
45 (0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x1F), (0x7F, 0x84),
46 (0x86, 0x9F), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF)]
48 # For wide builds, we have more.
49 if sys.maxunicode >= 0x10000:
50 illegal_chars_u.extend(
51 [(0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF),
52 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF),
53 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF),
54 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF),
55 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF),
56 (0x10FFFE, 0x10FFFF)])
58 # Build up an array of range expressions.
60 "%s-%s" % (six.unichr(low), six.unichr(high))
61 for (low, high) in illegal_chars_u]
64 return re.compile(six.u('[%s]') % six.u('').join(illegal_ranges))
67 def _quote_attribute(text):
68 """Returns the given text in a manner safe for usage in an XML attribute.
70 @param text the text that should appear within an XML attribute.
71 @return the attribute-escaped version of the input text.
73 return xml.sax.saxutils.quoteattr(text)
75 def _replace_invalid_xml(self, str_or_unicode):
76 """Replaces invalid XML characters with a '?'.
78 @param str_or_unicode a string to replace invalid XML
79 characters within. Can be unicode or not. If not unicode,
80 assumes it is a byte string in utf-8 encoding.
82 @returns a utf-8-encoded byte string with invalid
83 XML replaced with '?'.
85 # Get the content into unicode
86 if isinstance(str_or_unicode, str):
87 unicode_content = str_or_unicode.decode('utf-8')
89 unicode_content = str_or_unicode
90 return self.invalid_xml_re.sub(
91 six.u('?'), unicode_content).encode('utf-8')
95 """@return arg parser used to parse formatter-specific options."""
96 parser = super(XunitFormatter, cls).arg_parser()
98 # These are valid choices for results mapping.
99 results_mapping_choices = [
100 XunitFormatter.RM_IGNORE,
101 XunitFormatter.RM_SUCCESS,
102 XunitFormatter.RM_FAILURE,
103 XunitFormatter.RM_PASSTHRU]
105 "--assert-on-unknown-events",
107 help=('cause unknown test events to generate '
108 'a python assert. Default is to ignore.'))
110 "--ignore-skip-name",
114 dest='ignore_skip_name_patterns',
115 help=('a python regex pattern, where '
116 'any skipped test with a test method name where regex '
117 'matches (via search) will be ignored for xUnit test '
118 'result purposes. Can be specified multiple times.'))
120 "--ignore-skip-reason",
124 dest='ignore_skip_reason_patterns',
125 help=('a python regex pattern, where '
126 'any skipped test with a skip reason where the regex '
127 'matches (via search) will be ignored for xUnit test '
128 'result purposes. Can be specified multiple times.'))
130 "--xpass", action="store", choices=results_mapping_choices,
131 default=XunitFormatter.RM_FAILURE,
132 help=('specify mapping from unexpected success to jUnit/xUnit '
135 "--xfail", action="store", choices=results_mapping_choices,
136 default=XunitFormatter.RM_IGNORE,
137 help=('specify mapping from expected failure to jUnit/xUnit '
142 def _build_regex_list_from_patterns(patterns):
143 """Builds a list of compiled regular expressions from option value.
145 @param patterns contains a list of regular expression
148 @return list of compiled regular expressions, empty if no
152 if patterns is not None:
153 for pattern in patterns:
154 regex_list.append(re.compile(pattern))
157 def __init__(self, out_file, options, file_is_stream):
158 """Initializes the XunitFormatter instance.
159 @param out_file file-like object where formatted output is written.
160 @param options specifies a dictionary of options for the
163 # Initialize the parent
164 super(XunitFormatter, self).__init__(out_file, options, file_is_stream)
165 self.text_encoding = "UTF-8"
166 self.invalid_xml_re = XunitFormatter._build_illegal_xml_regex()
167 self.total_test_count = 0
168 self.ignore_skip_name_regexes = (
169 XunitFormatter._build_regex_list_from_patterns(
170 options.ignore_skip_name_patterns))
171 self.ignore_skip_reason_regexes = (
172 XunitFormatter._build_regex_list_from_patterns(
173 options.ignore_skip_reason_patterns))
180 "unexpected_successes": [],
181 "expected_failures": [],
185 self.status_handlers = {
186 EventBuilder.STATUS_SUCCESS: self._handle_success,
187 EventBuilder.STATUS_FAILURE: self._handle_failure,
188 EventBuilder.STATUS_ERROR: self._handle_error,
189 EventBuilder.STATUS_SKIP: self._handle_skip,
190 EventBuilder.STATUS_EXPECTED_FAILURE:
191 self._handle_expected_failure,
192 EventBuilder.STATUS_EXPECTED_TIMEOUT:
193 self._handle_expected_timeout,
194 EventBuilder.STATUS_UNEXPECTED_SUCCESS:
195 self._handle_unexpected_success,
196 EventBuilder.STATUS_EXCEPTIONAL_EXIT:
197 self._handle_exceptional_exit,
198 EventBuilder.STATUS_TIMEOUT:
203 EventBuilder.TYPE_TEST_RESULT,
204 EventBuilder.TYPE_JOB_RESULT}
206 def handle_event(self, test_event):
207 super(XunitFormatter, self).handle_event(test_event)
209 event_type = test_event["event"]
210 if event_type is None:
213 if event_type == "terminate":
214 # Process all the final result events into their
216 for result_event in self.result_events.values():
217 self._process_test_result(result_event)
218 self._finish_output()
220 # This is an unknown event.
221 if self.options.assert_on_unknown_events:
222 raise Exception("unknown event type {} from {}\n".format(
223 event_type, test_event))
225 def _handle_success(self, test_event):
226 """Handles a test success.
227 @param test_event the test event to handle.
229 result = self._common_add_testcase_entry(test_event)
231 self.elements["successes"].append(result)
233 def _handle_failure(self, test_event):
234 """Handles a test failure.
235 @param test_event the test event to handle.
237 message = self._replace_invalid_xml(test_event["issue_message"])
238 backtrace = self._replace_invalid_xml(
239 "".join(test_event.get("issue_backtrace", [])))
241 result = self._common_add_testcase_entry(
244 '<failure type={} message={}><![CDATA[{}]]></failure>'.format(
245 XunitFormatter._quote_attribute(test_event["issue_class"]),
246 XunitFormatter._quote_attribute(message),
250 self.elements["failures"].append(result)
252 def _handle_error_build(self, test_event):
253 """Handles a test error.
254 @param test_event the test event to handle.
256 message = self._replace_invalid_xml(test_event["issue_message"])
257 build_issue_description = self._replace_invalid_xml(
258 BuildError.format_build_error(
259 test_event.get("build_command", "<None>"),
260 test_event.get("build_error", "<None>")))
262 result = self._common_add_testcase_entry(
265 '<error type={} message={}><![CDATA[{}]]></error>'.format(
266 XunitFormatter._quote_attribute(test_event["issue_class"]),
267 XunitFormatter._quote_attribute(message),
268 build_issue_description)
271 self.elements["errors"].append(result)
273 def _handle_error_standard(self, test_event):
274 """Handles a test error.
275 @param test_event the test event to handle.
277 message = self._replace_invalid_xml(test_event["issue_message"])
278 backtrace = self._replace_invalid_xml(
279 "".join(test_event.get("issue_backtrace", [])))
281 result = self._common_add_testcase_entry(
284 '<error type={} message={}><![CDATA[{}]]></error>'.format(
285 XunitFormatter._quote_attribute(test_event["issue_class"]),
286 XunitFormatter._quote_attribute(message),
290 self.elements["errors"].append(result)
292 def _handle_error(self, test_event):
293 if test_event.get("issue_phase", None) == "build":
294 self._handle_error_build(test_event)
296 self._handle_error_standard(test_event)
298 def _handle_exceptional_exit(self, test_event):
299 """Handles an exceptional exit.
300 @param test_event the test method or job result event to handle.
302 if "test_name" in test_event:
303 name = test_event["test_name"]
305 name = test_event.get("test_filename", "<unknown test/filename>")
307 message_text = "ERROR: {} ({}): {}".format(
308 test_event.get("exception_code", 0),
309 test_event.get("exception_description", ""),
311 message = self._replace_invalid_xml(message_text)
313 result = self._common_add_testcase_entry(
316 '<error type={} message={}></error>'.format(
318 XunitFormatter._quote_attribute(message))
321 self.elements["errors"].append(result)
323 def _handle_timeout(self, test_event):
324 """Handles a test method or job timeout.
325 @param test_event the test method or job result event to handle.
327 if "test_name" in test_event:
328 name = test_event["test_name"]
330 name = test_event.get("test_filename", "<unknown test/filename>")
332 message_text = "TIMEOUT: {}".format(name)
333 message = self._replace_invalid_xml(message_text)
335 result = self._common_add_testcase_entry(
338 '<error type={} message={}></error>'.format(
340 XunitFormatter._quote_attribute(message))
343 self.elements["errors"].append(result)
346 def _ignore_based_on_regex_list(test_event, test_key, regex_list):
347 """Returns whether to ignore a test event based on patterns.
349 @param test_event the test event dictionary to check.
350 @param test_key the key within the dictionary to check.
351 @param regex_list a list of zero or more regexes. May contain
352 zero or more compiled regexes.
354 @return True if any o the regex list match based on the
355 re.search() method; false otherwise.
357 for regex in regex_list:
358 match = regex.search(test_event.get(test_key, ''))
363 def _handle_skip(self, test_event):
364 """Handles a skipped test.
365 @param test_event the test event to handle.
368 # Are we ignoring this test based on test name?
369 if XunitFormatter._ignore_based_on_regex_list(
370 test_event, 'test_name', self.ignore_skip_name_regexes):
373 # Are we ignoring this test based on skip reason?
374 if XunitFormatter._ignore_based_on_regex_list(
375 test_event, 'skip_reason', self.ignore_skip_reason_regexes):
378 # We're not ignoring this test. Process the skip.
379 reason = self._replace_invalid_xml(test_event.get("skip_reason", ""))
380 result = self._common_add_testcase_entry(
382 inner_content='<skipped message={} />'.format(
383 XunitFormatter._quote_attribute(reason)))
385 self.elements["skips"].append(result)
387 def _handle_expected_failure(self, test_event):
388 """Handles a test that failed as expected.
389 @param test_event the test event to handle.
391 if self.options.xfail == XunitFormatter.RM_PASSTHRU:
392 # This is not a natively-supported junit/xunit
393 # testcase mode, so it might fail a validating
394 # test results viewer.
395 if "bugnumber" in test_event:
396 bug_id_attribute = 'bug-id={} '.format(
397 XunitFormatter._quote_attribute(test_event["bugnumber"]))
399 bug_id_attribute = ''
401 result = self._common_add_testcase_entry(
404 '<expected-failure {}type={} message={} />'.format(
406 XunitFormatter._quote_attribute(
407 test_event["issue_class"]),
408 XunitFormatter._quote_attribute(
409 test_event["issue_message"]))
412 self.elements["expected_failures"].append(result)
413 elif self.options.xfail == XunitFormatter.RM_SUCCESS:
414 result = self._common_add_testcase_entry(test_event)
416 self.elements["successes"].append(result)
417 elif self.options.xfail == XunitFormatter.RM_FAILURE:
418 result = self._common_add_testcase_entry(
420 inner_content='<failure type={} message={} />'.format(
421 XunitFormatter._quote_attribute(test_event["issue_class"]),
422 XunitFormatter._quote_attribute(
423 test_event["issue_message"])))
425 self.elements["failures"].append(result)
426 elif self.options.xfail == XunitFormatter.RM_IGNORE:
430 "unknown xfail option: {}".format(self.options.xfail))
433 def _handle_expected_timeout(test_event):
434 """Handles expected_timeout.
435 @param test_event the test event to handle.
437 # We don't do anything with expected timeouts, not even report.
440 def _handle_unexpected_success(self, test_event):
441 """Handles a test that passed but was expected to fail.
442 @param test_event the test event to handle.
444 if self.options.xpass == XunitFormatter.RM_PASSTHRU:
445 # This is not a natively-supported junit/xunit
446 # testcase mode, so it might fail a validating
447 # test results viewer.
448 result = self._common_add_testcase_entry(
450 inner_content="<unexpected-success />")
452 self.elements["unexpected_successes"].append(result)
453 elif self.options.xpass == XunitFormatter.RM_SUCCESS:
454 # Treat the xpass as a success.
455 result = self._common_add_testcase_entry(test_event)
457 self.elements["successes"].append(result)
458 elif self.options.xpass == XunitFormatter.RM_FAILURE:
459 # Treat the xpass as a failure.
460 if "bugnumber" in test_event:
461 message = "unexpected success (bug_id:{})".format(
462 test_event["bugnumber"])
464 message = "unexpected success (bug_id:none)"
465 result = self._common_add_testcase_entry(
467 inner_content='<failure type={} message={} />'.format(
468 XunitFormatter._quote_attribute("unexpected_success"),
469 XunitFormatter._quote_attribute(message)))
471 self.elements["failures"].append(result)
472 elif self.options.xpass == XunitFormatter.RM_IGNORE:
473 # Ignore the xpass result as far as xUnit reporting goes.
476 raise Exception("unknown xpass option: {}".format(
479 def _process_test_result(self, test_event):
480 """Processes the test_event known to be a test result.
482 This categorizes the event appropriately and stores the data needed
483 to generate the final xUnit report. This method skips events that
484 cannot be represented in xUnit output.
486 if "status" not in test_event:
487 raise Exception("test event dictionary missing 'status' key")
489 status = test_event["status"]
490 if status not in self.status_handlers:
491 raise Exception("test event status '{}' unsupported".format(
494 # Call the status handler for the test result.
495 self.status_handlers[status](test_event)
497 def _common_add_testcase_entry(self, test_event, inner_content=None):
498 """Registers a testcase result, and returns the text created.
500 The caller is expected to manage failure/skip/success counts
501 in some kind of appropriate way. This call simply constructs
502 the XML and appends the returned result to the self.all_results
505 @param test_event the test event dictionary.
507 @param inner_content if specified, gets included in the <testcase>
508 inner section, at the point before stdout and stderr would be
509 included. This is where a <failure/>, <skipped/>, <error/>, etc.
512 @return the text of the xml testcase element.
516 test_class = test_event.get("test_class", "<no_class>")
517 test_name = test_event.get("test_name", "<no_test_method>")
518 event_time = test_event["event_time"]
519 time_taken = self.elapsed_time_for_test(
520 test_class, test_name, event_time)
522 # Plumb in stdout/stderr once we shift over to only test results.
526 # Formulate the output xml.
527 if not inner_content:
530 '<testcase classname="{}" name="{}" time="{:.3f}">'
531 '{}{}{}</testcase>'.format(
539 # Save the result, update total test count.
541 self.total_test_count += 1
542 self.elements["all"].append(result)
546 def _finish_output_no_lock(self):
547 """Flushes out the report of test executions to form valid xml output.
549 xUnit output is in XML. The reporting system cannot complete the
550 formatting of the output without knowing when there is no more input.
551 This call addresses notification of the completed test run and thus is
552 when we can finish off the report output.
555 # Figure out the counts line for the testsuite. If we have
556 # been counting either unexpected successes or expected
557 # failures, we'll output those in the counts, at the risk of
558 # being invalidated by a validating test results viewer.
559 # These aren't counted by default so they won't show up unless
560 # the user specified a formatter option to include them.
561 xfail_count = len(self.elements["expected_failures"])
562 xpass_count = len(self.elements["unexpected_successes"])
563 if xfail_count > 0 or xpass_count > 0:
564 extra_testsuite_attributes = (
565 ' expected-failures="{}"'
566 ' unexpected-successes="{}"'.format(xfail_count, xpass_count))
568 extra_testsuite_attributes = ""
572 '<?xml version="1.0" encoding="{}"?>\n'
574 '<testsuite name="{}" tests="{}" errors="{}" failures="{}" '
575 'skip="{}"{}>\n'.format(
578 self.total_test_count,
579 len(self.elements["errors"]),
580 len(self.elements["failures"]),
581 len(self.elements["skips"]),
582 extra_testsuite_attributes))
584 # Output each of the test result entries.
585 for result in self.elements["all"]:
586 self.out_file.write(result + '\n')
588 # Close off the test suite.
589 self.out_file.write('</testsuite></testsuites>\n')
591 def _finish_output(self):
592 """Finish writing output as all incoming events have arrived."""
594 self._finish_output_no_lock()