1 #!/usr/bin/env @PYTHON_SHEBANG@
4 # This file and its contents are supplied under the terms of the
5 # Common Development and Distribution License ("CDDL"), version 1.0.
6 # You may only use this file in accordance with the terms of version
9 # A full copy of the text of the CDDL should have accompanied this
10 # source. A copy of the CDDL is also available via the Internet at
11 # http://www.illumos.org/license/CDDL.
15 # Copyright (c) 2012, 2018 by Delphix. All rights reserved.
16 # Copyright (c) 2019 Datto Inc.
18 # This script must remain compatible with Python 3.6+.
27 from datetime import datetime
28 from optparse import OptionParser
29 from pwd import getpwnam
30 from pwd import getpwuid
31 from select import select
32 from subprocess import PIPE
33 from subprocess import Popen
34 from subprocess import check_output
35 from threading import Timer
36 from time import time, CLOCK_MONOTONIC
37 from os.path import exists
39 BASEDIR = '/var/tmp/test_results'
40 TESTDIR = '/usr/share/zfs/'
41 KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak'
51 from time import monotonic as monotonic_time
53 class timespec(ctypes.Structure):
55 ('tv_sec', ctypes.c_long),
56 ('tv_nsec', ctypes.c_long)
59 librt = ctypes.CDLL('librt.so.1', use_errno=True)
60 clock_gettime = librt.clock_gettime
61 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)]
65 if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
66 errno_ = ctypes.get_errno()
67 raise OSError(errno_, os.strerror(errno_))
68 return t.tv_sec + t.tv_nsec * 1e-9
73 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0}
77 self.returncode = None
84 def done(self, proc, killed, reran):
86 Finalize the results of this Cmd.
89 m, s = divmod(monotonic_time() - self.starttime, 60)
90 self.runtime = '%02d:%02d' % (m, s)
91 self.returncode = proc.returncode
93 Result.runresults['RERAN'] += 1
95 self.result = 'KILLED'
96 Result.runresults['KILLED'] += 1
97 elif len(self.kmemleak) > 0:
99 Result.runresults['FAIL'] += 1
100 elif self.returncode == 0:
102 Result.runresults['PASS'] += 1
103 elif self.returncode == 4:
105 Result.runresults['SKIP'] += 1
106 elif self.returncode != 0:
108 Result.runresults['FAIL'] += 1
111 class Output(object):
113 This class is a slightly modified version of the 'Stream' class found
114 here: http://goo.gl/aSGfv
116 def __init__(self, stream):
122 return self.stream.fileno()
124 def read(self, drain=0):
126 Read from the file descriptor. If 'drain' set, read until EOF.
128 while self._read() is not None:
134 Read up to 4k of data from this output stream. Collect the output
135 up to the last newline, and append it to any leftover data from a
136 previous call. The lines are stored as a (timestamp, data) tuple
137 for easy sorting/merging later.
140 buf = os.read(fd, 4096)
147 buf = self._buf + buf
148 tmp, rest = buf.rsplit(b'\n', 1)
151 rows = tmp.split(b'\n')
152 self.lines += [(now, r) for r in rows]
158 def __init__(self, pathname, identifier=None, outputdir=None,
159 timeout=None, user=None, tags=None):
160 self.pathname = pathname
161 self.identifier = identifier
162 self.outputdir = outputdir or 'BASEDIR'
164 The timeout for tests is measured in wall-clock time
166 self.timeout = timeout
167 self.user = user or ''
170 self.result = Result()
172 if self.timeout is None:
182 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
184 def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False):
186 Kill a running command due to timeout, or ^C from the keyboard. If
187 sudo is required, this user was verified previously.
190 do_sudo = len(self.user) != 0
193 cmd = [SUDO, KILL, signal, str(proc.pid)]
204 If this is not a user-initiated kill and the test has not been
205 reran before we consider if the test needs to be reran:
206 If the test has spent some time hibernating and didn't run the whole
207 length of time before being timed out we will rerun the test.
209 if keyboard_interrupt is False and self.reran is None:
210 runtime = monotonic_time() - self.result.starttime
211 if int(self.timeout) > runtime:
214 self.run(options, dryrun=False, kmemleak=kmemleak)
217 def update_cmd_privs(self, cmd, user):
219 If a user has been specified to run this Cmd and we're not already
220 running as that user, prepend the appropriate sudo command to run
223 me = getpwuid(os.getuid())
225 if not user or user is me:
226 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
228 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
232 if not os.path.isfile(cmd):
233 if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
235 if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
238 ret = '%s -E -u %s %s' % (SUDO, user, cmd)
239 return ret.split(' ')
241 def collect_output(self, proc):
243 Read from stdout/stderr as data becomes available, until the
244 process is no longer running. Return the lines from the stdout and
245 stderr Output objects.
247 out = Output(proc.stdout)
248 err = Output(proc.stderr)
250 while proc.returncode is None:
252 res = select([out, err], [], [], .1)
258 return out.lines, err.lines
260 def run(self, options, dryrun=None, kmemleak=None):
262 This is the main function that runs each individual test.
263 Determine whether or not the command requires sudo, and modify it
264 if needed. Run the command, and update the result object.
267 dryrun = options.dryrun
272 kmemleak = options.kmemleak
274 privcmd = self.update_cmd_privs(self.pathname, self.user)
277 if not os.path.isdir(self.outputdir):
278 os.makedirs(self.outputdir, mode=0o777)
284 Log each test we run to /dev/kmsg (on Linux), so if there's a kernel
285 warning we'll be able to match it up to a particular test.
287 if options.kmsg is True and exists("/dev/kmsg"):
289 kp = Popen([SUDO, "sh", "-c",
290 f"echo ZTS run {self.pathname} > /dev/kmsg"])
295 self.result.starttime = monotonic_time()
298 cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"'
299 check_output(cmd, shell=True)
301 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
302 # Allow a special timeout value of 0 to mean infinity
303 if int(self.timeout) == 0:
304 self.timeout = sys.maxsize / (10 ** 9)
306 int(self.timeout), self.kill_cmd, [proc, options, kmemleak]
311 self.result.stdout, self.result.stderr = self.collect_output(proc)
314 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"'
315 check_output(cmd, shell=True)
316 cmd = f'{SUDO} cat {KMEMLEAK_FILE}'
317 self.result.kmemleak = check_output(cmd, shell=True)
318 except KeyboardInterrupt:
319 self.kill_cmd(proc, options, kmemleak, True)
320 fail('\nRun terminated at user request.')
324 if self.reran is not False:
325 self.result.done(proc, self.killed, self.reran)
329 Initialize enough of the test result that we can log a skipped
333 Result.runresults['SKIP'] += 1
334 self.result.stdout = self.result.stderr = []
335 self.result.starttime = monotonic_time()
336 m, s = divmod(monotonic_time() - self.result.starttime, 60)
337 self.result.runtime = '%02d:%02d' % (m, s)
338 self.result.result = 'SKIP'
340 def log(self, options, suppress_console=False):
342 This function is responsible for writing all output. This includes
343 the console output, the logfile of all results (with timestamped
344 merged stdout and stderr), and for each test, the unmodified
345 stdout/stderr/merged in its own file.
348 logname = getpwuid(os.getuid()).pw_name
350 if self.reran is True:
352 user = ' (run as %s)' % (self.user if len(self.user) else logname)
354 msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
356 msga = 'Test: %s%s ' % (self.pathname, user)
357 msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
358 pad = ' ' * (80 - (len(msga) + len(msgb)))
359 result_line = msga + pad + msgb
361 # The result line is always written to the log file. If -q was
362 # specified only failures are written to the console, otherwise
363 # the result line is written to the console. The console output
364 # may be suppressed by calling log() with suppress_console=True.
365 write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
366 if not suppress_console:
367 if not options.quiet:
368 write_log(result_line, LOG_OUT)
369 elif options.quiet and self.result.result != 'PASS':
370 write_log(result_line, LOG_OUT)
372 lines = sorted(self.result.stdout + self.result.stderr,
375 # Write timestamped output (stdout and stderr) to the logfile
376 for dt, line in lines:
377 timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
379 write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
381 # Write the separate stdout/stderr/merged files, if the data exists
382 if len(self.result.stdout):
383 with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
384 for _, line in self.result.stdout:
385 os.write(out.fileno(), b'%s\n' % line)
386 if len(self.result.stderr):
387 with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
388 for _, line in self.result.stderr:
389 os.write(err.fileno(), b'%s\n' % line)
390 if len(self.result.stdout) and len(self.result.stderr):
391 with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
392 for _, line in lines:
393 os.write(merged.fileno(), b'%s\n' % line)
394 if len(self.result.kmemleak):
395 with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem:
396 kmem.write(self.result.kmemleak)
400 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
401 'post_user', 'failsafe', 'failsafe_user', 'tags']
403 def __init__(self, pathname,
404 pre=None, pre_user=None, post=None, post_user=None,
405 failsafe=None, failsafe_user=None, tags=None, **kwargs):
406 super(Test, self).__init__(pathname, **kwargs)
408 self.pre_user = pre_user or ''
409 self.post = post or ''
410 self.post_user = post_user or ''
411 self.failsafe = failsafe or ''
412 self.failsafe_user = failsafe_user or ''
413 self.tags = tags or []
416 post_user = pre_user = failsafe_user = ''
417 if len(self.pre_user):
418 pre_user = ' (as %s)' % (self.pre_user)
419 if len(self.post_user):
420 post_user = ' (as %s)' % (self.post_user)
421 if len(self.failsafe_user):
422 failsafe_user = ' (as %s)' % (self.failsafe_user)
433 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
434 self.pre, pre_user, self.post, post_user, self.failsafe,
435 failsafe_user, self.tags)
439 Check the pre/post/failsafe scripts, user and Test. Omit the Test from
440 this run if there are any problems.
442 files = [self.pre, self.pathname, self.post, self.failsafe]
443 users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
445 for f in [f for f in files if len(f)]:
446 if not verify_file(f):
447 write_log("Warning: Test '%s' not added to this run because"
448 " it failed verification.\n" % f, LOG_ERR)
451 for user in [user for user in users if len(user)]:
452 if not verify_user(user):
453 write_log("Not adding Test '%s' to this run.\n" %
454 self.pathname, LOG_ERR)
459 def run(self, options, dryrun=None, kmemleak=None):
461 Create Cmd instances for the pre/post/failsafe scripts. If the pre
462 script doesn't pass, skip this Test. Run the post script regardless.
463 If the Test is killed, also run the failsafe script.
465 odir = os.path.join(self.outputdir, os.path.basename(self.pre))
466 pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
467 timeout=self.timeout, user=self.pre_user)
468 test = Cmd(self.pathname, identifier=self.identifier,
469 outputdir=self.outputdir, timeout=self.timeout,
471 odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
472 failsafe = Cmd(self.failsafe, identifier=self.identifier,
473 outputdir=odir, timeout=self.timeout,
474 user=self.failsafe_user)
475 odir = os.path.join(self.outputdir, os.path.basename(self.post))
476 posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
477 timeout=self.timeout, user=self.post_user)
480 if len(pretest.pathname):
481 pretest.run(options, kmemleak=False)
482 cont = pretest.result.result == 'PASS'
486 test.run(options, kmemleak=kmemleak)
487 if test.result.result == 'KILLED' and len(failsafe.pathname):
488 failsafe.run(options, kmemleak=False)
489 failsafe.log(options, suppress_console=True)
495 if len(posttest.pathname):
496 posttest.run(options, kmemleak=False)
497 posttest.log(options)
500 class TestGroup(Test):
501 props = Test.props + ['tests']
503 def __init__(self, pathname, tests=None, **kwargs):
504 super(TestGroup, self).__init__(pathname, **kwargs)
505 self.tests = tests or []
508 post_user = pre_user = failsafe_user = ''
509 if len(self.pre_user):
510 pre_user = ' (as %s)' % (self.pre_user)
511 if len(self.post_user):
512 post_user = ' (as %s)' % (self.post_user)
513 if len(self.failsafe_user):
514 failsafe_user = ' (as %s)' % (self.failsafe_user)
526 ''' % (self.pathname, self.identifier, self.outputdir, self.tests,
527 self.timeout, self.user, self.pre, pre_user, self.post, post_user,
528 self.failsafe, failsafe_user, self.tags)
530 def filter(self, keeplist):
531 self.tests = [x for x in self.tests if x in keeplist]
535 Check the pre/post/failsafe scripts, user and tests in this TestGroup.
536 Omit the TestGroup entirely, or simply delete the relevant tests in the
537 group, if that's all that's required.
539 # If the pre/post/failsafe scripts are relative pathnames, convert to
540 # absolute, so they stand a chance of passing verification.
541 if len(self.pre) and not os.path.isabs(self.pre):
542 self.pre = os.path.join(self.pathname, self.pre)
543 if len(self.post) and not os.path.isabs(self.post):
544 self.post = os.path.join(self.pathname, self.post)
545 if len(self.failsafe) and not os.path.isabs(self.failsafe):
546 self.post = os.path.join(self.pathname, self.post)
548 auxfiles = [self.pre, self.post, self.failsafe]
549 users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
551 for f in [f for f in auxfiles if len(f)]:
552 if f != self.failsafe and self.pathname != os.path.dirname(f):
553 write_log("Warning: TestGroup '%s' not added to this run. "
554 "Auxiliary script '%s' exists in a different "
555 "directory.\n" % (self.pathname, f), LOG_ERR)
558 if not verify_file(f):
559 write_log("Warning: TestGroup '%s' not added to this run. "
560 "Auxiliary script '%s' failed verification.\n" %
561 (self.pathname, f), LOG_ERR)
564 for user in [user for user in users if len(user)]:
565 if not verify_user(user):
566 write_log("Not adding TestGroup '%s' to this run.\n" %
567 self.pathname, LOG_ERR)
570 # If one of the tests is invalid, delete it, log it, and drive on.
571 for test in self.tests:
572 if not verify_file(os.path.join(self.pathname, test)):
573 del self.tests[self.tests.index(test)]
574 write_log("Warning: Test '%s' removed from TestGroup '%s' "
575 "because it failed verification.\n" %
576 (test, self.pathname), LOG_ERR)
578 return len(self.tests) != 0
580 def run(self, options, dryrun=None, kmemleak=None):
582 Create Cmd instances for the pre/post/failsafe scripts. If the pre
583 script doesn't pass, skip all the tests in this TestGroup. Run the
584 post script regardless. Run the failsafe script when a test is killed.
586 # tags assigned to this test group also include the test names
587 if options.tags and not set(self.tags).intersection(set(options.tags)):
590 odir = os.path.join(self.outputdir, os.path.basename(self.pre))
591 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
592 user=self.pre_user, identifier=self.identifier)
593 odir = os.path.join(self.outputdir, os.path.basename(self.post))
594 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
595 user=self.post_user, identifier=self.identifier)
598 if len(pretest.pathname):
599 pretest.run(options, dryrun=dryrun, kmemleak=False)
600 cont = pretest.result.result == 'PASS'
603 for fname in self.tests:
604 odir = os.path.join(self.outputdir, fname)
605 test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
606 timeout=self.timeout, user=self.user,
607 identifier=self.identifier)
608 odir = os.path.join(odir, os.path.basename(self.failsafe))
609 failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
610 user=self.failsafe_user, identifier=self.identifier)
612 test.run(options, dryrun=dryrun, kmemleak=kmemleak)
613 if test.result.result == 'KILLED' and len(failsafe.pathname):
614 failsafe.run(options, dryrun=dryrun, kmemleak=False)
615 failsafe.log(options, suppress_console=True)
621 if len(posttest.pathname):
622 posttest.run(options, dryrun=dryrun, kmemleak=False)
623 posttest.log(options)
626 class TestRun(object):
627 props = ['quiet', 'outputdir']
629 def __init__(self, options):
632 self.starttime = time()
633 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
634 self.outputdir = os.path.join(options.outputdir, self.timestamp)
635 self.setup_logging(options)
637 ('outputdir', BASEDIR),
646 ('failsafe_user', ''),
651 s = 'TestRun:\n outputdir: %s\n' % self.outputdir
653 for key in sorted(self.tests.keys()):
654 s += '%s%s' % (self.tests[key].__str__(), '\n')
656 for key in sorted(self.testgroups.keys()):
657 s += '%s%s' % (self.testgroups[key].__str__(), '\n')
660 def addtest(self, pathname, options):
662 Create a new Test, and apply any properties that were passed in
663 from the command line. If it passes verification, add it to the
666 test = Test(pathname)
667 for prop in Test.props:
668 setattr(test, prop, getattr(options, prop))
671 self.tests[pathname] = test
673 def addtestgroup(self, dirname, filenames, options):
675 Create a new TestGroup, and apply any properties that were passed
676 in from the command line. If it passes verification, add it to the
679 if dirname not in self.testgroups:
680 testgroup = TestGroup(dirname)
681 for prop in Test.props:
682 setattr(testgroup, prop, getattr(options, prop))
684 # Prevent pre/post/failsafe scripts from running as regular tests
685 for f in [testgroup.pre, testgroup.post, testgroup.failsafe]:
687 del filenames[filenames.index(f)]
689 self.testgroups[dirname] = testgroup
690 self.testgroups[dirname].tests = sorted(filenames)
694 def filter(self, keeplist):
695 for group in list(self.testgroups.keys()):
696 if group not in keeplist:
697 del self.testgroups[group]
700 g = self.testgroups[group]
702 if g.pre and os.path.basename(g.pre) in keeplist[group]:
705 g.filter(keeplist[group])
707 for test in list(self.tests.keys()):
708 directory, base = os.path.split(test)
709 if directory not in keeplist or base not in keeplist[directory]:
712 def read(self, options):
714 Read in the specified runfiles, and apply the TestRun properties
715 listed in the 'DEFAULT' section to our TestRun. Then read each
716 section, and apply the appropriate properties to the Test or
717 TestGroup. Properties from individual sections override those set
718 in the 'DEFAULT' section. If the Test or TestGroup passes
719 verification, add it to the TestRun.
721 config = configparser.RawConfigParser()
722 parsed = config.read(options.runfiles)
723 failed = options.runfiles - set(parsed)
725 files = ' '.join(sorted(failed))
726 fail("Couldn't read config files: %s" % files)
728 for opt in TestRun.props:
729 if config.has_option('DEFAULT', opt):
730 setattr(self, opt, config.get('DEFAULT', opt))
731 self.outputdir = os.path.join(self.outputdir, self.timestamp)
733 testdir = options.testdir
735 for section in config.sections():
736 if 'tests' in config.options(section):
737 parts = section.split(':', 1)
738 sectiondir = parts[0]
739 identifier = parts[1] if len(parts) == 2 else None
740 if os.path.isdir(sectiondir):
741 pathname = sectiondir
742 elif os.path.isdir(os.path.join(testdir, sectiondir)):
743 pathname = os.path.join(testdir, sectiondir)
745 pathname = sectiondir
747 testgroup = TestGroup(os.path.abspath(pathname),
748 identifier=identifier)
749 for prop in TestGroup.props:
750 for sect in ['DEFAULT', section]:
751 if config.has_option(sect, prop):
753 setattr(testgroup, prop,
754 eval(config.get(sect, prop)))
755 elif prop == 'failsafe':
756 failsafe = config.get(sect, prop)
757 setattr(testgroup, prop,
758 os.path.join(testdir, failsafe))
760 setattr(testgroup, prop,
761 config.get(sect, prop))
763 # Repopulate tests using eval to convert the string to a list
764 testgroup.tests = eval(config.get(section, 'tests'))
766 if testgroup.verify():
767 self.testgroups[section] = testgroup
770 for prop in Test.props:
771 for sect in ['DEFAULT', section]:
772 if config.has_option(sect, prop):
773 if prop == 'failsafe':
774 failsafe = config.get(sect, prop)
776 os.path.join(testdir, failsafe))
778 setattr(test, prop, config.get(sect, prop))
781 self.tests[section] = test
783 def write(self, options):
785 Create a configuration file for editing and later use. The
786 'DEFAULT' section of the config file is created from the
787 properties that were specified on the command line. Tests are
788 simply added as sections that inherit everything from the
789 'DEFAULT' section. TestGroups are the same, except they get an
790 option including all the tests to run in that directory.
793 defaults = dict([(prop, getattr(options, prop)) for prop, _ in
795 config = configparser.RawConfigParser(defaults)
797 for test in sorted(self.tests.keys()):
798 config.add_section(test)
799 for prop in Test.props:
800 if prop not in self.props:
801 config.set(test, prop,
802 getattr(self.tests[test], prop))
804 for testgroup in sorted(self.testgroups.keys()):
805 config.add_section(testgroup)
806 config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
807 for prop in TestGroup.props:
808 if prop not in self.props:
809 config.set(testgroup, prop,
810 getattr(self.testgroups[testgroup], prop))
813 with open(options.template, 'w') as f:
814 return config.write(f)
816 fail('Could not open \'%s\' for writing.' % options.template)
818 def complete_outputdirs(self):
820 Collect all the pathnames for Tests, and TestGroups. Work
821 backwards one pathname component at a time, to create a unique
822 directory name in which to deposit test output. Tests will be able
823 to write output files directly in the newly modified outputdir.
824 TestGroups will be able to create one subdirectory per test in the
825 outputdir, and are guaranteed uniqueness because a group can only
826 contain files in one directory. Pre and post tests will create a
827 directory rooted at the outputdir of the Test or TestGroup in
828 question for their output. Failsafe scripts will create a directory
829 rooted at the outputdir of each Test for their output.
833 tmp_dict = dict(list(self.tests.items()) +
834 list(self.testgroups.items()))
835 total = len(tmp_dict)
836 base = self.outputdir
841 for testfile in list(tmp_dict.keys()):
842 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
843 if uniq not in paths:
845 tmp_dict[testfile].outputdir = os.path.join(base, uniq)
848 done = total == len(paths)
850 def setup_logging(self, options):
852 This function creates the output directory and gets a file object
853 for the logfile. This function must be called before write_log()
856 if options.dryrun is True:
860 if not options.template:
863 os.makedirs(self.outputdir, mode=0o777)
865 filename = os.path.join(self.outputdir, 'log')
866 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
870 def run(self, options):
872 Walk through all the Tests and TestGroups, calling run().
875 os.chdir(self.outputdir)
877 fail('Could not change to directory %s' % self.outputdir)
878 # make a symlink to the output for the currently running test
879 logsymlink = os.path.join(self.outputdir, '../current')
880 if os.path.islink(logsymlink):
881 os.unlink(logsymlink)
882 if not os.path.exists(logsymlink):
883 os.symlink(self.outputdir, logsymlink)
885 write_log('Could not make a symlink to directory %s\n' %
886 self.outputdir, LOG_ERR)
889 cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"'
890 check_output(cmd, shell=True)
893 while iteration < options.iterations:
894 for test in sorted(self.tests.keys()):
895 self.tests[test].run(options)
896 for testgroup in sorted(self.testgroups.keys()):
897 self.testgroups[testgroup].run(options)
901 if Result.total == 0:
904 print('\nResults Summary')
905 for key in list(Result.runresults.keys()):
906 if Result.runresults[key] != 0:
907 print('%s\t% 4d' % (key, Result.runresults[key]))
909 m, s = divmod(time() - self.starttime, 60)
911 print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
912 print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
913 float(Result.total)) * 100))
914 print('Log directory:\t%s' % self.outputdir)
916 if Result.runresults['FAIL'] > 0:
919 if Result.runresults['KILLED'] > 0:
922 if Result.runresults['RERAN'] > 0:
928 def write_log(msg, target):
930 Write the provided message to standard out, standard error or
931 the logfile. If specifying LOG_FILE, then `msg` must be a bytes
932 like object. This way we can still handle output from tests that
933 may be in unexpected encodings.
935 if target == LOG_OUT:
936 os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
937 elif target == LOG_ERR:
938 os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
939 elif target == LOG_FILE:
940 os.write(LOG_FILE_OBJ.fileno(), msg)
942 fail('log_msg called with unknown target "%s"' % target)
945 def verify_file(pathname):
947 Verify that the supplied pathname is an executable regular file.
949 if os.path.isdir(pathname) or os.path.islink(pathname):
952 for ext in '', '.ksh', '.sh':
953 script_path = pathname + ext
954 if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
960 def verify_user(user):
962 Verify that the specified user exists on this system, and can execute
963 sudo without being prompted for a password.
965 testcmd = [SUDO, '-n', '-u', user, TRUE]
967 if user in Cmd.verified_users:
973 write_log("Warning: user '%s' does not exist.\n" % user,
979 if p.returncode != 0:
980 write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
984 Cmd.verified_users.append(user)
989 def find_tests(testrun, options):
991 For the given list of pathnames, add files as Tests. For directories,
992 if do_groups is True, add the directory as a TestGroup. If False,
993 recursively search for executable files.
996 for p in sorted(options.pathnames):
998 for dirname, _, filenames in os.walk(p):
999 if options.do_groups:
1000 testrun.addtestgroup(dirname, filenames, options)
1002 for f in sorted(filenames):
1003 testrun.addtest(os.path.join(dirname, f), options)
1005 testrun.addtest(p, options)
1008 def filter_tests(testrun, options):
1010 fh = open(options.logfile, "r")
1011 except Exception as e:
1016 line = fh.readline()
1019 m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line)
1022 group, test = m.group(1, 2)
1024 failed[group].append(test)
1026 failed[group] = [test]
1029 testrun.filter(failed)
1032 def fail(retstr, ret=1):
1033 print('%s: %s' % (sys.argv[0], retstr))
1037 def kmemleak_cb(option, opt_str, value, parser):
1038 if not os.path.exists(KMEMLEAK_FILE):
1039 fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " +
1040 "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.")
1042 setattr(parser.values, option.dest, True)
1045 def options_cb(option, opt_str, value, parser):
1046 path_options = ['outputdir', 'template', 'testdir', 'logfile']
1048 if opt_str in parser.rargs:
1049 fail('%s may only be specified once.' % opt_str)
1051 if option.dest == 'runfiles':
1052 parser.values.cmd = 'rdconfig'
1053 value = set(os.path.abspath(p) for p in value.split(','))
1054 if option.dest == 'tags':
1055 value = [x.strip() for x in value.split(',')]
1057 if option.dest in path_options:
1058 setattr(parser.values, option.dest, os.path.abspath(value))
1060 setattr(parser.values, option.dest, value)
1064 parser = OptionParser()
1065 parser.add_option('-c', action='callback', callback=options_cb,
1066 type='string', dest='runfiles', metavar='runfiles',
1067 help='Specify tests to run via config files.')
1068 parser.add_option('-d', action='store_true', default=False, dest='dryrun',
1069 help='Dry run. Print tests, but take no other action.')
1070 parser.add_option('-l', action='callback', callback=options_cb,
1071 default=None, dest='logfile', metavar='logfile',
1073 help='Read logfile and re-run tests which failed.')
1074 parser.add_option('-g', action='store_true', default=False,
1075 dest='do_groups', help='Make directories TestGroups.')
1076 parser.add_option('-o', action='callback', callback=options_cb,
1077 default=BASEDIR, dest='outputdir', type='string',
1078 metavar='outputdir', help='Specify an output directory.')
1079 parser.add_option('-i', action='callback', callback=options_cb,
1080 default=TESTDIR, dest='testdir', type='string',
1081 metavar='testdir', help='Specify a test directory.')
1082 parser.add_option('-K', action='store_true', default=False, dest='kmsg',
1083 help='Log tests names to /dev/kmsg')
1084 parser.add_option('-m', action='callback', callback=kmemleak_cb,
1085 default=False, dest='kmemleak',
1086 help='Enable kmemleak reporting (Linux only)')
1087 parser.add_option('-p', action='callback', callback=options_cb,
1088 default='', dest='pre', metavar='script',
1089 type='string', help='Specify a pre script.')
1090 parser.add_option('-P', action='callback', callback=options_cb,
1091 default='', dest='post', metavar='script',
1092 type='string', help='Specify a post script.')
1093 parser.add_option('-q', action='store_true', default=False, dest='quiet',
1094 help='Silence on the console during a test run.')
1095 parser.add_option('-s', action='callback', callback=options_cb,
1096 default='', dest='failsafe', metavar='script',
1097 type='string', help='Specify a failsafe script.')
1098 parser.add_option('-S', action='callback', callback=options_cb,
1099 default='', dest='failsafe_user',
1100 metavar='failsafe_user', type='string',
1101 help='Specify a user to execute the failsafe script.')
1102 parser.add_option('-t', action='callback', callback=options_cb, default=60,
1103 dest='timeout', metavar='seconds', type='int',
1104 help='Timeout (in seconds) for an individual test.')
1105 parser.add_option('-u', action='callback', callback=options_cb,
1106 default='', dest='user', metavar='user', type='string',
1107 help='Specify a different user name to run as.')
1108 parser.add_option('-w', action='callback', callback=options_cb,
1109 default=None, dest='template', metavar='template',
1110 type='string', help='Create a new config file.')
1111 parser.add_option('-x', action='callback', callback=options_cb, default='',
1112 dest='pre_user', metavar='pre_user', type='string',
1113 help='Specify a user to execute the pre script.')
1114 parser.add_option('-X', action='callback', callback=options_cb, default='',
1115 dest='post_user', metavar='post_user', type='string',
1116 help='Specify a user to execute the post script.')
1117 parser.add_option('-T', action='callback', callback=options_cb, default='',
1118 dest='tags', metavar='tags', type='string',
1119 help='Specify tags to execute specific test groups.')
1120 parser.add_option('-I', action='callback', callback=options_cb, default=1,
1121 dest='iterations', metavar='iterations', type='int',
1122 help='Number of times to run the test run.')
1123 (options, pathnames) = parser.parse_args()
1125 if options.runfiles and len(pathnames):
1126 fail('Extraneous arguments.')
1128 options.pathnames = [os.path.abspath(path) for path in pathnames]
1134 options = parse_args()
1136 testrun = TestRun(options)
1138 if options.runfiles:
1139 testrun.read(options)
1141 find_tests(testrun, options)
1144 filter_tests(testrun, options)
1146 if options.template:
1147 testrun.write(options)
1150 testrun.complete_outputdirs()
1151 testrun.run(options)
1152 exit(testrun.summary())
1155 if __name__ == '__main__':