3 # Licensed to the Apache Software Foundation (ASF) under one
4 # or more contributor license agreements. See the NOTICE file
5 # distributed with this work for additional information
6 # regarding copyright ownership. The ASF licenses this file
7 # to you under the Apache License, Version 2.0 (the
8 # "License"); you may not use this file except in compliance
9 # with the License. You may obtain a copy of the License at
11 # http://www.apache.org/licenses/LICENSE-2.0
13 # Unless required by applicable law or agreed to in writing,
14 # software distributed under the License is distributed on an
15 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 # KIND, either express or implied. See the License for the
17 # specific language governing permissions and limitations
22 Driver for running the tests on Windows.
24 For a list of options, run this script with the --help option.
27 # $HeadURL: https://svn.apache.org/repos/asf/subversion/branches/1.10.x/win-tests.py $
28 # $LastChangedRevision: 1813897 $
30 import os, sys, subprocess
41 import ConfigParser as configparser
47 my_getopt = getopt.gnu_getopt
48 except AttributeError:
49 my_getopt = getopt.getopt
52 "print usage, exit the script"
54 print("Driver for running the tests on Windows.")
55 print("Usage: python win-tests.py [option] [test-path]")
57 print("Valid options:")
58 print(" -r, --release : test the Release configuration")
59 print(" -d, --debug : test the Debug configuration (default)")
60 print(" --bin=PATH : use the svn binaries installed in PATH")
61 print(" -u URL, --url=URL : run ra_dav or ra_svn tests against URL;")
62 print(" will start svnserve for ra_svn tests")
63 print(" -v, --verbose : talk more")
64 print(" -f, --fs-type=type : filesystem type to use (fsfs is default)")
65 print(" -c, --cleanup : cleanup after running a test")
66 print(" -t, --test=TEST : Run the TEST test (all is default); use")
67 print(" TEST#n to run a particular test number,")
68 print(" multiples also accepted e.g. '2,4-7'")
69 print(" --log-level=LEVEL : Set log level to LEVEL (E.g. DEBUG)")
70 print(" --log-to-stdout : Write log results to stdout")
72 print(" --svnserve-args=list : comma-separated list of arguments for")
74 print(" default is '-d,-r,<test-path-root>'")
75 print(" --asp.net-hack : use '_svn' instead of '.svn' for the admin")
77 print(" --httpd-dir : location where Apache HTTPD is installed")
78 print(" --httpd-port : port for Apache HTTPD; random port number")
79 print(" will be used, if not specified")
80 print(" --httpd-daemon : Run Apache httpd as daemon")
81 print(" --httpd-service : Run Apache httpd as Windows service (default)")
82 print(" --httpd-no-log : Disable httpd logging")
83 print(" --http-short-circuit : Use SVNPathAuthz short_circuit on HTTP server")
84 print(" --disable-http-v2 : Do not advertise support for HTTPv2 on server")
85 print(" --disable-bulk-updates : Disable bulk updates on HTTP server")
86 print(" --ssl-cert : Path to SSL server certificate to trust.")
87 print(" --https : Run Apache httpd with an https setup.")
88 print(" --http2 : Enable http2 in Apache Httpd (>= 2.4.17).")
89 print(" --mod-deflate : Enable mod_deflate in Apache Httpd.")
90 print(" --global-scheduler : Enable global scheduler.")
91 print(" --exclusive-wc-locks : Enable exclusive working copy locks")
92 print(" --memcached-dir=DIR : Run memcached from dir")
93 print(" --memcached-server= : Enable usage of the specified memcached server")
95 print(" --skip-c-tests : Skip all C tests")
96 print(" --dump-load-cross-check: Run the dump load cross check after every test")
98 print(" --javahl : Run the javahl tests instead of the normal tests")
99 print(" --swig=language : Run the swig perl/python/ruby tests instead of")
100 print(" the normal tests")
101 print(" --list : print test doc strings only")
102 print(" --milestone-filter=RE : RE is a regular expression pattern that (when")
103 print(" used with --list) limits the tests listed to")
104 print(" those with an associated issue in the tracker")
105 print(" which has a target milestone that matches RE.")
106 print(" --mode-filter=TYPE : limit tests to expected TYPE = XFAIL, SKIP, PASS,")
107 print(" or 'ALL' (default)")
108 print(" --enable-sasl : enable Cyrus SASL authentication for")
110 print(" -p, --parallel : run multiple tests in parallel")
111 print(" --server-minor-version : the minor version of the server being")
113 print(" --config-file : Configuration file for tests")
114 print(" --fsfs-sharding : Specify shard size (for fsfs)")
115 print(" --fsfs-packing : Run 'svnadmin pack' automatically")
116 print(" --fsfs-compression=VAL : Set compression type to VAL (for fsfs)")
117 print(" -q, --quiet : Deprecated; this is the default.")
118 print(" Use --set-log-level instead.")
122 CMDLINE_TEST_SCRIPT_PATH = 'subversion/tests/cmdline/'
123 CMDLINE_TEST_SCRIPT_NATIVE_PATH = CMDLINE_TEST_SCRIPT_PATH.replace('/', os.sep)
125 sys.path.insert(0, os.path.join('build', 'generator'))
126 sys.path.insert(1, 'build')
128 import gen_win_dependencies
130 version_header = os.path.join('subversion', 'include', 'svn_version.h')
131 cp = configparser.ConfigParser()
132 cp.read('gen-make.opts')
133 gen_obj = gen_win_dependencies.GenDependenciesBase('build.conf', version_header,
135 opts, args = my_getopt(sys.argv[1:], 'hrdvqct:pu:f:',
136 ['release', 'debug', 'verbose', 'quiet', 'cleanup',
137 'test=', 'url=', 'svnserve-args=', 'fs-type=', 'asp.net-hack',
138 'httpd-dir=', 'httpd-port=', 'httpd-daemon', 'https',
139 'httpd-server', 'http-short-circuit', 'httpd-no-log',
140 'disable-http-v2', 'disable-bulk-updates', 'help',
141 'fsfs-packing', 'fsfs-sharding=', 'javahl', 'swig=',
142 'list', 'enable-sasl', 'bin=', 'parallel', 'http2',
143 'mod-deflate', 'global-scheduler',
144 'config-file=', 'server-minor-version=', 'log-level=',
145 'log-to-stdout', 'mode-filter=', 'milestone-filter=',
146 'ssl-cert=', 'exclusive-wc-locks', 'memcached-server=',
147 'skip-c-tests', 'dump-load-cross-check', 'memcached-dir=',
151 print('Warning: non-option arguments after the first one will be ignored')
153 # Interpret the options and set parameters
154 base_url, fs_type, verbose, cleanup = None, None, None, None
155 global_scheduler = None
156 repo_loc = 'local repository.'
159 faillog = 'fails.log'
168 use_mod_deflate = False
169 http_short_circuit = False
170 advertise_httpv2 = True
171 http_bulk_updates = True
173 milestone_filter = None
181 server_minor_version = None
188 exclusive_wc_locks = None
190 memcached_server = None
193 dump_load_cross_check = None
194 fsfs_compression = None
195 fsfs_dir_deltification = None
197 for opt, val in opts:
198 if opt in ('-h', '--help'):
200 elif opt in ('-u', '--url'):
202 elif opt in ('-f', '--fs-type'):
204 elif opt in ('-v', '--verbose'):
206 log_level = logging.DEBUG
207 elif opt in ('-c', '--cleanup'):
209 elif opt in ('-t', '--test'):
210 tests_to_run.append(val)
211 elif opt in ['-r', '--release']:
213 elif opt in ['-d', '--debug']:
215 elif opt == '--svnserve-args':
216 svnserve_args = val.split(',')
218 elif opt == '--asp.net-hack':
219 os.environ['SVN_ASP_DOT_NET_HACK'] = opt
220 elif opt == '--httpd-dir':
221 abs_httpd_dir = os.path.abspath(val)
223 elif opt == '--httpd-port':
224 httpd_port = int(val)
225 elif opt == '--httpd-daemon':
227 elif opt == '--httpd-service':
229 elif opt == '--httpd-no-log':
231 elif opt == '--https':
233 elif opt == '--http2':
235 elif opt == '--mod-deflate':
237 elif opt == '--http-short-circuit':
238 http_short_circuit = True
239 elif opt == '--disable-http-v2':
240 advertise_httpv2 = False
241 elif opt == '--disable-bulk-updates':
242 http_bulk_updates = False
243 elif opt == '--fsfs-sharding':
244 fsfs_sharding = int(val)
245 elif opt == '--fsfs-packing':
247 elif opt == '--javahl':
249 elif opt == '--global-scheduler':
251 elif opt == '--swig':
252 if val not in ['perl', 'python', 'ruby']:
253 sys.stderr.write('Running \'%s\' swig tests not supported (yet).\n'
256 elif opt == '--list':
258 elif opt == '--milestone-filter':
259 milestone_filter = val
260 elif opt == '--mode-filter':
262 elif opt == '--enable-sasl':
264 base_url = "svn://localhost/"
265 elif opt == '--server-minor-version':
266 server_minor_version = int(val)
269 elif opt in ('-p', '--parallel'):
271 elif opt in ('--config-file'):
273 elif opt == '--log-to-stdout':
275 elif opt == '--log-level':
276 log_level = getattr(logging, val, None) or int(val)
277 elif opt == '--ssl-cert':
279 elif opt == '--exclusive-wc-locks':
280 exclusive_wc_locks = 1
281 elif opt == '--memcached-server':
282 memcached_server = val
283 elif opt == '--skip-c-tests':
285 elif opt == '--dump-load-cross-check':
286 dump_load_cross_check = 1
287 elif opt == '--memcached-dir':
290 elif opt == '--fsfs-compression':
291 fsfs_compression = val
292 elif opt == '--fsfs-dir-deltification':
293 fsfs_dir_deltification = val
295 # Calculate the source and test directory names
296 abs_srcdir = os.path.abspath("")
297 abs_objdir = os.path.join(abs_srcdir, objdir)
299 abs_builddir = abs_objdir
302 abs_builddir = os.path.abspath(args[0])
305 # Default to fsfs explicitly
310 all_tests = gen_obj.test_progs + gen_obj.bdb_test_progs \
311 + gen_obj.scripts + gen_obj.bdb_scripts
313 all_tests = gen_obj.test_progs + gen_obj.scripts
315 client_tests = [x for x in all_tests if x.startswith(CMDLINE_TEST_SCRIPT_PATH)]
319 httpd_port = random.randrange(1024, 30000)
326 base_url = '%s://localhost:%d' % (scheme, httpd_port)
329 repo_loc = 'remote repository ' + base_url + '.'
330 if base_url[:4] == 'http':
331 log = 'dav-tests.log'
332 faillog = 'dav-fails.log'
333 elif base_url[:3] == 'svn':
334 log = 'svn-tests.log'
335 faillog = 'svn-fails.log'
338 # Don't know this scheme, but who're we to judge whether it's
340 log = 'url-tests.log'
341 faillog = 'url-fails.log'
343 # Have to move the executables where the tests expect them to be
344 copied_execs = [] # Store copied exec files to avoid the final dir scan
346 def create_target_dir(dirname):
347 tgt_dir = os.path.join(abs_builddir, dirname)
348 if not os.path.exists(tgt_dir):
350 print("mkdir: %s" % tgt_dir)
353 def copy_changed_file(src, tgt=None, to_dir=None, cleanup=True):
354 if not os.path.isfile(src):
355 print('Could not find ' + src)
358 if to_dir and not tgt:
359 tgt = os.path.join(to_dir, os.path.basename(src))
360 elif not tgt or (tgt and to_dir):
361 raise RuntimeError("Using 'tgt' *or* 'to_dir' is required" % (tgt,))
362 elif tgt and os.path.isdir(tgt):
363 raise RuntimeError("'%s' is a directory. Use to_dir=" % (tgt,))
365 if os.path.exists(tgt):
366 assert os.path.isfile(tgt)
367 if filecmp.cmp(src, tgt):
369 print("same: %s" % src)
370 print(" and: %s" % tgt)
373 print("copy: %s" % src)
374 print(" to: %s" % tgt)
375 shutil.copy(src, tgt)
378 copied_execs.append(tgt)
381 "Move DLLs to a known location and set env vars"
383 debug = (objdir == 'Debug')
385 for lib in gen_obj._libraries.values():
388 name, dir = lib.debug_dll_name, lib.debug_dll_dir
390 name, dir = lib.dll_name, lib.dll_dir
393 src = os.path.join(dir, name)
394 if os.path.exists(src):
395 copy_changed_file(src, to_dir=abs_builddir, cleanup=False)
397 for name in lib.extra_bin:
398 src = os.path.join(dir, name)
399 copy_changed_file(src, to_dir=abs_builddir)
402 # Copy the Subversion library DLLs
403 for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL):
404 if isinstance(i, gen_base.TargetLib) and i.msvc_export:
405 src = os.path.join(abs_objdir, i.filename)
406 if os.path.isfile(src):
407 copy_changed_file(src, to_dir=abs_builddir,
410 # Copy the Apache modules
411 if run_httpd and cp.has_option('options', '--with-httpd'):
412 mod_dav_svn_path = os.path.join(abs_objdir, 'subversion',
413 'mod_dav_svn', 'mod_dav_svn.so')
414 mod_authz_svn_path = os.path.join(abs_objdir, 'subversion',
415 'mod_authz_svn', 'mod_authz_svn.so')
416 mod_dontdothat_path = os.path.join(abs_objdir, 'tools', 'server-side',
417 'mod_dontdothat', 'mod_dontdothat.so')
419 copy_changed_file(mod_dav_svn_path, to_dir=abs_builddir, cleanup=False)
420 copy_changed_file(mod_authz_svn_path, to_dir=abs_builddir, cleanup=False)
421 copy_changed_file(mod_dontdothat_path, to_dir=abs_builddir, cleanup=False)
423 os.environ['PATH'] = abs_builddir + os.pathsep + os.environ['PATH']
426 path = os.path.normpath(path)
427 parts = path.split(os.path.sep)
428 drive = parts[0].upper()
430 path = drive + os.path.sep
432 dirs = os.listdir(path)
434 if dir.lower() == part.lower():
435 path = os.path.join(path, dir)
440 "Run svnserve for ra_svn tests"
441 def __init__(self, svnserve_args, objdir, abs_objdir, abs_builddir):
442 self.args = svnserve_args
443 self.name = 'svnserve.exe'
445 self.path = os.path.join(abs_objdir,
446 'subversion', 'svnserve', self.name)
447 self.root = os.path.join(abs_builddir, CMDLINE_TEST_SCRIPT_NATIVE_PATH)
451 "Stop svnserve when the object is deleted"
454 def _quote(self, arg):
456 return '"' + arg + '"'
462 args = [self.name, '-d', '-r', self.root]
464 args = [self.name] + self.args
465 print('Starting %s %s' % (self.kind, self.name))
467 env = os.environ.copy()
468 env['SVN_DBG_STACKTRACES_TO_STDERR'] = 'y'
469 self.proc = subprocess.Popen([self.path] + args[1:], env=env)
472 if self.proc is not None:
474 print('Stopping %s' % self.name)
476 if self.proc.returncode is None:
479 except AttributeError:
481 print('Svnserve.stop not implemented')
484 "Run httpd for DAV tests"
485 def __init__(self, abs_httpd_dir, abs_objdir, abs_builddir, abs_srcdir,
486 httpd_port, service, use_ssl, use_http2, use_mod_deflate,
487 no_log, httpv2, short_circuit, bulk_updates):
488 self.name = 'apache.exe'
489 self.httpd_port = httpd_port
490 self.httpd_dir = abs_httpd_dir
493 self.httpv2_option = 'on'
495 self.httpv2_option = 'off'
498 self.bulkupdates_option = 'on'
500 self.bulkupdates_option = 'off'
502 self.service = service
504 self.path = os.path.join(self.httpd_dir, 'bin', self.name)
507 self.path_authz_option = 'short_circuit'
509 self.path_authz_option = 'on'
511 if not os.path.exists(self.path):
512 self.name = 'httpd.exe'
513 self.path = os.path.join(self.httpd_dir, 'bin', self.name)
514 if not os.path.exists(self.path):
515 raise RuntimeError("Could not find a valid httpd binary!")
517 self.root_dir = os.path.join(CMDLINE_TEST_SCRIPT_NATIVE_PATH, 'httpd')
518 self.root = os.path.join(abs_builddir, self.root_dir)
519 self.authz_file = os.path.join(abs_builddir,
520 CMDLINE_TEST_SCRIPT_NATIVE_PATH,
521 'svn-test-work', 'authz')
522 self.dontdothat_file = os.path.join(abs_builddir,
523 CMDLINE_TEST_SCRIPT_NATIVE_PATH,
524 'svn-test-work', 'dontdothat')
525 self.certfile = os.path.join(abs_builddir,
526 CMDLINE_TEST_SCRIPT_NATIVE_PATH,
527 'svn-test-work', 'cert.pem')
528 self.certkeyfile = os.path.join(abs_builddir,
529 CMDLINE_TEST_SCRIPT_NATIVE_PATH,
530 'svn-test-work', 'cert-key.pem')
531 self.httpd_config = os.path.join(self.root, 'httpd.conf')
532 self.httpd_users = os.path.join(self.root, 'users')
533 self.httpd_mime_types = os.path.join(self.root, 'mime.types')
534 self.httpd_groups = os.path.join(self.root, 'groups')
535 self.abs_builddir = abs_builddir
536 self.abs_objdir = abs_objdir
537 self.abs_srcdir = abs_srcdir
538 self.service_name = 'svn-test-httpd-' + str(httpd_port)
541 self.httpd_args = [self.name, '-n', self._quote(self.service_name),
542 '-f', self._quote(self.httpd_config)]
544 self.httpd_args = [self.name, '-f', self._quote(self.httpd_config)]
546 create_target_dir(self.root_dir)
548 self._create_users_file()
549 self._create_groups_file()
550 self._create_mime_types_file()
551 self._create_dontdothat_file()
554 self._create_cert_files()
557 version_vals = gen_obj._libraries['httpd'].version.split('.')
558 self.httpd_ver = float('%s.%s' % (version_vals[0], version_vals[1]))
560 # Create httpd config file
561 fp = open(self.httpd_config, 'w')
563 # Limit the number of threads (default = 64)
565 fp.write('<IfModule mpm_winnt.c>\n')
566 fp.write('ThreadsPerChild 16\n')
567 fp.write('</IfModule>\n')
570 fp.write('ServerRoot ' + self._quote(self.root) + '\n')
571 fp.write('DocumentRoot ' + self._quote(self.root) + '\n')
572 fp.write('ServerName localhost\n')
573 fp.write('PidFile pid\n')
574 fp.write('ErrorLog log\n')
575 fp.write('Listen ' + str(self.httpd_port) + '\n')
578 fp.write('LogFormat "%h %l %u %t \\"%r\\" %>s %b" common\n')
579 fp.write('Customlog log common\n')
580 fp.write('LogLevel Debug\n')
582 fp.write('LogLevel Crit\n')
584 # Write LoadModule for minimal system module
586 fp.write(self._sys_module('ssl_module', 'mod_ssl.so'))
588 fp.write(self._sys_module('http2_module', 'mod_http2.so'))
590 fp.write(self._sys_module('deflate_module', 'mod_deflate.so'))
591 fp.write(self._sys_module('dav_module', 'mod_dav.so'))
592 if self.httpd_ver >= 2.3:
593 fp.write(self._sys_module('access_compat_module', 'mod_access_compat.so'))
594 fp.write(self._sys_module('authz_core_module', 'mod_authz_core.so'))
595 fp.write(self._sys_module('authz_user_module', 'mod_authz_user.so'))
596 fp.write(self._sys_module('authn_core_module', 'mod_authn_core.so'))
597 if self.httpd_ver >= 2.2:
598 fp.write(self._sys_module('auth_basic_module', 'mod_auth_basic.so'))
599 fp.write(self._sys_module('authn_file_module', 'mod_authn_file.so'))
600 fp.write(self._sys_module('authz_groupfile_module', 'mod_authz_groupfile.so'))
601 fp.write(self._sys_module('authz_host_module', 'mod_authz_host.so'))
603 fp.write(self._sys_module('auth_module', 'mod_auth.so'))
604 fp.write(self._sys_module('alias_module', 'mod_alias.so'))
605 fp.write(self._sys_module('mime_module', 'mod_mime.so'))
606 fp.write(self._sys_module('log_config_module', 'mod_log_config.so'))
608 # Write LoadModule for Subversion modules
609 fp.write(self._svn_module('dav_svn_module', 'mod_dav_svn.so'))
610 fp.write(self._svn_module('authz_svn_module', 'mod_authz_svn.so'))
612 # And for mod_dontdothat
613 fp.write(self._svn_module('dontdothat_module', 'mod_dontdothat.so'))
616 fp.write('SSLEngine on\n')
617 fp.write('SSLProtocol All -SSLv2 -SSLv3\n')
618 fp.write('SSLCertificateFile %s\n' % self._quote(self.certfile))
619 fp.write('SSLCertificateKeyFile %s\n' % self._quote(self.certkeyfile))
621 if use_ssl and use_http2:
622 fp.write('Protocols h2 http/1.1\n')
624 fp.write('Protocols h2c http/1.1\n')
625 fp.write('H2Direct on\n')
628 fp.write('SetOutputFilter DEFLATE\n')
630 # Don't handle .htaccess, symlinks, etc.
631 fp.write('<Directory />\n')
632 fp.write('AllowOverride None\n')
633 fp.write('Options None\n')
634 fp.write('</Directory>\n\n')
636 # Define two locations for repositories
637 fp.write(self._svn_repo('repositories'))
638 fp.write(self._svn_repo('local_tmp'))
639 fp.write(self._svn_authz_repo())
641 # And two redirects for the redirect tests
642 fp.write('RedirectMatch permanent ^/svn-test-work/repositories/'
643 'REDIRECT-PERM-(.*)$ /svn-test-work/repositories/$1\n')
644 fp.write('RedirectMatch ^/svn-test-work/repositories/'
645 'REDIRECT-TEMP-(.*)$ /svn-test-work/repositories/$1\n')
647 fp.write('TypesConfig ' + self._quote(self.httpd_mime_types) + '\n')
648 fp.write('HostNameLookups Off\n')
653 "Stop httpd when the object is deleted"
656 def _quote(self, arg):
658 return '"' + arg + '"'
662 def _create_users_file(self):
664 htpasswd = os.path.join(self.httpd_dir, 'bin', 'htpasswd.exe')
665 # Create the cheapest to compare password form for our testsuite
666 os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-bcp', self.httpd_users,
667 'jrandom', 'rayjandom'])
668 os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-bp', self.httpd_users,
669 'jconstant', 'rayjandom'])
670 os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-bp', self.httpd_users,
671 '__dumpster__', '__loadster__'])
672 os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-bp', self.httpd_users,
673 'JRANDOM', 'rayjandom'])
674 os.spawnv(os.P_WAIT, htpasswd, ['htpasswd.exe', '-bp', self.httpd_users,
675 'JCONSTANT', 'rayjandom'])
677 def _create_groups_file(self):
678 "Create groups for mod_authz_svn tests"
679 fp = open(self.httpd_groups, 'w')
680 fp.write('random: jrandom\n')
681 fp.write('constant: jconstant\n')
684 def _create_mime_types_file(self):
685 "Create empty mime.types file"
686 fp = open(self.httpd_mime_types, 'w')
689 def _create_dontdothat_file(self):
690 "Create empty mime.types file"
691 # If the tests have not previously been run or were cleaned
692 # up, then 'svn-test-work' does not exist yet.
693 parent_dir = os.path.dirname(self.dontdothat_file)
694 if not os.path.exists(parent_dir):
695 os.makedirs(parent_dir)
697 fp = open(self.dontdothat_file, 'w')
698 fp.write('[recursive-actions]\n')
699 fp.write('/ = deny\n')
702 def _create_cert_files(self):
703 "Create certificate files"
704 # The unix build uses certificates encoded in davautocheck.sh
705 # Let's just read them from there
707 sh_path = os.path.join(self.abs_srcdir, 'subversion', 'tests', 'cmdline',
709 sh = open(sh_path).readlines()
711 def cert_extract(lines, what):
713 pattern = r'cat\s*\>\s*' + re.escape(what) + r'\s*\<\<([A-Z_a-z0-9]+)'
717 if i.startswith(exit_marker):
721 m = re.match(pattern, i)
723 exit_marker = m.groups(1)
725 cert_file = cert_extract(sh, '"$SSL_CERTIFICATE_FILE"')
726 cert_key = cert_extract(sh, '"$SSL_CERTIFICATE_KEY_FILE"')
727 open(self.certfile, 'w').write(''.join(cert_file))
728 open(self.certkeyfile, 'w').write(''.join(cert_key))
730 def _sys_module(self, name, path):
731 full_path = os.path.join(self.httpd_dir, 'modules', path)
732 return 'LoadModule ' + name + " " + self._quote(full_path) + '\n'
734 def _svn_module(self, name, path):
735 full_path = os.path.join(self.abs_builddir, path)
736 return 'LoadModule ' + name + ' ' + self._quote(full_path) + '\n'
738 def _svn_repo(self, name):
739 path = os.path.join(self.abs_builddir,
740 CMDLINE_TEST_SCRIPT_NATIVE_PATH,
741 'svn-test-work', name)
742 location = '/svn-test-work/' + name
743 ddt_location = '/ddt-test-work/' + name
745 '<Location ' + location + '>\n' \
747 ' SVNParentPath ' + self._quote(path) + '\n' \
748 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
749 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
750 ' SVNAllowBulkUpdates ' + self.bulkupdates_option + '\n' \
751 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
752 ' AuthType Basic\n' \
753 ' AuthName "Subversion Repository"\n' \
754 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
755 ' Require valid-user\n' \
757 '<Location ' + ddt_location + '>\n' \
759 ' SVNParentPath ' + self._quote(path) + '\n' \
760 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
761 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
762 ' SVNAllowBulkUpdates ' + self.bulkupdates_option + '\n' \
763 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
764 ' AuthType Basic\n' \
765 ' AuthName "Subversion Repository"\n' \
766 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
767 ' Require valid-user\n' \
768 ' DontDoThatConfigFile ' + self._quote(self.dontdothat_file) + '\n' \
771 def _svn_authz_repo(self):
772 local_tmp = os.path.join(self.abs_builddir,
773 CMDLINE_TEST_SCRIPT_NATIVE_PATH,
774 'svn-test-work', 'local_tmp')
776 '<Location /authz-test-work/anon>' + '\n' \
778 ' SVNParentPath ' + local_tmp + '\n' \
779 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
780 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
781 ' SVNListParentPath On' + '\n' \
782 ' <IfModule mod_authz_core.c>' + '\n' \
783 ' Require all granted' + '\n' \
784 ' </IfModule>' + '\n' \
785 ' <IfModule !mod_authz_core.c>' + '\n' \
786 ' Allow from all' + '\n' \
787 ' </IfModule>' + '\n' \
788 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
789 '</Location>' + '\n' \
790 '<Location /authz-test-work/mixed>' + '\n' \
792 ' SVNParentPath ' + local_tmp + '\n' \
793 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
794 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
795 ' SVNListParentPath On' + '\n' \
796 ' AuthType Basic' + '\n' \
797 ' AuthName "Subversion Repository"' + '\n' \
798 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
799 ' Require valid-user' + '\n' \
800 ' Satisfy Any' + '\n' \
801 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
802 '</Location>' + '\n' \
803 '<Location /authz-test-work/mixed-noauthwhenanon>' + '\n' \
805 ' SVNParentPath ' + local_tmp + '\n' \
806 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
807 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
808 ' SVNListParentPath On' + '\n' \
809 ' AuthType Basic' + '\n' \
810 ' AuthName "Subversion Repository"' + '\n' \
811 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
812 ' Require valid-user' + '\n' \
813 ' AuthzSVNNoAuthWhenAnonymousAllowed On' + '\n' \
814 ' SVNPathAuthz On' + '\n' \
815 '</Location>' + '\n' \
816 '<Location /authz-test-work/authn>' + '\n' \
818 ' SVNParentPath ' + local_tmp + '\n' \
819 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
820 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
821 ' SVNListParentPath On' + '\n' \
822 ' AuthType Basic' + '\n' \
823 ' AuthName "Subversion Repository"' + '\n' \
824 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
825 ' Require valid-user' + '\n' \
826 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
827 '</Location>' + '\n' \
828 '<Location /authz-test-work/authn-anonoff>' + '\n' \
830 ' SVNParentPath ' + local_tmp + '\n' \
831 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
832 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
833 ' SVNListParentPath On' + '\n' \
834 ' AuthType Basic' + '\n' \
835 ' AuthName "Subversion Repository"' + '\n' \
836 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
837 ' Require valid-user' + '\n' \
838 ' AuthzSVNAnonymous Off' + '\n' \
839 ' SVNPathAuthz On' + '\n' \
840 '</Location>' + '\n' \
841 '<Location /authz-test-work/authn-lcuser>' + '\n' \
843 ' SVNParentPath ' + local_tmp + '\n' \
844 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
845 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
846 ' SVNListParentPath On' + '\n' \
847 ' AuthType Basic' + '\n' \
848 ' AuthName "Subversion Repository"' + '\n' \
849 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
850 ' Require valid-user' + '\n' \
851 ' AuthzForceUsernameCase Lower' + '\n' \
852 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
853 '</Location>' + '\n' \
854 '<Location /authz-test-work/authn-lcuser>' + '\n' \
856 ' SVNParentPath ' + local_tmp + '\n' \
857 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
858 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
859 ' SVNListParentPath On' + '\n' \
860 ' AuthType Basic' + '\n' \
861 ' AuthName "Subversion Repository"' + '\n' \
862 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
863 ' Require valid-user' + '\n' \
864 ' AuthzForceUsernameCase Lower' + '\n' \
865 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
866 '</Location>' + '\n' \
867 '<Location /authz-test-work/authn-group>' + '\n' \
869 ' SVNParentPath ' + local_tmp + '\n' \
870 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
871 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
872 ' SVNListParentPath On' + '\n' \
873 ' AuthType Basic' + '\n' \
874 ' AuthName "Subversion Repository"' + '\n' \
875 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
876 ' AuthGroupFile ' + self._quote(self.httpd_groups) + '\n' \
877 ' Require group random' + '\n' \
878 ' AuthzSVNAuthoritative Off' + '\n' \
879 ' SVNPathAuthz On' + '\n' \
880 '</Location>' + '\n' \
881 '<IfModule mod_authz_core.c>' + '\n' \
882 '<Location /authz-test-work/sallrany>' + '\n' \
884 ' SVNParentPath ' + local_tmp + '\n' \
885 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
886 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
887 ' SVNListParentPath On' + '\n' \
888 ' AuthType Basic' + '\n' \
889 ' AuthName "Subversion Repository"' + '\n' \
890 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
891 ' AuthzSendForbiddenOnFailure On' + '\n' \
892 ' Satisfy All' + '\n' \
893 ' <RequireAny>' + '\n' \
894 ' Require valid-user' + '\n' \
895 ' Require expr req(\'ALLOW\') == \'1\'' + '\n' \
896 ' </RequireAny>' + '\n' \
897 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
898 '</Location>' + '\n' \
899 '<Location /authz-test-work/sallrall>'+ '\n' \
901 ' SVNParentPath ' + local_tmp + '\n' \
902 ' AuthzSVNAccessFile ' + self._quote(self.authz_file) + '\n' \
903 ' SVNAdvertiseV2Protocol ' + self.httpv2_option + '\n' \
904 ' SVNListParentPath On' + '\n' \
905 ' AuthType Basic' + '\n' \
906 ' AuthName "Subversion Repository"' + '\n' \
907 ' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
908 ' AuthzSendForbiddenOnFailure On' + '\n' \
909 ' Satisfy All' + '\n' \
910 ' <RequireAll>' + '\n' \
911 ' Require valid-user' + '\n' \
912 ' Require expr req(\'ALLOW\') == \'1\'' + '\n' \
913 ' </RequireAll>' + '\n' \
914 ' SVNPathAuthz ' + self.path_authz_option + '\n' \
915 '</Location>' + '\n' \
916 '</IfModule>' + '\n' \
920 self._start_service()
924 # Avoid output from starting and preparing between test results
934 def _start_service(self):
935 "Install and start HTTPD service"
936 print('Installing service %s' % self.service_name)
937 os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'install'])
938 print('Starting service %s' % self.service_name)
939 os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'start'])
941 def _stop_service(self):
942 "Stop and uninstall HTTPD service"
943 os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'stop'])
944 os.spawnv(os.P_WAIT, self.path, self.httpd_args + ['-k', 'uninstall'])
946 def _start_daemon(self):
947 "Start HTTPD as daemon"
948 print('Starting httpd as daemon')
949 print(self.httpd_args)
950 self.proc = subprocess.Popen([self.path] + self.httpd_args[1:])
952 def _stop_daemon(self):
953 "Stop the HTTPD daemon"
954 if self.proc is not None:
956 print('Stopping %s' % self.name)
958 if self.proc.returncode is None:
961 except AttributeError:
963 print('Httpd.stop_daemon not implemented')
966 "Run memcached for tests"
967 def __init__(self, abs_memcached_dir, memcached_server):
968 self.name = 'memcached.exe'
970 self.memcached_host, self.memcached_port = memcached_server.split(':')
971 self.memcached_dir = abs_memcached_dir
974 self.path = os.path.join(self.memcached_dir, self.name)
976 self.memcached_args = [
978 '-p', self.memcached_port,
979 '-l', self.memcached_host
983 "Stop memcached when the object is deleted"
987 "Start memcached as daemon"
988 print('Starting %s as daemon' % self.name)
989 print(self.memcached_args)
990 self.proc = subprocess.Popen([self.path] + self.memcached_args)
994 if self.proc is not None:
996 print('Stopping %s' % self.name)
998 if self.proc.returncode is None:
1001 except AttributeError:
1004 # Move the binaries to the test directory
1005 create_target_dir(abs_builddir)
1008 for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL):
1009 if isinstance(i, gen_base.TargetExe):
1010 src = os.path.join(abs_objdir, i.filename)
1012 if os.path.isfile(src):
1013 dst = os.path.join(abs_builddir, i.filename)
1014 create_target_dir(os.path.dirname(dst))
1015 copy_changed_file(src, dst)
1017 # Create the base directory for Python tests
1018 create_target_dir(CMDLINE_TEST_SCRIPT_NATIVE_PATH)
1020 # Ensure the tests directory is correctly cased
1021 abs_builddir = fix_case(abs_builddir)
1028 # No need to start any servers if we are only listing the tests.
1031 memcached = Memcached(memcached_dir, memcached_server)
1035 daemon = Svnserve(svnserve_args, objdir, abs_objdir, abs_builddir)
1038 daemon = Httpd(abs_httpd_dir, abs_objdir, abs_builddir, abs_srcdir,
1039 httpd_port, httpd_service, use_ssl, use_http2,
1040 use_mod_deflate, httpd_no_log, advertise_httpv2,
1041 http_short_circuit, http_bulk_updates)
1043 if use_ssl and not ssl_cert:
1044 ssl_cert = daemon.certfile
1046 # Start service daemon, if any
1050 # Find the full path and filename of any test that is specified just by
1052 if len(tests_to_run) != 0:
1054 for t in tests_to_run:
1057 t, tns = t.split('#')
1059 test = [x for x in all_tests if x.split('/')[-1] == t]
1060 if not test and not (t.endswith('-test.exe') or t.endswith('_tests.py')):
1061 # The lengths of '-test.exe' and of '_tests.py' are both 9.
1062 test = [x for x in all_tests if x.split('/')[-1][:-9] == t]
1065 print("Skipping test '%s', test not found." % t)
1067 tests.append('%s#%s' % (test[0], tns))
1071 tests_to_run = tests
1073 tests_to_run = all_tests
1077 print('Listing %s configuration on %s' % (objdir, repo_loc))
1079 print('Testing %s configuration on %s' % (objdir, repo_loc))
1080 sys.path.insert(0, os.path.join(abs_srcdir, 'build'))
1082 if not test_javahl and not test_swig:
1086 fail_log_file = None
1088 log_file = os.path.join(abs_builddir, log)
1089 fail_log_file = os.path.join(abs_builddir, faillog)
1092 httpd_version = gen_obj._libraries['httpd'].version
1094 httpd_version = None
1096 opts, args = run_tests.create_parser().parse_args([])
1098 opts.fs_type = fs_type
1099 opts.global_scheduler = global_scheduler
1100 opts.http_library = 'serf'
1101 opts.server_minor_version = server_minor_version
1102 opts.cleanup = cleanup
1103 opts.enable_sasl = enable_sasl
1104 opts.parallel = parallel
1105 opts.config_file = config_file
1106 opts.fsfs_sharding = fsfs_sharding
1107 opts.fsfs_packing = fsfs_packing
1108 opts.list_tests = list_tests
1109 opts.svn_bin = svn_bin
1110 opts.mode_filter = mode_filter
1111 opts.milestone_filter = milestone_filter
1112 opts.httpd_version = httpd_version
1113 opts.set_log_level = log_level
1114 opts.ssl_cert = ssl_cert
1115 opts.exclusive_wc_locks = exclusive_wc_locks
1116 opts.memcached_server = memcached_server
1117 opts.skip_c_tests = skip_c_tests
1118 opts.dump_load_cross_check = dump_load_cross_check
1119 opts.fsfs_compression = fsfs_compression
1120 opts.fsfs_dir_deltification = fsfs_dir_deltification
1121 th = run_tests.TestHarness(abs_srcdir, abs_builddir,
1122 log_file, fail_log_file, opts)
1123 old_cwd = os.getcwd()
1125 os.chdir(abs_builddir)
1126 failed = th.run(tests_to_run)
1137 for path in os.environ["PATH"].split(os.pathsep):
1138 if os.path.isfile(os.path.join(path, 'java.exe')):
1139 java_exe = os.path.join(path, 'java.exe')
1142 if not java_exe and 'java_sdk' in gen_obj._libraries:
1143 jdk = gen_obj._libraries['java_sdk']
1145 if os.path.isfile(os.path.join(jdk.lib_dir, '../bin/java.exe')):
1146 java_exe = os.path.join(jdk.lib_dir, '../bin/java.exe')
1149 print('Java not found. Skipping Java tests')
1151 args = (os.path.abspath(java_exe),)
1152 if (objdir == 'Debug'):
1153 args = args + ('-Xcheck:jni',)
1156 args = args + ('-Dtest.cleanup=1',)
1159 '-Dtest.rootdir=' + os.path.join(abs_builddir, 'javahl'),
1160 '-Dtest.srcdir=' + os.path.join(abs_srcdir,
1161 'subversion/bindings/javahl'),
1163 '-Dtest.fstype=' + fs_type ,
1166 '-Djava.library.path='
1167 + os.path.join(abs_objdir,
1168 'subversion/bindings/javahl/native'),
1170 os.path.join(abs_srcdir, 'subversion/bindings/javahl/classes') +';' +
1175 print('Running org.apache.subversion tests:')
1178 r = subprocess.call(args + tuple(['org.apache.subversion.javahl.RunTests']))
1182 print('[Test runner reported failure]')
1185 print('Running org.tigris.subversion tests:')
1187 r = subprocess.call(args + tuple(['org.tigris.subversion.javahl.RunTests']))
1191 print('[Test runner reported failure]')
1193 elif test_swig == 'perl':
1195 swig_dir = os.path.join(abs_builddir, 'swig')
1196 swig_pl_dir = os.path.join(swig_dir, 'p5lib')
1197 swig_pl_svn = os.path.join(swig_pl_dir, 'SVN')
1198 swig_pl_auto_svn = os.path.join(swig_pl_dir, 'auto', 'SVN')
1200 create_target_dir(swig_pl_svn)
1202 for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL):
1203 if isinstance(i, gen_base.TargetSWIG) and i.lang == 'perl':
1204 mod_dir = os.path.join(swig_pl_auto_svn, '_' + i.name[5:].capitalize())
1205 create_target_dir(mod_dir)
1206 copy_changed_file(os.path.join(abs_objdir, i.filename), to_dir=mod_dir)
1208 elif isinstance(i, gen_base.TargetSWIGLib) and i.lang == 'perl':
1209 copy_changed_file(os.path.join(abs_objdir, i.filename),
1210 to_dir=abs_builddir)
1212 pm_src = os.path.join(abs_srcdir, 'subversion', 'bindings', 'swig', 'perl',
1217 for root, dirs, files in os.walk(pm_src):
1219 if name.endswith('.pm'):
1220 fn = os.path.join(root, name)
1221 copy_changed_file(fn, to_dir=swig_pl_svn)
1222 elif name.endswith('.t'):
1223 tests.append(os.path.relpath(os.path.join(root, name), pm_src))
1225 perl5lib = swig_pl_dir
1226 if 'PERL5LIB' in os.environ:
1227 perl5lib += os.pathsep + os.environ['PERL5LIB']
1229 perl_exe = 'perl.exe'
1231 print('-- Running Swig Perl tests --')
1233 old_cwd = os.getcwd()
1237 os.environ['PERL5LIB'] = perl5lib
1238 os.environ["SVN_DBG_NO_ABORT_ON_ERROR_LEAK"] = 'YES'
1240 r = subprocess.call([
1242 '-MExtUtils::Command::MM',
1243 '-e', 'test_harness()'
1249 print('[Test runner reported failure]')
1251 elif test_swig == 'python':
1253 swig_dir = os.path.join(abs_builddir, 'swig')
1254 swig_py_dir = os.path.join(swig_dir, 'pylib')
1255 swig_py_libsvn = os.path.join(swig_py_dir, 'libsvn')
1256 swig_py_svn = os.path.join(swig_py_dir, 'svn')
1258 create_target_dir(swig_py_libsvn)
1259 create_target_dir(swig_py_svn)
1261 for i in gen_obj.graph.get_all_sources(gen_base.DT_INSTALL):
1262 if (isinstance(i, gen_base.TargetSWIG)
1263 or isinstance(i, gen_base.TargetSWIGLib)) and i.lang == 'python':
1265 src = os.path.join(abs_objdir, i.filename)
1266 copy_changed_file(src, to_dir=swig_py_libsvn)
1268 py_src = os.path.join(abs_srcdir, 'subversion', 'bindings', 'swig', 'python')
1270 for py_file in os.listdir(py_src):
1271 if py_file.endswith('.py'):
1272 copy_changed_file(os.path.join(py_src, py_file),
1273 to_dir=swig_py_libsvn)
1275 py_src_svn = os.path.join(py_src, 'svn')
1276 for py_file in os.listdir(py_src_svn):
1277 if py_file.endswith('.py'):
1278 copy_changed_file(os.path.join(py_src_svn, py_file),
1281 print('-- Running Swig Python tests --')
1284 pythonpath = swig_py_dir
1285 if 'PYTHONPATH' in os.environ:
1286 pythonpath += os.pathsep + os.environ['PYTHONPATH']
1288 python_exe = 'python.exe'
1289 old_cwd = os.getcwd()
1291 os.environ['PYTHONPATH'] = pythonpath
1293 r = subprocess.call([
1295 os.path.join(py_src, 'tests', 'run_all.py')
1301 print('[Test runner reported failure]')
1304 elif test_swig == 'ruby':
1307 if 'ruby' not in gen_obj._libraries:
1308 print('Ruby not found. Skipping Ruby tests')
1310 ruby_lib = gen_obj._libraries['ruby']
1312 ruby_exe = 'ruby.exe'
1313 ruby_subdir = os.path.join('subversion', 'bindings', 'swig', 'ruby')
1315 '-I', os.path.join(abs_srcdir, ruby_subdir),
1316 os.path.join(abs_srcdir, ruby_subdir, 'test', 'run-test.rb'),
1320 print('-- Running Swig Ruby tests --')
1322 old_cwd = os.getcwd()
1324 os.chdir(ruby_subdir)
1326 os.environ["BUILD_TYPE"] = objdir
1327 os.environ["SVN_DBG_NO_ABORT_ON_ERROR_LEAK"] = 'YES'
1328 r = subprocess.call([ruby_exe] + ruby_args)
1335 print('[Test runner reported failure]')
1339 print('Unknown Swig binding type: ' + str(test_swig))
1342 # Stop service daemon, if any
1349 # Remove the execs again
1350 for tgt in copied_execs:
1352 if os.path.isfile(tgt):
1354 print("kill: %s" % tgt)
1357 traceback.print_exc(file=sys.stdout)