2 """Test zstd interoperability between versions"""
4 # ################################################################
5 # Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
8 # This source code is licensed under both the BSD-style license (found in the
9 # LICENSE file in the root directory of this source tree) and the GPLv2 (found
10 # in the COPYING file in the root directory of this source tree).
11 # ################################################################
20 from subprocess import Popen, PIPE
22 repo_url = 'https://github.com/facebook/zstd.git'
23 tmp_dir_name = 'tests/versionsTest'
26 test_dat_src = 'README.md'
29 dict_source = 'dict_source'
30 dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
31 dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
34 def execute(command, print_output=False, print_error=True, param_shell=False):
35 popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell)
36 stdout_lines, stderr_lines = popen.communicate()
37 stderr_lines = stderr_lines.decode("utf-8")
38 stdout_lines = stdout_lines.decode("utf-8")
42 if popen.returncode is not None and popen.returncode != 0:
43 if not print_output and print_error:
45 return popen.returncode
48 def proc(cmd_args, pipe=True, dummy=False):
52 subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE)
54 subproc = Popen(cmd_args)
55 return subproc.communicate()
58 def make(args, pipe=True):
59 return proc([make_cmd] + args, pipe)
62 def git(args, pipe=True):
63 return proc([git_cmd] + args, pipe)
67 stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]'])
68 tags = stdout.decode('utf-8').split()
72 def create_dict(tag, dict_source_path):
73 dict_name = 'dict.' + tag
74 if not os.path.isfile(dict_name):
75 cFiles = glob.glob(dict_source_path + "/*.c")
76 hFiles = glob.glob(dict_source_path + "/*.h")
78 result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
80 result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
82 print(dict_name + ' created')
84 print('ERROR: creating of ' + dict_name + ' failed')
86 print(dict_name + ' already exists')
89 def dict_compress_sample(tag, sample):
90 dict_name = 'dict.' + tag
91 DEVNULL = open(os.devnull, 'wb')
92 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0:
93 os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
94 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0:
95 os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
96 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0:
97 os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
98 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
99 os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
100 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
101 os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
102 # zstdFiles = glob.glob("*.zst*")
104 print(tag + " : dict compression completed")
107 def compress_sample(tag, sample):
108 DEVNULL = open(os.devnull, 'wb')
109 if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0:
110 os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
111 if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0:
112 os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
113 if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0:
114 os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
115 if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
116 os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
117 if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
118 os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
119 # zstdFiles = glob.glob("*.zst*")
121 print(tag + " : compression completed")
124 # http://stackoverflow.com/a/19711609/2132223
125 def sha1_of_file(filepath):
126 with open(filepath, 'rb') as f:
127 return hashlib.sha1(f.read()).hexdigest()
130 def remove_duplicates():
131 list_of_zst = sorted(glob.glob('*.zst'))
132 for i, ref_zst in enumerate(list_of_zst):
133 if not os.path.isfile(ref_zst):
135 for j in range(i + 1, len(list_of_zst)):
136 compared_zst = list_of_zst[j]
137 if not os.path.isfile(compared_zst):
139 if filecmp.cmp(ref_zst, compared_zst):
140 os.remove(compared_zst)
141 print('duplicated : {} == {}'.format(ref_zst, compared_zst))
144 def decompress_zst(tag):
146 list_zst = sorted(glob.glob('*_nodict.zst'))
147 for file_zst in list_zst:
148 print(file_zst, end=' ')
150 file_dec = file_zst + '_d64_' + tag + '.dec'
152 params = ['./zstd.' + tag, '-df', file_zst, file_dec]
154 params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
155 if execute(params) == 0:
156 if not filecmp.cmp(file_dec, test_dat):
162 print('command does not work')
167 def decompress_dict(tag):
169 list_zst = sorted(glob.glob('*_dictio.zst'))
170 for file_zst in list_zst:
171 dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst"
172 if head in dict_tag: # find vdevel
175 dict_tag = dict_tag[dict_tag.rfind('v'):]
176 if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
178 dict_name = 'dict.' + dict_tag
179 print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
180 file_dec = file_zst + '_d64_' + tag + '.dec'
182 params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
184 params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
185 if execute(params) == 0:
186 if not filecmp.cmp(file_dec, test_dat):
192 print('command does not work')
197 if __name__ == '__main__':
199 base_dir = os.getcwd() + '/..' # /path/to/zstd
200 tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest
201 clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd
202 dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source
203 programs_dir = base_dir + '/programs' # /path/to/zstd/programs
204 os.makedirs(tmp_dir, exist_ok=True)
206 # since Travis clones limited depth, we should clone full repository
207 if not os.path.isdir(clone_dir):
208 git(['clone', repo_url, clone_dir])
210 shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
212 # Retrieve all release tags
213 print('Retrieve all release tags :')
215 alltags = get_git_tags() + [head]
216 tags = [t for t in alltags if t >= 'v0.5.0']
219 # Build all release zstd
222 dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd.<TAG>
223 if not os.path.isfile(dst_zstd) or tag == head:
225 r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/<TAG>
226 os.makedirs(r_dir, exist_ok=True)
228 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
230 os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
231 make(['clean', 'dictBuilder'], False)
232 shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
233 os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs
234 make(['clean', 'zstd'], False)
236 os.chdir(programs_dir)
237 make(['zstd'], False)
238 shutil.copy2('zstd', dst_zstd)
240 # remove any remaining *.zst and *.dec from previous test
242 for compressed in glob.glob("*.zst"):
243 os.remove(compressed)
244 for dec in glob.glob("*.dec"):
247 # copy *.c and *.h to a temporary directory ("dict_source")
248 if not os.path.isdir(dict_source_path):
249 os.mkdir(dict_source_path)
250 print('cp ' + dict_files + ' ' + dict_source_path)
251 execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
253 print('Compress test.dat by all released zstd')
259 create_dict(tag, dict_source_path)
260 dict_compress_sample(tag, test_dat)
262 error_code += decompress_dict(tag)
263 compress_sample(tag, test_dat)
265 error_code += decompress_zst(tag)
268 print('Enumerate different compressed files')
269 zstds = sorted(glob.glob('*.zst'))
271 print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
274 print('====== ERROR !!! =======')