25 println "roundTripTest: ./datagen $1 $proba | $ZSTD -v$cLevel | $ZSTD -d$dLevel"
26 ./datagen $1 $proba | $MD5SUM > tmp1
27 ./datagen $1 $proba | $ZSTD --ultra -v$cLevel | $ZSTD -d$dLevel | $MD5SUM > tmp2
45 rm -f tmp.zstd tmp.md5.1 tmp.md5.2
46 println "fileRoundTripTest: ./datagen $1 $local_p > tmp && $ZSTD -v$local_c -c tmp | $ZSTD -d$local_d"
47 ./datagen $1 $local_p > tmp
48 < tmp $MD5SUM > tmp.md5.1
49 $ZSTD --ultra -v$local_c -c tmp | $ZSTD -d$local_d | $MD5SUM > tmp.md5.2
50 $DIFF -q tmp.md5.1 tmp.md5.2
54 dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1"
62 SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
63 PRGDIR="$SCRIPT_DIR/../programs"
64 TESTDIR="$SCRIPT_DIR/../tests"
68 if [ -t 0 ] && [ -t 1 ]
76 GNU) DEVDEVICE="/dev/random" ;;
77 *) DEVDEVICE="/dev/zero" ;;
88 Darwin) MD5SUM="md5 -r" ;;
89 FreeBSD) MD5SUM="gmd5sum" ;;
90 OpenBSD) MD5SUM="md5" ;;
96 SunOS) DIFF="gdiff" ;;
99 println "\nStarting playTests.sh isWindows=$isWindows ZSTD='$ZSTD'"
101 [ -n "$ZSTD" ] || die "ZSTD variable must be defined!"
103 if echo hello | $ZSTD -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
112 println "\n===> simple tests "
115 println "test : basic compression "
116 $ZSTD -f tmp # trivial compression case, creates tmp.zst
117 println "test : basic decompression"
118 $ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
119 println "test : too large compression level => auto-fix"
120 $ZSTD -99 -f tmp # too large compression level, automatic sized down
121 $ZSTD -5000000000 -f tmp && die "too large numeric value : must fail"
122 println "test : --fast aka negative compression levels"
123 $ZSTD --fast -f tmp # == -1
124 $ZSTD --fast=3 -f tmp # == -3
125 $ZSTD --fast=200000 -f tmp # too low compression level, automatic fixed
126 $ZSTD --fast=5000000000 -f tmp && die "too large numeric value : must fail"
127 $ZSTD -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
128 println "test : too large numeric argument"
129 $ZSTD --fast=9999999999 -f tmp && die "should have refused numeric value"
130 println "test : set compression level with environment variable ZSTD_CLEVEL"
131 ZSTD_CLEVEL=12 $ZSTD -f tmp # positive compression level
132 ZSTD_CLEVEL=-12 $ZSTD -f tmp # negative compression level
133 ZSTD_CLEVEL=+12 $ZSTD -f tmp # valid: verbose '+' sign
134 ZSTD_CLEVEL='' $ZSTD -f tmp # empty env var, warn and revert to default setting
135 ZSTD_CLEVEL=- $ZSTD -f tmp # malformed env var, warn and revert to default setting
136 ZSTD_CLEVEL=a $ZSTD -f tmp # malformed env var, warn and revert to default setting
137 ZSTD_CLEVEL=+a $ZSTD -f tmp # malformed env var, warn and revert to default setting
138 ZSTD_CLEVEL=3a7 $ZSTD -f tmp # malformed env var, warn and revert to default setting
139 ZSTD_CLEVEL=50000000000 $ZSTD -f tmp # numeric value too large, warn and revert to default setting
140 println "test : override ZSTD_CLEVEL with command line option"
141 ZSTD_CLEVEL=12 $ZSTD --fast=3 -f tmp # overridden by command line option
142 println "test : compress to stdout"
143 $ZSTD tmp -c > tmpCompressed
144 $ZSTD tmp --stdout > tmpCompressed # long command format
145 println "test : compress to named file"
147 $ZSTD tmp -o tmpCompressed
148 test -f tmpCompressed # file must be created
149 println "test : -o must be followed by filename (must fail)"
150 $ZSTD tmp -of tmpCompressed && die "-o must be followed by filename "
151 println "test : force write, correct order"
152 $ZSTD tmp -fo tmpCompressed
153 println "test : forgotten argument"
155 $ZSTD tmp2 -fo && die "-o must be followed by filename "
156 println "test : implied stdout when input is stdin"
157 println bob | $ZSTD | $ZSTD -d
158 if [ "$isTerminal" = true ]; then
159 println "test : compressed data to terminal"
160 println bob | $ZSTD && die "should have refused : compressed data to terminal"
161 println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)"
162 $ZSTD -d > $INTOVOID && die "should have refused : compressed data from terminal"
164 println "test : null-length file roundtrip"
165 println -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
166 println "test : ensure small file doesn't add 3-bytes null block"
168 $ZSTD tmp1 -c | wc -c | grep "14"
169 $ZSTD < tmp1 | wc -c | grep "14"
170 println "test : decompress file with wrong suffix (must fail)"
171 $ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
172 $ZSTD -df tmp && die "should have refused : wrong extension"
173 println "test : decompress into stdout"
174 $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
175 $ZSTD --decompress tmpCompressed -c > tmpResult
176 $ZSTD --decompress tmpCompressed --stdout > tmpResult
177 println "test : decompress from stdin into stdout"
178 $ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout
179 $ZSTD -dc - < tmp.zst > $INTOVOID
180 $ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used
181 $ZSTD -d - < tmp.zst > $INTOVOID
182 println "test : impose memory limitation (must fail)"
183 $ZSTD -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
184 $ZSTD -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
185 $ZSTD -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
186 $ZSTD -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command
187 println "test : overwrite protection"
188 $ZSTD -q tmp && die "overwrite check failed!"
189 println "test : force overwrite"
192 println "test : overwrite readonly file"
193 rm -f tmpro tmpro.zst
194 println foo > tmpro.zst
197 $ZSTD -q tmpro && die "should have refused to overwrite read-only file"
199 println "test: --no-progress flag"
200 $ZSTD tmpro -c --no-progress | $ZSTD -d -f -o "$INTOVOID" --no-progress
201 $ZSTD tmpro -cv --no-progress | $ZSTD -dv -f -o "$INTOVOID" --no-progress
202 rm -f tmpro tmpro.zst
203 println "test: overwrite input file (must fail)"
204 $ZSTD tmp -fo tmp && die "zstd compression overwrote the input file"
205 $ZSTD tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
206 println "test: detect that input file does not exist"
207 $ZSTD nothere && die "zstd hasn't detected that input file does not exist"
208 println "test: --[no-]compress-literals"
209 $ZSTD tmp -c --no-compress-literals -1 | $ZSTD -t
210 $ZSTD tmp -c --no-compress-literals --fast=1 | $ZSTD -t
211 $ZSTD tmp -c --no-compress-literals -19 | $ZSTD -t
212 $ZSTD tmp -c --compress-literals -1 | $ZSTD -t
213 $ZSTD tmp -c --compress-literals --fast=1 | $ZSTD -t
214 $ZSTD tmp -c --compress-literals -19 | $ZSTD -t
215 $ZSTD -b --fast=1 -i1e1 tmp --compress-literals
216 $ZSTD -b --fast=1 -i1e1 tmp --no-compress-literals
218 println "test : file removal"
220 test ! -f tmp # tmp should no longer be present
221 $ZSTD -f -d --rm tmp.zst
222 test ! -f tmp.zst # tmp.zst should no longer be present
223 println "test : should quietly not remove non-regular file"
225 $ZSTD tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
226 grep -v "Refusing to remove non-regular file" tmplog
228 $ZSTD tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file"
229 println "test : --rm on stdin"
230 println a | $ZSTD --rm > $INTOVOID # --rm should remain silent
232 $ZSTD -f tmp && die "tmp not present : should have failed"
233 test ! -f tmp.zst # tmp.zst should not be created
234 println "test : -d -f do not delete destination when source is not present"
235 touch tmp # create destination file
236 $ZSTD -d -f tmp.zst && die "attempt to decompress a non existing file"
237 test -f tmp # destination file should still be present
238 println "test : -f do not delete destination when source is not present"
239 rm tmp # erase source file
240 touch tmp.zst # create destination file
241 $ZSTD -f tmp && die "attempt to compress a non existing file"
242 test -f tmp.zst # destination file should still be present
246 println "test : compress multiple files"
249 $ZSTD tmp1 tmp2 -o "$INTOVOID" -f
250 $ZSTD tmp1 tmp2 -c | $ZSTD -t
251 $ZSTD tmp1 tmp2 -o tmp.zst
255 $ZSTD -t tmp1.zst tmp2.zst
256 $ZSTD -dc tmp1.zst tmp2.zst
257 $ZSTD tmp1.zst tmp2.zst -o "$INTOVOID" -f
258 $ZSTD -d tmp1.zst tmp2.zst -o tmp
260 $ZSTD tmp1 tmp2 -f -o tmpexists
261 $ZSTD tmp1 tmp2 -o tmpexists && die "should have refused to overwrite"
263 if [ "$?" -eq 139 ]; then
264 die "should not have segfaulted"
269 println "\n===> Advanced compression parameters "
270 println "Hello world!" | $ZSTD --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!"
271 println "Hello world!" | $ZSTD --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!"
272 println "Hello world!" | $ZSTD --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!"
273 println "Hello world!" | $ZSTD --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist
274 test ! -f tmp.zst # tmp.zst should not be created
276 roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
277 roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
278 roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
279 roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
280 roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
281 roundTripTest -g64K "19 --zstd=strat=9" # btultra2
284 println "\n===> Pass-Through mode "
285 println "Hello world 1!" | $ZSTD -df
286 println "Hello world 2!" | $ZSTD -dcf
287 println "Hello world 3!" > tmp1
291 println "\n===> frame concatenation "
293 println "hello " > hello.tmp
294 println "world!" > world.tmp
295 cat hello.tmp world.tmp > helloworld.tmp
296 $ZSTD -c hello.tmp > hello.zstd
297 $ZSTD -c world.tmp > world.zstd
298 cat hello.zstd world.zstd > helloworld.zstd
299 $ZSTD -dc helloworld.zstd > result.tmp
301 $DIFF helloworld.tmp result.tmp
302 println "frame concatenation without checksum"
303 $ZSTD -c hello.tmp > hello.zstd --no-check
304 $ZSTD -c world.tmp > world.zstd --no-check
305 cat hello.zstd world.zstd > helloworld.zstd
306 $ZSTD -dc helloworld.zstd > result.tmp
307 $DIFF helloworld.tmp result.tmp
308 println "testing zstdcat symlink"
310 ./zstdcat helloworld.zstd > result.tmp
311 $DIFF helloworld.tmp result.tmp
312 ln -s helloworld.zstd helloworld.link.zstd
313 ./zstdcat helloworld.link.zstd > result.tmp
314 $DIFF helloworld.tmp result.tmp
317 println "testing zcat symlink"
319 ./zcat helloworld.zstd > result.tmp
320 $DIFF helloworld.tmp result.tmp
321 ./zcat helloworld.link.zstd > result.tmp
322 $DIFF helloworld.tmp result.tmp
325 println "frame concatenation tests completed"
328 if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] ; then
329 println "\n**** flush write error test **** "
331 println "println foo | $ZSTD > /dev/full"
332 println foo | $ZSTD > /dev/full && die "write error not detected!"
333 println "println foo | $ZSTD | $ZSTD -d > /dev/full"
334 println foo | $ZSTD | $ZSTD -d > /dev/full && die "write error not detected!"
339 if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then
341 println "\n===> symbolic link test "
343 rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
344 println "hello world" > hello.tmp
345 ln -s hello.tmp world.tmp
346 ln -s hello.tmp world2.tmp
347 $ZSTD world.tmp hello.tmp || true
348 test -f hello.tmp.zst # regular file should have been compressed!
349 test ! -f world.tmp.zst # symbolic link should not have been compressed!
350 $ZSTD world.tmp || true
351 test ! -f world.tmp.zst # symbolic link should not have been compressed!
352 $ZSTD world.tmp world2.tmp || true
353 test ! -f world.tmp.zst # symbolic link should not have been compressed!
354 test ! -f world2.tmp.zst # symbolic link should not have been compressed!
355 $ZSTD world.tmp hello.tmp -f
356 test -f world.tmp.zst # symbolic link should have been compressed with --force
357 rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
362 println "\n===> test sparse file support "
364 ./datagen -g5M -P100 > tmpSparse
365 $ZSTD tmpSparse -c | $ZSTD -dv -o tmpSparseRegen
366 $DIFF -s tmpSparse tmpSparseRegen
367 $ZSTD tmpSparse -c | $ZSTD -dv --sparse -c > tmpOutSparse
368 $DIFF -s tmpSparse tmpOutSparse
369 $ZSTD tmpSparse -c | $ZSTD -dv --no-sparse -c > tmpOutNoSparse
370 $DIFF -s tmpSparse tmpOutNoSparse
371 ls -ls tmpSparse* # look at file size and block size on disk
372 ./datagen -s1 -g1200007 -P100 | $ZSTD | $ZSTD -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks)
373 ./datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
374 ls -ls tmpSparseOdd # look at file size and block size on disk
375 println "\n Sparse Compatibility with Console :"
376 println "Hello World 1 !" | $ZSTD | $ZSTD -d -c
377 println "Hello World 2 !" | $ZSTD | $ZSTD -d | cat
378 println "\n Sparse Compatibility with Append :"
379 ./datagen -P100 -g1M > tmpSparse1M
380 cat tmpSparse1M tmpSparse1M > tmpSparse2M
381 $ZSTD -v -f tmpSparse1M -o tmpSparseCompressed
382 $ZSTD -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
383 $ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
384 ls -ls tmpSparse* # look at file size and block size on disk
385 $DIFF tmpSparse2M tmpSparseRegenerated
389 println "\n===> multiple files tests "
391 ./datagen -s1 > tmp1 2> $INTOVOID
392 ./datagen -s2 -g100K > tmp2 2> $INTOVOID
393 ./datagen -s3 -g1M > tmp3 2> $INTOVOID
394 println "compress tmp* : "
398 println "decompress tmp* : "
401 println "compress tmp* into stdout > tmpall : "
402 $ZSTD -c tmp1 tmp2 tmp3 > tmpall
403 ls -ls tmp* # check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
404 println "decompress tmpall* into stdout > tmpdec : "
406 $ZSTD -dc tmpall* > tmpdec
407 ls -ls tmp* # check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
408 println "compress multiple files including a missing one (notHere) : "
409 $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
412 println "\n===> dictionary tests "
414 println "- test with raw dict (content only) "
416 ./datagen -g1M | $MD5SUM > tmp1
417 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
419 println "- Create first dictionary "
420 TESTFILE="$PRGDIR"/zstdcli.c
421 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
423 println "- Test dictionary compression with tmpDict as an input file and dictionary"
424 $ZSTD -f tmpDict -D tmpDict && die "compression error not detected!"
425 println "- Dictionary compression roundtrip"
426 $ZSTD -f tmp -D tmpDict
427 $ZSTD -d tmp.zst -D tmpDict -fo result
428 $DIFF "$TESTFILE" result
429 println "- Dictionary compression with btlazy2 strategy"
430 $ZSTD -f tmp -D tmpDict --zstd=strategy=6
431 $ZSTD -d tmp.zst -D tmpDict -fo result
432 $DIFF "$TESTFILE" result
435 println "- Test dictionary compression with multithreading "
436 ./datagen -g5M | $ZSTD -T2 -D tmpDict | $ZSTD -t -D tmpDict # fails with v1.3.2
438 println "- Create second (different) dictionary "
439 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
440 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
441 println "- Create dictionary with short dictID"
442 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
443 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
444 println "- Create dictionary with wrong dictID parameter order (must fail)"
445 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
446 println "- Create dictionary with size limit"
447 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v
448 println "- Create dictionary with small size limit"
449 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v
450 println "- Create dictionary with wrong parameter order (must fail)"
451 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
452 println "- Compress without dictID"
453 $ZSTD -f tmp -D tmpDict1 --no-dictID
454 $ZSTD -d tmp.zst -D tmpDict -fo result
455 $DIFF "$TESTFILE" result
456 println "- Compress with wrong argument order (must fail)"
457 $ZSTD tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name "
458 println "- Compress multiple files with dictionary"
461 cp "$TESTDIR"/*.c dirTestDict
462 cp "$PRGDIR"/*.c dirTestDict
463 cp "$PRGDIR"/*.h dirTestDict
464 $MD5SUM dirTestDict/* > tmph1
465 $ZSTD -f --rm dirTestDict/* -D tmpDictC
466 $ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default
468 Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5
469 *) $MD5SUM -c tmph1 ;;
472 println "- dictionary builder on bogus input"
473 println "Hello World" > tmp
474 $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : not enough input source"
475 ./datagen -P0 -g10M > tmp
476 $ZSTD --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise"
477 println "- Test -o before --train"
478 rm -f tmpDict dictionary
479 $ZSTD -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c
481 $ZSTD --train "$TESTDIR"/*.c "$PRGDIR"/*.c
486 println "\n===> fastCover dictionary builder : advanced options "
487 TESTFILE="$PRGDIR"/zstdcli.c
489 println "- Create first dictionary"
490 $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
492 $ZSTD -f tmp -D tmpDict
493 $ZSTD -d tmp.zst -D tmpDict -fo result
494 $DIFF "$TESTFILE" result
495 println "- Create second (different) dictionary"
496 $ZSTD --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
497 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
498 $ZSTD --train-fastcover=k=56,d=8 && die "Create dictionary without input file"
499 println "- Create dictionary with short dictID"
500 $ZSTD --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
501 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
502 println "- Create dictionaries with shrink-dict flag enabled"
503 $ZSTD --train-fastcover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
504 $ZSTD --train-fastcover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
505 $ZSTD --train-fastcover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
506 println "- Create dictionary with size limit"
507 $ZSTD --train-fastcover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
508 println "- Compare size of dictionary from 90% training samples with 80% training samples"
509 $ZSTD --train-fastcover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
510 $ZSTD --train-fastcover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
511 println "- Create dictionary using all samples for both training and testing"
512 $ZSTD --train-fastcover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
513 println "- Create dictionary using f=16"
514 $ZSTD --train-fastcover=f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
515 $ZSTD --train-fastcover=accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
516 println "- Create dictionary using accel=2"
517 $ZSTD --train-fastcover=accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
518 println "- Create dictionary using accel=10"
519 $ZSTD --train-fastcover=accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
520 println "- Create dictionary with multithreading"
521 $ZSTD --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
522 println "- Test -o before --train-fastcover"
523 rm -f tmpDict dictionary
524 $ZSTD -o tmpDict --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
526 $ZSTD --train-fastcover "$TESTDIR"/*.c "$PRGDIR"/*.c
531 println "\n===> legacy dictionary builder "
533 TESTFILE="$PRGDIR"/zstdcli.c
535 println "- Create first dictionary"
536 $ZSTD --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
538 $ZSTD -f tmp -D tmpDict
539 $ZSTD -d tmp.zst -D tmpDict -fo result
540 $DIFF "$TESTFILE" result
541 $ZSTD --train-legacy=s=8 && die "Create dictionary without input files (should error)"
542 println "- Create second (different) dictionary"
543 $ZSTD --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
544 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
545 println "- Create dictionary with short dictID"
546 $ZSTD --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
547 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
548 println "- Create dictionary with size limit"
549 $ZSTD --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
550 println "- Test -o before --train-legacy"
551 rm -f tmpDict dictionary
552 $ZSTD -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c
554 $ZSTD --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c
559 println "\n===> integrity tests "
561 println "test one file (tmp1.zst) "
565 $ZSTD --test tmp1.zst
566 println "test multiple files (*.zst) "
568 println "test bad files (*) "
569 $ZSTD -t ./* && die "bad files not detected !"
570 $ZSTD -t tmp1 && die "bad file not detected !"
572 $ZSTD -t tmp2.zst && die "bad file not detected !"
574 $ZSTD -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad
575 println "test --rm and --test combined "
576 $ZSTD -t --rm tmp1.zst
577 test -f tmp1.zst # check file is still present
578 split -b16384 tmp1.zst tmpSplit.
579 $ZSTD -t tmpSplit.* && die "bad file not detected !"
580 ./datagen | $ZSTD -c | $ZSTD -t
584 println "\n===> golden files tests "
586 $ZSTD -t -r "$TESTDIR/files"
587 $ZSTD -c -r "$TESTDIR/files" | $ZSTD -t
590 println "\n===> benchmark mode tests "
592 println "bench one file"
595 println "bench multiple levels"
597 println "bench negative level"
598 $ZSTD -bi0 --fast tmp1
599 println "with recursive and quiet modes"
601 println "benchmark decompression only"
603 $ZSTD -b -d -i1 tmp1.zst
605 println "\n===> zstd compatibility tests "
609 $ZSTD --format=zstd -f tmp
612 println "\n===> gzip compatibility tests "
615 $ZSTD --format=gzip -V || GZIPMODE=0
616 if [ $GZIPMODE -eq 1 ]; then
617 println "gzip support detected"
620 if [ $GZIPEXE -eq 1 ]; then
622 $ZSTD --format=gzip -f tmp
625 $ZSTD -d -f -v tmp.gz
628 println "gzip binary not detected"
631 println "gzip mode not supported"
635 println "\n===> gzip frame tests "
637 if [ $GZIPMODE -eq 1 ]; then
639 $ZSTD -f --format=gzip tmp
641 cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
642 truncateLastByte tmp.gz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
645 println "gzip mode not supported"
648 if [ $GZIPMODE -eq 1 ]; then
651 $ZSTD --format=gzip --format=zstd -f tmp
655 println "\n===> xz compatibility tests "
658 $ZSTD --format=xz -V || LZMAMODE=0
659 if [ $LZMAMODE -eq 1 ]; then
660 println "xz support detected"
662 xz -Q -V && lzma -Q -V || XZEXE=0
663 if [ $XZEXE -eq 1 ]; then
664 println "Testing zstd xz and lzma support"
666 $ZSTD --format=lzma -f tmp
667 $ZSTD --format=xz -f tmp
671 lzma -Q -f -k --lzma1 tmp
672 $ZSTD -d -f -v tmp.xz
673 $ZSTD -d -f -v tmp.lzma
675 println "Creating symlinks"
680 println "Testing xz and lzma symlinks"
686 println "Testing unxz and unlzma symlinks"
691 rm xz unxz lzma unlzma
694 println "xz binary not detected"
697 println "xz mode not supported"
701 println "\n===> xz frame tests "
703 if [ $LZMAMODE -eq 1 ]; then
705 $ZSTD -f --format=xz tmp
706 $ZSTD -f --format=lzma tmp
708 cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
709 truncateLastByte tmp.xz | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
710 truncateLastByte tmp.lzma | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
713 println "xz mode not supported"
716 println "\n===> lz4 compatibility tests "
719 $ZSTD --format=lz4 -V || LZ4MODE=0
720 if [ $LZ4MODE -eq 1 ]; then
721 println "lz4 support detected"
724 if [ $LZ4EXE -eq 1 ]; then
726 $ZSTD --format=lz4 -f tmp
729 $ZSTD -d -f -v tmp.lz4
732 println "lz4 binary not detected"
735 println "lz4 mode not supported"
739 println "\n===> lz4 frame tests "
741 if [ $LZ4MODE -eq 1 ]; then
743 $ZSTD -f --format=lz4 tmp
745 cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | $ZSTD -d -f -o tmp
746 truncateLastByte tmp.lz4 | $ZSTD -t > $INTOVOID && die "incomplete frame not detected !"
749 println "lz4 mode not supported"
752 println "\n===> suffix list test"
754 ! $ZSTD -d tmp.abc 2> tmplg
756 if [ $GZIPMODE -ne 1 ]; then
757 grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed"
760 if [ $LZMAMODE -ne 1 ]; then
761 grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed"
762 grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed"
765 if [ $LZ4MODE -ne 1 ]; then
766 grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed"
769 println "\n===> zstd round-trip tests "
772 roundTripTest -g15K # TableID==3
773 roundTripTest -g127K # TableID==2
774 roundTripTest -g255K # TableID==1
775 roundTripTest -g522K # TableID==0
776 roundTripTest -g519K 6 # greedy, hash chain
777 roundTripTest -g517K 16 # btlazy2
778 roundTripTest -g516K 19 # btopt
780 fileRoundTripTest -g500K
782 println "\n===> zstd long distance matching round-trip tests "
783 roundTripTest -g0 "2 --single-thread --long"
784 roundTripTest -g1000K "1 --single-thread --long"
785 roundTripTest -g517K "6 --single-thread --long"
786 roundTripTest -g516K "16 --single-thread --long"
787 roundTripTest -g518K "19 --single-thread --long"
788 fileRoundTripTest -g5M "3 --single-thread --long"
791 roundTripTest -g96K "5 --single-thread"
794 println "\n===> zstdmt round-trip tests "
795 roundTripTest -g4M "1 -T0"
796 roundTripTest -g8M "3 -T2"
797 roundTripTest -g8000K "2 --threads=2"
798 fileRoundTripTest -g4M "19 -T2 -B1M"
800 println "\n===> zstdmt long distance matching round-trip tests "
801 roundTripTest -g8M "3 --long=24 -T2"
803 println "\n===> ovLog tests "
804 ./datagen -g2MB > tmp
805 refSize=$($ZSTD tmp -6 -c --zstd=wlog=18 | wc -c)
806 ov9Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c)
807 ov1Size=$($ZSTD tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c)
808 if [ "$refSize" -eq "$ov9Size" ]; then
809 echo ov9Size should be different from refSize
812 if [ "$refSize" -eq "$ov1Size" ]; then
813 echo ov1Size should be different from refSize
816 if [ "$ov9Size" -ge "$ov1Size" ]; then
817 echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size"
822 println "\n===> no multithreading, skipping zstdmt tests "
827 println "\n===> zstd --list/-l single frame tests "
833 $ZSTD -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header
835 $ZSTD --list -v ./*.zst
837 println "\n===> zstd --list/-l multiple frame tests "
838 cat tmp1.zst tmp2.zst > tmp12.zst
839 cat tmp12.zst tmp3.zst > tmp123.zst
843 println "\n===> zstd --list/-l error detection tests "
844 $ZSTD -l tmp1 tmp1.zst && die "-l must fail on non-zstd file"
845 $ZSTD --list tmp* && die "-l must fail on non-zstd file"
846 $ZSTD -lv tmp1* && die "-l must fail on non-zstd file"
847 $ZSTD --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file"
849 println "test : detect truncated compressed file "
850 TEST_DATA_FILE=truncatable-input.txt
851 FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst
852 TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst
853 ./datagen -g50000 > $TEST_DATA_FILE
854 $ZSTD -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE
855 dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE
856 $ZSTD --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file"
859 rm $FULL_COMPRESSED_FILE
860 rm $TRUNCATED_COMPRESSED_FILE
862 println "\n===> zstd --list/-l errors when presented with stdin / no files"
863 $ZSTD -l && die "-l must fail on empty list of files"
864 $ZSTD -l - && die "-l does not work on stdin"
865 $ZSTD -l < tmp1.zst && die "-l does not work on stdin"
866 $ZSTD -l - < tmp1.zst && die "-l does not work on stdin"
867 $ZSTD -l - tmp1.zst && die "-l does not work on stdin"
868 $ZSTD -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin"
869 $ZSTD -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty
871 println "\n===> zstd --list/-l test with null files "
875 $ZSTD -l tmp5* && die "-l must fail on non-zstd file"
876 $ZSTD -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header
877 $ZSTD -lv tmp5* && die "-l must fail on non-zstd file"
879 println "\n===> zstd --list/-l test with no content size field "
880 ./datagen -g513K | $ZSTD > tmp6.zst
882 $ZSTD -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file"
884 println "\n===> zstd --list/-l test with no checksum "
885 $ZSTD -f --no-check tmp1
892 println "\n===> zstd long distance matching tests "
893 roundTripTest -g0 " --single-thread --long"
894 roundTripTest -g9M "2 --single-thread --long"
895 # Test parameter parsing
896 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB"
897 roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB"
898 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB"
899 roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
904 println "\n===> adaptive mode "
905 roundTripTest -g270000000 " --adapt"
906 roundTripTest -g27000000 " --adapt=min=1,max=4"
907 println "===> test: --adapt must fail on incoherent bounds "
909 $ZSTD -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"
911 println "\n===> rsyncable mode "
912 roundTripTest -g10M " --rsyncable"
913 roundTripTest -g10M " --rsyncable -B100K"
914 println "===> test: --rsyncable must fail with --single-thread"
915 $ZSTD -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
919 if [ "$1" != "--test-large-data" ]; then
920 println "Skipping large data tests"
925 #############################################################################
927 println "\n===> large files tests "
929 roundTripTest -g270000000 1
930 roundTripTest -g250000000 2
931 roundTripTest -g230000000 3
933 roundTripTest -g140000000 -P60 4
934 roundTripTest -g130000000 -P62 5
935 roundTripTest -g120000000 -P65 6
937 roundTripTest -g70000000 -P70 7
938 roundTripTest -g60000000 -P71 8
939 roundTripTest -g50000000 -P73 9
941 roundTripTest -g35000000 -P75 10
942 roundTripTest -g30000000 -P76 11
943 roundTripTest -g25000000 -P78 12
945 roundTripTest -g18000013 -P80 13
946 roundTripTest -g18000014 -P80 14
947 roundTripTest -g18000015 -P81 15
948 roundTripTest -g18000016 -P84 16
949 roundTripTest -g18000017 -P88 17
950 roundTripTest -g18000018 -P94 18
951 roundTripTest -g18000019 -P96 19
953 roundTripTest -g5000000000 -P99 1
954 roundTripTest -g1700000000 -P0 "1 --zstd=strategy=6" # ensure btlazy2 can survive an overflow rescale
956 fileRoundTripTest -g4193M -P99 1
959 println "\n===> zstd long, long distance matching round-trip tests "
960 roundTripTest -g270000000 "1 --single-thread --long"
961 roundTripTest -g130000000 -P60 "5 --single-thread --long"
962 roundTripTest -g35000000 -P70 "8 --single-thread --long"
963 roundTripTest -g18000001 -P80 "18 --single-thread --long"
964 # Test large window logs
965 roundTripTest -g700M -P50 "1 --single-thread --long=29"
966 roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28"
971 println "\n===> zstdmt long round-trip tests "
972 roundTripTest -g80000000 -P99 "19 -T2" " "
973 roundTripTest -g5000000000 -P99 "1 -T2" " "
974 roundTripTest -g500000000 -P97 "1 -T999" " "
975 fileRoundTripTest -g4103M -P98 " -T0" " "
976 roundTripTest -g400000000 -P97 "1 --long=24 -T2" " "
977 # Exposes the bug in https://github.com/facebook/zstd/pull/1678
978 # This test fails on 4 different travis builds at the time of writing
979 # because it needs to allocate 8 GB of memory.
980 # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000"
982 println "\n**** no multithreading, skipping zstdmt tests **** "
986 println "\n===> cover dictionary builder : advanced options "
988 TESTFILE="$PRGDIR"/zstdcli.c
990 println "- Create first dictionary"
991 $ZSTD --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
993 $ZSTD -f tmp -D tmpDict
994 $ZSTD -d tmp.zst -D tmpDict -fo result
995 $DIFF "$TESTFILE" result
996 $ZSTD --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
997 println "- Create second (different) dictionary"
998 $ZSTD --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
999 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
1000 println "- Create dictionary using shrink-dict flag"
1001 $ZSTD --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
1002 $ZSTD --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
1003 $ZSTD --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
1004 println "- Create dictionary with short dictID"
1005 $ZSTD --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
1006 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
1007 println "- Create dictionary with size limit"
1008 $ZSTD --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
1009 println "- Compare size of dictionary from 90% training samples with 80% training samples"
1010 $ZSTD --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1011 $ZSTD --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1012 println "- Create dictionary using all samples for both training and testing"
1013 $ZSTD --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
1014 println "- Test -o before --train-cover"
1015 rm -f tmpDict dictionary
1016 $ZSTD -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
1018 $ZSTD --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
1020 rm -f tmp* dictionary