1 # ################################################################
2 # Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
5 # This source code is licensed under both the BSD-style license (found in the
6 # LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 # in the COPYING file in the root directory of this source tree).
8 # ################################################################
9 # datagen : Synthetic and parametrable data generator, for tests
10 # fullbench : Precisely measure speed for each zstd inner functions
11 # fullbench32: Same as fullbench, but forced to compile in 32-bits mode
12 # fuzzer : Test tool, to check zstd integrity on target platform
13 # fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
14 # paramgrill : parameter tester for zstd
15 # test-zstd-speed.py : script for testing zstd speed difference between commits
16 # versionsTest : compatibility test between zstd versions stored on Github (v0.1+)
17 # zstreamtest : Fuzzer test tool for zstd streaming API
18 # zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
19 # ##########################################################################
24 TESTARTEFACT := versionsTest
27 DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL)
28 CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
29 -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
30 ifeq ($(OS),Windows_NT) # MinGW assumed
31 CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
34 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
35 -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
36 -Wstrict-prototypes -Wundef -Wformat-security \
37 -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
38 -Wredundant-decls -Wmissing-prototypes
39 CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
40 FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
43 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
44 ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
45 ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
46 ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
47 ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
48 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
50 ZSTD_F1 := $(wildcard $(ZSTD_FILES))
51 ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1))
52 ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1))
53 ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2))
54 ZSTD_OBJECTS := $(ZSTD_OBJ3:.c=.o)
56 ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1))
57 ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1))
58 ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2))
59 ZSTDMT_OBJECTS := $(ZSTDMT_OBJ3:.c=.o)
61 # Define *.exe as extension for Windows systems
62 ifneq (,$(filter Windows%,$(OS)))
64 MULTITHREAD_CPP = -DZSTD_MULTITHREAD
68 MULTITHREAD_CPP = -DZSTD_MULTITHREAD
69 MULTITHREAD_LD = -pthread
71 MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
74 ZSTREAM_TESTTIME ?= -T90s
76 ZSTDRTTEST = --test-large-data
77 DECODECORPUS_TESTTIME ?= -T30
79 .PHONY: default all all32 allnothread dll clean test test32 test-all versionsTest
82 @echo $(ZSTDMT_OBJECTS)
84 all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \
87 all32: fullbench32 fuzzer32 zstreamtest32
89 allnothread: MULTITHREAD_CPP=
90 allnothread: MULTITHREAD_LD=
91 allnothread: fullbench fuzzer paramgrill datagen decodecorpus
93 dll: fuzzer-dll zstreamtest-dll
95 PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update
96 zstd zstd32 zstd-nolegacy:
97 $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
100 $(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)"
104 $(MAKE) -C $(ZSTDDIR) libzstd
106 .PHONY: zstd-staticLib
108 $(MAKE) -C $(ZSTDDIR) libzstd.a
110 zstdm_%.o : $(ZSTDDIR)/common/%.c
111 $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
113 zstdc_%.o : $(ZSTDDIR)/compress/%.c
114 $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
116 zstdd_%.o : $(ZSTDDIR)/decompress/%.c
117 $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
119 zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP)
121 zstdmt_m_%.o : $(ZSTDDIR)/common/%.c
122 $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
124 zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c
125 $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
127 zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
128 $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
130 fullbench32: CPPFLAGS += -m32
131 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
132 fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD)
133 fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
134 fullbench fullbench32 : $(ZSTD_FILES)
135 fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
136 $(CC) $(FLAGS) $^ -o $@$(EXT)
138 fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
139 fullbench-lib : zstd-staticLib
140 fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c
141 $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a
143 # note : broken : requires unavailable symbols
144 fullbench-dll : zstd-dll
145 fullbench-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
146 fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
147 # $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
148 $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT)
150 fuzzer : CPPFLAGS += $(MULTITHREAD_CPP)
151 fuzzer : LDFLAGS += $(MULTITHREAD_LD)
152 fuzzer32: CFLAGS += -m32
153 fuzzer : $(ZSTDMT_OBJECTS)
154 fuzzer32: $(ZSTD_FILES)
155 fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
156 $(CC) $(FLAGS) $^ -o $@$(EXT)
158 fuzzer-dll : zstd-dll
159 fuzzer-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
160 fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c fuzzer.c
161 $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
163 zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
164 zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
165 zbufftest : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
166 $(CC) $(FLAGS) $^ -o $@$(EXT)
168 zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
169 zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
170 zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
171 $(CC) $(FLAGS) $^ -o $@$(EXT)
173 zbufftest-dll : zstd-dll
174 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
175 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
176 zbufftest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
177 zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
178 $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
180 ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c seqgen.c zstreamtest.c
181 ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
182 ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
183 zstreamtest32 : CFLAGS += -m32
184 zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
185 zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
186 zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES)
187 zstreamtest32 : $(ZSTREAMFILES)
188 zstreamtest zstreamtest32 :
189 $(CC) $(FLAGS) $^ -o $@$(EXT)
191 zstreamtest_asan : CFLAGS += -fsanitize=address
192 zstreamtest_asan : $(ZSTREAMFILES)
193 $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
195 zstreamtest_tsan : CFLAGS += -fsanitize=thread
196 zstreamtest_tsan : $(ZSTREAMFILES)
197 $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
199 zstreamtest-dll : zstd-dll
200 zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
201 zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
202 zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
203 $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
205 paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements
206 paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c
207 $(CC) $(FLAGS) $^ -lm -o $@$(EXT)
209 datagen : $(PRGDIR)/datagen.c datagencli.c
210 $(CC) $(FLAGS) $^ -o $@$(EXT)
212 roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
213 $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
215 longmatch : $(ZSTD_OBJECTS) longmatch.c
216 $(CC) $(FLAGS) $^ -o $@$(EXT)
218 invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
219 $(CC) $(FLAGS) $^ -o $@$(EXT)
221 legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4
222 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
223 $(CC) $(FLAGS) $^ -o $@$(EXT)
225 decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) decodecorpus.c
226 $(CC) $(FLAGS) $^ -o $@$(EXT) -lm
228 symbols : symbols.c zstd-dll
229 ifneq (,$(filter Windows%,$(OS)))
230 cp $(ZSTDDIR)/dll/libzstd.dll .
231 $(CC) $(FLAGS) $< -o $@$(EXT) -DZSTD_DLL_IMPORT=1 libzstd.dll
233 $(CC) $(FLAGS) $< -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so # broken on Mac
236 poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
237 $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
241 $(PYTHON) test-zstd-versions.py
243 checkTag: checkTag.c $(ZSTDDIR)/zstd.h
244 $(CC) $(FLAGS) $< -o $@$(EXT)
247 $(MAKE) -C $(ZSTDDIR) clean
248 $(MAKE) -C $(PRGDIR) clean
249 @$(RM) -fR $(TESTARTEFACT)
250 @$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
251 $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
252 fullbench$(EXT) fullbench32$(EXT) \
253 fullbench-lib$(EXT) fullbench-dll$(EXT) \
254 fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
255 fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT) \
256 zstreamtest$(EXT) zstreamtest32$(EXT) \
257 datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
258 symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
259 decodecorpus$(EXT) checkTag$(EXT)
260 @echo Cleaning completed
263 #----------------------------------------------------------------------------------
264 #make valgrindTest is validated only for Linux, macOS, BSD, Hurd and Solaris targets
265 #----------------------------------------------------------------------------------
266 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
269 valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
270 valgrindTest: zstd datagen fuzzer fullbench
271 @echo "\n ---- valgrind tests : memory analyzer ----"
272 $(VALGRIND) ./datagen -g50M > $(VOID)
273 $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
274 ./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID)
275 ./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
276 ./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
277 $(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
278 ./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
280 $(VALGRIND) ./fuzzer -T1mn -t1
281 $(VALGRIND) ./fullbench -i1
286 ifneq (,$(filter MSYS%,$(shell uname)))
291 #-----------------------------------------------------------------------------
292 # make tests validated only for below targets
293 #-----------------------------------------------------------------------------
294 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
297 ifneq (,$(filter $(shell uname),SunOS))
303 @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
306 shortest: ZSTDRTTEST=
310 fuzztest: test-fuzzer test-zstream test-decodecorpus
313 test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
318 test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
320 test-all: test test32 valgrindTest test-decodecorpus-cli
323 .PHONY: test-zstd test-zstd32 test-zstd-nolegacy
324 test-zstd: ZSTD = $(PRGDIR)/zstd
327 test-zstd32: ZSTD = $(PRGDIR)/zstd32
330 test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
331 test-zstd-nolegacy: zstd-nolegacy
333 test-zstd test-zstd32 test-zstd-nolegacy: datagen
335 ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
339 $(PRGDIR)/zstd -f README.md test-zstd-speed.py
340 gzip -f README.md test-zstd-speed.py
341 cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst
342 cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz
343 $(PRGDIR)/zstd -df README.md.gz -o README2.md
344 $(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz
345 $(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz
346 $(DIFF) -q zstd_gz gz_zstd
347 echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
348 echo Hello World GZIP | gzip -c - >hello.gz
349 echo Hello World TEXT >hello.txt
350 cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz
351 $(PRGDIR)/zstd -dcf hello.*
352 $(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
353 $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt
355 test-fullbench: fullbench datagen
356 $(QEMU_SYS) ./fullbench -i1
357 $(QEMU_SYS) ./fullbench -i1 -P0
359 test-fullbench32: fullbench32 datagen
360 $(QEMU_SYS) ./fullbench32 -i1
361 $(QEMU_SYS) ./fullbench32 -i1 -P0
364 $(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
366 test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0
367 test-fuzzer-stackmode: test-fuzzer
369 test-fuzzer32: fuzzer32
370 $(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
372 test-zbuff: zbufftest
373 $(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
375 test-zbuff32: zbufftest32
376 $(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
378 test-zstream: zstreamtest
379 $(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
380 $(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
381 $(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
383 test-zstream32: zstreamtest32
384 $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
386 test-longmatch: longmatch
387 $(QEMU_SYS) ./longmatch
389 test-invalidDictionaries: invalidDictionaries
390 $(QEMU_SYS) ./invalidDictionaries
392 test-symbols: symbols
393 $(QEMU_SYS) ./symbols
398 test-decodecorpus: decodecorpus
399 $(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
401 test-decodecorpus-cli: decodecorpus
402 @echo "\n ---- decodecorpus basic cli tests ----"
404 ./decodecorpus -n5 -otestdir -ptestdir
406 $(ZSTD) -d z000000.zst -o tmp0 && \
407 $(ZSTD) -d z000001.zst -o tmp1 && \
408 $(ZSTD) -d z000002.zst -o tmp2 && \
409 $(ZSTD) -d z000003.zst -o tmp3 && \
410 $(ZSTD) -d z000004.zst -o tmp4 && \
411 diff z000000 tmp0 && \
412 diff z000001 tmp1 && \
413 diff z000002 tmp2 && \
414 diff z000003 tmp3 && \
415 diff z000004 tmp4 && \
418 @echo "\n ---- decodecorpus dictionary cli tests ----"
419 ./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB
421 $(ZSTD) -d z000000.zst -D dictionary -o tmp0 && \
422 $(ZSTD) -d z000001.zst -D dictionary -o tmp1 && \
423 $(ZSTD) -d z000002.zst -D dictionary -o tmp2 && \
424 $(ZSTD) -d z000003.zst -D dictionary -o tmp3 && \
425 $(ZSTD) -d z000004.zst -D dictionary -o tmp4 && \
426 diff z000000 tmp0 && \
427 diff z000001 tmp1 && \
428 diff z000002 tmp2 && \
429 diff z000003 tmp3 && \
430 diff z000004 tmp4 && \
435 $(QEMU_SYS) ./poolTests
437 test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
438 test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
439 test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
440 test-lz4: zstd decodecorpus datagen
441 [ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
442 [ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
453 $(ZSTD) --format=lz4 < tmp | \
456 $(ZSTD_LZ4) < tmp | \
460 $(ZSTD) --format=lz4 < tmp | \
468 ./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null