1 # $Id: Makefile,v 1.107 2020/11/02 00:40:25 sjg Exp $
3 # $NetBSD: Makefile,v 1.181 2020/11/01 19:02:22 rillig Exp $
5 # Unit tests for make(1)
7 # The main targets are:
12 # run 'all', and compare to expected results
14 # move generated output to expected results
19 # The make program to be tested.
24 # Each feature should get its own set of tests in its own suitably
25 # named makefile (*.mk), with its own set of expected results (*.exp),
26 # and it should be added to the TESTS list.
28 # A few *.mk files are helper files for other tests (such as include-sub.mk)
29 # and are thus not added to TESTS. Such files must be ignored in
30 # src/tests/usr.bin/make/t_make.sh.
33 # Each test is in a sub-makefile.
34 # Keep the list sorted.
35 # Any test that is commented out must be ignored in
36 # src/tests/usr.bin/make/t_make.sh as well.
38 TESTS+= archive-suffix
42 TESTS+= cond-cmp-numeric
43 TESTS+= cond-cmp-numeric-eq
44 TESTS+= cond-cmp-numeric-ge
45 TESTS+= cond-cmp-numeric-gt
46 TESTS+= cond-cmp-numeric-le
47 TESTS+= cond-cmp-numeric-lt
48 TESTS+= cond-cmp-numeric-ne
49 TESTS+= cond-cmp-string
50 TESTS+= cond-cmp-unary
52 TESTS+= cond-func-commands
53 TESTS+= cond-func-defined
54 TESTS+= cond-func-empty
55 TESTS+= cond-func-exists
56 TESTS+= cond-func-make
57 TESTS+= cond-func-target
63 TESTS+= cond-op-parentheses
65 TESTS+= cond-token-number
66 TESTS+= cond-token-plain
67 TESTS+= cond-token-string
68 TESTS+= cond-token-var
69 TESTS+= cond-undef-lint
72 TESTS+= counter-append
75 TESTS+= dep-colon-bug-cross-file
76 TESTS+= dep-double-colon
77 TESTS+= dep-double-colon-indep
91 TESTS+= depsrc-nometa_cmp
93 TESTS+= depsrc-notmain
94 TESTS+= depsrc-optional
96 TESTS+= depsrc-precious
97 TESTS+= depsrc-recursive
100 TESTS+= depsrc-usebefore
101 TESTS+= depsrc-usebefore-double-colon
105 TESTS+= deptgt-default
106 TESTS+= deptgt-delete_on_error
108 TESTS+= deptgt-end-jobs
110 TESTS+= deptgt-ignore
111 TESTS+= deptgt-interrupt
113 TESTS+= deptgt-makeflags
114 TESTS+= deptgt-no_parallel
115 TESTS+= deptgt-nopath
116 TESTS+= deptgt-notparallel
117 TESTS+= deptgt-objdir
120 TESTS+= deptgt-path-suffix
122 TESTS+= deptgt-precious
124 TESTS+= deptgt-silent
126 TESTS+= deptgt-suffixes
128 TESTS+= dir-expand-path
130 TESTS+= directive-dinclude
131 TESTS+= directive-elif
132 TESTS+= directive-elifdef
133 TESTS+= directive-elifmake
134 TESTS+= directive-elifndef
135 TESTS+= directive-elifnmake
136 TESTS+= directive-else
137 TESTS+= directive-endif
138 TESTS+= directive-error
139 TESTS+= directive-export
140 TESTS+= directive-export-env
141 TESTS+= directive-export-gmake
142 TESTS+= directive-export-literal
143 TESTS+= directive-for
144 TESTS+= directive-for-generating-endif
145 TESTS+= directive-hyphen-include
147 TESTS+= directive-ifdef
148 TESTS+= directive-ifmake
149 TESTS+= directive-ifndef
150 TESTS+= directive-ifnmake
151 TESTS+= directive-include
152 TESTS+= directive-include-fatal
153 TESTS+= directive-info
154 TESTS+= directive-sinclude
155 TESTS+= directive-undef
156 TESTS+= directive-unexport
157 TESTS+= directive-unexport-env
158 TESTS+= directive-warning
165 TESTS+= # escape # broken by reverting POSIX changes
169 TESTS+= export-variants
172 TESTS+= hanoi-include
175 #TESTS+= job-output-long-lines
177 TESTS+= make-exported
184 TESTS+= opt-backwards
187 TESTS+= opt-debug-all
188 TESTS+= opt-debug-archive
189 TESTS+= opt-debug-curdir
190 TESTS+= opt-debug-cond
191 TESTS+= opt-debug-dir
192 TESTS+= opt-debug-errors
193 TESTS+= opt-debug-file
194 TESTS+= opt-debug-for
195 TESTS+= opt-debug-graph1
196 TESTS+= opt-debug-graph2
197 TESTS+= opt-debug-graph3
198 TESTS+= opt-debug-hash
199 #TESTS+= opt-debug-jobs
200 TESTS+= opt-debug-lint
201 TESTS+= opt-debug-loud
202 TESTS+= opt-debug-meta
203 TESTS+= opt-debug-making
204 TESTS+= opt-debug-no-rm
205 TESTS+= opt-debug-parse
206 TESTS+= opt-debug-suff
207 TESTS+= opt-debug-targets
208 TESTS+= opt-debug-varraw
209 TESTS+= opt-debug-var
210 TESTS+= opt-debug-x-trace
215 TESTS+= opt-include-dir
217 TESTS+= opt-jobs-internal
218 TESTS+= opt-keep-going
219 TESTS+= opt-m-include-dir
220 TESTS+= opt-no-action
221 TESTS+= opt-no-action-at-all
226 TESTS+= opt-tracefile
227 TESTS+= opt-var-expanded
228 TESTS+= opt-var-literal
229 TESTS+= opt-warnings-as-errors
230 TESTS+= opt-where-am-i
231 TESTS+= opt-x-reduce-exported
236 TESTS+= # posix1 # broken by reverting POSIX changes
242 TESTS+= sh-jobs-error
243 TESTS+= sh-leading-at
244 TESTS+= sh-leading-hyphen
245 TESTS+= sh-leading-plus
246 TESTS+= sh-meta-chars
247 TESTS+= sh-multi-line
248 TESTS+= sh-single-line
253 TESTS+= suff-add-later
254 TESTS+= suff-clear-regular
255 TESTS+= suff-clear-single
259 TESTS+= suff-transform-endless
260 TESTS+= suff-transform-expand
261 TESTS+= suff-transform-select
266 TESTS+= use-inference
268 TESTS+= var-class-cmdline
269 TESTS+= var-class-env
270 TESTS+= var-class-global
271 TESTS+= var-class-local
272 TESTS+= var-class-local-legacy
274 TESTS+= var-op-append
275 TESTS+= var-op-assign
276 TESTS+= var-op-default
277 TESTS+= var-op-expand
280 TESTS+= var-recursive
286 TESTS+= varmod-assign
287 TESTS+= varmod-defined
289 TESTS+= varmod-exclam-shell
290 TESTS+= varmod-extension
291 TESTS+= varmod-gmtime
294 TESTS+= varmod-ifelse
295 TESTS+= varmod-l-name-to-value
296 TESTS+= varmod-localtime
299 TESTS+= varmod-match-escape
300 TESTS+= varmod-no-match
302 TESTS+= varmod-order-reverse
303 TESTS+= varmod-order-shuffle
306 TESTS+= varmod-quote-dollar
308 TESTS+= varmod-remember
310 TESTS+= varmod-select-words
313 TESTS+= varmod-subst-regex
316 TESTS+= varmod-to-abs
317 TESTS+= varmod-to-lower
318 TESTS+= varmod-to-many-words
319 TESTS+= varmod-to-one-word
320 TESTS+= varmod-to-separator
321 TESTS+= varmod-to-upper
322 TESTS+= varmod-undefined
323 TESTS+= varmod-unique
325 TESTS+= varname-dollar
326 TESTS+= varname-dot-alltargets
327 TESTS+= varname-dot-curdir
328 TESTS+= varname-dot-includes
329 TESTS+= varname-dot-includedfromdir
330 TESTS+= varname-dot-includedfromfile
331 TESTS+= varname-dot-libs
332 TESTS+= varname-dot-make-dependfile
333 TESTS+= varname-dot-make-expand_variables
334 TESTS+= varname-dot-make-exported
335 TESTS+= varname-dot-make-jobs
336 TESTS+= varname-dot-make-jobs-prefix
337 TESTS+= varname-dot-make-level
338 TESTS+= varname-dot-make-makefile_preference
339 TESTS+= varname-dot-make-makefiles
340 TESTS+= varname-dot-make-meta-bailiwick
341 TESTS+= varname-dot-make-meta-created
342 TESTS+= varname-dot-make-meta-files
343 TESTS+= varname-dot-make-meta-ignore_filter
344 TESTS+= varname-dot-make-meta-ignore_paths
345 TESTS+= varname-dot-make-meta-ignore_patterns
346 TESTS+= varname-dot-make-meta-prefix
347 TESTS+= varname-dot-make-mode
348 TESTS+= varname-dot-make-path_filemon
349 TESTS+= varname-dot-make-pid
350 TESTS+= varname-dot-make-ppid
351 TESTS+= varname-dot-make-save_dollars
352 TESTS+= varname-dot-makeoverrides
353 TESTS+= varname-dot-newline
354 TESTS+= varname-dot-objdir
355 TESTS+= varname-dot-parsedir
356 TESTS+= varname-dot-parsefile
357 TESTS+= varname-dot-path
358 TESTS+= varname-dot-shell
359 TESTS+= varname-dot-targets
360 TESTS+= varname-empty
362 TESTS+= varname-make_print_var_on_error
363 TESTS+= varname-make_print_var_on_error-jobs
364 TESTS+= varname-makefile
365 TESTS+= varname-makeflags
367 TESTS+= varname-vpath
368 TESTS+= varparse-dynamic
370 TESTS+= varparse-undef-partial
374 # Additional environment variables for some of the tests.
375 # The base environment is -i PATH="$PATH".
376 ENV.envfirst= FROM_ENV=value-from-env
377 ENV.varmisc= FROM_ENV=env
378 ENV.varmisc+= FROM_ENV_BEFORE=env
379 ENV.varmisc+= FROM_ENV_AFTER=env
380 ENV.varmod-localtime+= TZ=Europe/Berlin
382 # Override make flags for some of the tests; default is -k.
383 # If possible, write ".MAKEFLAGS: -dv" in the test .mk file instead of
384 # settings FLAGS.test=-dv here, since that is closer to the test code.
385 FLAGS.cond-func-make= via-cmdline
386 FLAGS.directive-ifmake= first second
387 FLAGS.doterror= # none
391 FLAGS.opt-keep-going= -k
392 FLAGS.opt-no-action= -n
394 FLAGS.opt-var-expanded= -v VAR -v VALUE
395 FLAGS.opt-var-literal= -V VAR -V VALUE
396 FLAGS.opt-warnings-as-errors= -W
399 FLAGS.sh-leading-plus= -n
400 FLAGS.varname-empty= -dv '$${:U}=cmdline-u' '=cmline-plain'
402 # Some tests need extra postprocessing.
404 -e '/^[^=_A-Za-z0-9]*=/d'
405 # these all share the same requirement
406 .for t in export-all export-env
407 SED_CMDS.$t= ${SED_CMDS.export}
409 SED_CMDS.job-output-long-lines= \
410 ${:D Job separators on their own line are ok. } \
411 -e '/^--- job-[ab] ---$$/d' \
412 ${:D Plain output lines are ok as well. } \
413 ${:D They may come in multiples of 1024 or as 10000. } \
416 ${:D The following lines should rather not occur since the job } \
417 ${:D marker should always be at the beginning of the line. } \
418 -e '/^aa*--- job-b ---$$/d' \
419 -e '/^bb*--- job-a ---$$/d'
420 SED_CMDS.opt-debug-graph1= \
421 -e 's,${.CURDIR},CURDIR,'
422 SED_CMDS.opt-debug-graph1+= \
423 -e '/Global Variables:/,/Suffixes:/d'
424 SED_CMDS.sh-dots= -e 's,^.*\.\.\.:.*,<normalized: ...: not found>,'
425 SED_CMDS.opt-debug-jobs= -e 's,([0-9][0-9]*),(<pid>),'
426 SED_CMDS.opt-debug-jobs+= -e 's,pid [0-9][0-9]*,pid <pid>,'
427 SED_CMDS.opt-debug-jobs+= -e 's,Process [0-9][0-9]*,Process <pid>,'
428 SED_CMDS.opt-debug-jobs+= -e 's,JobFinish: [0-9][0-9]*,JobFinish: <pid>,'
429 # The "-q" may be there or not, see jobs.c, variable shells.
430 SED_CMDS.opt-debug-jobs+= -e 's,^\(.Command: sh\) -q,\1,'
431 SED_CMDS.varmod-subst-regex+= \
432 -e 's,\(Regex compilation error:\).*,\1 (details omitted),'
433 SED_CMDS.varmod-edge+= -e 's, line [0-9]*:, line omitted:,'
434 SED_CMDS.varshell+= -e 's,^${.SHELL:T}: ,,'
435 SED_CMDS.varshell+= -e '/command/s,No such.*,not found,'
436 SED_CMDS.varname-dot-parsedir= -e '/in some cases/ s,^make: "[^"]*,make: "<normalized>,'
437 SED_CMDS.varname-dot-parsefile= -e '/in some cases/ s,^make: "[^"]*,make: "<normalized>,'
438 SED_CMDS.varname-dot-shell= -e 's, = /[^ ]*, = (details omitted),g'
439 SED_CMDS.varname-dot-shell+= -e 's,"/[^" ]*","(details omitted)",g'
440 SED_CMDS.varname-dot-shell+= -e 's,\[/[^] ]*\],[(details omitted)],g'
442 # Some tests need an additional round of postprocessing.
443 POSTPROC.deptgt-suffixes= \
444 ${TOOL_SED} -n -e '/^\#\*\*\* Suffixes/,/^\#\*/p'
445 POSTPROC.varname= ${TOOL_SED} -n -e '/^MAGIC/p' -e '/^ORDER_/p'
446 POSTPROC.varname-empty= ${TOOL_SED} -n -e '/^Var_Set/p' -e '/^out:/p'
448 # Some tests reuse other tests, which makes them unnecessarily fragile.
449 export-all.rawout: export.mk
450 unexport.rawout: export.mk
451 unexport-env.rawout: export.mk
453 # End of the configuration section.
457 .-include "Makefile.inc"
458 .-include "Makefile.config"
460 UNIT_TESTS:= ${.PARSEDIR}
463 .if ${USE_ABSOLUTE_TESTNAMES:Uno} == yes
464 OUTFILES= ${TESTS:@test@${.CURDIR:tA}/${test}.out@}
466 OUTFILES= ${TESTS:=.out}
471 CLEANFILES= *.rawout *.out *.status *.tmp *.core *.tmp
472 CLEANFILES+= obj*.[och] lib*.a # posix1.mk
473 CLEANFILES+= issue* .[ab]* # suffixes.mk
474 CLEANDIRS= dir dummy # posix1.mk
486 .if defined(.PARSEDIR)
487 # ensure consistent results from sort(1)
493 .if ${.MAKE.MODE:Unormal:Mmeta} != ""
494 # we don't need the noise
498 MAKE_TEST_ENV?= MALLOC_OPTIONS="JA" # for jemalloc
500 # Each test is run in a sub-make, to keep the tests for interfering with
501 # each other, and because they use different environment variables and
502 # command line options.
503 .SUFFIXES: .mk .rawout .out
505 @${_MKMSG_TEST:Uecho '# test '} ${.PREFIX}
508 env -i PATH="$$PATH" ${MAKE_TEST_ENV} ${ENV.${.PREFIX:T}} \
510 -r -C ${.CURDIR} -f ${.IMPSRC} \
511 ${FLAGS.${.PREFIX:T}:U-k} \
512 > ${.TARGET}.tmp 2>&1 \
513 && status=$$? || status=$$?; \
514 echo $$status > ${.TARGET:R}.status
515 @mv ${.TARGET}.tmp ${.TARGET}
517 # Postprocess the test output so that the results can be compared.
519 # always pretend .MAKE was called 'make'
520 _SED_CMDS+= -e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,'
521 _SED_CMDS+= -e 's,${TEST_MAKE:S,.,\\.,g},make,'
522 # replace anything after 'stopped in' with unit-tests
523 _SED_CMDS+= -e '/stopped/s, /.*, unit-tests,'
524 # strip ${.CURDIR}/ from the output
525 _SED_CMDS+= -e 's,${.CURDIR:S,.,\\.,g}/,,g'
526 _SED_CMDS+= -e 's,${UNIT_TESTS:S,.,\\.,g}/,,g'
529 @${TOOL_SED} ${_SED_CMDS} ${SED_CMDS.${.PREFIX:T}} \
530 < ${.IMPSRC} > ${.TARGET}.tmp1
531 @${POSTPROC.${.PREFIX:T}:Ucat} < ${.TARGET}.tmp1 > ${.TARGET}.tmp2
533 @echo "exit status `cat ${.TARGET:R}.status`" >> ${.TARGET}.tmp2
534 @mv ${.TARGET}.tmp2 ${.TARGET}
536 # Compare all output files
537 test: ${OUTFILES} .PHONY
539 for test in ${TESTS}; do \
540 ${TOOL_DIFF} ${DIFF_FLAGS} ${UNIT_TESTS}/$${test}.exp $${test}.out \
541 || failed="$${failed}$${failed:+ }$${test}" ; \
543 if [ -n "$${failed}" ]; then \
544 echo "Failed tests: $${failed}" ; false ; \
546 echo "All tests passed" ; \
550 @for test in ${TESTS}; do \
551 cmp -s ${UNIT_TESTS}/$${test}.exp $${test}.out \
552 || { echo "Replacing $${test}.exp" ; \
553 cp $${test}.out ${UNIT_TESTS}/$${test}.exp ; } \
556 .if exists(${TEST_MAKE})
557 ${TESTS:=.rawout}: ${TEST_MAKE}
558 # in meta mode, we *know* if a target script is impacted
559 # by a makefile change.
560 .if ${.MAKE.MODE:Unormal:Mmeta} == ""
561 ${TESTS:=.rawout}: ${.PARSEDIR}/Makefile