Merge branch 'tr/test-v-and-v-subtest-only'
Allows N instances of tests run in parallel, each running 1/N parts of the test suite under Valgrind, to speed things up. * tr/test-v-and-v-subtest-only: perf-lib: fix start/stop of perf tests test-lib: support running tests under valgrind in parallel test-lib: allow prefixing a custom string before "ok N" etc. test-lib: valgrind for only tests matching a pattern test-lib: verbose mode for only tests matching a pattern test-lib: self-test that --verbose works test-lib: rearrange start/end of test_expect_* and test_skip test-lib: refactor $GIT_SKIP_TESTS matching test-lib: enable MALLOC_* for the actual tests
This commit is contained in:
Junio C Hamano
commit
04f2ddda84
10
t/README
10
t/README
@ -76,6 +76,11 @@ appropriately before running "make".
|
||||
command being run and their output if any are also
|
||||
output.
|
||||
|
||||
--verbose-only=<pattern>::
|
||||
Like --verbose, but the effect is limited to tests with
|
||||
numbers matching <pattern>. The number matched against is
|
||||
simply the running count of the test within the file.
|
||||
|
||||
--debug::
|
||||
This may help the person who is developing a new test.
|
||||
It causes the command defined with test_debug to run.
|
||||
@ -121,6 +126,11 @@ appropriately before running "make".
|
||||
the 't/valgrind/' directory and use the commands under
|
||||
't/valgrind/bin/'.
|
||||
|
||||
--valgrind-only=<pattern>::
|
||||
Like --valgrind, but the effect is limited to tests with
|
||||
numbers matching <pattern>. The number matched against is
|
||||
simply the running count of the test within the file.
|
||||
|
||||
--tee::
|
||||
In addition to printing the test output to the terminal,
|
||||
write it to files named 't/test-results/$TEST_NAME.out'.
|
||||
|
||||
@ -150,6 +150,7 @@ exit $ret' >&3 2>&4
|
||||
|
||||
|
||||
test_perf () {
|
||||
test_start_
|
||||
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
|
||||
test "$#" = 2 ||
|
||||
error "bug in the test script: not 2 or 3 parameters to test-expect-success"
|
||||
@ -187,7 +188,7 @@ test_perf () {
|
||||
base="$perf_results_dir"/"$perf_results_prefix$(basename "$0" .sh)"."$test_count"
|
||||
"$TEST_DIRECTORY"/perf/min_time.perl test_time.* >"$base".times
|
||||
fi
|
||||
echo >&3 ""
|
||||
test_finish_
|
||||
}
|
||||
|
||||
# We extend test_done to print timings at the end (./run disables this
|
||||
|
||||
@ -47,8 +47,13 @@ test_expect_failure 'pretend we have a known breakage' '
|
||||
|
||||
run_sub_test_lib_test () {
|
||||
name="$1" descr="$2" # stdin is the body of the test code
|
||||
shift 2
|
||||
mkdir "$name" &&
|
||||
(
|
||||
# Pretend we're a test harness. This prevents
|
||||
# test-lib from writing the counts to a file that will
|
||||
# later be summarized, showing spurious "failed" tests
|
||||
export HARNESS_ACTIVE=t &&
|
||||
cd "$name" &&
|
||||
cat >"$name.sh" <<-EOF &&
|
||||
#!$SHELL_PATH
|
||||
@ -65,7 +70,7 @@ run_sub_test_lib_test () {
|
||||
cat >>"$name.sh" &&
|
||||
chmod +x "$name.sh" &&
|
||||
export TEST_DIRECTORY &&
|
||||
./"$name.sh" >out 2>err
|
||||
./"$name.sh" "$@" >out 2>err
|
||||
)
|
||||
}
|
||||
|
||||
@ -215,6 +220,60 @@ test_expect_success 'pretend we have a mix of all possible results' "
|
||||
EOF
|
||||
"
|
||||
|
||||
test_expect_success 'test --verbose' '
|
||||
test_must_fail run_sub_test_lib_test \
|
||||
test-verbose "test verbose" --verbose <<-\EOF &&
|
||||
test_expect_success "passing test" true
|
||||
test_expect_success "test with output" "echo foo"
|
||||
test_expect_success "failing test" false
|
||||
test_done
|
||||
EOF
|
||||
mv test-verbose/out test-verbose/out+
|
||||
grep -v "^Initialized empty" test-verbose/out+ >test-verbose/out &&
|
||||
check_sub_test_lib_test test-verbose <<-\EOF
|
||||
> expecting success: true
|
||||
> Z
|
||||
> ok 1 - passing test
|
||||
> Z
|
||||
> expecting success: echo foo
|
||||
> foo
|
||||
> Z
|
||||
> ok 2 - test with output
|
||||
> Z
|
||||
> expecting success: false
|
||||
> Z
|
||||
> not ok 3 - failing test
|
||||
> # false
|
||||
> Z
|
||||
> # failed 1 among 3 test(s)
|
||||
> 1..3
|
||||
EOF
|
||||
'
|
||||
|
||||
test_expect_success 'test --verbose-only' '
|
||||
test_must_fail run_sub_test_lib_test \
|
||||
test-verbose-only-2 "test verbose-only=2" \
|
||||
--verbose-only=2 <<-\EOF &&
|
||||
test_expect_success "passing test" true
|
||||
test_expect_success "test with output" "echo foo"
|
||||
test_expect_success "failing test" false
|
||||
test_done
|
||||
EOF
|
||||
check_sub_test_lib_test test-verbose-only-2 <<-\EOF
|
||||
> ok 1 - passing test
|
||||
> Z
|
||||
> expecting success: echo foo
|
||||
> foo
|
||||
> Z
|
||||
> ok 2 - test with output
|
||||
> Z
|
||||
> not ok 3 - failing test
|
||||
> # false
|
||||
> # failed 1 among 3 test(s)
|
||||
> 1..3
|
||||
EOF
|
||||
'
|
||||
|
||||
test_set_prereq HAVEIT
|
||||
haveit=no
|
||||
test_expect_success HAVEIT 'test runs if prerequisite is satisfied' '
|
||||
|
||||
@ -343,6 +343,7 @@ test_declared_prereq () {
|
||||
}
|
||||
|
||||
test_expect_failure () {
|
||||
test_start_
|
||||
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
|
||||
test "$#" = 2 ||
|
||||
error "bug in the test script: not 2 or 3 parameters to test-expect-failure"
|
||||
@ -357,10 +358,11 @@ test_expect_failure () {
|
||||
test_known_broken_failure_ "$1"
|
||||
fi
|
||||
fi
|
||||
echo >&3 ""
|
||||
test_finish_
|
||||
}
|
||||
|
||||
test_expect_success () {
|
||||
test_start_
|
||||
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
|
||||
test "$#" = 2 ||
|
||||
error "bug in the test script: not 2 or 3 parameters to test-expect-success"
|
||||
@ -375,7 +377,7 @@ test_expect_success () {
|
||||
test_failure_ "$@"
|
||||
fi
|
||||
fi
|
||||
echo >&3 ""
|
||||
test_finish_
|
||||
}
|
||||
|
||||
# test_external runs external test scripts that provide continuous
|
||||
|
||||
246
t/test-lib.sh
246
t/test-lib.sh
@ -184,6 +184,9 @@ do
|
||||
help=t; shift ;;
|
||||
-v|--v|--ve|--ver|--verb|--verbo|--verbos|--verbose)
|
||||
verbose=t; shift ;;
|
||||
--verbose-only=*)
|
||||
verbose_only=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
-q|--q|--qu|--qui|--quie|--quiet)
|
||||
# Ignore --quiet under a TAP::Harness. Saying how many tests
|
||||
# passed without the ok/not ok details is always an error.
|
||||
@ -198,17 +201,39 @@ do
|
||||
--valgrind=*)
|
||||
valgrind=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
--valgrind-only=*)
|
||||
valgrind_only=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
--valgrind-parallel=*)
|
||||
valgrind_parallel=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
--valgrind-only-stride=*)
|
||||
valgrind_only_stride=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
--valgrind-only-offset=*)
|
||||
valgrind_only_offset=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
--tee)
|
||||
shift ;; # was handled already
|
||||
--root=*)
|
||||
root=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
--statusprefix=*)
|
||||
statusprefix=$(expr "z$1" : 'z[^=]*=\(.*\)')
|
||||
shift ;;
|
||||
*)
|
||||
echo "error: unknown test option '$1'" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
test -n "$valgrind" && verbose=t
|
||||
if test -n "$valgrind_only" || test -n "$valgrind_only_stride"
|
||||
then
|
||||
test -z "$valgrind" && valgrind=memcheck
|
||||
test -z "$verbose" && verbose_only="$valgrind_only"
|
||||
elif test -n "$valgrind"
|
||||
then
|
||||
verbose=t
|
||||
fi
|
||||
|
||||
if test -n "$color"
|
||||
then
|
||||
@ -303,12 +328,12 @@ trap 'die' EXIT
|
||||
|
||||
test_ok_ () {
|
||||
test_success=$(($test_success + 1))
|
||||
say_color "" "ok $test_count - $@"
|
||||
say_color "" "${statusprefix}ok $test_count - $@"
|
||||
}
|
||||
|
||||
test_failure_ () {
|
||||
test_failure=$(($test_failure + 1))
|
||||
say_color error "not ok $test_count - $1"
|
||||
say_color error "${statusprefix}not ok $test_count - $1"
|
||||
shift
|
||||
echo "$@" | sed -e 's/^/# /'
|
||||
test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
|
||||
@ -316,18 +341,83 @@ test_failure_ () {
|
||||
|
||||
test_known_broken_ok_ () {
|
||||
test_fixed=$(($test_fixed+1))
|
||||
say_color error "ok $test_count - $@ # TODO known breakage vanished"
|
||||
say_color error "${statusprefix}ok $test_count - $@ # TODO known breakage vanished"
|
||||
}
|
||||
|
||||
test_known_broken_failure_ () {
|
||||
test_broken=$(($test_broken+1))
|
||||
say_color warn "not ok $test_count - $@ # TODO known breakage"
|
||||
say_color warn "${statusprefix}not ok $test_count - $@ # TODO known breakage"
|
||||
}
|
||||
|
||||
test_debug () {
|
||||
test "$debug" = "" || eval "$1"
|
||||
}
|
||||
|
||||
match_pattern_list () {
|
||||
arg="$1"
|
||||
shift
|
||||
test -z "$*" && return 1
|
||||
for pattern_
|
||||
do
|
||||
case "$arg" in
|
||||
$pattern_)
|
||||
return 0
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
maybe_teardown_verbose () {
|
||||
test -z "$verbose_only" && return
|
||||
exec 4>/dev/null 3>/dev/null
|
||||
verbose=
|
||||
}
|
||||
|
||||
last_verbose=t
|
||||
maybe_setup_verbose () {
|
||||
test -z "$verbose_only" && return
|
||||
if match_pattern_list $test_count $verbose_only ||
|
||||
{ test -n "$valgrind_only_stride" &&
|
||||
expr $test_count "%" $valgrind_only_stride - $valgrind_only_offset = 0 >/dev/null; }
|
||||
then
|
||||
exec 4>&2 3>&1
|
||||
# Emit a delimiting blank line when going from
|
||||
# non-verbose to verbose. Within verbose mode the
|
||||
# delimiter is printed by test_expect_*. The choice
|
||||
# of the initial $last_verbose is such that before
|
||||
# test 1, we do not print it.
|
||||
test -z "$last_verbose" && echo >&3 ""
|
||||
verbose=t
|
||||
else
|
||||
exec 4>/dev/null 3>/dev/null
|
||||
verbose=
|
||||
fi
|
||||
last_verbose=$verbose
|
||||
}
|
||||
|
||||
maybe_teardown_valgrind () {
|
||||
test -z "$GIT_VALGRIND" && return
|
||||
GIT_VALGRIND_ENABLED=
|
||||
}
|
||||
|
||||
maybe_setup_valgrind () {
|
||||
test -z "$GIT_VALGRIND" && return
|
||||
if test -z "$valgrind_only" && test -z "$valgrind_only_stride"
|
||||
then
|
||||
GIT_VALGRIND_ENABLED=t
|
||||
return
|
||||
fi
|
||||
GIT_VALGRIND_ENABLED=
|
||||
if match_pattern_list $test_count $valgrind_only
|
||||
then
|
||||
GIT_VALGRIND_ENABLED=t
|
||||
elif test -n "$valgrind_only_stride" &&
|
||||
expr $test_count "%" $valgrind_only_stride - $valgrind_only_offset = 0 >/dev/null
|
||||
then
|
||||
GIT_VALGRIND_ENABLED=t
|
||||
fi
|
||||
}
|
||||
|
||||
test_eval_ () {
|
||||
# This is a separate function because some tests use
|
||||
# "return" to end a test_expect_success block early.
|
||||
@ -337,8 +427,10 @@ test_eval_ () {
|
||||
test_run_ () {
|
||||
test_cleanup=:
|
||||
expecting_failure=$2
|
||||
setup_malloc_check
|
||||
test_eval_ "$1"
|
||||
eval_ret=$?
|
||||
teardown_malloc_check
|
||||
|
||||
if test -z "$immediate" || test $eval_ret = 0 || test -n "$expecting_failure"
|
||||
then
|
||||
@ -353,17 +445,24 @@ test_run_ () {
|
||||
return "$eval_ret"
|
||||
}
|
||||
|
||||
test_skip () {
|
||||
test_start_ () {
|
||||
test_count=$(($test_count+1))
|
||||
maybe_setup_verbose
|
||||
maybe_setup_valgrind
|
||||
}
|
||||
|
||||
test_finish_ () {
|
||||
echo >&3 ""
|
||||
maybe_teardown_valgrind
|
||||
maybe_teardown_verbose
|
||||
}
|
||||
|
||||
test_skip () {
|
||||
to_skip=
|
||||
for skp in $GIT_SKIP_TESTS
|
||||
do
|
||||
case $this_test.$test_count in
|
||||
$skp)
|
||||
to_skip=t
|
||||
break
|
||||
esac
|
||||
done
|
||||
if match_pattern_list $this_test.$test_count $GIT_SKIP_TESTS
|
||||
then
|
||||
to_skip=t
|
||||
fi
|
||||
if test -z "$to_skip" && test -n "$test_prereq" &&
|
||||
! test_have_prereq "$test_prereq"
|
||||
then
|
||||
@ -377,8 +476,8 @@ test_skip () {
|
||||
of_prereq=" of $test_prereq"
|
||||
fi
|
||||
|
||||
say_color skip >&3 "skipping test: $@"
|
||||
say_color skip "ok $test_count # skip $1 (missing $missing_prereq${of_prereq})"
|
||||
say_color skip >&3 "${statusprefix}skipping test: $@"
|
||||
say_color skip "${statusprefix}ok $test_count # skip $1 (missing $missing_prereq${of_prereq})"
|
||||
: true
|
||||
;;
|
||||
*)
|
||||
@ -395,6 +494,8 @@ test_at_end_hook_ () {
|
||||
test_done () {
|
||||
GIT_EXIT_OK=t
|
||||
|
||||
# Note: t0000 relies on $HARNESS_ACTIVE disabling the .counts
|
||||
# output file
|
||||
if test -z "$HARNESS_ACTIVE"
|
||||
then
|
||||
test_results_dir="$TEST_OUTPUT_DIRECTORY/test-results"
|
||||
@ -414,11 +515,11 @@ test_done () {
|
||||
|
||||
if test "$test_fixed" != 0
|
||||
then
|
||||
say_color error "# $test_fixed known breakage(s) vanished; please update test(s)"
|
||||
say_color error "${statusprefix}# $test_fixed known breakage(s) vanished; please update test(s)"
|
||||
fi
|
||||
if test "$test_broken" != 0
|
||||
then
|
||||
say_color warn "# still have $test_broken known breakage(s)"
|
||||
say_color warn "${statusprefix}# still have $test_broken known breakage(s)"
|
||||
fi
|
||||
if test "$test_broken" != 0 || test "$test_fixed" != 0
|
||||
then
|
||||
@ -441,9 +542,9 @@ test_done () {
|
||||
then
|
||||
if test $test_remaining -gt 0
|
||||
then
|
||||
say_color pass "# passed all $msg"
|
||||
say_color pass "${statusprefix}# passed all $msg"
|
||||
fi
|
||||
say "1..$test_count$skip_all"
|
||||
say "${statusprefix}1..$test_count$skip_all"
|
||||
fi
|
||||
|
||||
test -d "$remove_trash" &&
|
||||
@ -457,8 +558,8 @@ test_done () {
|
||||
*)
|
||||
if test $test_external_has_tap -eq 0
|
||||
then
|
||||
say_color error "# failed $test_failure among $msg"
|
||||
say "1..$test_count"
|
||||
say_color error "${statusprefix}# failed $test_failure among $msg"
|
||||
say "${statusprefix}1..$test_count"
|
||||
fi
|
||||
|
||||
exit 1 ;;
|
||||
@ -466,6 +567,9 @@ test_done () {
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
# Set up a directory that we can put in PATH which redirects all git
|
||||
# calls to 'valgrind git ...'.
|
||||
if test -n "$valgrind"
|
||||
then
|
||||
make_symlink () {
|
||||
@ -513,31 +617,43 @@ then
|
||||
make_symlink "$symlink_target" "$GIT_VALGRIND/bin/$base" || exit
|
||||
}
|
||||
|
||||
# override all git executables in TEST_DIRECTORY/..
|
||||
GIT_VALGRIND=$TEST_DIRECTORY/valgrind
|
||||
mkdir -p "$GIT_VALGRIND"/bin
|
||||
for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-*
|
||||
do
|
||||
make_valgrind_symlink $file
|
||||
done
|
||||
# special-case the mergetools loadables
|
||||
make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools"
|
||||
OLDIFS=$IFS
|
||||
IFS=:
|
||||
for path in $PATH
|
||||
do
|
||||
ls "$path"/git-* 2> /dev/null |
|
||||
while read file
|
||||
# In the case of --valgrind-parallel, we only need to do the
|
||||
# wrapping once, in the main script. The worker children all
|
||||
# have $valgrind_only_stride set, so we can skip based on that.
|
||||
if test -z "$valgrind_only_stride"
|
||||
then
|
||||
# override all git executables in TEST_DIRECTORY/..
|
||||
GIT_VALGRIND=$TEST_DIRECTORY/valgrind
|
||||
mkdir -p "$GIT_VALGRIND"/bin
|
||||
for file in $GIT_BUILD_DIR/git* $GIT_BUILD_DIR/test-*
|
||||
do
|
||||
make_valgrind_symlink "$file"
|
||||
make_valgrind_symlink $file
|
||||
done
|
||||
done
|
||||
IFS=$OLDIFS
|
||||
# special-case the mergetools loadables
|
||||
make_symlink "$GIT_BUILD_DIR"/mergetools "$GIT_VALGRIND/bin/mergetools"
|
||||
OLDIFS=$IFS
|
||||
IFS=:
|
||||
for path in $PATH
|
||||
do
|
||||
ls "$path"/git-* 2> /dev/null |
|
||||
while read file
|
||||
do
|
||||
make_valgrind_symlink "$file"
|
||||
done
|
||||
done
|
||||
IFS=$OLDIFS
|
||||
fi
|
||||
PATH=$GIT_VALGRIND/bin:$PATH
|
||||
GIT_EXEC_PATH=$GIT_VALGRIND/bin
|
||||
export GIT_VALGRIND
|
||||
GIT_VALGRIND_MODE="$valgrind"
|
||||
export GIT_VALGRIND_MODE
|
||||
GIT_VALGRIND_ENABLED=t
|
||||
if test -n "$valgrind_only" || test -n "$valgrind_only_stride"
|
||||
then
|
||||
GIT_VALGRIND_ENABLED=
|
||||
fi
|
||||
export GIT_VALGRIND_ENABLED
|
||||
elif test -n "$GIT_TEST_INSTALLED"
|
||||
then
|
||||
GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) ||
|
||||
@ -622,21 +738,53 @@ then
|
||||
else
|
||||
mkdir -p "$TRASH_DIRECTORY"
|
||||
fi
|
||||
|
||||
# Gross hack to spawn N sub-instances of the tests in parallel, and
|
||||
# summarize the results. Note that if this is enabled, the script
|
||||
# terminates at the end of this 'if' block.
|
||||
if test -n "$valgrind_parallel"
|
||||
then
|
||||
for i in $(test_seq 1 $valgrind_parallel)
|
||||
do
|
||||
root="$TRASH_DIRECTORY/vgparallel-$i"
|
||||
mkdir "$root"
|
||||
TEST_OUTPUT_DIRECTORY="$root" \
|
||||
${SHELL_PATH} "$0" \
|
||||
--root="$root" --statusprefix="[$i] " \
|
||||
--valgrind="$valgrind" \
|
||||
--valgrind-only-stride="$valgrind_parallel" \
|
||||
--valgrind-only-offset="$i" &
|
||||
pids="$pids $!"
|
||||
done
|
||||
trap "kill $pids" INT TERM HUP
|
||||
wait $pids
|
||||
trap - INT TERM HUP
|
||||
for i in $(test_seq 1 $valgrind_parallel)
|
||||
do
|
||||
root="$TRASH_DIRECTORY/vgparallel-$i"
|
||||
eval "$(cat "$root/test-results/$(basename "$0" .sh)"-*.counts |
|
||||
sed 's/^\([a-z][a-z]*\) \([0-9][0-9]*\)/inner_\1=\2/')"
|
||||
test_count=$(expr $test_count + $inner_total)
|
||||
test_success=$(expr $test_success + $inner_success)
|
||||
test_fixed=$(expr $test_fixed + $inner_fixed)
|
||||
test_broken=$(expr $test_broken + $inner_broken)
|
||||
test_failure=$(expr $test_failure + $inner_failed)
|
||||
done
|
||||
test_done
|
||||
fi
|
||||
|
||||
# Use -P to resolve symlinks in our working directory so that the cwd
|
||||
# in subprocesses like git equals our $PWD (for pathname comparisons).
|
||||
cd -P "$TRASH_DIRECTORY" || exit 1
|
||||
|
||||
this_test=${0##*/}
|
||||
this_test=${this_test%%-*}
|
||||
for skp in $GIT_SKIP_TESTS
|
||||
do
|
||||
case "$this_test" in
|
||||
$skp)
|
||||
say_color info >&3 "skipping test $this_test altogether"
|
||||
skip_all="skip all tests in $this_test"
|
||||
test_done
|
||||
esac
|
||||
done
|
||||
if match_pattern_list "$this_test" $GIT_SKIP_TESTS
|
||||
then
|
||||
say_color info >&3 "skipping test $this_test altogether"
|
||||
skip_all="skip all tests in $this_test"
|
||||
test_done
|
||||
fi
|
||||
|
||||
# Provide an implementation of the 'yes' utility
|
||||
yes () {
|
||||
|
||||
@ -4,6 +4,9 @@ base=$(basename "$0")
|
||||
|
||||
TOOL_OPTIONS='--leak-check=no'
|
||||
|
||||
test -z "$GIT_VALGRIND_ENABLED" &&
|
||||
exec "$GIT_VALGRIND"/../../"$base" "$@"
|
||||
|
||||
case "$GIT_VALGRIND_MODE" in
|
||||
memcheck-fast)
|
||||
;;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user