undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'as/test-tweaks'

Browse Source 
Output from the tests is coloured using "green is okay, yellow is
questionable, red is bad and blue is informative" scheme.

* as/test-tweaks:
  tests: paint unexpectedly fixed known breakages in bold red
  tests: test the test framework more thoroughly
  tests: refactor mechanics of testing in a sub test-lib
  tests: change info messages from yellow/brown to cyan
  tests: paint skipped tests in blue
  tests: paint known breakages in yellow
  tests: test number comes first in 'not ok $count - $message'
This commit is contained in:
Junio C Hamano 2013-01-03 10:29:12 -08:00
commit 6fedcd8188
2 changed files with 183 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
t/t0000-basic.sh
View File

@ -45,39 +45,176 @@ test_expect_failure 'pretend we have a known breakage' '
false false
' '
test_expect_success 'pretend we have fixed a known breakage (run in sub test-lib)' " run_sub_test_lib_test () {
mkdir passing-todo && name="$1" descr="$2" # stdin is the body of the test code
(cd passing-todo && mkdir "$name" &&
cat >passing-todo.sh <<-EOF && (
#!$SHELL_PATH cd "$name" &&
cat >"$name.sh" <<-EOF &&
#!$SHELL_PATH
test_description='A passing TODO test test_description='$descr (run in sub test-lib)
This is run in a sub test-lib so that we do not get incorrect This is run in a sub test-lib so that we do not get incorrect
passing metrics passing metrics
' '
# Point to the t/test-lib.sh, which isn't in ../ as usual # Point to the t/test-lib.sh, which isn't in ../ as usual
TEST_DIRECTORY=\"$TEST_DIRECTORY\" . "\$TEST_DIRECTORY"/test-lib.sh
. \"\$TEST_DIRECTORY\"/test-lib.sh EOF
cat >>"$name.sh" &&
chmod +x "$name.sh" &&
export TEST_DIRECTORY &&
./"$name.sh" >out 2>err
)
}
test_expect_failure 'pretend we have fixed a known breakage' ' check_sub_test_lib_test () {
: name="$1" # stdin is the expected output from the test
' (
cd "$name" &&
! test -s err &&
sed -e 's/^> //' -e 's/Z$//' >expect &&
test_cmp expect out
)
}
test_expect_success 'pretend we have a fully passing test suite' "
run_sub_test_lib_test full-pass '3 passing tests' <<-\\EOF &&
for i in 1 2 3
do
test_expect_success \"passing test #\$i\" 'true'
done
test_done test_done
EOF EOF
chmod +x passing-todo.sh && check_sub_test_lib_test full-pass <<-\\EOF
./passing-todo.sh >out 2>err && > ok 1 - passing test #1
! test -s err && > ok 2 - passing test #2
sed -e 's/^> //' >expect <<-\\EOF && > ok 3 - passing test #3
> ok 1 - pretend we have fixed a known breakage # TODO known breakage > # passed all 3 test(s)
> # fixed 1 known breakage(s) > 1..3
> # passed all 1 test(s) EOF
"
test_expect_success 'pretend we have a partially passing test suite' "
test_must_fail run_sub_test_lib_test \
partial-pass '2/3 tests passing' <<-\\EOF &&
test_expect_success 'passing test #1' 'true'
test_expect_success 'failing test #2' 'false'
test_expect_success 'passing test #3' 'true'
test_done
EOF
check_sub_test_lib_test partial-pass <<-\\EOF
> ok 1 - passing test #1
> not ok 2 - failing test #2
# false
> ok 3 - passing test #3
> # failed 1 among 3 test(s)
> 1..3
EOF
"
test_expect_success 'pretend we have a known breakage' "
run_sub_test_lib_test failing-todo 'A failing TODO test' <<-\\EOF &&
test_expect_success 'passing test' 'true'
test_expect_failure 'pretend we have a known breakage' 'false'
test_done
EOF
check_sub_test_lib_test failing-todo <<-\\EOF
> ok 1 - passing test
> not ok 2 - pretend we have a known breakage # TODO known breakage
> # still have 1 known breakage(s)
> # passed all remaining 1 test(s)
> 1..2
EOF
"
test_expect_success 'pretend we have fixed a known breakage' "
run_sub_test_lib_test passing-todo 'A passing TODO test' <<-\\EOF &&
test_expect_failure 'pretend we have fixed a known breakage' 'true'
test_done
EOF
check_sub_test_lib_test passing-todo <<-\\EOF
> ok 1 - pretend we have fixed a known breakage # TODO known breakage vanished
> # 1 known breakage(s) vanished; please update test(s)
> 1..1 > 1..1
EOF EOF
test_cmp expect out)
" "
test_expect_success 'pretend we have fixed one of two known breakages (run in sub test-lib)' "
run_sub_test_lib_test partially-passing-todos \
'2 TODO tests, one passing' <<-\\EOF &&
test_expect_failure 'pretend we have a known breakage' 'false'
test_expect_success 'pretend we have a passing test' 'true'
test_expect_failure 'pretend we have fixed another known breakage' 'true'
test_done
EOF
check_sub_test_lib_test partially-passing-todos <<-\\EOF
> not ok 1 - pretend we have a known breakage # TODO known breakage
> ok 2 - pretend we have a passing test
> ok 3 - pretend we have fixed another known breakage # TODO known breakage vanished
> # 1 known breakage(s) vanished; please update test(s)
> # still have 1 known breakage(s)
> # passed all remaining 1 test(s)
> 1..3
EOF
"
test_expect_success 'pretend we have a pass, fail, and known breakage' "
test_must_fail run_sub_test_lib_test \
mixed-results1 'mixed results #1' <<-\\EOF &&
test_expect_success 'passing test' 'true'
test_expect_success 'failing test' 'false'
test_expect_failure 'pretend we have a known breakage' 'false'
test_done
EOF
check_sub_test_lib_test mixed-results1 <<-\\EOF
> ok 1 - passing test
> not ok 2 - failing test
> # false
> not ok 3 - pretend we have a known breakage # TODO known breakage
> # still have 1 known breakage(s)
> # failed 1 among remaining 2 test(s)
> 1..3
EOF
"
test_expect_success 'pretend we have a mix of all possible results' "
test_must_fail run_sub_test_lib_test \
mixed-results2 'mixed results #2' <<-\\EOF &&
test_expect_success 'passing test' 'true'
test_expect_success 'passing test' 'true'
test_expect_success 'passing test' 'true'
test_expect_success 'passing test' 'true'
test_expect_success 'failing test' 'false'
test_expect_success 'failing test' 'false'
test_expect_success 'failing test' 'false'
test_expect_failure 'pretend we have a known breakage' 'false'
test_expect_failure 'pretend we have a known breakage' 'false'
test_expect_failure 'pretend we have fixed a known breakage' 'true'
test_done
EOF
check_sub_test_lib_test mixed-results2 <<-\\EOF
> ok 1 - passing test
> ok 2 - passing test
> ok 3 - passing test
> ok 4 - passing test
> not ok 5 - failing test
> # false
> not ok 6 - failing test
> # false
> not ok 7 - failing test
> # false
> not ok 8 - pretend we have a known breakage # TODO known breakage
> not ok 9 - pretend we have a known breakage # TODO known breakage
> ok 10 - pretend we have fixed a known breakage # TODO known breakage vanished
> # 1 known breakage(s) vanished; please update test(s)
> # still have 2 known breakage(s)
> # failed 3 among remaining 7 test(s)
> 1..10
EOF
"
test_set_prereq HAVEIT test_set_prereq HAVEIT
haveit=no haveit=no
test_expect_success HAVEIT 'test runs if prerequisite is satisfied' ' test_expect_success HAVEIT 'test runs if prerequisite is satisfied' '
@ -159,19 +296,8 @@ then
fi fi
test_expect_success 'tests clean up even on failures' " test_expect_success 'tests clean up even on failures' "
mkdir failing-cleanup && test_must_fail run_sub_test_lib_test \
( failing-cleanup 'Failing tests with cleanup commands' <<-\\EOF &&
cd failing-cleanup &&
cat >failing-cleanup.sh <<-EOF &&
#!$SHELL_PATH
test_description='Failing tests with cleanup commands'
# Point to the t/test-lib.sh, which isn't in ../ as usual
TEST_DIRECTORY=\"$TEST_DIRECTORY\"
. \"\$TEST_DIRECTORY\"/test-lib.sh
test_expect_success 'tests clean up even after a failure' ' test_expect_success 'tests clean up even after a failure' '
touch clean-after-failure && touch clean-after-failure &&
test_when_finished rm clean-after-failure && test_when_finished rm clean-after-failure &&
@ -181,29 +307,21 @@ test_expect_success 'tests clean up even on failures' "
test_when_finished \"(exit 2)\" test_when_finished \"(exit 2)\"
' '
test_done test_done
EOF EOF
check_sub_test_lib_test failing-cleanup <<-\\EOF
chmod +x failing-cleanup.sh && > not ok 1 - tests clean up even after a failure
test_must_fail ./failing-cleanup.sh >out 2>err &&
! test -s err &&
! test -f \"trash directory.failing-cleanup/clean-after-failure\" &&
sed -e 's/Z$//' -e 's/^> //' >expect <<-\\EOF &&
> not ok - 1 tests clean up even after a failure
> # Z > # Z
> # touch clean-after-failure && > # touch clean-after-failure &&
> # test_when_finished rm clean-after-failure && > # test_when_finished rm clean-after-failure &&
> # (exit 1) > # (exit 1)
> # Z > # Z
> not ok - 2 failure to clean up causes the test to fail > not ok 2 - failure to clean up causes the test to fail
> # Z > # Z
> # test_when_finished \"(exit 2)\" > # test_when_finished \"(exit 2)\"
> # Z > # Z
> # failed 2 among 2 test(s) > # failed 2 among 2 test(s)
> 1..2 > 1..2
EOF EOF
test_cmp expect out
)
" "
################################################################ ################################################################

29
t/test-lib.sh
View File

@ -212,11 +212,13 @@ then
error) error)
tput bold; tput setaf 1;; # bold red tput bold; tput setaf 1;; # bold red
skip) skip)
tput bold; tput setaf 2;; # bold green tput setaf 4;; # blue
warn)
tput setaf 3;; # brown/yellow
pass) pass)
tput setaf 2;; # green tput setaf 2;; # green
info) info)
tput setaf 3;; # brown tput setaf 6;; # cyan
*) *)
test -n "$quiet" && return;; test -n "$quiet" && return;;
esac esac
@ -298,7 +300,7 @@ test_ok_ () {
test_failure_ () { test_failure_ () {
test_failure=$(($test_failure + 1)) test_failure=$(($test_failure + 1))
say_color error "not ok - $test_count $1" say_color error "not ok $test_count - $1"
shift shift
echo "$@" | sed -e 's/^/# /' echo "$@" | sed -e 's/^/# /'
test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; } test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
@ -306,12 +308,12 @@ test_failure_ () {
test_known_broken_ok_ () { test_known_broken_ok_ () {
test_fixed=$(($test_fixed+1)) test_fixed=$(($test_fixed+1))
say_color "" "ok $test_count - $@ # TODO known breakage" say_color error "ok $test_count - $@ # TODO known breakage vanished"
} }
test_known_broken_failure_ () { test_known_broken_failure_ () {
test_broken=$(($test_broken+1)) test_broken=$(($test_broken+1))
say_color skip "not ok $test_count - $@ # TODO known breakage" say_color warn "not ok $test_count - $@ # TODO known breakage"
} }
test_debug () { test_debug () {
@ -404,13 +406,18 @@ test_done () {
if test "$test_fixed" != 0 if test "$test_fixed" != 0
then then
say_color pass "# fixed $test_fixed known breakage(s)" say_color error "# $test_fixed known breakage(s) vanished; please update test(s)"
fi fi
if test "$test_broken" != 0 if test "$test_broken" != 0
then then
say_color error "# still have $test_broken known breakage(s)" say_color warn "# still have $test_broken known breakage(s)"
msg="remaining $(($test_count-$test_broken)) test(s)" fi
if test "$test_broken" != 0 || test "$test_fixed" != 0
then
test_remaining=$(( $test_count - $test_broken - $test_fixed ))
msg="remaining $test_remaining test(s)"
else else
test_remaining=$test_count
msg="$test_count test(s)" msg="$test_count test(s)"
fi fi
case "$test_failure" in case "$test_failure" in
@ -424,7 +431,7 @@ test_done () {
if test $test_external_has_tap -eq 0 if test $test_external_has_tap -eq 0
then then
if test $test_count -gt 0 if test $test_remaining -gt 0
then then
say_color pass "# passed all $msg" say_color pass "# passed all $msg"
fi fi
@ -615,7 +622,7 @@ for skp in $GIT_SKIP_TESTS
do do
case "$this_test" in case "$this_test" in
$skp) $skp)
say_color skip >&3 "skipping test $this_test altogether" say_color info >&3 "skipping test $this_test altogether"
skip_all="skip all tests in $this_test" skip_all="skip all tests in $this_test"
test_done test_done
esac esac