t/perf: add iteration setup mechanism to perf-lib
Tests that affect the repo in stateful ways are easier to write if we can run setup steps outside of the measured portion of perf iteration. This change adds a "--setup 'setup-script'" parameter to test_perf. To make invocations easier to understand, I also moved the prerequisites to a new --prereq parameter. The setup facility will be used in the upcoming perf tests for batch mode, but it already helps in some existing tests, like t5302 and t7820. Signed-off-by: Neeraj Singh <neerajsi@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Neeraj Singh
Junio C Hamano
parent
d42bab442d
commit
5dccd9155f
@ -36,7 +36,8 @@ do
|
||||
else
|
||||
prereq=""
|
||||
fi
|
||||
test_perf $prereq "$engine log$GIT_PERF_4220_LOG_OPTS --grep='$pattern'" "
|
||||
test_perf "$engine log$GIT_PERF_4220_LOG_OPTS --grep='$pattern'" \
|
||||
--prereq "$prereq" "
|
||||
git -c grep.patternType=$engine log --pretty=format:%h$GIT_PERF_4220_LOG_OPTS --grep='$pattern' >'out.$engine' || :
|
||||
"
|
||||
done
|
||||
|
||||
@ -26,7 +26,8 @@ do
|
||||
else
|
||||
prereq=""
|
||||
fi
|
||||
test_perf $prereq "$engine log$GIT_PERF_4221_LOG_OPTS --grep='$pattern'" "
|
||||
test_perf "$engine log$GIT_PERF_4221_LOG_OPTS --grep='$pattern'" \
|
||||
--prereq "$prereq" "
|
||||
git -c grep.patternType=$engine log --pretty=format:%h$GIT_PERF_4221_LOG_OPTS --grep='$pattern' >'out.$engine' || :
|
||||
"
|
||||
done
|
||||
|
||||
@ -26,9 +26,8 @@ test_expect_success 'set up thread-counting tests' '
|
||||
done
|
||||
'
|
||||
|
||||
test_perf PERF_EXTRA 'index-pack 0 threads' '
|
||||
rm -rf repo.git &&
|
||||
git init --bare repo.git &&
|
||||
test_perf 'index-pack 0 threads' --prereq PERF_EXTRA \
|
||||
--setup 'rm -rf repo.git && git init --bare repo.git' '
|
||||
GIT_DIR=repo.git git index-pack --threads=1 --stdin < $PACK
|
||||
'
|
||||
|
||||
@ -36,17 +35,15 @@ for t in $threads
|
||||
do
|
||||
THREADS=$t
|
||||
export THREADS
|
||||
test_perf PERF_EXTRA "index-pack $t threads" '
|
||||
rm -rf repo.git &&
|
||||
git init --bare repo.git &&
|
||||
test_perf "index-pack $t threads" --prereq PERF_EXTRA \
|
||||
--setup 'rm -rf repo.git && git init --bare repo.git' '
|
||||
GIT_DIR=repo.git GIT_FORCE_THREADS=1 \
|
||||
git index-pack --threads=$THREADS --stdin <$PACK
|
||||
'
|
||||
done
|
||||
|
||||
test_perf 'index-pack default number of threads' '
|
||||
rm -rf repo.git &&
|
||||
git init --bare repo.git &&
|
||||
test_perf 'index-pack default number of threads' \
|
||||
--setup 'rm -rf repo.git && git init --bare repo.git' '
|
||||
GIT_DIR=repo.git git index-pack --stdin < $PACK
|
||||
'
|
||||
|
||||
|
||||
@ -60,18 +60,6 @@ then
|
||||
esac
|
||||
fi
|
||||
|
||||
if test -n "$GIT_PERF_7519_DROP_CACHE"
|
||||
then
|
||||
# When using GIT_PERF_7519_DROP_CACHE, GIT_PERF_REPEAT_COUNT must be 1 to
|
||||
# generate valid results. Otherwise the caching that happens for the nth
|
||||
# run will negate the validity of the comparisons.
|
||||
if test "$GIT_PERF_REPEAT_COUNT" -ne 1
|
||||
then
|
||||
echo "warning: Setting GIT_PERF_REPEAT_COUNT=1" >&2
|
||||
GIT_PERF_REPEAT_COUNT=1
|
||||
fi
|
||||
fi
|
||||
|
||||
trace_start() {
|
||||
if test -n "$GIT_PERF_7519_TRACE"
|
||||
then
|
||||
@ -167,10 +155,10 @@ setup_for_fsmonitor() {
|
||||
|
||||
test_perf_w_drop_caches () {
|
||||
if test -n "$GIT_PERF_7519_DROP_CACHE"; then
|
||||
test-tool drop-caches
|
||||
test_perf "$1" --setup "test-tool drop-caches" "$2"
|
||||
else
|
||||
test_perf "$@"
|
||||
fi
|
||||
|
||||
test_perf "$@"
|
||||
}
|
||||
|
||||
test_fsmonitor_suite() {
|
||||
|
||||
@ -49,13 +49,15 @@ do
|
||||
fi
|
||||
if ! test_have_prereq PERF_GREP_ENGINES_THREADS
|
||||
then
|
||||
test_perf $prereq "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern'" "
|
||||
test_perf "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern'" \
|
||||
--prereq "$prereq" "
|
||||
git -c grep.patternType=$engine grep$GIT_PERF_7820_GREP_OPTS -- '$pattern' >'out.$engine' || :
|
||||
"
|
||||
else
|
||||
for threads in $GIT_PERF_GREP_THREADS
|
||||
do
|
||||
test_perf PTHREADS,$prereq "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern' with $threads threads" "
|
||||
test_perf "$engine grep$GIT_PERF_7820_GREP_OPTS '$pattern' with $threads threads"
|
||||
--prereq PTHREADS,$prereq "
|
||||
git -c grep.patternType=$engine -c grep.threads=$threads grep$GIT_PERF_7820_GREP_OPTS -- '$pattern' >'out.$engine.$threads' || :
|
||||
"
|
||||
done
|
||||
|
||||
@ -189,19 +189,39 @@ exit $ret' >&3 2>&4
|
||||
}
|
||||
|
||||
test_wrapper_ () {
|
||||
test_wrapper_func_=$1; shift
|
||||
local test_wrapper_func_="$1"; shift
|
||||
local test_title_="$1"; shift
|
||||
test_start_
|
||||
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
|
||||
test "$#" = 2 ||
|
||||
BUG "not 2 or 3 parameters to test-expect-success"
|
||||
test_prereq=
|
||||
test_perf_setup_=
|
||||
while test $# != 0
|
||||
do
|
||||
case $1 in
|
||||
--prereq)
|
||||
test_prereq=$2
|
||||
shift
|
||||
;;
|
||||
--setup)
|
||||
test_perf_setup_=$2
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
test "$#" = 1 || BUG "test_wrapper_ needs 2 positional parameters"
|
||||
export test_prereq
|
||||
if ! test_skip "$@"
|
||||
export test_perf_setup_
|
||||
|
||||
if ! test_skip "$test_title_" "$@"
|
||||
then
|
||||
base=$(basename "$0" .sh)
|
||||
echo "$test_count" >>"$perf_results_dir"/$base.subtests
|
||||
echo "$1" >"$perf_results_dir"/$base.$test_count.descr
|
||||
base="$perf_results_dir"/"$PERF_RESULTS_PREFIX$(basename "$0" .sh)"."$test_count"
|
||||
"$test_wrapper_func_" "$@"
|
||||
"$test_wrapper_func_" "$test_title_" "$@"
|
||||
fi
|
||||
|
||||
test_finish_
|
||||
@ -214,6 +234,16 @@ test_perf_ () {
|
||||
echo "perf $test_count - $1:"
|
||||
fi
|
||||
for i in $(test_seq 1 $GIT_PERF_REPEAT_COUNT); do
|
||||
if test -n "$test_perf_setup_"
|
||||
then
|
||||
say >&3 "setup: $test_perf_setup_"
|
||||
if ! test_eval_ $test_perf_setup_
|
||||
then
|
||||
test_failure_ "$test_perf_setup_"
|
||||
break
|
||||
fi
|
||||
|
||||
fi
|
||||
say >&3 "running: $2"
|
||||
if test_run_perf_ "$2"
|
||||
then
|
||||
@ -237,11 +267,24 @@ test_perf_ () {
|
||||
rm test_time.*
|
||||
}
|
||||
|
||||
# Usage: test_perf 'title' [options] 'perf-test'
|
||||
# Run the performance test script specified in perf-test with
|
||||
# optional prerequisite and setup steps.
|
||||
# Options:
|
||||
# --prereq prerequisites: Skip the test if prequisites aren't met
|
||||
# --setup "setup-steps": Run setup steps prior to each measured iteration
|
||||
#
|
||||
test_perf () {
|
||||
test_wrapper_ test_perf_ "$@"
|
||||
}
|
||||
|
||||
test_size_ () {
|
||||
if test -n "$test_perf_setup_"
|
||||
then
|
||||
say >&3 "setup: $test_perf_setup_"
|
||||
test_eval_ $test_perf_setup_
|
||||
fi
|
||||
|
||||
say >&3 "running: $2"
|
||||
if test_eval_ "$2" 3>"$base".result; then
|
||||
test_ok_ "$1"
|
||||
@ -250,6 +293,14 @@ test_size_ () {
|
||||
fi
|
||||
}
|
||||
|
||||
# Usage: test_size 'title' [options] 'size-test'
|
||||
# Run the size test script specified in size-test with optional
|
||||
# prerequisites and setup steps. Returns the numeric value
|
||||
# returned by size-test.
|
||||
# Options:
|
||||
# --prereq prerequisites: Skip the test if prequisites aren't met
|
||||
# --setup "setup-steps": Run setup steps prior to the size measurement
|
||||
|
||||
test_size () {
|
||||
test_wrapper_ test_size_ "$@"
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user