From 94fcf0e85207979e3a7b35ad348cdd6b7ab42058 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 10 Nov 2022 19:06:01 +0000 Subject: [PATCH 1/5] cache-tree: add perf test comparing update and prime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a performance test comparing the execution times of 'prime_cache_tree()' and 'cache_tree_update(_, WRITE_TREE_SILENT | WRITE_TREE_REPAIR)'. The goal of comparing these two is to identify which is the faster method for rebuilding an invalid cache tree, ultimately to remove one when both are (reundantly) called in immediate succession. Both methods are fast, so the new tests in 'p0090-cache-tree.sh' must call each tested function multiple times to ensure the reported times (to 0.01s resolution) convey the differences between them. The tests compare the timing of a 'test-tool cache-tree' run as a no-op (to capture a baseline for the overhead associated with running the tool), 'cache_tree_update()', and 'prime_cache_tree()' on four scenarios: - A completely valid cache tree - A cache tree with 2 invalid paths - A cache tree with 50 invalid paths - A completely empty cache tree Example results: Test this tree ----------------------------------------------------------- 0090.2: no-op, clean 1.27(0.48+0.52) 0090.3: prime_cache_tree, clean 2.02(0.83+0.85) 0090.4: cache_tree_update, clean 1.30(0.49+0.54) 0090.5: no-op, invalidate 2 1.29(0.48+0.54) 0090.6: prime_cache_tree, invalidate 2 1.98(0.81+0.83) 0090.7: cache_tree_update, invalidate 2 2.12(0.94+0.86) 0090.8: no-op, invalidate 50 1.32(0.50+0.55) 0090.9: prime_cache_tree, invalidate 50 2.10(0.86+0.89) 0090.10: cache_tree_update, invalidate 50 2.35(1.14+0.90) 0090.11: no-op, empty 1.33(0.50+0.54) 0090.12: prime_cache_tree, empty 2.04(0.84+0.87) 0090.13: cache_tree_update, empty 2.51(1.27+0.92) These timings show that, while 'cache_tree_update()' is faster when the cache tree is completely valid, it is equal to or slower than 'prime_cache_tree()' when there are any invalid paths. Since the redundant calls are mostly in scenarios where the cache tree will be at least partially invalid (e.g., 'git reset --hard'), 'prime_cache_tree()' will likely perform better than 'cache_tree_update()' in typical cases. Helped-by: SZEDER Gábor Signed-off-by: Victoria Dye Signed-off-by: Taylor Blau --- Makefile | 1 + t/helper/test-cache-tree.c | 64 ++++++++++++++++++++++++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + t/perf/p0090-cache-tree.sh | 36 +++++++++++++++++++++ 5 files changed, 103 insertions(+) create mode 100644 t/helper/test-cache-tree.c create mode 100755 t/perf/p0090-cache-tree.sh diff --git a/Makefile b/Makefile index 4927379184..3639c7c2a9 100644 --- a/Makefile +++ b/Makefile @@ -723,6 +723,7 @@ TEST_BUILTINS_OBJS += test-advise.o TEST_BUILTINS_OBJS += test-bitmap.o TEST_BUILTINS_OBJS += test-bloom.o TEST_BUILTINS_OBJS += test-bundle-uri.o +TEST_BUILTINS_OBJS += test-cache-tree.o TEST_BUILTINS_OBJS += test-chmtime.o TEST_BUILTINS_OBJS += test-config.o TEST_BUILTINS_OBJS += test-crontab.o diff --git a/t/helper/test-cache-tree.c b/t/helper/test-cache-tree.c new file mode 100644 index 0000000000..93051b25f5 --- /dev/null +++ b/t/helper/test-cache-tree.c @@ -0,0 +1,64 @@ +#include "test-tool.h" +#include "cache.h" +#include "tree.h" +#include "cache-tree.h" +#include "parse-options.h" + +static char const * const test_cache_tree_usage[] = { + N_("test-tool cache-tree (control|prime|update)"), + NULL +}; + +int cmd__cache_tree(int argc, const char **argv) +{ + struct object_id oid; + struct tree *tree; + int empty = 0; + int invalidate_qty = 0; + int i; + + struct option options[] = { + OPT_BOOL(0, "empty", &empty, + N_("clear the cache tree before each iteration")), + OPT_INTEGER_F(0, "invalidate", &invalidate_qty, + N_("number of entries in the cache tree to invalidate (default 0)"), + PARSE_OPT_NONEG), + OPT_END() + }; + + setup_git_directory(); + + argc = parse_options(argc, argv, NULL, options, test_cache_tree_usage, 0); + + if (read_cache() < 0) + die(_("unable to read index file")); + + oidcpy(&oid, &the_index.cache_tree->oid); + tree = parse_tree_indirect(&oid); + if (!tree) + die(_("not a tree object: %s"), oid_to_hex(&oid)); + + if (empty) { + /* clear the cache tree & allocate a new one */ + cache_tree_free(&the_index.cache_tree); + the_index.cache_tree = cache_tree(); + } else if (invalidate_qty) { + /* invalidate the specified number of unique paths */ + float f_interval = (float)the_index.cache_nr / invalidate_qty; + int interval = f_interval < 1.0 ? 1 : (int)f_interval; + for (i = 0; i < invalidate_qty && i * interval < the_index.cache_nr; i++) + cache_tree_invalidate_path(&the_index, the_index.cache[i * interval]->name); + } + + if (argc != 1) + usage_with_options(test_cache_tree_usage, options); + else if (!strcmp(argv[0], "prime")) + prime_cache_tree(the_repository, &the_index, tree); + else if (!strcmp(argv[0], "update")) + cache_tree_update(&the_index, WRITE_TREE_SILENT | WRITE_TREE_REPAIR); + /* use "control" subcommand to specify no-op */ + else if (!!strcmp(argv[0], "control")) + die(_("Unhandled subcommand '%s'"), argv[0]); + + return 0; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index 01cda9358d..547a3be1c8 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -14,6 +14,7 @@ static struct test_cmd cmds[] = { { "bitmap", cmd__bitmap }, { "bloom", cmd__bloom }, { "bundle-uri", cmd__bundle_uri }, + { "cache-tree", cmd__cache_tree }, { "chmtime", cmd__chmtime }, { "config", cmd__config }, { "crontab", cmd__crontab }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index ca2948066f..e44e1d896d 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -8,6 +8,7 @@ int cmd__advise_if_enabled(int argc, const char **argv); int cmd__bitmap(int argc, const char **argv); int cmd__bloom(int argc, const char **argv); int cmd__bundle_uri(int argc, const char **argv); +int cmd__cache_tree(int argc, const char **argv); int cmd__chmtime(int argc, const char **argv); int cmd__config(int argc, const char **argv); int cmd__crontab(int argc, const char **argv); diff --git a/t/perf/p0090-cache-tree.sh b/t/perf/p0090-cache-tree.sh new file mode 100755 index 0000000000..a8eabca2c4 --- /dev/null +++ b/t/perf/p0090-cache-tree.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +test_description="Tests performance of cache tree update operations" + +. ./perf-lib.sh + +test_perf_large_repo +test_checkout_worktree + +count=100 + +test_expect_success 'setup cache tree' ' + git write-tree +' + +test_cache_tree () { + test_perf "$1, $3" " + for i in \$(test_seq $count) + do + test-tool cache-tree $4 $2 + done + " +} + +test_cache_tree_update_functions () { + test_cache_tree 'no-op' 'control' "$1" "$2" + test_cache_tree 'prime_cache_tree' 'prime' "$1" "$2" + test_cache_tree 'cache_tree_update' 'update' "$1" "$2" +} + +test_cache_tree_update_functions "clean" "" +test_cache_tree_update_functions "invalidate 2" "--invalidate 2" +test_cache_tree_update_functions "invalidate 50" "--invalidate 50" +test_cache_tree_update_functions "empty" "--empty" + +test_done From 68fcd48bafa1ef51b1ba40a41cb0fcdbad7acce1 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 10 Nov 2022 19:06:02 +0000 Subject: [PATCH 2/5] unpack-trees: add 'skip_cache_tree_update' option Add (disabled by default) option to skip the 'cache_tree_update()' at the end of 'unpack_trees()'. In many cases, this cache tree update is redundant because the caller of 'unpack_trees()' immediately follows it with 'prime_cache_tree()', rebuilding the entire cache tree from scratch. While these operations aren't the most expensive part of operations like 'git reset', the duplicate calls still create a minor unnecessary slowdown. Introduce an option for callers to skip the 'cache_tree_update()' in 'unpack_trees()' if it is redundant (that is, if 'prime_cache_tree()' is called afterwards). At the moment, no 'unpack_trees()' callers use the new option; they will be updated in subsequent patches. Signed-off-by: Victoria Dye Signed-off-by: Taylor Blau --- unpack-trees.c | 3 ++- unpack-trees.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/unpack-trees.c b/unpack-trees.c index bae812156c..8a762aa077 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -2043,7 +2043,8 @@ int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options if (!ret) { if (git_env_bool("GIT_TEST_CHECK_CACHE_TREE", 0)) cache_tree_verify(the_repository, &o->result); - if (!cache_tree_fully_valid(o->result.cache_tree)) + if (!o->skip_cache_tree_update && + !cache_tree_fully_valid(o->result.cache_tree)) cache_tree_update(&o->result, WRITE_TREE_SILENT | WRITE_TREE_REPAIR); diff --git a/unpack-trees.h b/unpack-trees.h index efb9edfbb2..6ab0d74c84 100644 --- a/unpack-trees.h +++ b/unpack-trees.h @@ -71,7 +71,8 @@ struct unpack_trees_options { quiet, exiting_early, show_all_errors, - dry_run; + dry_run, + skip_cache_tree_update; enum unpack_trees_reset_type reset; const char *prefix; int cache_bottom; From 0e47bca0f7592f8053ffcc530d8afa1d2e364563 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 10 Nov 2022 19:06:03 +0000 Subject: [PATCH 3/5] reset: use 'skip_cache_tree_update' option Enable the 'skip_cache_tree_update' option in the variants that call 'prime_cache_tree()' after 'unpack_trees()' (specifically, 'git reset --mixed' and 'git reset --hard'). This avoids redundantly rebuilding the cache tree in both 'cache_tree_update()' at the end of 'unpack_trees()' and in 'prime_cache_tree()', resulting in a small (but consistent) performance improvement. From the newly-added 'p7102-reset.sh' test: Test before after -------------------------------------------------------------------- 7102.1: reset --hard (...) 2.11(0.40+1.54) 1.97(0.38+1.47) -6.6% Signed-off-by: Victoria Dye Signed-off-by: Taylor Blau --- builtin/reset.c | 2 ++ t/perf/p7102-reset.sh | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100755 t/perf/p7102-reset.sh diff --git a/builtin/reset.c b/builtin/reset.c index fdce6f8c85..ab02777482 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -73,9 +73,11 @@ static int reset_index(const char *ref, const struct object_id *oid, int reset_t case HARD: opts.update = 1; opts.reset = UNPACK_RESET_OVERWRITE_UNTRACKED; + opts.skip_cache_tree_update = 1; break; case MIXED: opts.reset = UNPACK_RESET_PROTECT_UNTRACKED; + opts.skip_cache_tree_update = 1; /* but opts.update=0, so working tree not updated */ break; default: diff --git a/t/perf/p7102-reset.sh b/t/perf/p7102-reset.sh new file mode 100755 index 0000000000..9b039e8691 --- /dev/null +++ b/t/perf/p7102-reset.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +test_description='performance of reset' +. ./perf-lib.sh + +test_perf_default_repo +test_checkout_worktree + +test_perf 'reset --hard with change in tree' ' + base=$(git rev-parse HEAD) && + test_commit --no-tag A && + new=$(git rev-parse HEAD) && + + for i in $(test_seq 10) + do + git reset --hard $new && + git reset --hard $base || return $? + done +' + +test_done From dc5d40f5bc4d93dcc57ee82c5ca8d1369055d8cb Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 10 Nov 2022 19:06:04 +0000 Subject: [PATCH 4/5] read-tree: use 'skip_cache_tree_update' option When running 'read-tree' with a single tree and no prefix, 'prime_cache_tree()' is called after the tree is unpacked. In that situation, skip a redundant call to 'cache_tree_update()' in 'unpack_trees()' by enabling the 'skip_cache_tree_update' unpack option. Removing the redundant cache tree update provides a substantial performance improvement to 'git read-tree ', as shown by a test added to 'p0006-read-tree-checkout.sh': Test before after ---------------------------------------------------------------------- read-tree br_ballast_plus_1 3.94(1.80+1.57) 3.00(1.14+1.28) -23.9% Note that the 'read-tree' in 't1022-read-tree-partial-clone.sh' is updated to read two trees, rather than one. The test was first introduced in d3da223f221 (cache-tree: prefetch in partial clone read-tree, 2021-07-23) to exercise the 'cache_tree_update()' code path, as used in 'git merge'. Since this patch drops the call to 'cache_tree_update()' in single-tree 'git read-tree', change the test to use the two-tree variant so that 'cache_tree_update()' is called as intended. Signed-off-by: Victoria Dye Signed-off-by: Taylor Blau --- builtin/read-tree.c | 4 ++++ t/perf/p0006-read-tree-checkout.sh | 8 ++++++++ t/t1022-read-tree-partial-clone.sh | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/builtin/read-tree.c b/builtin/read-tree.c index f4cbe460b9..45c6652444 100644 --- a/builtin/read-tree.c +++ b/builtin/read-tree.c @@ -249,6 +249,10 @@ int cmd_read_tree(int argc, const char **argv, const char *cmd_prefix) if (opts.debug_unpack) opts.fn = debug_merge; + /* If we're going to prime_cache_tree later, skip cache tree update */ + if (nr_trees == 1 && !opts.prefix) + opts.skip_cache_tree_update = 1; + cache_tree_free(&active_cache_tree); for (i = 0; i < nr_trees; i++) { struct tree *tree = trees[i]; diff --git a/t/perf/p0006-read-tree-checkout.sh b/t/perf/p0006-read-tree-checkout.sh index c481c012d2..325566e18e 100755 --- a/t/perf/p0006-read-tree-checkout.sh +++ b/t/perf/p0006-read-tree-checkout.sh @@ -49,6 +49,14 @@ test_perf "read-tree br_base br_ballast ($nr_files)" ' git read-tree -n -m br_base br_ballast ' +test_perf "read-tree br_ballast_plus_1 ($nr_files)" ' + # Run read-tree 100 times for clearer performance results & comparisons + for i in $(test_seq 100) + do + git read-tree -n -m br_ballast_plus_1 || return 1 + done +' + test_perf "switch between br_base br_ballast ($nr_files)" ' git checkout -q br_base && git checkout -q br_ballast diff --git a/t/t1022-read-tree-partial-clone.sh b/t/t1022-read-tree-partial-clone.sh index a9953b6a71..da53971635 100755 --- a/t/t1022-read-tree-partial-clone.sh +++ b/t/t1022-read-tree-partial-clone.sh @@ -19,7 +19,7 @@ test_expect_success 'read-tree in partial clone prefetches in one batch' ' git -C server config uploadpack.allowfilter 1 && git -C server config uploadpack.allowanysha1inwant 1 && git clone --bare --filter=blob:none "file://$(pwd)/server" client && - GIT_TRACE_PACKET="$(pwd)/trace" git -C client read-tree $TREE && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client read-tree $TREE $TREE && # "done" marks the end of negotiation (once per fetch). Expect that # only one fetch occurs. From 652bd0211d456be052ff3e884a5e29b74ff824c0 Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 10 Nov 2022 19:06:05 +0000 Subject: [PATCH 5/5] rebase: use 'skip_cache_tree_update' option Enable the 'skip_cache_tree_update' option in both 'do_reset()' ('sequencer.c') and 'reset_head()' ('reset.c'). Both of these callers invoke 'prime_cache_tree()' after 'unpack_trees()', so we can remove an unnecessary cache tree rebuild by skipping 'cache_tree_update()'. When testing with 'p3400-rebase.sh' and 'p3404-rebase-interactive.sh', the performance change of this update was negligible, likely due to the operation being dominated by more expensive operations (like checking out trees). However, since the change doesn't harm performance, it's worth keeping this 'unpack_trees()' usage consistent with others that subsequently invoke 'prime_cache_tree()'. Signed-off-by: Victoria Dye Signed-off-by: Taylor Blau --- reset.c | 1 + sequencer.c | 1 + 2 files changed, 2 insertions(+) diff --git a/reset.c b/reset.c index e3383a9334..5ded23611f 100644 --- a/reset.c +++ b/reset.c @@ -128,6 +128,7 @@ int reset_head(struct repository *r, const struct reset_head_opts *opts) unpack_tree_opts.update = 1; unpack_tree_opts.merge = 1; unpack_tree_opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ + unpack_tree_opts.skip_cache_tree_update = 1; init_checkout_metadata(&unpack_tree_opts.meta, switch_to_branch, oid, NULL); if (reset_hard) unpack_tree_opts.reset = UNPACK_RESET_PROTECT_UNTRACKED; diff --git a/sequencer.c b/sequencer.c index f0f1af4d47..0c60800ad1 100644 --- a/sequencer.c +++ b/sequencer.c @@ -3748,6 +3748,7 @@ static int do_reset(struct repository *r, unpack_tree_opts.merge = 1; unpack_tree_opts.update = 1; unpack_tree_opts.preserve_ignored = 0; /* FIXME: !overwrite_ignore */ + unpack_tree_opts.skip_cache_tree_update = 1; init_checkout_metadata(&unpack_tree_opts.meta, name, &oid, NULL); if (repo_read_index_unmerged(r)) {