2021-03-30 15:10:45 +02:00
|
|
|
#!/bin/sh
|
|
|
|
|
|
|
|
test_description="test performance of Git operations using the index"
|
|
|
|
|
|
|
|
. ./perf-lib.sh
|
|
|
|
|
|
|
|
test_perf_default_repo
|
|
|
|
|
2021-06-29 04:13:02 +02:00
|
|
|
SPARSE_CONE=f2/f4
|
2021-03-30 15:10:45 +02:00
|
|
|
|
|
|
|
test_expect_success 'setup repo and indexes' '
|
|
|
|
git reset --hard HEAD &&
|
|
|
|
|
|
|
|
# Remove submodules from the example repo, because our
|
|
|
|
# duplication of the entire repo creates an unlikely data shape.
|
|
|
|
if git config --file .gitmodules --get-regexp "submodule.*.path" >modules
|
|
|
|
then
|
|
|
|
git rm $(awk "{print \$2}" modules) &&
|
|
|
|
git commit -m "remove submodules" || return 1
|
|
|
|
fi &&
|
|
|
|
|
|
|
|
echo bogus >a &&
|
|
|
|
cp a b &&
|
|
|
|
git add a b &&
|
|
|
|
git commit -m "level 0" &&
|
|
|
|
BLOB=$(git rev-parse HEAD:a) &&
|
|
|
|
OLD_COMMIT=$(git rev-parse HEAD) &&
|
|
|
|
OLD_TREE=$(git rev-parse HEAD^{tree}) &&
|
|
|
|
|
2021-06-29 04:13:02 +02:00
|
|
|
for i in $(test_seq 1 3)
|
2021-03-30 15:10:45 +02:00
|
|
|
do
|
|
|
|
cat >in <<-EOF &&
|
|
|
|
100755 blob $BLOB a
|
|
|
|
040000 tree $OLD_TREE f1
|
|
|
|
040000 tree $OLD_TREE f2
|
|
|
|
040000 tree $OLD_TREE f3
|
|
|
|
040000 tree $OLD_TREE f4
|
|
|
|
EOF
|
|
|
|
NEW_TREE=$(git mktree <in) &&
|
|
|
|
NEW_COMMIT=$(git commit-tree $NEW_TREE -p $OLD_COMMIT -m "level $i") &&
|
|
|
|
OLD_TREE=$NEW_TREE &&
|
|
|
|
OLD_COMMIT=$NEW_COMMIT || return 1
|
|
|
|
done &&
|
|
|
|
|
|
|
|
git sparse-checkout init --cone &&
|
2021-06-29 04:13:02 +02:00
|
|
|
git sparse-checkout set $SPARSE_CONE &&
|
|
|
|
git checkout -b wide $OLD_COMMIT &&
|
|
|
|
|
|
|
|
for l2 in f1 f2 f3 f4
|
|
|
|
do
|
|
|
|
echo more bogus >>$SPARSE_CONE/$l2/a &&
|
|
|
|
git commit -a -m "edit $SPARSE_CONE/$l2/a" || return 1
|
|
|
|
done &&
|
|
|
|
|
2021-06-29 04:13:03 +02:00
|
|
|
git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v3 &&
|
2021-03-30 15:10:45 +02:00
|
|
|
(
|
2021-06-29 04:13:03 +02:00
|
|
|
cd full-v3 &&
|
2021-03-30 15:10:45 +02:00
|
|
|
git sparse-checkout init --cone &&
|
|
|
|
git sparse-checkout set $SPARSE_CONE &&
|
|
|
|
git config index.version 3 &&
|
2021-06-29 04:13:02 +02:00
|
|
|
git update-index --index-version=3 &&
|
|
|
|
git checkout HEAD~4
|
2021-03-30 15:10:45 +02:00
|
|
|
) &&
|
2021-06-29 04:13:03 +02:00
|
|
|
git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . full-v4 &&
|
2021-03-30 15:10:45 +02:00
|
|
|
(
|
2021-06-29 04:13:03 +02:00
|
|
|
cd full-v4 &&
|
2021-03-30 15:10:45 +02:00
|
|
|
git sparse-checkout init --cone &&
|
|
|
|
git sparse-checkout set $SPARSE_CONE &&
|
|
|
|
git config index.version 4 &&
|
2021-06-29 04:13:02 +02:00
|
|
|
git update-index --index-version=4 &&
|
|
|
|
git checkout HEAD~4
|
2021-03-30 15:11:04 +02:00
|
|
|
) &&
|
2021-06-29 04:13:03 +02:00
|
|
|
git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . sparse-v3 &&
|
2021-03-30 15:11:04 +02:00
|
|
|
(
|
2021-06-29 04:13:03 +02:00
|
|
|
cd sparse-v3 &&
|
2021-03-30 15:11:04 +02:00
|
|
|
git sparse-checkout init --cone --sparse-index &&
|
|
|
|
git sparse-checkout set $SPARSE_CONE &&
|
|
|
|
git config index.version 3 &&
|
2021-06-29 04:13:02 +02:00
|
|
|
git update-index --index-version=3 &&
|
|
|
|
git checkout HEAD~4
|
2021-03-30 15:11:04 +02:00
|
|
|
) &&
|
2021-06-29 04:13:03 +02:00
|
|
|
git -c core.sparseCheckoutCone=true clone --branch=wide --sparse . sparse-v4 &&
|
2021-03-30 15:11:04 +02:00
|
|
|
(
|
2021-06-29 04:13:03 +02:00
|
|
|
cd sparse-v4 &&
|
2021-03-30 15:11:04 +02:00
|
|
|
git sparse-checkout init --cone --sparse-index &&
|
|
|
|
git sparse-checkout set $SPARSE_CONE &&
|
|
|
|
git config index.version 4 &&
|
2021-06-29 04:13:02 +02:00
|
|
|
git update-index --index-version=4 &&
|
|
|
|
git checkout HEAD~4
|
2021-03-30 15:10:45 +02:00
|
|
|
)
|
|
|
|
'
|
|
|
|
|
|
|
|
test_perf_on_all () {
|
|
|
|
command="$@"
|
2021-06-29 04:13:03 +02:00
|
|
|
for repo in full-v3 full-v4 \
|
|
|
|
sparse-v3 sparse-v4
|
2021-03-30 15:10:45 +02:00
|
|
|
do
|
|
|
|
test_perf "$command ($repo)" "
|
|
|
|
(
|
|
|
|
cd $repo &&
|
|
|
|
echo >>$SPARSE_CONE/a &&
|
|
|
|
$command
|
|
|
|
)
|
|
|
|
"
|
|
|
|
done
|
|
|
|
}
|
|
|
|
|
|
|
|
test_perf_on_all git status
|
2022-05-11 01:32:27 +02:00
|
|
|
test_perf_on_all 'git stash && git stash pop'
|
|
|
|
test_perf_on_all 'echo >>new && git stash -u && git stash pop'
|
2021-03-30 15:10:45 +02:00
|
|
|
test_perf_on_all git add -A
|
|
|
|
test_perf_on_all git add .
|
|
|
|
test_perf_on_all git commit -a -m A
|
2021-06-29 04:13:02 +02:00
|
|
|
test_perf_on_all git checkout -f -
|
2022-05-23 15:48:45 +02:00
|
|
|
test_perf_on_all "git sparse-checkout add f2/f3/f1 && git sparse-checkout set $SPARSE_CONE"
|
2021-11-29 16:52:39 +01:00
|
|
|
test_perf_on_all git reset
|
|
|
|
test_perf_on_all git reset --hard
|
|
|
|
test_perf_on_all git reset -- does-not-exist
|
2021-12-06 16:56:00 +01:00
|
|
|
test_perf_on_all git diff
|
|
|
|
test_perf_on_all git diff --cached
|
2021-12-06 16:56:01 +01:00
|
|
|
test_perf_on_all git blame $SPARSE_CONE/a
|
|
|
|
test_perf_on_all git blame $SPARSE_CONE/f3/a
|
2022-03-01 21:24:27 +01:00
|
|
|
test_perf_on_all git read-tree -mu HEAD
|
2022-01-11 19:05:01 +01:00
|
|
|
test_perf_on_all git checkout-index -f --all
|
update-index: add tests for sparse-checkout compatibility
Introduce tests for a variety of `git update-index` use cases, including
performance scenarios. Tests are intended to exercise `update-index` with
options that change the commands interaction with the index (e.g.,
`--again`) and with files/directories inside and outside a sparse checkout
cone.
Of note is that these tests clearly establish the behavior of `git
update-index --add` with untracked, outside-of-cone files. Unlike `git add`,
which fails with an error when provided with such files, `update-index`
succeeds in adding them to the index. Additionally, the `skip-worktree` flag
is *not* automatically added to the new entry. Although this is pre-existing
behavior, there are a couple of reasons to avoid changing it in favor of
consistency with e.g. `git add`:
* `update-index` is low-level command for modifying the index; while it can
perform operations similar to those of `add`, it traditionally has fewer
"guardrails" preventing a user from doing something they may not want to
do (in this case, adding an outside-of-cone, non-`skip-worktree` file to
the index)
* `update-index` typically only exits with an error code if it is incapable
of performing an operation (e.g., if an internal function call fails);
adding a new file outside the sparse checkout definition is still a valid
operation, albeit an inadvisable one
* `update-index` does not implicitly set flags (e.g., `skip-worktree`) when
creating new index entries with `--add`; if flags need to be updated,
options like `--[no-]skip-worktree` allow a user to intentionally set them
All this to say that, while there are valid reasons to consider changing the
treatment of outside-of-cone files in `update-index`, there are also
sufficient reasons for leaving it as-is.
Co-authored-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Victoria Dye <vdye@github.com>
Reviewed-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-01-11 19:05:04 +01:00
|
|
|
test_perf_on_all git update-index --add --remove $SPARSE_CONE/a
|
2022-08-07 06:13:35 +02:00
|
|
|
test_perf_on_all "git rm -f $SPARSE_CONE/a && git checkout HEAD -- $SPARSE_CONE/a"
|
builtin/grep.c: integrate with sparse index
Turn on sparse index and remove ensure_full_index().
Before this patch, `git-grep` utilizes the ensure_full_index() method to
expand the index and search all the entries. Because this method
requires walking all the trees and constructing the index, it is the
slow part within the whole command.
To achieve better performance, this patch uses grep_tree() to search the
sparse directory entries and get rid of the ensure_full_index() method.
Why grep_tree() is a better choice over ensure_full_index()?
1) grep_tree() is as correct as ensure_full_index(). grep_tree() looks
into every sparse-directory entry (represented by a tree) recursively
when looping over the index, and the result of doing so matches the
result of expanding the index.
2) grep_tree() utilizes pathspecs to limit the scope of searching.
ensure_full_index() always expands the index, which means it will
always walk all the trees and blobs in the repo without caring if
the user only wants a subset of the content, i.e. using a pathspec.
On the other hand, grep_tree() will only search the contents that
match the pathspec, and thus possibly walking fewer trees.
3) grep_tree() does not construct and copy back a new index, while
ensure_full_index() does. This also saves some time.
----------------
Performance test
- Summary:
p2000 tests demonstrate a ~71% execution time reduction for
`git grep --cached bogus -- "f2/f1/f1/*"` using tree-walking logic.
However, notice that this result varies depending on the pathspec
given. See below "Command used for testing" for more details.
Test HEAD~ HEAD
-------------------------------------------------------
2000.78: git grep ... (full-v3) 0.35 0.39 (≈)
2000.79: git grep ... (full-v4) 0.36 0.30 (≈)
2000.80: git grep ... (sparse-v3) 0.88 0.23 (-73.8%)
2000.81: git grep ... (sparse-v4) 0.83 0.26 (-68.6%)
- Command used for testing:
git grep --cached bogus -- "f2/f1/f1/*"
The reason for specifying a pathspec is that, if we don't specify a
pathspec, then grep_tree() will walk all the trees and blobs to find the
pattern, and the time consumed doing so is not too different from using
the original ensure_full_index() method, which also spends most of the
time walking trees. However, when a pathspec is specified, this latest
logic will only walk the area of trees enclosed by the pathspec, and the
time consumed is reasonably a lot less.
Generally speaking, because the performance gain is acheived by walking
less trees, which are specified by the pathspec, the HEAD time v.s.
HEAD~ time in sparse-v[3|4], should be proportional to
"pathspec enclosed area" v.s. "all area", respectively. Namely, the
wider the <pathspec> is encompassing, the less the performance
difference between HEAD~ and HEAD, and vice versa.
That is, if we don't specify a pathspec, the performance difference [1]
is indistinguishable: both methods walk all the trees and take generally
same amount of time (even with the index construction time included for
ensure_full_index()).
[1] Performance test result without pathspec (hence walking all trees):
Command used:
git grep --cached bogus
Test HEAD~ HEAD
---------------------------------------------------
2000.78: git grep ... (full-v3) 6.17 5.19 (≈)
2000.79: git grep ... (full-v4) 6.19 5.46 (≈)
2000.80: git grep ... (sparse-v3) 6.57 6.44 (≈)
2000.81: git grep ... (sparse-v4) 6.65 6.28 (≈)
--------------------------
NEEDSWORK about submodules
There are a few NEEDSWORKs that belong to improvements beyond this
topic. See the NEEDSWORK in builtin/grep.c::grep_submodule() for
more context. The other two NEEDSWORKs in t1092 are also relative.
Suggested-by: Derrick Stolee <derrickstolee@github.com>
Helped-by: Derrick Stolee <derrickstolee@github.com>
Helped-by: Victoria Dye <vdye@github.com>
Helped-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Shaoxuan Yuan <shaoxuan.yuan02@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-09-23 06:18:42 +02:00
|
|
|
test_perf_on_all git grep --cached --sparse bogus -- "f2/f1/f1/*"
|
2021-03-30 15:10:45 +02:00
|
|
|
|
|
|
|
test_done
|