7f514b7a5e
When a MIDX contains the new `RIDX` chunk, ensure that the reverse index is read from it instead of the on-disk .rev file. Since we need to encode the object order in the MIDX itself for correctness reasons, there is no point in storing the same data again outside of the MIDX. So, this patch stops writing separate .rev files, and reads it out of the MIDX itself. This is possible to do with relatively little new code, since the format of the RIDX chunk is identical to the data in the .rev file. In other words, we can implement this by pointing the `revindex_data` field at the reverse index chunk of the MIDX instead of the .rev file without any other changes. Note that we have two knobs that are adjusted for the new tests: GIT_TEST_MIDX_WRITE_REV and GIT_TEST_MIDX_READ_RIDX. The former controls whether the MIDX .rev is written at all, and the latter controls whether we read the MIDX's RIDX chunk. Both are necessary to ensure that the test added at the beginning of this series continues to work. This is because we always need to write the RIDX chunk in the MIDX in order to change its checksum, but we want to make sure reading the existing .rev file still works (since the RIDX chunk takes precedence by default). Arguably this isn't a very interesting mode to test, because the precedence rules mean that we'll always read the RIDX chunk over the .rev file. But it makes it impossible for a user to induce corruption in their repository by adjusting the test knobs (since if we had an either/or knob they could stop writing the RIDX chunk, allowing them to tweak the MIDX's object order without changing its checksum). Signed-off-by: Taylor Blau <me@ttaylorr.com> Reviewed-by: Derrick Stolee <dstolee@microsoft.com> Reviewed-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
452 lines
13 KiB
Bash
452 lines
13 KiB
Bash
# Helpers for scripts testing bitmap functionality; see t5310 for
|
|
# example usage.
|
|
|
|
objdir=.git/objects
|
|
midx=$objdir/pack/multi-pack-index
|
|
|
|
# Compare a file containing rev-list bitmap traversal output to its non-bitmap
|
|
# counterpart. You can't just use test_cmp for this, because the two produce
|
|
# subtly different output:
|
|
#
|
|
# - regular output is in traversal order, whereas bitmap is split by type,
|
|
# with non-packed objects at the end
|
|
#
|
|
# - regular output has a space and the pathname appended to non-commit
|
|
# objects; bitmap output omits this
|
|
#
|
|
# This function normalizes and compares the two. The second file should
|
|
# always be the bitmap output.
|
|
test_bitmap_traversal () {
|
|
if test "$1" = "--no-confirm-bitmaps"
|
|
then
|
|
shift
|
|
elif cmp "$1" "$2"
|
|
then
|
|
echo >&2 "identical raw outputs; are you sure bitmaps were used?"
|
|
return 1
|
|
fi &&
|
|
cut -d' ' -f1 "$1" | sort >"$1.normalized" &&
|
|
sort "$2" >"$2.normalized" &&
|
|
test_cmp "$1.normalized" "$2.normalized" &&
|
|
rm -f "$1.normalized" "$2.normalized"
|
|
}
|
|
|
|
# To ensure the logic for "maximal commits" is exercised, make
|
|
# the repository a bit more complicated.
|
|
#
|
|
# other second
|
|
# * *
|
|
# (99 commits) (99 commits)
|
|
# * *
|
|
# |\ /|
|
|
# | * octo-other octo-second * |
|
|
# |/|\_________ ____________/|\|
|
|
# | \ \/ __________/ |
|
|
# | | ________/\ / |
|
|
# * |/ * merge-right *
|
|
# | _|__________/ \____________ |
|
|
# |/ | \|
|
|
# (l1) * * merge-left * (r1)
|
|
# | / \________________________ |
|
|
# |/ \|
|
|
# (l2) * * (r2)
|
|
# \___________________________ |
|
|
# \|
|
|
# * (base)
|
|
#
|
|
# We only push bits down the first-parent history, which
|
|
# makes some of these commits unimportant!
|
|
#
|
|
# The important part for the maximal commit algorithm is how
|
|
# the bitmasks are extended. Assuming starting bit positions
|
|
# for second (bit 0) and other (bit 1), the bitmasks at the
|
|
# end should be:
|
|
#
|
|
# second: 1 (maximal, selected)
|
|
# other: 01 (maximal, selected)
|
|
# (base): 11 (maximal)
|
|
#
|
|
# This complicated history was important for a previous
|
|
# version of the walk that guarantees never walking a
|
|
# commit multiple times. That goal might be important
|
|
# again, so preserve this complicated case. For now, this
|
|
# test will guarantee that the bitmaps are computed
|
|
# correctly, even with the repeat calculations.
|
|
setup_bitmap_history() {
|
|
test_expect_success 'setup repo with moderate-sized history' '
|
|
test_commit_bulk --id=file 10 &&
|
|
git branch -M second &&
|
|
git checkout -b other HEAD~5 &&
|
|
test_commit_bulk --id=side 10 &&
|
|
|
|
# add complicated history setup, including merges and
|
|
# ambiguous merge-bases
|
|
|
|
git checkout -b merge-left other~2 &&
|
|
git merge second~2 -m "merge-left" &&
|
|
|
|
git checkout -b merge-right second~1 &&
|
|
git merge other~1 -m "merge-right" &&
|
|
|
|
git checkout -b octo-second second &&
|
|
git merge merge-left merge-right -m "octopus-second" &&
|
|
|
|
git checkout -b octo-other other &&
|
|
git merge merge-left merge-right -m "octopus-other" &&
|
|
|
|
git checkout other &&
|
|
git merge octo-other -m "pull octopus" &&
|
|
|
|
git checkout second &&
|
|
git merge octo-second -m "pull octopus" &&
|
|
|
|
# Remove these branches so they are not selected
|
|
# as bitmap tips
|
|
git branch -D merge-left &&
|
|
git branch -D merge-right &&
|
|
git branch -D octo-other &&
|
|
git branch -D octo-second &&
|
|
|
|
# add padding to make these merges less interesting
|
|
# and avoid having them selected for bitmaps
|
|
test_commit_bulk --id=file 100 &&
|
|
git checkout other &&
|
|
test_commit_bulk --id=side 100 &&
|
|
git checkout second &&
|
|
|
|
bitmaptip=$(git rev-parse second) &&
|
|
blob=$(echo tagged-blob | git hash-object -w --stdin) &&
|
|
git tag tagged-blob $blob
|
|
'
|
|
}
|
|
|
|
rev_list_tests_head () {
|
|
test_expect_success "counting commits via bitmap ($state, $branch)" '
|
|
git rev-list --count $branch >expect &&
|
|
git rev-list --use-bitmap-index --count $branch >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success "counting partial commits via bitmap ($state, $branch)" '
|
|
git rev-list --count $branch~5..$branch >expect &&
|
|
git rev-list --use-bitmap-index --count $branch~5..$branch >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success "counting commits with limit ($state, $branch)" '
|
|
git rev-list --count -n 1 $branch >expect &&
|
|
git rev-list --use-bitmap-index --count -n 1 $branch >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success "counting non-linear history ($state, $branch)" '
|
|
git rev-list --count other...second >expect &&
|
|
git rev-list --use-bitmap-index --count other...second >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success "counting commits with limiting ($state, $branch)" '
|
|
git rev-list --count $branch -- 1.t >expect &&
|
|
git rev-list --use-bitmap-index --count $branch -- 1.t >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success "counting objects via bitmap ($state, $branch)" '
|
|
git rev-list --count --objects $branch >expect &&
|
|
git rev-list --use-bitmap-index --count --objects $branch >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success "enumerate commits ($state, $branch)" '
|
|
git rev-list --use-bitmap-index $branch >actual &&
|
|
git rev-list $branch >expect &&
|
|
test_bitmap_traversal --no-confirm-bitmaps expect actual
|
|
'
|
|
|
|
test_expect_success "enumerate --objects ($state, $branch)" '
|
|
git rev-list --objects --use-bitmap-index $branch >actual &&
|
|
git rev-list --objects $branch >expect &&
|
|
test_bitmap_traversal expect actual
|
|
'
|
|
|
|
test_expect_success "bitmap --objects handles non-commit objects ($state, $branch)" '
|
|
git rev-list --objects --use-bitmap-index $branch tagged-blob >actual &&
|
|
grep $blob actual
|
|
'
|
|
}
|
|
|
|
rev_list_tests () {
|
|
state=$1
|
|
|
|
for branch in "second" "other"
|
|
do
|
|
rev_list_tests_head
|
|
done
|
|
}
|
|
|
|
basic_bitmap_tests () {
|
|
tip="$1"
|
|
test_expect_success 'rev-list --test-bitmap verifies bitmaps' "
|
|
git rev-list --test-bitmap "${tip:-HEAD}"
|
|
"
|
|
|
|
rev_list_tests 'full bitmap'
|
|
|
|
test_expect_success 'clone from bitmapped repository' '
|
|
rm -fr clone.git &&
|
|
git clone --no-local --bare . clone.git &&
|
|
git rev-parse HEAD >expect &&
|
|
git --git-dir=clone.git rev-parse HEAD >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success 'partial clone from bitmapped repository' '
|
|
test_config uploadpack.allowfilter true &&
|
|
rm -fr partial-clone.git &&
|
|
git clone --no-local --bare --filter=blob:none . partial-clone.git &&
|
|
(
|
|
cd partial-clone.git &&
|
|
pack=$(echo objects/pack/*.pack) &&
|
|
git verify-pack -v "$pack" >have &&
|
|
awk "/blob/ { print \$1 }" <have >blobs &&
|
|
# we expect this single blob because of the direct ref
|
|
git rev-parse refs/tags/tagged-blob >expect &&
|
|
test_cmp expect blobs
|
|
)
|
|
'
|
|
|
|
test_expect_success 'setup further non-bitmapped commits' '
|
|
test_commit_bulk --id=further 10
|
|
'
|
|
|
|
rev_list_tests 'partial bitmap'
|
|
|
|
test_expect_success 'fetch (partial bitmap)' '
|
|
git --git-dir=clone.git fetch origin second:second &&
|
|
git rev-parse HEAD >expect &&
|
|
git --git-dir=clone.git rev-parse HEAD >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success 'enumerating progress counts pack-reused objects' '
|
|
count=$(git rev-list --objects --all --count) &&
|
|
git repack -adb &&
|
|
|
|
# check first with only reused objects; confirm that our
|
|
# progress showed the right number, and also that we did
|
|
# pack-reuse as expected. Check only the final "done"
|
|
# line of the meter (there may be an arbitrary number of
|
|
# intermediate lines ending with CR).
|
|
GIT_PROGRESS_DELAY=0 \
|
|
git pack-objects --all --stdout --progress \
|
|
</dev/null >/dev/null 2>stderr &&
|
|
grep "Enumerating objects: $count, done" stderr &&
|
|
grep "pack-reused $count" stderr &&
|
|
|
|
# now the same but with one non-reused object
|
|
git commit --allow-empty -m "an extra commit object" &&
|
|
GIT_PROGRESS_DELAY=0 \
|
|
git pack-objects --all --stdout --progress \
|
|
</dev/null >/dev/null 2>stderr &&
|
|
grep "Enumerating objects: $((count+1)), done" stderr &&
|
|
grep "pack-reused $count" stderr
|
|
'
|
|
}
|
|
|
|
# have_delta <obj> <expected_base>
|
|
#
|
|
# Note that because this relies on cat-file, it might find _any_ copy of an
|
|
# object in the repository. The caller is responsible for making sure
|
|
# there's only one (e.g., via "repack -ad", or having just fetched a copy).
|
|
have_delta () {
|
|
echo $2 >expect &&
|
|
echo $1 | git cat-file --batch-check="%(deltabase)" >actual &&
|
|
test_cmp expect actual
|
|
}
|
|
|
|
midx_checksum () {
|
|
test-tool read-midx --checksum "$1"
|
|
}
|
|
|
|
# midx_pack_source <obj>
|
|
midx_pack_source () {
|
|
test-tool read-midx --show-objects .git/objects | grep "^$1 " | cut -f2
|
|
}
|
|
|
|
test_rev_exists () {
|
|
commit="$1"
|
|
kind="$2"
|
|
|
|
test_expect_success "reverse index exists ($kind)" '
|
|
GIT_TRACE2_EVENT=$(pwd)/event.trace \
|
|
git rev-list --test-bitmap "$commit" &&
|
|
|
|
if test "rev" = "$kind"
|
|
then
|
|
test_path_is_file $midx-$(midx_checksum $objdir).rev
|
|
fi &&
|
|
grep "\"category\":\"load_midx_revindex\",\"key\":\"source\",\"value\":\"$kind\"" event.trace
|
|
'
|
|
}
|
|
|
|
midx_bitmap_core () {
|
|
rev_kind="${1:-midx}"
|
|
|
|
setup_bitmap_history
|
|
|
|
test_expect_success 'create single-pack midx with bitmaps' '
|
|
git repack -ad &&
|
|
git multi-pack-index write --bitmap &&
|
|
test_path_is_file $midx &&
|
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap
|
|
'
|
|
|
|
test_rev_exists HEAD "$rev_kind"
|
|
|
|
basic_bitmap_tests
|
|
|
|
test_expect_success 'create new additional packs' '
|
|
for i in $(test_seq 1 16)
|
|
do
|
|
test_commit "$i" &&
|
|
git repack -d || return 1
|
|
done &&
|
|
|
|
git checkout -b other2 HEAD~8 &&
|
|
for i in $(test_seq 1 8)
|
|
do
|
|
test_commit "side-$i" &&
|
|
git repack -d || return 1
|
|
done &&
|
|
git checkout second
|
|
'
|
|
|
|
test_expect_success 'create multi-pack midx with bitmaps' '
|
|
git multi-pack-index write --bitmap &&
|
|
|
|
ls $objdir/pack/pack-*.pack >packs &&
|
|
test_line_count = 25 packs &&
|
|
|
|
test_path_is_file $midx &&
|
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap
|
|
'
|
|
|
|
test_rev_exists HEAD "$rev_kind"
|
|
|
|
basic_bitmap_tests
|
|
|
|
test_expect_success '--no-bitmap is respected when bitmaps exist' '
|
|
git multi-pack-index write --bitmap &&
|
|
|
|
test_commit respect--no-bitmap &&
|
|
git repack -d &&
|
|
|
|
test_path_is_file $midx &&
|
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
|
|
|
|
git multi-pack-index write --no-bitmap &&
|
|
|
|
test_path_is_file $midx &&
|
|
test_path_is_missing $midx-$(midx_checksum $objdir).bitmap &&
|
|
test_path_is_missing $midx-$(midx_checksum $objdir).rev
|
|
'
|
|
|
|
test_expect_success 'setup midx with base from later pack' '
|
|
# Write a and b so that "a" is a delta on top of base "b", since Git
|
|
# prefers to delete contents out of a base rather than add to a shorter
|
|
# object.
|
|
test_seq 1 128 >a &&
|
|
test_seq 1 130 >b &&
|
|
|
|
git add a b &&
|
|
git commit -m "initial commit" &&
|
|
|
|
a=$(git rev-parse HEAD:a) &&
|
|
b=$(git rev-parse HEAD:b) &&
|
|
|
|
# In the first pack, "a" is stored as a delta to "b".
|
|
p1=$(git pack-objects .git/objects/pack/pack <<-EOF
|
|
$a
|
|
$b
|
|
EOF
|
|
) &&
|
|
|
|
# In the second pack, "a" is missing, and "b" is not a delta nor base to
|
|
# any other object.
|
|
p2=$(git pack-objects .git/objects/pack/pack <<-EOF
|
|
$b
|
|
$(git rev-parse HEAD)
|
|
$(git rev-parse HEAD^{tree})
|
|
EOF
|
|
) &&
|
|
|
|
git prune-packed &&
|
|
# Use the second pack as the preferred source, so that "b" occurs
|
|
# earlier in the MIDX object order, rendering "a" unusable for pack
|
|
# reuse.
|
|
git multi-pack-index write --bitmap --preferred-pack=pack-$p2.idx &&
|
|
|
|
have_delta $a $b &&
|
|
test $(midx_pack_source $a) != $(midx_pack_source $b)
|
|
'
|
|
|
|
rev_list_tests 'full bitmap with backwards delta'
|
|
|
|
test_expect_success 'clone with bitmaps enabled' '
|
|
git clone --no-local --bare . clone-reverse-delta.git &&
|
|
test_when_finished "rm -fr clone-reverse-delta.git" &&
|
|
|
|
git rev-parse HEAD >expect &&
|
|
git --git-dir=clone-reverse-delta.git rev-parse HEAD >actual &&
|
|
test_cmp expect actual
|
|
'
|
|
|
|
test_expect_success 'changing the preferred pack does not corrupt bitmaps' '
|
|
rm -fr repo &&
|
|
git init repo &&
|
|
test_when_finished "rm -fr repo" &&
|
|
(
|
|
cd repo &&
|
|
|
|
test_commit A &&
|
|
test_commit B &&
|
|
|
|
git rev-list --objects --no-object-names HEAD^ >A.objects &&
|
|
git rev-list --objects --no-object-names HEAD^.. >B.objects &&
|
|
|
|
A=$(git pack-objects $objdir/pack/pack <A.objects) &&
|
|
B=$(git pack-objects $objdir/pack/pack <B.objects) &&
|
|
|
|
cat >indexes <<-EOF &&
|
|
pack-$A.idx
|
|
pack-$B.idx
|
|
EOF
|
|
|
|
git multi-pack-index write --bitmap --stdin-packs \
|
|
--preferred-pack=pack-$A.pack <indexes &&
|
|
git rev-list --test-bitmap A &&
|
|
|
|
git multi-pack-index write --bitmap --stdin-packs \
|
|
--preferred-pack=pack-$B.pack <indexes &&
|
|
git rev-list --test-bitmap A
|
|
)
|
|
'
|
|
}
|
|
|
|
midx_bitmap_partial_tests () {
|
|
rev_kind="${1:-midx}"
|
|
|
|
test_expect_success 'setup partial bitmaps' '
|
|
test_commit packed &&
|
|
git repack &&
|
|
test_commit loose &&
|
|
git multi-pack-index write --bitmap 2>err &&
|
|
test_path_is_file $midx &&
|
|
test_path_is_file $midx-$(midx_checksum $objdir).bitmap
|
|
'
|
|
|
|
test_rev_exists HEAD~ "$rev_kind"
|
|
|
|
basic_bitmap_tests HEAD~
|
|
}
|