undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pack-bitmap-write: learn pack.writeBitmapLookupTable and add tests

Browse Source 
Teach Git to provide a way for users to enable/disable bitmap lookup
table extension by providing a config option named 'writeBitmapLookupTable'.
Default is false.

Also add test to verify writting of lookup table.

Mentored-by: Taylor Blau <me@ttaylorr.com>
Co-Mentored-by: Kaartic Sivaraam <kaartic.sivaraam@gmail.com>
Co-Authored-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Abhradeep Chakraborty <chakrabortyabhradeep79@gmail.com>
Reviewed-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Abhradeep Chakraborty 2022-08-14 16:55:09 +00:00 committed by Junio C Hamano
parent 93eb41e240
commit 76f14b777c
9 changed files with 772 additions and 622 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Documentation/config/pack.txt
View File

@ -164,6 +164,13 @@ When writing a multi-pack reachability bitmap, no new namehashes are
computed; instead, any namehashes stored in an existing bitmap are
permuted into their appropriate location when writing a new bitmap.
pack.writeBitmapLookupTable::
When true, Git will include a "lookup table" section in the
bitmap index (if one is written). This table is used to defer
loading individual bitmaps as late as possible. This can be
beneficial in repositories that have relatively large bitmap
indexes. Defaults to false.
pack.writeReverseIndex::
When true, git will write a corresponding .rev file (see:
link:../technical/pack-format.html[Documentation/technical/pack-format.txt])

7
builtin/multi-pack-index.c
View File

@ -87,6 +87,13 @@ static int git_multi_pack_index_write_config(const char *var, const char *value,
opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
}
if (!strcmp(var, "pack.writebitmaplookuptable")) {
if (git_config_bool(var, value))
opts.flags |= MIDX_WRITE_BITMAP_LOOKUP_TABLE;
else
opts.flags &= ~MIDX_WRITE_BITMAP_LOOKUP_TABLE;
}
/*
* We should never make a fall-back call to 'git_default_config', since
* this was already called in 'cmd_multi_pack_index()'.

8
builtin/pack-objects.c
View File

@ -3148,6 +3148,14 @@ static int git_pack_config(const char *k, const char *v, void *cb)
else
write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;
}
if (!strcmp(k, "pack.writebitmaplookuptable")) {
if (git_config_bool(k, v))
write_bitmap_options |= BITMAP_OPT_LOOKUP_TABLE;
else
write_bitmap_options &= ~BITMAP_OPT_LOOKUP_TABLE;
}
if (!strcmp(k, "pack.usebitmaps")) {
use_bitmap_index_default = git_config_bool(k, v);
return 0;

3
midx.c
View File

@ -1070,6 +1070,9 @@ static int write_midx_bitmap(const char *midx_name,
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
options |= BITMAP_OPT_HASH_CACHE;
if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
options |= BITMAP_OPT_LOOKUP_TABLE;
/*
* Build the MIDX-order index based on pdata.objects (which is already
* in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of

1
midx.h
View File

@ -47,6 +47,7 @@ struct multi_pack_index {
#define MIDX_WRITE_REV_INDEX (1 << 1)
#define MIDX_WRITE_BITMAP (1 << 2)
#define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
#define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
const unsigned char *get_midx_checksum(struct multi_pack_index *m);
void get_midx_filename(struct strbuf *out, const char *object_dir);

50
t/t5310-pack-bitmaps.sh
View File

@ -26,6 +26,20 @@ has_any () {
grep -Ff "$1" "$2"
}
test_bitmap_cases () {
writeLookupTable=false
for i in "$@"
do
case "$i" in
"pack.writeBitmapLookupTable") writeLookupTable=true;;
esac
done
test_expect_success 'setup test repository' '
rm -fr * .git &&
git init &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"'
'
setup_bitmap_history
test_expect_success 'setup writing bitmaps during repack' '
@ -43,17 +57,6 @@ test_expect_success 'full repack creates bitmaps' '
basic_bitmap_tests
test_expect_success 'incremental repack fails when bitmaps are requested' '
test_commit more-1 &&
test_must_fail git repack -d 2>err &&
test_i18ngrep "Incremental repacks are incompatible with bitmap" err
'
test_expect_success 'incremental repack can disable bitmaps' '
test_commit more-2 &&
git repack -d --no-write-bitmap-index
'
test_expect_success 'pack-objects respects --local (non-local loose)' '
git init --bare alt.git &&
echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
@ -200,6 +203,7 @@ test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
git clone --bare . compat-us.git &&
(
cd compat-us.git &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
git repack -adb &&
# jgit gc will barf if it does not like our bitmaps
jgit gc
@ -364,6 +368,7 @@ test_expect_success 'pack.preferBitmapTips' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
# create enough commits that not all are receive bitmap
# coverage even if they are all at the tip of some reference.
@ -403,6 +408,7 @@ test_expect_success 'complains about multiple pack bitmaps' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit base &&
@ -424,5 +430,27 @@ test_expect_success 'complains about multiple pack bitmaps' '
grep "ignoring extra bitmap file" err
)
'
}
test_bitmap_cases
test_expect_success 'incremental repack fails when bitmaps are requested' '
test_commit more-1 &&
test_must_fail git repack -d 2>err &&
test_i18ngrep "Incremental repacks are incompatible with bitmap" err
'
test_expect_success 'incremental repack can disable bitmaps' '
test_commit more-2 &&
git repack -d --no-write-bitmap-index
'
test_bitmap_cases "pack.writeBitmapLookupTable"
test_expect_success 'verify writing bitmap lookup table when enabled' '
GIT_TRACE2_EVENT="$(pwd)/trace2" \
git repack -ad &&
grep "\"label\":\"writing_lookup_table\"" trace2
'
test_done

17
t/t5311-pack-bitmaps-shallow.sh
View File

@ -17,7 +17,20 @@ test_description='check bitmap operation with shallow repositories'
# the tree for A. But in a shallow one, we've grafted away
# A, and fetching A to B requires that the other side send
# us the tree for file=1.
test_shallow_bitmaps () {
writeLookupTable=false
for i in "$@"
do
case $i in
"pack.writeBitmapLookupTable") writeLookupTable=true;;
esac
done
test_expect_success 'setup shallow repo' '
rm -rf * .git &&
git init &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
echo 1 >file &&
git add file &&
git commit -m orig &&
@ -35,5 +48,9 @@ test_expect_success 'turn on bitmaps in the parent' '
test_expect_success 'shallow fetch from bitmapped repo' '
(cd shallow.git && git fetch)
'
}
test_shallow_bitmaps
test_shallow_bitmaps "pack.writeBitmapLookupTable"
test_done

69
t/t5326-multi-pack-bitmaps.sh
View File

@ -15,17 +15,24 @@ GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
sane_unset GIT_TEST_MIDX_WRITE_REV
sane_unset GIT_TEST_MIDX_READ_RIDX
midx_bitmap_core
bitmap_reuse_tests() {
from=$1
to=$2
writeLookupTable=false
for i in $3-${$#}
do
case $i in
"pack.writeBitmapLookupTable") writeLookupTable=true;;
esac
done
test_expect_success "setup pack reuse tests ($from -> $to)" '
rm -fr repo &&
git init repo &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit_bulk 16 &&
git tag old-tip &&
@ -43,6 +50,7 @@ bitmap_reuse_tests() {
test_expect_success "build bitmap from existing ($from -> $to)" '
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit_bulk --id=further 16 &&
git tag new-tip &&
@ -59,6 +67,7 @@ bitmap_reuse_tests() {
test_expect_success "verify resulting bitmaps ($from -> $to)" '
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
git for-each-ref &&
git rev-list --test-bitmap refs/tags/old-tip &&
git rev-list --test-bitmap refs/tags/new-tip
@ -66,9 +75,31 @@ bitmap_reuse_tests() {
'
}
bitmap_reuse_tests 'pack' 'MIDX'
bitmap_reuse_tests 'MIDX' 'pack'
bitmap_reuse_tests 'MIDX' 'MIDX'
test_midx_bitmap_cases () {
writeLookupTable=false
writeBitmapLookupTable=
for i in "$@"
do
case $i in
"pack.writeBitmapLookupTable")
writeLookupTable=true
writeBitmapLookupTable="$i"
;;
esac
done
test_expect_success 'setup test_repository' '
rm -rf * .git &&
git init &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"'
'
midx_bitmap_core
bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
test_expect_success 'missing object closure fails gracefully' '
rm -fr repo &&
@ -76,6 +107,7 @@ test_expect_success 'missing object closure fails gracefully' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit loose &&
test_commit packed &&
@ -100,6 +132,7 @@ test_expect_success 'removing a MIDX clears stale bitmaps' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit base &&
git repack &&
git multi-pack-index write --bitmap &&
@ -124,6 +157,7 @@ test_expect_success 'pack.preferBitmapTips' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit_bulk --message="%s" 103 &&
@ -161,6 +195,7 @@ test_expect_success 'writing a bitmap with --refs-snapshot' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit one &&
test_commit two &&
@ -202,6 +237,7 @@ test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit_bulk --message="%s" 103 &&
@ -242,6 +278,7 @@ test_expect_success 'hash-cache values are propagated from pack bitmaps' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit base &&
test_commit base2 &&
@ -272,6 +309,7 @@ test_expect_success 'no .bitmap is written without any objects' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
cat >packs <<-EOF &&
@ -294,6 +332,7 @@ test_expect_success 'graceful fallback when missing reverse index' '
test_when_finished "rm -fr repo" &&
(
cd repo &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
test_commit base &&
@ -306,5 +345,25 @@ test_expect_success 'graceful fallback when missing reverse index' '
! grep "ignoring extra bitmap file" err
)
'
}
test_midx_bitmap_cases
test_midx_bitmap_cases "pack.writeBitmapLookupTable"
test_expect_success 'multi-pack-index write writes lookup table if enabled' '
rm -fr repo &&
git init repo &&
test_when_finished "rm -fr repo" &&
(
cd repo &&
test_commit base &&
git config pack.writeBitmapLookupTable true &&
git repack -ad &&
GIT_TRACE2_EVENT="$(pwd)/trace" \
git multi-pack-index write --bitmap &&
grep "\"label\":\"writing_lookup_table\"" trace
)
'
test_done

20
t/t5327-multi-pack-bitmaps-rev.sh
View File

@ -17,7 +17,27 @@ GIT_TEST_MIDX_READ_RIDX=0
export GIT_TEST_MIDX_WRITE_REV
export GIT_TEST_MIDX_READ_RIDX
test_midx_bitmap_rev () {
writeLookupTable=false
for i in "$@"
do
case $i in
"pack.writeBitmapLookupTable") writeLookupTable=true;;
esac
done
test_expect_success 'setup bitmap config' '
rm -rf * .git &&
git init &&
git config pack.writeBitmapLookupTable '"$writeLookupTable"'
'
midx_bitmap_core rev
midx_bitmap_partial_tests rev
}
test_midx_bitmap_rev
test_midx_bitmap_rev "pack.writeBitmapLookupTable"
test_done