2007-04-10 22:26:10 +02:00
|
|
|
#!/bin/sh
|
|
|
|
|
#
|
|
|
|
|
# Copyright (c) 2007 Nicolas Pitre
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
test_description='pack index with 64-bit offsets and object CRC'
|
|
|
|
|
. ./test-lib.sh
|
|
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'setup' \
|
|
|
|
|
'rm -rf .git
|
2007-07-03 07:52:14 +02:00
|
|
|
git init &&
|
2008-12-15 20:44:30 +01:00
|
|
|
git config pack.threads 1 &&
|
2007-05-02 18:59:55 +02:00
|
|
|
i=1 &&
|
2008-10-23 02:59:22 +02:00
|
|
|
while test $i -le 100
|
2007-04-10 22:26:10 +02:00
|
|
|
do
|
2008-10-23 02:59:22 +02:00
|
|
|
iii=`printf '%03i' $i`
|
|
|
|
|
test-genrandom "bar" 200 > wide_delta_$iii &&
|
|
|
|
|
test-genrandom "baz $iii" 50 >> wide_delta_$iii &&
|
|
|
|
|
test-genrandom "foo"$i 100 > deep_delta_$iii &&
|
|
|
|
|
test-genrandom "foo"`expr $i + 1` 100 >> deep_delta_$iii &&
|
|
|
|
|
test-genrandom "foo"`expr $i + 2` 100 >> deep_delta_$iii &&
|
|
|
|
|
echo $iii >file_$iii &&
|
|
|
|
|
test-genrandom "$iii" 8192 >>file_$iii &&
|
|
|
|
|
git update-index --add file_$iii deep_delta_$iii wide_delta_$iii &&
|
|
|
|
|
i=`expr $i + 1` || return 1
|
2007-04-10 22:26:10 +02:00
|
|
|
done &&
|
2007-04-24 07:05:22 +02:00
|
|
|
{ echo 101 && test-genrandom 100 8192; } >file_101 &&
|
2007-07-03 07:52:14 +02:00
|
|
|
git update-index --add file_101 &&
|
|
|
|
|
tree=`git write-tree` &&
|
|
|
|
|
commit=`git commit-tree $tree </dev/null` && {
|
2007-04-10 22:26:10 +02:00
|
|
|
echo $tree &&
|
2007-07-03 07:52:14 +02:00
|
|
|
git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/"
|
2007-04-10 22:26:10 +02:00
|
|
|
} >obj-list &&
|
2007-07-03 07:52:14 +02:00
|
|
|
git update-ref HEAD $commit'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'pack-objects with index version 1' \
|
2007-07-03 07:52:14 +02:00
|
|
|
'pack1=$(git pack-objects --index-version=1 test-1 <obj-list) &&
|
|
|
|
|
git verify-pack -v "test-1-${pack1}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'pack-objects with index version 2' \
|
2007-07-03 07:52:14 +02:00
|
|
|
'pack2=$(git pack-objects --index-version=2 test-2 <obj-list) &&
|
|
|
|
|
git verify-pack -v "test-2-${pack2}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'both packs should be identical' \
|
|
|
|
|
'cmp "test-1-${pack1}.pack" "test-2-${pack2}.pack"'
|
|
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'index v1 and index v2 should be different' \
|
2008-02-01 10:50:53 +01:00
|
|
|
'! cmp "test-1-${pack1}.idx" "test-2-${pack2}.idx"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'index-pack with index version 1' \
|
2008-09-03 10:59:29 +02:00
|
|
|
'git index-pack --index-version=1 -o 1.idx "test-1-${pack1}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'index-pack with index version 2' \
|
2008-09-03 10:59:29 +02:00
|
|
|
'git index-pack --index-version=2 -o 2.idx "test-1-${pack1}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'index-pack results should match pack-objects ones' \
|
|
|
|
|
'cmp "test-1-${pack1}.idx" "1.idx" &&
|
|
|
|
|
cmp "test-2-${pack2}.idx" "2.idx"'
|
|
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'index v2: force some 64-bit offsets with pack-objects' \
|
2007-11-13 21:04:58 +01:00
|
|
|
'pack3=$(git pack-objects --index-version=2,0x40000 test-3 <obj-list)'
|
|
|
|
|
|
|
|
|
|
have_64bits=
|
|
|
|
|
if msg=$(git verify-pack -v "test-3-${pack3}.pack" 2>&1) ||
|
2008-05-14 06:01:22 +02:00
|
|
|
! (echo "$msg" | grep "pack too large .* off_t")
|
2007-11-13 21:04:58 +01:00
|
|
|
then
|
|
|
|
|
have_64bits=t
|
|
|
|
|
else
|
|
|
|
|
say "skipping tests concerning 64-bit offsets"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
test "$have_64bits" &&
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'index v2: verify a pack with some 64-bit offsets' \
|
|
|
|
|
'git verify-pack -v "test-3-${pack3}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2007-11-13 21:04:58 +01:00
|
|
|
test "$have_64bits" &&
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'64-bit offsets: should be different from previous index v2 results' \
|
2008-02-01 10:50:53 +01:00
|
|
|
'! cmp "test-2-${pack2}.idx" "test-3-${pack3}.idx"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2007-11-13 21:04:58 +01:00
|
|
|
test "$have_64bits" &&
|
2007-04-10 22:26:10 +02:00
|
|
|
test_expect_success \
|
|
|
|
|
'index v2: force some 64-bit offsets with index-pack' \
|
2008-09-03 10:59:29 +02:00
|
|
|
'git index-pack --index-version=2,0x40000 -o 3.idx "test-1-${pack1}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2007-11-13 21:04:58 +01:00
|
|
|
test "$have_64bits" &&
|
2007-04-10 22:26:10 +02:00
|
|
|
test_expect_success \
|
|
|
|
|
'64-bit offsets: index-pack result should match pack-objects one' \
|
|
|
|
|
'cmp "test-3-${pack3}.idx" "3.idx"'
|
|
|
|
|
|
2008-10-23 02:59:22 +02:00
|
|
|
# returns the object number for given object in given pack index
|
|
|
|
|
index_obj_nr()
|
|
|
|
|
{
|
|
|
|
|
idx_file=$1
|
|
|
|
|
object_sha1=$2
|
|
|
|
|
nr=0
|
|
|
|
|
git show-index < $idx_file |
|
|
|
|
|
while read offs sha1 extra
|
|
|
|
|
do
|
|
|
|
|
nr=$(($nr + 1))
|
|
|
|
|
test "$sha1" = "$object_sha1" || continue
|
|
|
|
|
echo "$(($nr - 1))"
|
|
|
|
|
break
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# returns the pack offset for given object as found in given pack index
|
|
|
|
|
index_obj_offset()
|
|
|
|
|
{
|
|
|
|
|
idx_file=$1
|
|
|
|
|
object_sha1=$2
|
|
|
|
|
git show-index < $idx_file | grep $object_sha1 |
|
|
|
|
|
( read offs extra && echo "$offs" )
|
|
|
|
|
}
|
|
|
|
|
|
2007-04-10 22:26:10 +02:00
|
|
|
test_expect_success \
|
|
|
|
|
'[index v1] 1) stream pack to repository' \
|
2008-09-03 10:59:29 +02:00
|
|
|
'git index-pack --index-version=1 --stdin < "test-1-${pack1}.pack" &&
|
2007-07-03 07:52:14 +02:00
|
|
|
git prune-packed &&
|
|
|
|
|
git count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
|
2007-04-10 22:26:10 +02:00
|
|
|
cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
|
|
|
|
|
cmp "test-1-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
|
|
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'[index v1] 2) create a stealth corruption in a delta base reference' \
|
2008-10-23 02:59:22 +02:00
|
|
|
'# This test assumes file_101 is a delta smaller than 16 bytes.
|
|
|
|
|
# It should be against file_100 but we substitute its base for file_099
|
|
|
|
|
sha1_101=`git hash-object file_101` &&
|
|
|
|
|
sha1_099=`git hash-object file_099` &&
|
|
|
|
|
offs_101=`index_obj_offset 1.idx $sha1_101` &&
|
|
|
|
|
nr_099=`index_obj_nr 1.idx $sha1_099` &&
|
|
|
|
|
chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
|
|
|
|
|
dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($offs_101 + 1)) \
|
|
|
|
|
if=".git/objects/pack/pack-${pack1}.idx" \
|
|
|
|
|
skip=$((4 + 256 * 4 + $nr_099 * 24)) \
|
|
|
|
|
bs=1 count=20 conv=notrunc &&
|
|
|
|
|
git cat-file blob $sha1_101 > file_101_foo1'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'[index v1] 3) corrupted delta happily returned wrong data' \
|
2008-10-23 02:59:22 +02:00
|
|
|
'test -f file_101_foo1 && ! cmp file_101 file_101_foo1'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'[index v1] 4) confirm that the pack is actually corrupted' \
|
2008-07-12 17:47:52 +02:00
|
|
|
'test_must_fail git fsck --full $commit'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'[index v1] 5) pack-objects happily reuses corrupted data' \
|
2007-07-03 07:52:14 +02:00
|
|
|
'pack4=$(git pack-objects test-4 <obj-list) &&
|
2007-04-10 22:26:10 +02:00
|
|
|
test -f "test-4-${pack1}.pack"'
|
|
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'[index v1] 6) newly created pack is BAD !' \
|
2008-07-12 17:47:52 +02:00
|
|
|
'test_must_fail git verify-pack -v "test-4-${pack1}.pack"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'[index v2] 1) stream pack to repository' \
|
|
|
|
|
'rm -f .git/objects/pack/* &&
|
2008-09-03 10:59:29 +02:00
|
|
|
git index-pack --index-version=2 --stdin < "test-1-${pack1}.pack" &&
|
2007-07-03 07:52:14 +02:00
|
|
|
git prune-packed &&
|
|
|
|
|
git count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
|
2007-04-10 22:26:10 +02:00
|
|
|
cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
|
2007-11-15 18:24:17 +01:00
|
|
|
cmp "test-2-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
|
|
|
|
test_expect_success \
|
|
|
|
|
'[index v2] 2) create a stealth corruption in a delta base reference' \
|
2008-10-23 02:59:22 +02:00
|
|
|
'# This test assumes file_101 is a delta smaller than 16 bytes.
|
|
|
|
|
# It should be against file_100 but we substitute its base for file_099
|
|
|
|
|
sha1_101=`git hash-object file_101` &&
|
|
|
|
|
sha1_099=`git hash-object file_099` &&
|
|
|
|
|
offs_101=`index_obj_offset 1.idx $sha1_101` &&
|
|
|
|
|
nr_099=`index_obj_nr 1.idx $sha1_099` &&
|
|
|
|
|
chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
|
|
|
|
|
dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($offs_101 + 1)) \
|
|
|
|
|
if=".git/objects/pack/pack-${pack1}.idx" \
|
|
|
|
|
skip=$((8 + 256 * 4 + $nr_099 * 20)) \
|
|
|
|
|
bs=1 count=20 conv=notrunc &&
|
|
|
|
|
git cat-file blob $sha1_101 > file_101_foo2'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'[index v2] 3) corrupted delta happily returned wrong data' \
|
2008-10-23 02:59:22 +02:00
|
|
|
'test -f file_101_foo2 && ! cmp file_101 file_101_foo2'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'[index v2] 4) confirm that the pack is actually corrupted' \
|
2008-07-12 17:47:52 +02:00
|
|
|
'test_must_fail git fsck --full $commit'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2008-02-01 10:50:53 +01:00
|
|
|
test_expect_success \
|
2007-04-10 22:26:10 +02:00
|
|
|
'[index v2] 5) pack-objects refuses to reuse corrupted data' \
|
close another possibility for propagating pack corruption
Abstract
--------
With index v2 we have a per object CRC to allow quick and safe reuse of
pack data when repacking. This, however, doesn't currently prevent a
stealth corruption from being propagated into a new pack when _not_
reusing pack data as demonstrated by the modification to t5302 included
here.
The Context
-----------
The Git database is all checksummed with SHA1 hashes. Any kind of
corruption can be confirmed by verifying this per object hash against
corresponding data. However this can be costly to perform systematically
and therefore this check is often not performed at run time when
accessing the object database.
First, the loose object format is entirely compressed with zlib which
already provide a CRC verification of its own when inflating data. Any
disk corruption would be caught already in this case.
Then, packed objects are also compressed with zlib but only for their
actual payload. The object headers and delta base references are not
deflated for obvious performance reasons, however this leave them
vulnerable to potentially undetected disk corruptions. Object types
are often validated against the expected type when they're requested,
and deflated size must always match the size recorded in the object header,
so those cases are pretty much covered as well.
Where corruptions could go unnoticed is in the delta base reference.
Of course, in the OBJ_REF_DELTA case, the odds for a SHA1 reference to
get corrupted so it actually matches the SHA1 of another object with the
same size (the delta header stores the expected size of the base object
to apply against) are virtually zero. In the OBJ_OFS_DELTA case, the
reference is a pack offset which would have to match the start boundary
of a different base object but still with the same size, and although this
is relatively much more "probable" than in the OBJ_REF_DELTA case, the
probability is also about zero in absolute terms. Still, the possibility
exists as demonstrated in t5302 and is certainly greater than a SHA1
collision, especially in the OBJ_OFS_DELTA case which is now the default
when repacking.
Again, repacking by reusing existing pack data is OK since the per object
CRC provided by index v2 guards against any such corruptions. What t5302
failed to test is a full repack in such case.
The Solution
------------
As unlikely as this kind of stealth corruption can be in practice, it
certainly isn't acceptable to propagate it into a freshly created pack.
But, because this is so unlikely, we don't want to pay the run time cost
associated with extra validation checks all the time either. Furthermore,
consequences of such corruption in anything but repacking should be rather
visible, and even if it could be quite unpleasant, it still has far less
severe consequences than actively creating bad packs.
So the best compromize is to check packed object CRC when unpacking
objects, and only during the compression/writing phase of a repack, and
only when not streaming the result. The cost of this is minimal (less
than 1% CPU time), and visible only with a full repack.
Someone with a stats background could provide an objective evaluation of
this, but I suspect that it's bad RAM that has more potential for data
corruptions at this point, even in those cases where this extra check
is not performed. Still, it is best to prevent a known hole for
corruption when recreating object data into a new pack.
What about the streamed pack case? Well, any client receiving a pack
must always consider that pack as untrusty and perform full validation
anyway, hence no such stealth corruption could be propagated to remote
repositoryes already. It is therefore worthless doing local validation
in that case.
Signed-off-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2008-10-31 16:31:08 +01:00
|
|
|
'test_must_fail git pack-objects test-5 <obj-list &&
|
|
|
|
|
test_must_fail git pack-objects --no-reuse-object test-6 <obj-list'
|
2007-04-10 22:26:10 +02:00
|
|
|
|
2008-06-25 05:19:44 +02:00
|
|
|
test_expect_success \
|
|
|
|
|
'[index v2] 6) verify-pack detects CRC mismatch' \
|
|
|
|
|
'rm -f .git/objects/pack/* &&
|
2008-09-03 10:59:29 +02:00
|
|
|
git index-pack --index-version=2 --stdin < "test-1-${pack1}.pack" &&
|
2008-06-25 05:19:44 +02:00
|
|
|
git verify-pack ".git/objects/pack/pack-${pack1}.pack" &&
|
2008-10-23 02:59:22 +02:00
|
|
|
obj=`git hash-object file_001` &&
|
|
|
|
|
nr=`index_obj_nr ".git/objects/pack/pack-${pack1}.idx" $obj` &&
|
2008-06-25 05:19:44 +02:00
|
|
|
chmod +w ".git/objects/pack/pack-${pack1}.idx" &&
|
|
|
|
|
dd if=/dev/zero of=".git/objects/pack/pack-${pack1}.idx" conv=notrunc \
|
2008-10-23 02:59:22 +02:00
|
|
|
bs=1 count=4 seek=$((8 + 256 * 4 + `wc -l <obj-list` * 20 + $nr * 4)) &&
|
2008-06-25 05:19:44 +02:00
|
|
|
( while read obj
|
|
|
|
|
do git cat-file -p $obj >/dev/null || exit 1
|
|
|
|
|
done <obj-list ) &&
|
2008-07-12 17:47:52 +02:00
|
|
|
err=$(test_must_fail git verify-pack \
|
|
|
|
|
".git/objects/pack/pack-${pack1}.pack" 2>&1) &&
|
2008-06-25 05:19:44 +02:00
|
|
|
echo "$err" | grep "CRC mismatch"'
|
|
|
|
|
|
2008-10-21 03:17:07 +02:00
|
|
|
test_expect_success 'running index-pack in the object store' '
|
|
|
|
|
rm -f .git/objects/pack/* &&
|
|
|
|
|
cp test-1-${pack1}.pack .git/objects/pack/pack-${pack1}.pack &&
|
|
|
|
|
(
|
|
|
|
|
cd .git/objects/pack
|
|
|
|
|
git index-pack pack-${pack1}.pack
|
|
|
|
|
) &&
|
|
|
|
|
test -f .git/objects/pack/pack-${pack1}.idx
|
|
|
|
|
'
|
|
|
|
|
|
2007-04-10 22:26:10 +02:00
|
|
|
test_done
|