Merge branch 'ds/multi-pack-verify'
"git multi-pack-index" learned to detect corruption in the .midx file it uses, and this feature has been integrated into "git fsck". * ds/multi-pack-verify: fsck: verify multi-pack-index multi-pack-index: report progress during 'verify' multi-pack-index: verify object offsets multi-pack-index: fix 32-bit vs 64-bit size check multi-pack-index: verify oid lookup order multi-pack-index: verify oid fanout order multi-pack-index: verify missing pack multi-pack-index: verify packname order multi-pack-index: verify corrupt chunk lookup table multi-pack-index: verify bad header multi-pack-index: add 'verify' verb
This commit is contained in:
Junio C Hamano
commit
468b322137
@ -27,6 +27,10 @@ write::
|
||||
When given as the verb, write a new MIDX file to
|
||||
`<dir>/packs/multi-pack-index`.
|
||||
|
||||
verify::
|
||||
When given as the verb, verify the contents of the MIDX file
|
||||
at `<dir>/packs/multi-pack-index`.
|
||||
|
||||
|
||||
EXAMPLES
|
||||
--------
|
||||
@ -43,6 +47,12 @@ $ git multi-pack-index write
|
||||
$ git multi-pack-index --object-dir <alt> write
|
||||
-----------------------------------------------
|
||||
|
||||
* Verify the MIDX file for the packfiles in the current .git folder.
|
||||
+
|
||||
-----------------------------------------------
|
||||
$ git multi-pack-index verify
|
||||
-----------------------------------------------
|
||||
|
||||
|
||||
SEE ALSO
|
||||
--------
|
||||
|
||||
@ -848,5 +848,23 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)
|
||||
}
|
||||
}
|
||||
|
||||
if (!git_config_get_bool("core.multipackindex", &i) && i) {
|
||||
struct child_process midx_verify = CHILD_PROCESS_INIT;
|
||||
const char *midx_argv[] = { "multi-pack-index", "verify", NULL, NULL, NULL };
|
||||
|
||||
midx_verify.argv = midx_argv;
|
||||
midx_verify.git_cmd = 1;
|
||||
if (run_command(&midx_verify))
|
||||
errors_found |= ERROR_COMMIT_GRAPH;
|
||||
|
||||
prepare_alt_odb(the_repository);
|
||||
for (alt = the_repository->objects->alt_odb_list; alt; alt = alt->next) {
|
||||
midx_argv[2] = "--object-dir";
|
||||
midx_argv[3] = alt->path;
|
||||
if (run_command(&midx_verify))
|
||||
errors_found |= ERROR_COMMIT_GRAPH;
|
||||
}
|
||||
}
|
||||
|
||||
return errors_found;
|
||||
}
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#include "midx.h"
|
||||
|
||||
static char const * const builtin_multi_pack_index_usage[] = {
|
||||
N_("git multi-pack-index [--object-dir=<dir>] write"),
|
||||
N_("git multi-pack-index [--object-dir=<dir>] (write|verify)"),
|
||||
NULL
|
||||
};
|
||||
|
||||
@ -42,6 +42,8 @@ int cmd_multi_pack_index(int argc, const char **argv,
|
||||
|
||||
if (!strcmp(argv[0], "write"))
|
||||
return write_midx_file(opts.object_dir);
|
||||
if (!strcmp(argv[0], "verify"))
|
||||
return verify_midx_file(opts.object_dir);
|
||||
|
||||
die(_("unrecognized verb: %s"), argv[0]);
|
||||
}
|
||||
|
||||
113
midx.c
113
midx.c
@ -7,6 +7,7 @@
|
||||
#include "object-store.h"
|
||||
#include "sha1-lookup.h"
|
||||
#include "midx.h"
|
||||
#include "progress.h"
|
||||
|
||||
#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
|
||||
#define MIDX_VERSION 1
|
||||
@ -76,24 +77,18 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local
|
||||
m->local = local;
|
||||
|
||||
m->signature = get_be32(m->data);
|
||||
if (m->signature != MIDX_SIGNATURE) {
|
||||
error(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
|
||||
if (m->signature != MIDX_SIGNATURE)
|
||||
die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
|
||||
m->signature, MIDX_SIGNATURE);
|
||||
goto cleanup_fail;
|
||||
}
|
||||
|
||||
m->version = m->data[MIDX_BYTE_FILE_VERSION];
|
||||
if (m->version != MIDX_VERSION) {
|
||||
error(_("multi-pack-index version %d not recognized"),
|
||||
if (m->version != MIDX_VERSION)
|
||||
die(_("multi-pack-index version %d not recognized"),
|
||||
m->version);
|
||||
goto cleanup_fail;
|
||||
}
|
||||
|
||||
hash_version = m->data[MIDX_BYTE_HASH_VERSION];
|
||||
if (hash_version != MIDX_HASH_VERSION) {
|
||||
error(_("hash version %u does not match"), hash_version);
|
||||
goto cleanup_fail;
|
||||
}
|
||||
if (hash_version != MIDX_HASH_VERSION)
|
||||
die(_("hash version %u does not match"), hash_version);
|
||||
m->hash_len = MIDX_HASH_LEN;
|
||||
|
||||
m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
|
||||
@ -106,6 +101,9 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local
|
||||
uint64_t chunk_offset = get_be64(m->data + MIDX_HEADER_SIZE + 4 +
|
||||
MIDX_CHUNKLOOKUP_WIDTH * i);
|
||||
|
||||
if (chunk_offset >= m->data_len)
|
||||
die(_("invalid chunk offset (too large)"));
|
||||
|
||||
switch (chunk_id) {
|
||||
case MIDX_CHUNKID_PACKNAMES:
|
||||
m->chunk_pack_names = m->data + chunk_offset;
|
||||
@ -160,12 +158,10 @@ struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local
|
||||
|
||||
cur_pack_name += strlen(cur_pack_name) + 1;
|
||||
|
||||
if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0) {
|
||||
error(_("multi-pack-index pack names out of order: '%s' before '%s'"),
|
||||
if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
|
||||
die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
|
||||
m->pack_names[i - 1],
|
||||
m->pack_names[i]);
|
||||
goto cleanup_fail;
|
||||
}
|
||||
}
|
||||
|
||||
return m;
|
||||
@ -202,7 +198,8 @@ int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id)
|
||||
struct strbuf pack_name = STRBUF_INIT;
|
||||
|
||||
if (pack_int_id >= m->num_packs)
|
||||
BUG("bad pack-int-id");
|
||||
die(_("bad pack-int-id: %u (%u total packs"),
|
||||
pack_int_id, m->num_packs);
|
||||
|
||||
if (m->packs[pack_int_id])
|
||||
return 0;
|
||||
@ -241,7 +238,7 @@ static off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
|
||||
offset32 = get_be32(offset_data + sizeof(uint32_t));
|
||||
|
||||
if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
|
||||
if (sizeof(offset32) < sizeof(uint64_t))
|
||||
if (sizeof(off_t) < sizeof(uint64_t))
|
||||
die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
|
||||
|
||||
offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
|
||||
@ -928,3 +925,83 @@ void clear_midx_file(const char *object_dir)
|
||||
|
||||
free(midx);
|
||||
}
|
||||
|
||||
static int verify_midx_error;
|
||||
|
||||
static void midx_report(const char *fmt, ...)
|
||||
{
|
||||
va_list ap;
|
||||
verify_midx_error = 1;
|
||||
va_start(ap, fmt);
|
||||
vfprintf(stderr, fmt, ap);
|
||||
fprintf(stderr, "\n");
|
||||
va_end(ap);
|
||||
}
|
||||
|
||||
int verify_midx_file(const char *object_dir)
|
||||
{
|
||||
uint32_t i;
|
||||
struct progress *progress = NULL;
|
||||
struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
|
||||
verify_midx_error = 0;
|
||||
|
||||
if (!m)
|
||||
return 0;
|
||||
|
||||
for (i = 0; i < m->num_packs; i++) {
|
||||
if (prepare_midx_pack(m, i))
|
||||
midx_report("failed to load pack in position %d", i);
|
||||
}
|
||||
|
||||
for (i = 0; i < 255; i++) {
|
||||
uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
|
||||
uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i + 1]);
|
||||
|
||||
if (oid_fanout1 > oid_fanout2)
|
||||
midx_report(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
|
||||
i, oid_fanout1, oid_fanout2, i + 1);
|
||||
}
|
||||
|
||||
for (i = 0; i < m->num_objects - 1; i++) {
|
||||
struct object_id oid1, oid2;
|
||||
|
||||
nth_midxed_object_oid(&oid1, m, i);
|
||||
nth_midxed_object_oid(&oid2, m, i + 1);
|
||||
|
||||
if (oidcmp(&oid1, &oid2) >= 0)
|
||||
midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
|
||||
i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
|
||||
}
|
||||
|
||||
progress = start_progress(_("Verifying object offsets"), m->num_objects);
|
||||
for (i = 0; i < m->num_objects; i++) {
|
||||
struct object_id oid;
|
||||
struct pack_entry e;
|
||||
off_t m_offset, p_offset;
|
||||
|
||||
nth_midxed_object_oid(&oid, m, i);
|
||||
if (!fill_midx_entry(&oid, &e, m)) {
|
||||
midx_report(_("failed to load pack entry for oid[%d] = %s"),
|
||||
i, oid_to_hex(&oid));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (open_pack_index(e.p)) {
|
||||
midx_report(_("failed to load pack-index for packfile %s"),
|
||||
e.p->pack_name);
|
||||
break;
|
||||
}
|
||||
|
||||
m_offset = e.offset;
|
||||
p_offset = find_pack_entry_one(oid.hash, e.p);
|
||||
|
||||
if (m_offset != p_offset)
|
||||
midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
|
||||
i, oid_to_hex(&oid), m_offset, p_offset);
|
||||
|
||||
display_progress(progress, i + 1);
|
||||
}
|
||||
stop_progress(&progress);
|
||||
|
||||
return verify_midx_error;
|
||||
}
|
||||
|
||||
1
midx.h
1
midx.h
@ -43,5 +43,6 @@ int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, i
|
||||
|
||||
int write_midx_file(const char *object_dir);
|
||||
void clear_midx_file(const char *object_dir);
|
||||
int verify_midx_file(const char *object_dir);
|
||||
|
||||
#endif
|
||||
|
||||
@ -150,6 +150,125 @@ test_expect_success 'write midx with twelve packs' '
|
||||
|
||||
compare_results_with_midx "twelve packs"
|
||||
|
||||
test_expect_success 'verify multi-pack-index success' '
|
||||
git multi-pack-index verify --object-dir=$objdir
|
||||
'
|
||||
|
||||
# usage: corrupt_midx_and_verify <pos> <data> <objdir> <string>
|
||||
corrupt_midx_and_verify() {
|
||||
POS=$1 &&
|
||||
DATA="${2:-\0}" &&
|
||||
OBJDIR=$3 &&
|
||||
GREPSTR="$4" &&
|
||||
COMMAND="$5" &&
|
||||
if test -z "$COMMAND"
|
||||
then
|
||||
COMMAND="git multi-pack-index verify --object-dir=$OBJDIR"
|
||||
fi &&
|
||||
FILE=$OBJDIR/pack/multi-pack-index &&
|
||||
chmod a+w $FILE &&
|
||||
test_when_finished mv midx-backup $FILE &&
|
||||
cp $FILE midx-backup &&
|
||||
printf "$DATA" | dd of="$FILE" bs=1 seek="$POS" conv=notrunc &&
|
||||
test_must_fail $COMMAND 2>test_err &&
|
||||
grep -v "^+" test_err >err &&
|
||||
test_i18ngrep "$GREPSTR" err
|
||||
}
|
||||
|
||||
test_expect_success 'verify bad signature' '
|
||||
corrupt_midx_and_verify 0 "\00" $objdir \
|
||||
"multi-pack-index signature"
|
||||
'
|
||||
|
||||
HASH_LEN=20
|
||||
NUM_OBJECTS=74
|
||||
MIDX_BYTE_VERSION=4
|
||||
MIDX_BYTE_OID_VERSION=5
|
||||
MIDX_BYTE_CHUNK_COUNT=6
|
||||
MIDX_HEADER_SIZE=12
|
||||
MIDX_BYTE_CHUNK_ID=$MIDX_HEADER_SIZE
|
||||
MIDX_BYTE_CHUNK_OFFSET=$(($MIDX_HEADER_SIZE + 4))
|
||||
MIDX_NUM_CHUNKS=5
|
||||
MIDX_CHUNK_LOOKUP_WIDTH=12
|
||||
MIDX_OFFSET_PACKNAMES=$(($MIDX_HEADER_SIZE + \
|
||||
$MIDX_NUM_CHUNKS * $MIDX_CHUNK_LOOKUP_WIDTH))
|
||||
MIDX_BYTE_PACKNAME_ORDER=$(($MIDX_OFFSET_PACKNAMES + 2))
|
||||
MIDX_OFFSET_OID_FANOUT=$(($MIDX_OFFSET_PACKNAMES + 652))
|
||||
MIDX_OID_FANOUT_WIDTH=4
|
||||
MIDX_BYTE_OID_FANOUT_ORDER=$((MIDX_OFFSET_OID_FANOUT + 250 * $MIDX_OID_FANOUT_WIDTH + 1))
|
||||
MIDX_OFFSET_OID_LOOKUP=$(($MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH))
|
||||
MIDX_BYTE_OID_LOOKUP=$(($MIDX_OFFSET_OID_LOOKUP + 16 * $HASH_LEN))
|
||||
MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN))
|
||||
MIDX_OFFSET_WIDTH=8
|
||||
MIDX_BYTE_PACK_INT_ID=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 2))
|
||||
MIDX_BYTE_OFFSET=$(($MIDX_OFFSET_OBJECT_OFFSETS + 16 * $MIDX_OFFSET_WIDTH + 6))
|
||||
|
||||
test_expect_success 'verify bad version' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_VERSION "\00" $objdir \
|
||||
"multi-pack-index version"
|
||||
'
|
||||
|
||||
test_expect_success 'verify bad OID version' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_OID_VERSION "\02" $objdir \
|
||||
"hash version"
|
||||
'
|
||||
|
||||
test_expect_success 'verify truncated chunk count' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\01" $objdir \
|
||||
"missing required"
|
||||
'
|
||||
|
||||
test_expect_success 'verify extended chunk count' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_CHUNK_COUNT "\07" $objdir \
|
||||
"terminating multi-pack-index chunk id appears earlier than expected"
|
||||
'
|
||||
|
||||
test_expect_success 'verify missing required chunk' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_CHUNK_ID "\01" $objdir \
|
||||
"missing required"
|
||||
'
|
||||
|
||||
test_expect_success 'verify invalid chunk offset' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_CHUNK_OFFSET "\01" $objdir \
|
||||
"invalid chunk offset (too large)"
|
||||
'
|
||||
|
||||
test_expect_success 'verify packnames out of order' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "z" $objdir \
|
||||
"pack names out of order"
|
||||
'
|
||||
|
||||
test_expect_success 'verify packnames out of order' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_PACKNAME_ORDER "a" $objdir \
|
||||
"failed to load pack"
|
||||
'
|
||||
|
||||
test_expect_success 'verify oid fanout out of order' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_OID_FANOUT_ORDER "\01" $objdir \
|
||||
"oid fanout out of order"
|
||||
'
|
||||
|
||||
test_expect_success 'verify oid lookup out of order' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_OID_LOOKUP "\00" $objdir \
|
||||
"oid lookup out of order"
|
||||
'
|
||||
|
||||
test_expect_success 'verify incorrect pack-int-id' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_PACK_INT_ID "\07" $objdir \
|
||||
"bad pack-int-id"
|
||||
'
|
||||
|
||||
test_expect_success 'verify incorrect offset' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\07" $objdir \
|
||||
"incorrect object offset"
|
||||
'
|
||||
|
||||
test_expect_success 'git-fsck incorrect offset' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\07" $objdir \
|
||||
"incorrect object offset" \
|
||||
"git -c core.multipackindex=true fsck"
|
||||
'
|
||||
|
||||
test_expect_success 'repack removes multi-pack-index' '
|
||||
test_path_is_file $objdir/pack/multi-pack-index &&
|
||||
git repack -adf &&
|
||||
@ -187,7 +306,6 @@ test_expect_success 'multi-pack-index in an alternate' '
|
||||
|
||||
compare_results_with_midx "with alternate (remote midx)"
|
||||
|
||||
|
||||
# usage: corrupt_data <file> <pos> [<data>]
|
||||
corrupt_data () {
|
||||
file=$1
|
||||
@ -214,4 +332,20 @@ test_expect_success 'force some 64-bit offsets with pack-objects' '
|
||||
midx_read_expect 1 63 5 objects64 " large-offsets"
|
||||
'
|
||||
|
||||
test_expect_success 'verify multi-pack-index with 64-bit offsets' '
|
||||
git multi-pack-index verify --object-dir=objects64
|
||||
'
|
||||
|
||||
NUM_OBJECTS=63
|
||||
MIDX_OFFSET_OID_FANOUT=$((MIDX_OFFSET_PACKNAMES + 54))
|
||||
MIDX_OFFSET_OID_LOOKUP=$((MIDX_OFFSET_OID_FANOUT + 256 * $MIDX_OID_FANOUT_WIDTH))
|
||||
MIDX_OFFSET_OBJECT_OFFSETS=$(($MIDX_OFFSET_OID_LOOKUP + $NUM_OBJECTS * $HASH_LEN))
|
||||
MIDX_OFFSET_LARGE_OFFSETS=$(($MIDX_OFFSET_OBJECT_OFFSETS + $NUM_OBJECTS * $MIDX_OFFSET_WIDTH))
|
||||
MIDX_BYTE_LARGE_OFFSET=$(($MIDX_OFFSET_LARGE_OFFSETS + 3))
|
||||
|
||||
test_expect_success 'verify incorrect 64-bit offset' '
|
||||
corrupt_midx_and_verify $MIDX_BYTE_LARGE_OFFSET "\07" objects64 \
|
||||
"incorrect object offset"
|
||||
'
|
||||
|
||||
test_done
|
||||
|
||||
Loading…
Reference in New Issue
Block a user