undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'gb/idx'

Browse Source 
* gb/idx:
  Unify write_index_file functions
This commit is contained in:
Junio C Hamano 2007-06-05 21:36:51 -07:00
commit 7be003b026
4 changed files with 245 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

222
builtin-pack-objects.c
View File

@ -23,10 +23,9 @@ git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
[--stdout | base-name] [<ref-list | <object-list]"; [--stdout | base-name] [<ref-list | <object-list]";
struct object_entry { struct object_entry {
unsigned char sha1[20]; struct pack_idx_entry idx;
uint32_t crc32; /* crc of raw pack data for this object */
off_t offset; /* offset into the final pack file */
unsigned long size; /* uncompressed size */ unsigned long size; /* uncompressed size */
unsigned int hash; /* name hint hash */ unsigned int hash; /* name hint hash */
unsigned int depth; /* delta depth */ unsigned int depth; /* delta depth */
struct packed_git *in_pack; /* already in pack */ struct packed_git *in_pack; /* already in pack */
@ -66,7 +65,6 @@ static int allow_ofs_delta;
static const char *pack_tmp_name, *idx_tmp_name; static const char *pack_tmp_name, *idx_tmp_name;
static char tmpname[PATH_MAX]; static char tmpname[PATH_MAX];
static const char *base_name; static const char *base_name;
static unsigned char pack_file_sha1[20];
static int progress = 1; static int progress = 1;
static int window = 10; static int window = 10;
static uint32_t pack_size_limit; static uint32_t pack_size_limit;
@ -246,11 +244,11 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
{ {
unsigned long othersize, delta_size; unsigned long othersize, delta_size;
enum object_type type; enum object_type type;
void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize); void *otherbuf = read_sha1_file(entry->delta->idx.sha1, &type, &othersize);
void *delta_buf; void *delta_buf;
if (!otherbuf) if (!otherbuf)
die("unable to read %s", sha1_to_hex(entry->delta->sha1)); die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1));
delta_buf = diff_delta(otherbuf, othersize, delta_buf = diff_delta(otherbuf, othersize,
buf, size, &delta_size, 0); buf, size, &delta_size, 0);
if (!delta_buf || delta_size != entry->delta_size) if (!delta_buf || delta_size != entry->delta_size)
@ -379,11 +377,11 @@ static unsigned long write_object(struct sha1file *f,
/* yes if unlimited packfile */ /* yes if unlimited packfile */
!pack_size_limit ? 1 : !pack_size_limit ? 1 :
/* no if base written to previous pack */ /* no if base written to previous pack */
entry->delta->offset == (off_t)-1 ? 0 : entry->delta->idx.offset == (off_t)-1 ? 0 :
/* otherwise double-check written to this /* otherwise double-check written to this
* pack, like we do below * pack, like we do below
*/ */
entry->delta->offset ? 1 : 0; entry->delta->idx.offset ? 1 : 0;
if (!pack_to_stdout) if (!pack_to_stdout)
crc32_begin(f); crc32_begin(f);
@ -411,22 +409,22 @@ static unsigned long write_object(struct sha1file *f,
unsigned long maxsize; unsigned long maxsize;
void *out; void *out;
if (!usable_delta) { if (!usable_delta) {
buf = read_sha1_file(entry->sha1, &obj_type, &size); buf = read_sha1_file(entry->idx.sha1, &obj_type, &size);
if (!buf) if (!buf)
die("unable to read %s", sha1_to_hex(entry->sha1)); die("unable to read %s", sha1_to_hex(entry->idx.sha1));
} else if (entry->delta_data) { } else if (entry->delta_data) {
size = entry->delta_size; size = entry->delta_size;
buf = entry->delta_data; buf = entry->delta_data;
entry->delta_data = NULL; entry->delta_data = NULL;
obj_type = (allow_ofs_delta && entry->delta->offset) ? obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA; OBJ_OFS_DELTA : OBJ_REF_DELTA;
} else { } else {
buf = read_sha1_file(entry->sha1, &type, &size); buf = read_sha1_file(entry->idx.sha1, &type, &size);
if (!buf) if (!buf)
die("unable to read %s", sha1_to_hex(entry->sha1)); die("unable to read %s", sha1_to_hex(entry->idx.sha1));
buf = delta_against(buf, size, entry); buf = delta_against(buf, size, entry);
size = entry->delta_size; size = entry->delta_size;
obj_type = (allow_ofs_delta && entry->delta->offset) ? obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA; OBJ_OFS_DELTA : OBJ_REF_DELTA;
} }
/* compress the data to store and put compressed length in datalen */ /* compress the data to store and put compressed length in datalen */
@ -456,7 +454,7 @@ static unsigned long write_object(struct sha1file *f,
* encoding of the relative offset for the delta * encoding of the relative offset for the delta
* base from this object's position in the pack. * base from this object's position in the pack.
*/ */
off_t ofs = entry->offset - entry->delta->offset; off_t ofs = entry->idx.offset - entry->delta->idx.offset;
unsigned pos = sizeof(dheader) - 1; unsigned pos = sizeof(dheader) - 1;
dheader[pos] = ofs & 127; dheader[pos] = ofs & 127;
while (ofs >>= 7) while (ofs >>= 7)
@ -480,7 +478,7 @@ static unsigned long write_object(struct sha1file *f,
return 0; return 0;
} }
sha1write(f, header, hdrlen); sha1write(f, header, hdrlen);
sha1write(f, entry->delta->sha1, 20); sha1write(f, entry->delta->idx.sha1, 20);
hdrlen += 20; hdrlen += 20;
} else { } else {
if (limit && hdrlen + datalen + 20 >= limit) { if (limit && hdrlen + datalen + 20 >= limit) {
@ -501,7 +499,7 @@ static unsigned long write_object(struct sha1file *f,
off_t offset; off_t offset;
if (entry->delta) { if (entry->delta) {
obj_type = (allow_ofs_delta && entry->delta->offset) ? obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA; OBJ_OFS_DELTA : OBJ_REF_DELTA;
reused_delta++; reused_delta++;
} }
@ -511,11 +509,11 @@ static unsigned long write_object(struct sha1file *f,
datalen = revidx[1].offset - offset; datalen = revidx[1].offset - offset;
if (!pack_to_stdout && p->index_version > 1 && if (!pack_to_stdout && p->index_version > 1 &&
check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
die("bad packed object CRC for %s", sha1_to_hex(entry->sha1)); die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1));
offset += entry->in_pack_header_size; offset += entry->in_pack_header_size;
datalen -= entry->in_pack_header_size; datalen -= entry->in_pack_header_size;
if (obj_type == OBJ_OFS_DELTA) { if (obj_type == OBJ_OFS_DELTA) {
off_t ofs = entry->offset - entry->delta->offset; off_t ofs = entry->idx.offset - entry->delta->idx.offset;
unsigned pos = sizeof(dheader) - 1; unsigned pos = sizeof(dheader) - 1;
dheader[pos] = ofs & 127; dheader[pos] = ofs & 127;
while (ofs >>= 7) while (ofs >>= 7)
@ -529,7 +527,7 @@ static unsigned long write_object(struct sha1file *f,
if (limit && hdrlen + 20 + datalen + 20 >= limit) if (limit && hdrlen + 20 + datalen + 20 >= limit)
return 0; return 0;
sha1write(f, header, hdrlen); sha1write(f, header, hdrlen);
sha1write(f, entry->delta->sha1, 20); sha1write(f, entry->delta->idx.sha1, 20);
hdrlen += 20; hdrlen += 20;
} else { } else {
if (limit && hdrlen + datalen + 20 >= limit) if (limit && hdrlen + datalen + 20 >= limit)
@ -539,7 +537,7 @@ static unsigned long write_object(struct sha1file *f,
if (!pack_to_stdout && p->index_version == 1 && if (!pack_to_stdout && p->index_version == 1 &&
check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
die("corrupt packed object for %s", sha1_to_hex(entry->sha1)); die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1));
copy_pack_data(f, p, &w_curs, offset, datalen); copy_pack_data(f, p, &w_curs, offset, datalen);
unuse_pack(&w_curs); unuse_pack(&w_curs);
reused++; reused++;
@ -548,7 +546,7 @@ static unsigned long write_object(struct sha1file *f,
written_delta++; written_delta++;
written++; written++;
if (!pack_to_stdout) if (!pack_to_stdout)
entry->crc32 = crc32_end(f); entry->idx.crc32 = crc32_end(f);
return hdrlen + datalen; return hdrlen + datalen;
} }
@ -559,7 +557,7 @@ static off_t write_one(struct sha1file *f,
unsigned long size; unsigned long size;
/* offset is non zero if object is written already. */ /* offset is non zero if object is written already. */
if (e->offset || e->preferred_base) if (e->idx.offset || e->preferred_base)
return offset; return offset;
/* if we are deltified, write out base object first. */ /* if we are deltified, write out base object first. */
@ -569,10 +567,10 @@ static off_t write_one(struct sha1file *f,
return 0; return 0;
} }
e->offset = offset; e->idx.offset = offset;
size = write_object(f, e, offset); size = write_object(f, e, offset);
if (!size) { if (!size) {
e->offset = 0; e->idx.offset = 0;
return 0; return 0;
} }
written_list[nr_written++] = e; written_list[nr_written++] = e;
@ -589,8 +587,7 @@ static int open_object_dir_tmp(const char *path)
return mkstemp(tmpname); return mkstemp(tmpname);
} }
/* forward declarations for write_pack_file */ /* forward declaration for write_pack_file */
static void write_index_file(off_t last_obj_offset, unsigned char *sha1);
static int adjust_perm(const char *path, mode_t mode); static int adjust_perm(const char *path, mode_t mode);
static void write_pack_file(void) static void write_pack_file(void)
@ -607,6 +604,8 @@ static void write_pack_file(void)
written_list = xmalloc(nr_objects * sizeof(struct object_entry *)); written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
do { do {
unsigned char sha1[20];
if (pack_to_stdout) { if (pack_to_stdout) {
f = sha1fd(1, "<stdout>"); f = sha1fd(1, "<stdout>");
} else { } else {
@ -638,23 +637,23 @@ static void write_pack_file(void)
* If so, rewrite it like in fast-import * If so, rewrite it like in fast-import
*/ */
if (pack_to_stdout || nr_written == nr_remaining) { if (pack_to_stdout || nr_written == nr_remaining) {
sha1close(f, pack_file_sha1, 1); sha1close(f, sha1, 1);
} else { } else {
sha1close(f, pack_file_sha1, 0); sha1close(f, sha1, 0);
fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written); fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
close(f->fd); close(f->fd);
} }
if (!pack_to_stdout) { if (!pack_to_stdout) {
unsigned char object_list_sha1[20];
mode_t mode = umask(0); mode_t mode = umask(0);
umask(mode); umask(mode);
mode = 0444 & ~mode; mode = 0444 & ~mode;
write_index_file(last_obj_offset, object_list_sha1); idx_tmp_name = write_idx_file(NULL,
(struct pack_idx_entry **) written_list, nr_written, sha1);
snprintf(tmpname, sizeof(tmpname), "%s-%s.pack", snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
base_name, sha1_to_hex(object_list_sha1)); base_name, sha1_to_hex(sha1));
if (adjust_perm(pack_tmp_name, mode)) if (adjust_perm(pack_tmp_name, mode))
die("unable to make temporary pack file readable: %s", die("unable to make temporary pack file readable: %s",
strerror(errno)); strerror(errno));
@ -662,19 +661,19 @@ static void write_pack_file(void)
die("unable to rename temporary pack file: %s", die("unable to rename temporary pack file: %s",
strerror(errno)); strerror(errno));
snprintf(tmpname, sizeof(tmpname), "%s-%s.idx", snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
base_name, sha1_to_hex(object_list_sha1)); base_name, sha1_to_hex(sha1));
if (adjust_perm(idx_tmp_name, mode)) if (adjust_perm(idx_tmp_name, mode))
die("unable to make temporary index file readable: %s", die("unable to make temporary index file readable: %s",
strerror(errno)); strerror(errno));
if (rename(idx_tmp_name, tmpname)) if (rename(idx_tmp_name, tmpname))
die("unable to rename temporary index file: %s", die("unable to rename temporary index file: %s",
strerror(errno)); strerror(errno));
puts(sha1_to_hex(object_list_sha1)); puts(sha1_to_hex(sha1));
} }
/* mark written objects as written to previous pack */ /* mark written objects as written to previous pack */
for (j = 0; j < nr_written; j++) { for (j = 0; j < nr_written; j++) {
written_list[j]->offset = (off_t)-1; written_list[j]->idx.offset = (off_t)-1;
} }
nr_remaining -= nr_written; nr_remaining -= nr_written;
} while (nr_remaining && i < nr_objects); } while (nr_remaining && i < nr_objects);
@ -692,129 +691,12 @@ static void write_pack_file(void)
*/ */
for (j = 0; i < nr_objects; i++) { for (j = 0; i < nr_objects; i++) {
struct object_entry *e = objects + i; struct object_entry *e = objects + i;
j += !e->offset && !e->preferred_base; j += !e->idx.offset && !e->preferred_base;
} }
if (j) if (j)
die("wrote %u objects as expected but %u unwritten", written, j); die("wrote %u objects as expected but %u unwritten", written, j);
} }
static int sha1_sort(const void *_a, const void *_b)
{
const struct object_entry *a = *(struct object_entry **)_a;
const struct object_entry *b = *(struct object_entry **)_b;
return hashcmp(a->sha1, b->sha1);
}
static uint32_t index_default_version = 1;
static uint32_t index_off32_limit = 0x7fffffff;
static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
{
struct sha1file *f;
struct object_entry **sorted_by_sha, **list, **last;
uint32_t array[256];
uint32_t i, index_version;
SHA_CTX ctx;
int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
if (fd < 0)
die("unable to create %s: %s\n", tmpname, strerror(errno));
idx_tmp_name = xstrdup(tmpname);
f = sha1fd(fd, idx_tmp_name);
if (nr_written) {
sorted_by_sha = written_list;
qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
list = sorted_by_sha;
last = sorted_by_sha + nr_written;
} else
sorted_by_sha = list = last = NULL;
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
struct object_entry **next = list;
while (next < last) {
struct object_entry *entry = *next;
if (entry->sha1[0] != i)
break;
next++;
}
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * 4);
/* Compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/* Write the actual SHA1 entries. */
list = sorted_by_sha;
for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
if (index_version < 2) {
uint32_t offset = htonl(entry->offset);
sha1write(f, &offset, 4);
}
sha1write(f, entry->sha1, 20);
SHA1_Update(&ctx, entry->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
uint32_t crc32_val = htonl(entry->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_written; i++) {
struct object_entry *entry = *list++;
uint32_t offset = (entry->offset <= index_off32_limit) ?
entry->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct object_entry *entry = *list++;
uint64_t offset = entry->offset;
if (offset > index_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, pack_file_sha1, 20);
sha1close(f, NULL, 1);
SHA1_Final(sha1, &ctx);
}
static int locate_object_entry_hash(const unsigned char *sha1) static int locate_object_entry_hash(const unsigned char *sha1)
{ {
int i; int i;
@ -822,7 +704,7 @@ static int locate_object_entry_hash(const unsigned char *sha1)
memcpy(&ui, sha1, sizeof(unsigned int)); memcpy(&ui, sha1, sizeof(unsigned int));
i = ui % object_ix_hashsz; i = ui % object_ix_hashsz;
while (0 < object_ix[i]) { while (0 < object_ix[i]) {
if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1)) if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
return i; return i;
if (++i == object_ix_hashsz) if (++i == object_ix_hashsz)
i = 0; i = 0;
@ -854,7 +736,7 @@ static void rehash_objects(void)
object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz); object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
memset(object_ix, 0, sizeof(int) * object_ix_hashsz); memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
for (i = 0, oe = objects; i < nr_objects; i++, oe++) { for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
int ix = locate_object_entry_hash(oe->sha1); int ix = locate_object_entry_hash(oe->idx.sha1);
if (0 <= ix) if (0 <= ix)
continue; continue;
ix = -1 - ix; ix = -1 - ix;
@ -948,7 +830,7 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type,
entry = objects + nr_objects++; entry = objects + nr_objects++;
memset(entry, 0, sizeof(*entry)); memset(entry, 0, sizeof(*entry));
hashcpy(entry->sha1, sha1); hashcpy(entry->idx.sha1, sha1);
entry->hash = hash; entry->hash = hash;
if (type) if (type)
entry->type = type; entry->type = type;
@ -1268,13 +1150,13 @@ static void check_object(struct object_entry *entry)
ofs += 1; ofs += 1;
if (!ofs || MSB(ofs, 7)) if (!ofs || MSB(ofs, 7))
die("delta base offset overflow in pack for %s", die("delta base offset overflow in pack for %s",
sha1_to_hex(entry->sha1)); sha1_to_hex(entry->idx.sha1));
c = buf[used_0++]; c = buf[used_0++];
ofs = (ofs << 7) + (c & 127); ofs = (ofs << 7) + (c & 127);
} }
if (ofs >= entry->in_pack_offset) if (ofs >= entry->in_pack_offset)
die("delta base offset out of bound for %s", die("delta base offset out of bound for %s",
sha1_to_hex(entry->sha1)); sha1_to_hex(entry->idx.sha1));
ofs = entry->in_pack_offset - ofs; ofs = entry->in_pack_offset - ofs;
if (!no_reuse_delta && !entry->preferred_base) if (!no_reuse_delta && !entry->preferred_base)
base_ref = find_packed_object_name(p, ofs); base_ref = find_packed_object_name(p, ofs);
@ -1321,10 +1203,10 @@ static void check_object(struct object_entry *entry)
unuse_pack(&w_curs); unuse_pack(&w_curs);
} }
entry->type = sha1_object_info(entry->sha1, &entry->size); entry->type = sha1_object_info(entry->idx.sha1, &entry->size);
if (entry->type < 0) if (entry->type < 0)
die("unable to get type of object %s", die("unable to get type of object %s",
sha1_to_hex(entry->sha1)); sha1_to_hex(entry->idx.sha1));
} }
static int pack_offset_sort(const void *_a, const void *_b) static int pack_offset_sort(const void *_a, const void *_b)
@ -1334,7 +1216,7 @@ static int pack_offset_sort(const void *_a, const void *_b)
/* avoid filesystem trashing with loose objects */ /* avoid filesystem trashing with loose objects */
if (!a->in_pack && !b->in_pack) if (!a->in_pack && !b->in_pack)
return hashcmp(a->sha1, b->sha1); return hashcmp(a->idx.sha1, b->idx.sha1);
if (a->in_pack < b->in_pack) if (a->in_pack < b->in_pack)
return -1; return -1;
@ -1463,16 +1345,16 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
/* Load data if not already done */ /* Load data if not already done */
if (!trg->data) { if (!trg->data) {
trg->data = read_sha1_file(trg_entry->sha1, &type, &sz); trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
if (sz != trg_size) if (sz != trg_size)
die("object %s inconsistent object length (%lu vs %lu)", die("object %s inconsistent object length (%lu vs %lu)",
sha1_to_hex(trg_entry->sha1), sz, trg_size); sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
} }
if (!src->data) { if (!src->data) {
src->data = read_sha1_file(src_entry->sha1, &type, &sz); src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
if (sz != src_size) if (sz != src_size)
die("object %s inconsistent object length (%lu vs %lu)", die("object %s inconsistent object length (%lu vs %lu)",
sha1_to_hex(src_entry->sha1), sz, src_size); sha1_to_hex(src_entry->idx.sha1), sz, src_size);
} }
if (!src->index) { if (!src->index) {
src->index = create_delta_index(src->data, src_size); src->index = create_delta_index(src->data, src_size);
@ -1869,12 +1751,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
} }
if (!prefixcmp(arg, "--index-version=")) { if (!prefixcmp(arg, "--index-version=")) {
char *c; char *c;
index_default_version = strtoul(arg + 16, &c, 10); pack_idx_default_version = strtoul(arg + 16, &c, 10);
if (index_default_version > 2) if (pack_idx_default_version > 2)
die("bad %s", arg); die("bad %s", arg);
if (*c == ',') if (*c == ',')
index_off32_limit = strtoul(c+1, &c, 0); pack_idx_off32_limit = strtoul(c+1, &c, 0);
if (*c || index_off32_limit & 0x80000000) if (*c || pack_idx_off32_limit & 0x80000000)
die("bad %s", arg); die("bad %s", arg);
continue; continue;
} }

208
index-pack.c
View File

@ -13,13 +13,11 @@ static const char index_pack_usage[] =
struct object_entry struct object_entry
{ {
off_t offset; struct pack_idx_entry idx;
unsigned long size; unsigned long size;
unsigned int hdr_size; unsigned int hdr_size;
uint32_t crc32;
enum object_type type; enum object_type type;
enum object_type real_type; enum object_type real_type;
unsigned char sha1[20];
}; };
union delta_base { union delta_base {
@ -197,7 +195,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
unsigned shift; unsigned shift;
void *data; void *data;
obj->offset = consumed_bytes; obj->idx.offset = consumed_bytes;
input_crc32 = crc32(0, Z_NULL, 0); input_crc32 = crc32(0, Z_NULL, 0);
p = fill(1); p = fill(1);
@ -229,15 +227,15 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
while (c & 128) { while (c & 128) {
base_offset += 1; base_offset += 1;
if (!base_offset || MSB(base_offset, 7)) if (!base_offset || MSB(base_offset, 7))
bad_object(obj->offset, "offset value overflow for delta base object"); bad_object(obj->idx.offset, "offset value overflow for delta base object");
p = fill(1); p = fill(1);
c = *p; c = *p;
use(1); use(1);
base_offset = (base_offset << 7) + (c & 127); base_offset = (base_offset << 7) + (c & 127);
} }
delta_base->offset = obj->offset - base_offset; delta_base->offset = obj->idx.offset - base_offset;
if (delta_base->offset >= obj->offset) if (delta_base->offset >= obj->idx.offset)
bad_object(obj->offset, "delta base offset is out of bound"); bad_object(obj->idx.offset, "delta base offset is out of bound");
break; break;
case OBJ_COMMIT: case OBJ_COMMIT:
case OBJ_TREE: case OBJ_TREE:
@ -245,19 +243,19 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
case OBJ_TAG: case OBJ_TAG:
break; break;
default: default:
bad_object(obj->offset, "unknown object type %d", obj->type); bad_object(obj->idx.offset, "unknown object type %d", obj->type);
} }
obj->hdr_size = consumed_bytes - obj->offset; obj->hdr_size = consumed_bytes - obj->idx.offset;
data = unpack_entry_data(obj->offset, obj->size); data = unpack_entry_data(obj->idx.offset, obj->size);
obj->crc32 = input_crc32; obj->idx.crc32 = input_crc32;
return data; return data;
} }
static void *get_data_from_pack(struct object_entry *obj) static void *get_data_from_pack(struct object_entry *obj)
{ {
unsigned long from = obj[0].offset + obj[0].hdr_size; unsigned long from = obj[0].idx.offset + obj[0].hdr_size;
unsigned long len = obj[1].offset - from; unsigned long len = obj[1].idx.offset - from;
unsigned long rdy = 0; unsigned long rdy = 0;
unsigned char *src, *data; unsigned char *src, *data;
z_stream stream; z_stream stream;
@ -360,11 +358,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
&result_size); &result_size);
free(delta_data); free(delta_data);
if (!result) if (!result)
bad_object(delta_obj->offset, "failed to apply delta"); bad_object(delta_obj->idx.offset, "failed to apply delta");
sha1_object(result, result_size, type, delta_obj->sha1); sha1_object(result, result_size, type, delta_obj->idx.sha1);
nr_resolved_deltas++; nr_resolved_deltas++;
hashcpy(delta_base.sha1, delta_obj->sha1); hashcpy(delta_base.sha1, delta_obj->idx.sha1);
if (!find_delta_children(&delta_base, &first, &last)) { if (!find_delta_children(&delta_base, &first, &last)) {
for (j = first; j <= last; j++) { for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no; struct object_entry *child = objects + deltas[j].obj_no;
@ -374,7 +372,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
} }
memset(&delta_base, 0, sizeof(delta_base)); memset(&delta_base, 0, sizeof(delta_base));
delta_base.offset = delta_obj->offset; delta_base.offset = delta_obj->idx.offset;
if (!find_delta_children(&delta_base, &first, &last)) { if (!find_delta_children(&delta_base, &first, &last)) {
for (j = first; j <= last; j++) { for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no; struct object_entry *child = objects + deltas[j].obj_no;
@ -418,12 +416,12 @@ static void parse_pack_objects(unsigned char *sha1)
delta->obj_no = i; delta->obj_no = i;
delta++; delta++;
} else } else
sha1_object(data, obj->size, obj->type, obj->sha1); sha1_object(data, obj->size, obj->type, obj->idx.sha1);
free(data); free(data);
if (verbose) if (verbose)
display_progress(&progress, i+1); display_progress(&progress, i+1);
} }
objects[i].offset = consumed_bytes; objects[i].idx.offset = consumed_bytes;
if (verbose) if (verbose)
stop_progress(&progress); stop_progress(&progress);
@ -465,10 +463,10 @@ static void parse_pack_objects(unsigned char *sha1)
if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
continue; continue;
hashcpy(base.sha1, obj->sha1); hashcpy(base.sha1, obj->idx.sha1);
ref = !find_delta_children(&base, &ref_first, &ref_last); ref = !find_delta_children(&base, &ref_first, &ref_last);
memset(&base, 0, sizeof(base)); memset(&base, 0, sizeof(base));
base.offset = obj->offset; base.offset = obj->idx.offset;
ofs = !find_delta_children(&base, &ofs_first, &ofs_last); ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
if (!ref && !ofs) if (!ref && !ofs)
continue; continue;
@ -535,11 +533,11 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
} }
header[n++] = c; header[n++] = c;
write_or_die(output_fd, header, n); write_or_die(output_fd, header, n);
obj[0].crc32 = crc32(0, Z_NULL, 0); obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
obj[0].crc32 = crc32(obj[0].crc32, header, n); obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
obj[1].offset = obj[0].offset + n; obj[1].idx.offset = obj[0].idx.offset + n;
obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32); obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
hashcpy(obj->sha1, sha1); hashcpy(obj->idx.sha1, sha1);
} }
static int delta_pos_compare(const void *_a, const void *_b) static int delta_pos_compare(const void *_a, const void *_b)
@ -602,145 +600,6 @@ static void fix_unresolved_deltas(int nr_unresolved)
free(sorted_by_pos); free(sorted_by_pos);
} }
static uint32_t index_default_version = 1;
static uint32_t index_off32_limit = 0x7fffffff;
static int sha1_compare(const void *_a, const void *_b)
{
struct object_entry *a = *(struct object_entry **)_a;
struct object_entry *b = *(struct object_entry **)_b;
return hashcmp(a->sha1, b->sha1);
}
/*
* On entry *sha1 contains the pack content SHA1 hash, on exit it is
* the SHA1 hash of sorted object names.
*/
static const char *write_index_file(const char *index_name, unsigned char *sha1)
{
struct sha1file *f;
struct object_entry **sorted_by_sha, **list, **last;
uint32_t array[256];
int i, fd;
SHA_CTX ctx;
uint32_t index_version;
if (nr_objects) {
sorted_by_sha =
xcalloc(nr_objects, sizeof(struct object_entry *));
list = sorted_by_sha;
last = sorted_by_sha + nr_objects;
for (i = 0; i < nr_objects; ++i)
sorted_by_sha[i] = &objects[i];
qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
sha1_compare);
}
else
sorted_by_sha = list = last = NULL;
if (!index_name) {
static char tmpfile[PATH_MAX];
snprintf(tmpfile, sizeof(tmpfile),
"%s/tmp_idx_XXXXXX", get_object_directory());
fd = mkstemp(tmpfile);
index_name = xstrdup(tmpfile);
} else {
unlink(index_name);
fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
}
if (fd < 0)
die("unable to create %s: %s", index_name, strerror(errno));
f = sha1fd(fd, index_name);
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
struct object_entry **next = list;
while (next < last) {
struct object_entry *obj = *next;
if (obj->sha1[0] != i)
break;
next++;
}
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * 4);
/* compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/*
* Write the actual SHA1 entries..
*/
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
if (index_version < 2) {
uint32_t offset = htonl(obj->offset);
sha1write(f, &offset, 4);
}
sha1write(f, obj->sha1, 20);
SHA1_Update(&ctx, obj->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
uint32_t crc32_val = htonl(obj->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = *list++;
uint32_t offset = (obj->offset <= index_off32_limit) ?
obj->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct object_entry *obj = *list++;
uint64_t offset = obj->offset;
if (offset > index_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, sha1, 20);
sha1close(f, NULL, 1);
free(sorted_by_sha);
SHA1_Final(sha1, &ctx);
return index_name;
}
static void final(const char *final_pack_name, const char *curr_pack_name, static void final(const char *final_pack_name, const char *curr_pack_name,
const char *final_index_name, const char *curr_index_name, const char *final_index_name, const char *curr_index_name,
const char *keep_name, const char *keep_msg, const char *keep_name, const char *keep_msg,
@ -830,6 +689,7 @@ int main(int argc, char **argv)
const char *curr_index, *index_name = NULL; const char *curr_index, *index_name = NULL;
const char *keep_name = NULL, *keep_msg = NULL; const char *keep_name = NULL, *keep_msg = NULL;
char *index_name_buf = NULL, *keep_name_buf = NULL; char *index_name_buf = NULL, *keep_name_buf = NULL;
struct pack_idx_entry **idx_objects;
unsigned char sha1[20]; unsigned char sha1[20];
for (i = 1; i < argc; i++) { for (i = 1; i < argc; i++) {
@ -865,12 +725,12 @@ int main(int argc, char **argv)
index_name = argv[++i]; index_name = argv[++i];
} else if (!prefixcmp(arg, "--index-version=")) { } else if (!prefixcmp(arg, "--index-version=")) {
char *c; char *c;
index_default_version = strtoul(arg + 16, &c, 10); pack_idx_default_version = strtoul(arg + 16, &c, 10);
if (index_default_version > 2) if (pack_idx_default_version > 2)
die("bad %s", arg); die("bad %s", arg);
if (*c == ',') if (*c == ',')
index_off32_limit = strtoul(c+1, &c, 0); pack_idx_off32_limit = strtoul(c+1, &c, 0);
if (*c || index_off32_limit & 0x80000000) if (*c || pack_idx_off32_limit & 0x80000000)
die("bad %s", arg); die("bad %s", arg);
} else } else
usage(index_pack_usage); usage(index_pack_usage);
@ -940,7 +800,13 @@ int main(int argc, char **argv)
nr_deltas - nr_resolved_deltas); nr_deltas - nr_resolved_deltas);
} }
free(deltas); free(deltas);
curr_index = write_index_file(index_name, sha1);
idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
for (i = 0; i < nr_objects; i++)
idx_objects[i] = &objects[i].idx;
curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
free(idx_objects);
final(pack_name, curr_pack, final(pack_name, curr_pack,
index_name, curr_index, index_name, curr_index,
keep_name, keep_msg, keep_name, keep_msg,

142
pack-write.c
View File

@ -1,5 +1,147 @@
#include "cache.h" #include "cache.h"
#include "pack.h" #include "pack.h"
#include "csum-file.h"
uint32_t pack_idx_default_version = 1;
uint32_t pack_idx_off32_limit = 0x7fffffff;
static int sha1_compare(const void *_a, const void *_b)
{
struct pack_idx_entry *a = *(struct pack_idx_entry **)_a;
struct pack_idx_entry *b = *(struct pack_idx_entry **)_b;
return hashcmp(a->sha1, b->sha1);
}
/*
* On entry *sha1 contains the pack content SHA1 hash, on exit it is
* the SHA1 hash of sorted object names. The objects array passed in
* will be sorted by SHA1 on exit.
*/
const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
{
struct sha1file *f;
struct pack_idx_entry **sorted_by_sha, **list, **last;
off_t last_obj_offset = 0;
uint32_t array[256];
int i, fd;
SHA_CTX ctx;
uint32_t index_version;
if (nr_objects) {
sorted_by_sha = objects;
list = sorted_by_sha;
last = sorted_by_sha + nr_objects;
for (i = 0; i < nr_objects; ++i) {
if (objects[i]->offset > last_obj_offset)
last_obj_offset = objects[i]->offset;
}
qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
sha1_compare);
}
else
sorted_by_sha = list = last = NULL;
if (!index_name) {
static char tmpfile[PATH_MAX];
snprintf(tmpfile, sizeof(tmpfile),
"%s/tmp_idx_XXXXXX", get_object_directory());
fd = mkstemp(tmpfile);
index_name = xstrdup(tmpfile);
} else {
unlink(index_name);
fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
}
if (fd < 0)
die("unable to create %s: %s", index_name, strerror(errno));
f = sha1fd(fd, index_name);
/* if last object's offset is >= 2^31 we should use index V2 */
index_version = (last_obj_offset >> 31) ? 2 : pack_idx_default_version;
/* index versions 2 and above need a header */
if (index_version >= 2) {
struct pack_idx_header hdr;
hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
hdr.idx_version = htonl(index_version);
sha1write(f, &hdr, sizeof(hdr));
}
/*
* Write the first-level table (the list is sorted,
* but we use a 256-entry lookup to be able to avoid
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
struct pack_idx_entry **next = list;
while (next < last) {
struct pack_idx_entry *obj = *next;
if (obj->sha1[0] != i)
break;
next++;
}
array[i] = htonl(next - sorted_by_sha);
list = next;
}
sha1write(f, array, 256 * 4);
/* compute the SHA1 hash of sorted object names. */
SHA1_Init(&ctx);
/*
* Write the actual SHA1 entries..
*/
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct pack_idx_entry *obj = *list++;
if (index_version < 2) {
uint32_t offset = htonl(obj->offset);
sha1write(f, &offset, 4);
}
sha1write(f, obj->sha1, 20);
SHA1_Update(&ctx, obj->sha1, 20);
}
if (index_version >= 2) {
unsigned int nr_large_offset = 0;
/* write the crc32 table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct pack_idx_entry *obj = *list++;
uint32_t crc32_val = htonl(obj->crc32);
sha1write(f, &crc32_val, 4);
}
/* write the 32-bit offset table */
list = sorted_by_sha;
for (i = 0; i < nr_objects; i++) {
struct pack_idx_entry *obj = *list++;
uint32_t offset = (obj->offset <= pack_idx_off32_limit) ?
obj->offset : (0x80000000 | nr_large_offset++);
offset = htonl(offset);
sha1write(f, &offset, 4);
}
/* write the large offset table */
list = sorted_by_sha;
while (nr_large_offset) {
struct pack_idx_entry *obj = *list++;
uint64_t offset = obj->offset;
if (offset > pack_idx_off32_limit) {
uint32_t split[2];
split[0] = htonl(offset >> 32);
split[1] = htonl(offset & 0xffffffff);
sha1write(f, split, 8);
nr_large_offset--;
}
}
}
sha1write(f, sha1, 20);
sha1close(f, NULL, 1);
SHA1_Final(sha1, &ctx);
return index_name;
}
void fixup_pack_header_footer(int pack_fd, void fixup_pack_header_footer(int pack_fd,
unsigned char *pack_file_sha1, unsigned char *pack_file_sha1,

14
pack.h
View File

@ -34,6 +34,10 @@ struct pack_header {
*/ */
#define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */ #define PACK_IDX_SIGNATURE 0xff744f63 /* "\377tOc" */
/* These may be overridden by command-line parameters */
extern uint32_t pack_idx_default_version;
extern uint32_t pack_idx_off32_limit;
/* /*
* Packed object index header * Packed object index header
*/ */
@ -42,6 +46,16 @@ struct pack_idx_header {
uint32_t idx_version; uint32_t idx_version;
}; };
/*
* Common part of object structure used for write_idx_file
*/
struct pack_idx_entry {
unsigned char sha1[20];
uint32_t crc32;
off_t offset;
};
extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
extern int verify_pack(struct packed_git *, int); extern int verify_pack(struct packed_git *, int);
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t); extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);