Merge branch 'np/pack'

* np/pack: (27 commits)
  document --index-version for index-pack and pack-objects
  pack-objects: remove obsolete comments
  pack-objects: better check_object() performances
  add get_size_from_delta()
  pack-objects: make in_pack_header_size a variable of its own
  pack-objects: get rid of create_final_object_list()
  pack-objects: get rid of reuse_cached_pack
  pack-objects: clean up list sorting
  pack-objects: rework check_delta_limit usage
  pack-objects: equal objects in size should delta against newer objects
  pack-objects: optimize preferred base handling a bit
  clean up add_object_entry()
  tests for various pack index features
  use test-genrandom in tests instead of /dev/urandom
  simple random data generator for tests
  validate reused pack data with CRC when possible
  allow forcing index v2 and 64-bit offset treshold
  pack-redundant.c: learn about index v2
  show-index.c: learn about index v2
  sha1_file.c: learn about index version 2
  ...

.gitignore

@@ -149,6 +149,7 @@ test-chmtime
 test-date
 test-delta
 test-dump-cache-tree
+test-genrandom
 test-match-trees
 common-cmds.h
 *.tar.gz

Documentation/git-index-pack.txt

@@ -68,6 +68,11 @@ OPTIONS
 	message can later be searched for within all .keep files to
 	locate any which have outlived their usefulness.
 
+--index-version=<version>[,<offset>]::
+	This is intended to be used by the test suite only. It allows
+	to force the version for the generated pack index, and to force
+	64-bit index entries on objects located above the given offset.
+
 
 Note
 ----

Documentation/git-pack-objects.txt

@@ -138,6 +138,11 @@ base-name::
 	length, this option typically shrinks the resulting
 	packfile by 3-5 per-cent.
 
+--index-version=<version>[,<offset>]::
+	This is intended to be used by the test suite only. It allows
+	to force the version for the generated pack index, and to force
+	64-bit index entries on objects located above the given offset.
+
 
 Author
 ------

Makefile

@@ -933,7 +933,7 @@ endif
 
 export NO_SVN_TESTS
 
-test: all test-chmtime$X
+test: all test-chmtime$X test-genrandom$X
 	$(MAKE) -C t/ all
 
 test-date$X: test-date.c date.o ctype.o
@@ -954,6 +954,9 @@ test-match-trees$X: test-match-trees.o $(GITLIBS)
 test-chmtime$X: test-chmtime.c
 	$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $<
 
+test-genrandom$X: test-genrandom.c
+	$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $<
+
 check-sha1:: test-sha1$X
 	./test-sha1.sh
 
@@ -1042,7 +1045,7 @@ dist-doc:
 
 clean:
 	rm -f *.o mozilla-sha1/*.o arm/*.o ppc/*.o compat/*.o xdiff/*.o \
-		test-chmtime$X $(LIB_FILE) $(XDIFF_LIB)
+		test-chmtime$X test-genrandom$X $(LIB_FILE) $(XDIFF_LIB)
 	rm -f $(ALL_PROGRAMS) $(BUILT_INS) git$X
 	rm -f *.spec *.pyc *.pyo */*.pyc */*.pyo common-cmds.h TAGS tags
 	rm -rf autom4te.cache

builtin-count-objects.c

@@ -111,7 +111,7 @@ int cmd_count_objects(int ac, const char **av, const char *prefix)
 		for (p = packed_git; p; p = p->next) {
 			if (!p->pack_local)
 				continue;
-			packed += num_packed_objects(p);
+			packed += p->num_objects;
 			num_pack++;
 		}
 		printf("count: %lu\n", loose);

builtin-fsck.c

@@ -661,7 +661,7 @@ int cmd_fsck(int argc, char **argv, const char *prefix)
 			verify_pack(p, 0);
 
 		for (p = packed_git; p; p = p->next) {
-			uint32_t i, num = num_packed_objects(p);
+			uint32_t i, num = p->num_objects;
 			for (i = 0; i < num; i++)
 				fsck_sha1(nth_packed_object_sha1(p, i));
 		}

builtin-unpack-objects.c

@@ -13,7 +13,8 @@ static const char unpack_usage[] = "git-unpack-objects [-n] [-q] [-r] < pack-fil
 
 /* We always read in 4kB chunks. */
 static unsigned char buffer[4096];
-static unsigned long offset, len, consumed_bytes;
+static unsigned int offset, len;
+static off_t consumed_bytes;
 static SHA_CTX ctx;
 
 /*
@@ -49,6 +50,10 @@ static void use(int bytes)
 		die("used more bytes than were available");
 	len -= bytes;
 	offset += bytes;
+
+	/* make sure off_t is sufficiently large not to wrap */
+	if (consumed_bytes > consumed_bytes + bytes)
+		die("pack too large for current definition of off_t");
 	consumed_bytes += bytes;
 }
 
@@ -88,17 +93,17 @@ static void *get_data(unsigned long size)
 
 struct delta_info {
 	unsigned char base_sha1[20];
-	unsigned long base_offset;
+	unsigned nr;
+	off_t base_offset;
 	unsigned long size;
 	void *delta;
-	unsigned nr;
 	struct delta_info *next;
 };
 
 static struct delta_info *delta_list;
 
 static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
-			      unsigned long base_offset,
+			      off_t base_offset,
 			      void *delta, unsigned long size)
 {
 	struct delta_info *info = xmalloc(sizeof(*info));
@@ -113,7 +118,7 @@ static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
 }
 
 struct obj_info {
-	unsigned long offset;
+	off_t offset;
 	unsigned char sha1[20];
 };
 
@@ -200,7 +205,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 	} else {
 		unsigned base_found = 0;
 		unsigned char *pack, c;
-		unsigned long base_offset;
+		off_t base_offset;
 		unsigned lo, mid, hi;
 
 		pack = fill(1);
@@ -209,7 +214,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 		base_offset = c & 127;
 		while (c & 128) {
 			base_offset += 1;
-			if (!base_offset || base_offset & ~(~0UL >> 7))
+			if (!base_offset || MSB(base_offset, 7))
 				die("offset value overflow for delta base object");
 			pack = fill(1);
 			c = *pack;

cache.h

@@ -376,11 +376,12 @@ struct pack_window {
 extern struct packed_git {
 	struct packed_git *next;
 	struct pack_window *windows;
-	const void *index_data;
-	off_t index_size;
 	off_t pack_size;
-	time_t mtime;
+	const void *index_data;
+	size_t index_size;
+	uint32_t num_objects;
 	int index_version;
+	time_t mtime;
 	int pack_fd;
 	int pack_local;
 	unsigned char sha1[20];
@@ -431,11 +432,11 @@ extern void pack_report(void);
 extern unsigned char* use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
 extern void unuse_pack(struct pack_window **);
 extern struct packed_git *add_packed_git(const char *, int, int);
-extern uint32_t num_packed_objects(const struct packed_git *p);
 extern const unsigned char *nth_packed_object_sha1(const struct packed_git *, uint32_t);
 extern off_t find_pack_entry_one(const unsigned char *, struct packed_git *);
 extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *);
 extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
+extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
 extern const char *packed_object_info_detail(struct packed_git *, off_t, unsigned long *, unsigned long *, unsigned int *, unsigned char *);
 
 /* Dumb servers support */

csum-file.c

@@ -49,6 +49,8 @@ int sha1close(struct sha1file *f, unsigned char *result, int update)
 
 int sha1write(struct sha1file *f, void *buf, unsigned int count)
 {
+	if (f->do_crc)
+		f->crc32 = crc32(f->crc32, buf, count);
 	while (count) {
 		unsigned offset = f->offset;
 		unsigned left = sizeof(f->buffer) - offset;
@@ -91,6 +93,7 @@ struct sha1file *sha1create(const char *fmt, ...)
 	f->fd = fd;
 	f->error = 0;
 	f->offset = 0;
+	f->do_crc = 0;
 	SHA1_Init(&f->ctx);
 	return f;
 }
@@ -111,6 +114,7 @@ struct sha1file *sha1fd(int fd, const char *name)
 	f->fd = fd;
 	f->error = 0;
 	f->offset = 0;
+	f->do_crc = 0;
 	SHA1_Init(&f->ctx);
 	return f;
 }
@@ -143,4 +147,14 @@ int sha1write_compressed(struct sha1file *f, void *in, unsigned int size)
 	return size;
 }
 
+void crc32_begin(struct sha1file *f)
+{
+	f->crc32 = crc32(0, Z_NULL, 0);
+	f->do_crc = 1;
+}
 
+uint32_t crc32_end(struct sha1file *f)
+{
+	f->do_crc = 0;
+	return f->crc32;
+}

csum-file.h

@@ -7,6 +7,8 @@ struct sha1file {
 	unsigned int offset, namelen;
 	SHA_CTX ctx;
 	char name[PATH_MAX];
+	int do_crc;
+	uint32_t crc32;
 	unsigned char buffer[8192];
 };
 
@@ -15,5 +17,7 @@ extern struct sha1file *sha1create(const char *fmt, ...) __attribute__((format (
 extern int sha1close(struct sha1file *, unsigned char *, int);
 extern int sha1write(struct sha1file *, void *, unsigned int);
 extern int sha1write_compressed(struct sha1file *, void *, unsigned int);
+extern void crc32_begin(struct sha1file *);
+extern uint32_t crc32_end(struct sha1file *);
 
 #endif

git-compat-util.h

@@ -13,6 +13,14 @@
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
 
+#ifdef __GNUC__
+#define TYPEOF(x) (__typeof__(x))
+#else
+#define TYPEOF(x)
+#endif
+
+#define MSB(x, bits) ((x) & TYPEOF(x)(~0ULL << (sizeof(x) * 8 - (bits))))
+
 #if !defined(__APPLE__) && !defined(__FreeBSD__)
 #define _XOPEN_SOURCE 600 /* glibc2 and AIX 5.3L need 500, OpenBSD needs 600 for S_ISLNK() */
 #define _XOPEN_SOURCE_EXTENDED 1 /* AIX 5.3L needs this */

index-pack.c

@@ -12,9 +12,10 @@ static const char index_pack_usage[] =
 
 struct object_entry
 {
-	unsigned long offset;
+	off_t offset;
 	unsigned long size;
 	unsigned int hdr_size;
+	uint32_t crc32;
 	enum object_type type;
 	enum object_type real_type;
 	unsigned char sha1[20];
@@ -22,7 +23,7 @@ struct object_entry
 
 union delta_base {
 	unsigned char sha1[20];
-	unsigned long offset;
+	off_t offset;
 };
 
 /*
@@ -83,8 +84,10 @@ static unsigned display_progress(unsigned n, unsigned total, unsigned last_pc)
 
 /* We always read in 4kB chunks. */
 static unsigned char input_buffer[4096];
-static unsigned long input_offset, input_len, consumed_bytes;
+static unsigned int input_offset, input_len;
+static off_t consumed_bytes;
 static SHA_CTX input_ctx;
+static uint32_t input_crc32;
 static int input_fd, output_fd, pack_fd;
 
 /* Discard current buffer used content. */
@@ -127,8 +130,13 @@ static void use(int bytes)
 {
 	if (bytes > input_len)
 		die("used more bytes than were available");
+	input_crc32 = crc32(input_crc32, input_buffer + input_offset, bytes);
 	input_len -= bytes;
 	input_offset += bytes;
+
+	/* make sure off_t is sufficiently large not to wrap */
+	if (consumed_bytes > consumed_bytes + bytes)
+		die("pack too large for current definition of off_t");
 	consumed_bytes += bytes;
 }
 
@@ -216,10 +224,13 @@ static void *unpack_entry_data(unsigned long offset, unsigned long size)
 static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_base)
 {
 	unsigned char *p, c;
-	unsigned long size, base_offset;
+	unsigned long size;
+	off_t base_offset;
 	unsigned shift;
+	void *data;
 
 	obj->offset = consumed_bytes;
+	input_crc32 = crc32(0, Z_NULL, 0);
 
 	p = fill(1);
 	c = *p;
@@ -249,7 +260,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
 		base_offset = c & 127;
 		while (c & 128) {
 			base_offset += 1;
-			if (!base_offset || base_offset & ~(~0UL >> 7))
+			if (!base_offset || MSB(base_offset, 7))
 				bad_object(obj->offset, "offset value overflow for delta base object");
 			p = fill(1);
 			c = *p;
@@ -270,7 +281,9 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
 	}
 	obj->hdr_size = consumed_bytes - obj->offset;
 
-	return unpack_entry_data(obj->offset, obj->size);
+	data = unpack_entry_data(obj->offset, obj->size);
+	obj->crc32 = input_crc32;
+	return data;
 }
 
 static void *get_data_from_pack(struct object_entry *obj)
@@ -515,7 +528,7 @@ static void parse_pack_objects(unsigned char *sha1)
 		fputc('\n', stderr);
 }
 
-static int write_compressed(int fd, void *in, unsigned int size)
+static int write_compressed(int fd, void *in, unsigned int size, uint32_t *obj_crc)
 {
 	z_stream stream;
 	unsigned long maxsize;
@@ -536,6 +549,7 @@ static int write_compressed(int fd, void *in, unsigned int size)
 
 	size = stream.total_out;
 	write_or_die(fd, out, size);
+	*obj_crc = crc32(*obj_crc, out, size);
 	free(out);
 	return size;
 }
@@ -556,8 +570,10 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
 	}
 	header[n++] = c;
 	write_or_die(output_fd, header, n);
+	obj[0].crc32 = crc32(0, Z_NULL, 0);
+	obj[0].crc32 = crc32(obj[0].crc32, header, n);
 	obj[1].offset = obj[0].offset + n;
-	obj[1].offset += write_compressed(output_fd, buf, size);
+	obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
 	hashcpy(obj->sha1, sha1);
 }
 
@@ -655,6 +671,9 @@ static void readjust_pack_header_and_sha1(unsigned char *sha1)
 	write_or_die(output_fd, sha1, 20);
 }
 
+static uint32_t index_default_version = 1;
+static uint32_t index_off32_limit = 0x7fffffff;
+
 static int sha1_compare(const void *_a, const void *_b)
 {
 	struct object_entry *a = *(struct object_entry **)_a;
@@ -670,9 +689,10 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
 {
 	struct sha1file *f;
 	struct object_entry **sorted_by_sha, **list, **last;
-	unsigned int array[256];
+	uint32_t array[256];
 	int i, fd;
 	SHA_CTX ctx;
+	uint32_t index_version;
 
 	if (nr_objects) {
 		sorted_by_sha =
@@ -683,7 +703,6 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
 			sorted_by_sha[i] = &objects[i];
 		qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
 		      sha1_compare);
-
 	}
 	else
 		sorted_by_sha = list = last = NULL;
@@ -702,6 +721,17 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
 		die("unable to create %s: %s", index_name, strerror(errno));
 	f = sha1fd(fd, index_name);
 
+	/* if last object's offset is >= 2^31 we should use index V2 */
+	index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
+
+	/* index versions 2 and above need a header */
+	if (index_version >= 2) {
+		struct pack_idx_header hdr;
+		hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
+		hdr.idx_version = htonl(index_version);
+		sha1write(f, &hdr, sizeof(hdr));
+	}
+
 	/*
 	 * Write the first-level table (the list is sorted,
 	 * but we use a 256-entry lookup to be able to avoid
@@ -718,24 +748,61 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
 		array[i] = htonl(next - sorted_by_sha);
 		list = next;
 	}
-	sha1write(f, array, 256 * sizeof(int));
+	sha1write(f, array, 256 * 4);
 
-	/* recompute the SHA1 hash of sorted object names.
+	/* compute the SHA1 hash of sorted object names. */
-	 * currently pack-objects does not do this, but that
-	 * can be fixed.
-	 */
 	SHA1_Init(&ctx);
+
 	/*
 	 * Write the actual SHA1 entries..
 	 */
 	list = sorted_by_sha;
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *obj = *list++;
-		unsigned int offset = htonl(obj->offset);
+		if (index_version < 2) {
+			uint32_t offset = htonl(obj->offset);
 			sha1write(f, &offset, 4);
+		}
 		sha1write(f, obj->sha1, 20);
 		SHA1_Update(&ctx, obj->sha1, 20);
 	}
+
+	if (index_version >= 2) {
+		unsigned int nr_large_offset = 0;
+
+		/* write the crc32 table */
+		list = sorted_by_sha;
+		for (i = 0; i < nr_objects; i++) {
+			struct object_entry *obj = *list++;
+			uint32_t crc32_val = htonl(obj->crc32);
+			sha1write(f, &crc32_val, 4);
+		}
+
+		/* write the 32-bit offset table */
+		list = sorted_by_sha;
+		for (i = 0; i < nr_objects; i++) {
+			struct object_entry *obj = *list++;
+			uint32_t offset = (obj->offset <= index_off32_limit) ?
+				obj->offset : (0x80000000 | nr_large_offset++);
+			offset = htonl(offset);
+			sha1write(f, &offset, 4);
+		}
+
+		/* write the large offset table */
+		list = sorted_by_sha;
+		while (nr_large_offset) {
+			struct object_entry *obj = *list++;
+			uint64_t offset = obj->offset;
+			if (offset > index_off32_limit) {
+				uint32_t split[2];
+				split[0]	= htonl(offset >> 32);
+				split[1] = htonl(offset & 0xffffffff);
+				sha1write(f, split, 8);
+				nr_large_offset--;
+			}
+		}
+	}
+
 	sha1write(f, sha1, 20);
 	sha1close(f, NULL, 1);
 	free(sorted_by_sha);
@@ -865,6 +932,15 @@ int main(int argc, char **argv)
 				if (index_name || (i+1) >= argc)
 					usage(index_pack_usage);
 				index_name = argv[++i];
+			} else if (!prefixcmp(arg, "--index-version=")) {
+				char *c;
+				index_default_version = strtoul(arg + 16, &c, 10);
+				if (index_default_version > 2)
+					die("bad %s", arg);
+				if (*c == ',')
+					index_off32_limit = strtoul(c+1, &c, 0);
+				if (*c || index_off32_limit & 0x80000000)
+					die("bad %s", arg);
 			} else
 				usage(index_pack_usage);
 			continue;

pack-check.c

@@ -40,7 +40,7 @@ static int verify_packfile(struct packed_git *p,
 	 * have verified that nr_objects matches between idx and pack,
 	 * we do not do scan-streaming check on the pack file.
 	 */
-	nr_objects = num_packed_objects(p);
+	nr_objects = p->num_objects;
 	for (i = 0, err = 0; i < nr_objects; i++) {
 		const unsigned char *sha1;
 		void *data;
@@ -79,7 +79,7 @@ static void show_pack_info(struct packed_git *p)
 {
 	uint32_t nr_objects, i, chain_histogram[MAX_CHAIN];
 
-	nr_objects = num_packed_objects(p);
+	nr_objects = p->num_objects;
 	memset(chain_histogram, 0, sizeof(chain_histogram));
 
 	for (i = 0; i < nr_objects; i++) {

pack-redundant.c

@@ -247,16 +247,19 @@ static struct pack_list * pack_list_difference(const struct pack_list *A,
 
 static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 {
-	int p1_off, p2_off;
+	unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
 	const unsigned char *p1_base, *p2_base;
 	struct llist_item *p1_hint = NULL, *p2_hint = NULL;
 
-	p1_off = p2_off = 256 * 4 + 4;
 	p1_base = p1->pack->index_data;
 	p2_base = p2->pack->index_data;
+	p1_base += 256 * 4 + ((p1->pack->index_version < 2) ? 4 : 8);
+	p2_base += 256 * 4 + ((p2->pack->index_version < 2) ? 4 : 8);
+	p1_step = (p1->pack->index_version < 2) ? 24 : 20;
+	p2_step = (p2->pack->index_version < 2) ? 24 : 20;
 
-	while (p1_off <= p1->pack->index_size - 3 * 20 &&
+	while (p1_off < p1->pack->num_objects * p1_step &&
-	       p2_off <= p2->pack->index_size - 3 * 20)
+	       p2_off < p2->pack->num_objects * p2_step)
 	{
 		int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
 		/* cmp ~ p1 - p2 */
@@ -265,14 +268,14 @@ static void cmp_two_packs(struct pack_list *p1, struct pack_list *p2)
 					p1_base + p1_off, p1_hint);
 			p2_hint = llist_sorted_remove(p2->unique_objects,
 					p1_base + p1_off, p2_hint);
-			p1_off+=24;
+			p1_off += p1_step;
-			p2_off+=24;
+			p2_off += p2_step;
 			continue;
 		}
 		if (cmp < 0) { /* p1 has the object, p2 doesn't */
-			p1_off+=24;
+			p1_off += p1_step;
 		} else { /* p2 has the object, p1 doesn't */
-			p2_off+=24;
+			p2_off += p2_step;
 		}
 	}
 }
@@ -352,28 +355,31 @@ static int is_superset(struct pack_list *pl, struct llist *list)
 static size_t sizeof_union(struct packed_git *p1, struct packed_git *p2)
 {
 	size_t ret = 0;
-	int p1_off, p2_off;
+	unsigned long p1_off = 0, p2_off = 0, p1_step, p2_step;
 	const unsigned char *p1_base, *p2_base;
 
-	p1_off = p2_off = 256 * 4 + 4;
 	p1_base = p1->index_data;
 	p2_base = p2->index_data;
+	p1_base += 256 * 4 + ((p1->index_version < 2) ? 4 : 8);
+	p2_base += 256 * 4 + ((p2->index_version < 2) ? 4 : 8);
+	p1_step = (p1->index_version < 2) ? 24 : 20;
+	p2_step = (p2->index_version < 2) ? 24 : 20;
 
-	while (p1_off <= p1->index_size - 3 * 20 &&
+	while (p1_off < p1->num_objects * p1_step &&
-	       p2_off <= p2->index_size - 3 * 20)
+	       p2_off < p2->num_objects * p2_step)
 	{
 		int cmp = hashcmp(p1_base + p1_off, p2_base + p2_off);
 		/* cmp ~ p1 - p2 */
 		if (cmp == 0) {
 			ret++;
-			p1_off+=24;
+			p1_off += p1_step;
-			p2_off+=24;
+			p2_off += p2_step;
 			continue;
 		}
 		if (cmp < 0) { /* p1 has the object, p2 doesn't */
-			p1_off+=24;
+			p1_off += p1_step;
 		} else { /* p2 has the object, p1 doesn't */
-			p2_off+=24;
+			p2_off += p2_step;
 		}
 	}
 	return ret;
@@ -535,7 +541,7 @@ static void scan_alt_odb_packs(void)
 static struct pack_list * add_pack(struct packed_git *p)
 {
 	struct pack_list l;
-	size_t off;
+	unsigned long off = 0, step;
 	const unsigned char *base;
 
 	if (!p->pack_local && !(alt_odb || verbose))
@@ -544,11 +550,12 @@ static struct pack_list * add_pack(struct packed_git *p)
 	l.pack = p;
 	llist_init(&l.all_objects);
 
-	off = 256 * 4 + 4;
 	base = p->index_data;
-	while (off <= p->index_size - 3 * 20) {
+	base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
+	step = (p->index_version < 2) ? 24 : 20;
+	while (off < p->num_objects * step) {
 		llist_insert_back(l.all_objects, base + off);
-		off += 24;
+		off += step;
 	}
 	/* this list will be pruned in cmp_two_packs later */
 	l.unique_objects = llist_copy(l.all_objects);

sha1_file.c

@@ -437,7 +437,7 @@ static int check_packed_git_idx(const char *path,  struct packed_git *p)
 	void *idx_map;
 	struct pack_idx_header *hdr;
 	size_t idx_size;
-	uint32_t nr, i, *index;
+	uint32_t version, nr, i, *index;
 	int fd = open(path, O_RDONLY);
 	struct stat st;
 
@@ -455,21 +455,23 @@ static int check_packed_git_idx(const char *path,  struct packed_git *p)
 	idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
 	close(fd);
 
-	/* a future index format would start with this, as older git
-	 * binaries would fail the non-monotonic index check below.
-	 * give a nicer warning to the user if we can.
-	 */
 	hdr = idx_map;
 	if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
+		version = ntohl(hdr->idx_version);
+		if (version < 2 || version > 2) {
 			munmap(idx_map, idx_size);
-		return error("index file %s is a newer version"
+			return error("index file %s is version %d"
 				     " and is not supported by this binary"
 				     " (try upgrading GIT to a newer version)",
-			path);
+				     path, version);
 		}
+	} else
+		version = 1;
 
 	nr = 0;
 	index = idx_map;
+	if (version > 1)
+		index += 2;  /* skip index header */
 	for (i = 0; i < 256; i++) {
 		uint32_t n = ntohl(index[i]);
 		if (n < nr) {
@@ -479,6 +481,7 @@ static int check_packed_git_idx(const char *path,  struct packed_git *p)
 		nr = n;
 	}
 
+	if (version == 1) {
 		/*
 		 * Total size:
 		 *  - 256 index entries 4 bytes each
@@ -490,10 +493,39 @@ static int check_packed_git_idx(const char *path,  struct packed_git *p)
 			munmap(idx_map, idx_size);
 			return error("wrong index file size in %s", path);
 		}
+	} else if (version == 2) {
+		/*
+		 * Minimum size:
+		 *  - 8 bytes of header
+		 *  - 256 index entries 4 bytes each
+		 *  - 20-byte sha1 entry * nr
+		 *  - 4-byte crc entry * nr
+		 *  - 4-byte offset entry * nr
+		 *  - 20-byte SHA1 of the packfile
+		 *  - 20-byte SHA1 file checksum
+		 * And after the 4-byte offset table might be a
+		 * variable sized table containing 8-byte entries
+		 * for offsets larger than 2^31.
+		 */
+		unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
+		if (idx_size < min_size || idx_size > min_size + (nr - 1)*8) {
+			munmap(idx_map, idx_size);
+			return error("wrong index file size in %s", path);
+		}
+		if (idx_size != min_size) {
+			/* make sure we can deal with large pack offsets */
+			off_t x = 0x7fffffffUL, y = 0xffffffffUL;
+			if (x > (x + 1) || y > (y + 1)) {
+				munmap(idx_map, idx_size);
+				return error("pack too large for current definition of off_t in %s", path);
+			}
+		}
+	}
 
-	p->index_version = 1;
+	p->index_version = version;
 	p->index_data = idx_map;
 	p->index_size = idx_size;
+	p->num_objects = nr;
 	return 0;
 }
 
@@ -605,11 +637,11 @@ static int open_packed_git_1(struct packed_git *p)
 			p->pack_name, ntohl(hdr.hdr_version));
 
 	/* Verify the pack matches its index. */
-	if (num_packed_objects(p) != ntohl(hdr.hdr_entries))
+	if (p->num_objects != ntohl(hdr.hdr_entries))
 		return error("packfile %s claims to have %u objects"
-			" while index size indicates %u objects",
+			     " while index indicates %u objects",
 			     p->pack_name, ntohl(hdr.hdr_entries),
-			num_packed_objects(p));
+			     p->num_objects);
 	if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
 		return error("end of packfile %s is unavailable", p->pack_name);
 	if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
@@ -1128,6 +1160,43 @@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type
 	return unpack_sha1_rest(&stream, hdr, *size, sha1);
 }
 
+unsigned long get_size_from_delta(struct packed_git *p,
+				  struct pack_window **w_curs,
+			          off_t curpos)
+{
+	const unsigned char *data;
+	unsigned char delta_head[20], *in;
+	z_stream stream;
+	int st;
+
+	memset(&stream, 0, sizeof(stream));
+	stream.next_out = delta_head;
+	stream.avail_out = sizeof(delta_head);
+
+	inflateInit(&stream);
+	do {
+		in = use_pack(p, w_curs, curpos, &stream.avail_in);
+		stream.next_in = in;
+		st = inflate(&stream, Z_FINISH);
+		curpos += stream.next_in - in;
+	} while ((st == Z_OK || st == Z_BUF_ERROR) &&
+		 stream.total_out < sizeof(delta_head));
+	inflateEnd(&stream);
+	if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head))
+		die("delta data unpack-initial failed");
+
+	/* Examine the initial part of the delta to figure out
+	 * the result size.
+	 */
+	data = delta_head;
+
+	/* ignore base size */
+	get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
+
+	/* Read the result size */
+	return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
+}
+
 static off_t get_delta_base(struct packed_git *p,
 				    struct pack_window **w_curs,
 				    off_t *curpos,
@@ -1149,7 +1218,7 @@ static off_t get_delta_base(struct packed_git *p,
 		base_offset = c & 127;
 		while (c & 128) {
 			base_offset += 1;
-			if (!base_offset || base_offset & ~(~0UL >> 7))
+			if (!base_offset || MSB(base_offset, 7))
 				die("offset value overflow for delta base object");
 			c = base_info[used++];
 			base_offset = (base_offset << 7) + (c & 127);
@@ -1191,40 +1260,8 @@ static int packed_delta_info(struct packed_git *p,
 	 * based on a base with a wrong size.  This saves tons of
 	 * inflate() calls.
 	 */
-	if (sizep) {
+	if (sizep)
-		const unsigned char *data;
+		*sizep = get_size_from_delta(p, w_curs, curpos);
-		unsigned char delta_head[20], *in;
-		z_stream stream;
-		int st;
-
-		memset(&stream, 0, sizeof(stream));
-		stream.next_out = delta_head;
-		stream.avail_out = sizeof(delta_head);
-
-		inflateInit(&stream);
-		do {
-			in = use_pack(p, w_curs, curpos, &stream.avail_in);
-			stream.next_in = in;
-			st = inflate(&stream, Z_FINISH);
-			curpos += stream.next_in - in;
-		} while ((st == Z_OK || st == Z_BUF_ERROR)
-			&& stream.total_out < sizeof(delta_head));
-		inflateEnd(&stream);
-		if ((st != Z_STREAM_END) &&
-		    stream.total_out != sizeof(delta_head))
-			die("delta data unpack-initial failed");
-
-		/* Examine the initial part of the delta to figure out
-		 * the result size.
-		 */
-		data = delta_head;
-
-		/* ignore base size */
-		get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
-
-		/* Read the result size */
-		*sizep = get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
-	}
 
 	return type;
 }
@@ -1526,37 +1563,60 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
 	return data;
 }
 
-uint32_t num_packed_objects(const struct packed_git *p)
-{
-	/* See check_packed_git_idx() */
-	return (uint32_t)((p->index_size - 20 - 20 - 4*256) / 24);
-}
-
 const unsigned char *nth_packed_object_sha1(const struct packed_git *p,
 					    uint32_t n)
 {
 	const unsigned char *index = p->index_data;
-	index += 4 * 256;
+	if (n >= p->num_objects)
-	if (num_packed_objects(p) <= n)
 		return NULL;
+	index += 4 * 256;
+	if (p->index_version == 1) {
 		return index + 24 * n + 4;
+	} else {
+		index += 8;
+		return index + 20 * n;
+	}
+}
+
+static off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
+{
+	const unsigned char *index = p->index_data;
+	index += 4 * 256;
+	if (p->index_version == 1) {
+		return ntohl(*((uint32_t *)(index + 24 * n)));
+	} else {
+		uint32_t off;
+		index += 8 + p->num_objects * (20 + 4);
+		off = ntohl(*((uint32_t *)(index + 4 * n)));
+		if (!(off & 0x80000000))
+			return off;
+		index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
+		return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
+				   ntohl(*((uint32_t *)(index + 4)));
+	}
 }
 
 off_t find_pack_entry_one(const unsigned char *sha1,
 				  struct packed_git *p)
 {
 	const uint32_t *level1_ofs = p->index_data;
-	int hi = ntohl(level1_ofs[*sha1]);
-	int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
 	const unsigned char *index = p->index_data;
+	unsigned hi, lo;
 
+	if (p->index_version > 1) {
+		level1_ofs += 2;
+		index += 8;
+	}
 	index += 4 * 256;
+	hi = ntohl(level1_ofs[*sha1]);
+	lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
 
 	do {
-		int mi = (lo + hi) / 2;
+		unsigned mi = (lo + hi) / 2;
-		int cmp = hashcmp(index + 24 * mi + 4, sha1);
+		unsigned x = (p->index_version > 1) ? (mi * 20) : (mi * 24 + 4);
+		int cmp = hashcmp(index + x, sha1);
 		if (!cmp)
-			return ntohl(*((uint32_t *)((char *)index + (24 * mi))));
+			return nth_packed_object_offset(p, mi);
 		if (cmp > 0)
 			hi = mi;
 		else

sha1_name.c

@@ -76,7 +76,7 @@ static int find_short_packed_object(int len, const unsigned char *match, unsigne
 
 	prepare_packed_git();
 	for (p = packed_git; p && found < 2; p = p->next) {
-		uint32_t num = num_packed_objects(p);
+		uint32_t num = p->num_objects;
 		uint32_t first = 0, last = num;
 		while (first < last) {
 			uint32_t mid = (first + last) / 2;

show-index.c

@@ -1,14 +1,26 @@
 #include "cache.h"
+#include "pack.h"
 
 int main(int argc, char **argv)
 {
 	int i;
 	unsigned nr;
-	unsigned int entry[6];
+	unsigned int version;
 	static unsigned int top_index[256];
 
-	if (fread(top_index, sizeof(top_index), 1, stdin) != 1)
+	if (fread(top_index, 2 * 4, 1, stdin) != 1)
+		die("unable to read header");
+	if (top_index[0] == htonl(PACK_IDX_SIGNATURE)) {
+		version = ntohl(top_index[1]);
+		if (version < 2 || version > 2)
+			die("unknown index version");
+		if (fread(top_index, 256 * 4, 1, stdin) != 1)
 			die("unable to read index");
+	} else {
+		version = 1;
+		if (fread(&top_index[2], 254 * 4, 1, stdin) != 1)
+			die("unable to read index");
+	}
 	nr = 0;
 	for (i = 0; i < 256; i++) {
 		unsigned n = ntohl(top_index[i]);
@@ -16,13 +28,51 @@ int main(int argc, char **argv)
 			die("corrupt index file");
 		nr = n;
 	}
+	if (version == 1) {
 		for (i = 0; i < nr; i++) {
-		unsigned offset;
+			unsigned int offset, entry[6];
 
-		if (fread(entry, 24, 1, stdin) != 1)
+			if (fread(entry, 4 + 20, 1, stdin) != 1)
 				die("unable to read entry %u/%u", i, nr);
 			offset = ntohl(entry[0]);
 			printf("%u %s\n", offset, sha1_to_hex((void *)(entry+1)));
 		}
+	} else {
+		unsigned off64_nr = 0;
+		struct {
+			unsigned char sha1[20];
+			uint32_t crc;
+			uint32_t off;
+		} *entries = xmalloc(nr * sizeof(entries[0]));
+		for (i = 0; i < nr; i++)
+			if (fread(entries[i].sha1, 20, 1, stdin) != 1)
+				die("unable to read sha1 %u/%u", i, nr);
+		for (i = 0; i < nr; i++)
+			if (fread(&entries[i].crc, 4, 1, stdin) != 1)
+				die("unable to read crc %u/%u", i, nr);
+		for (i = 0; i < nr; i++)
+			if (fread(&entries[i].off, 4, 1, stdin) != 1)
+				die("unable to read 32b offset %u/%u", i, nr);
+		for (i = 0; i < nr; i++) {
+			uint64_t offset;
+			uint32_t off = ntohl(entries[i].off);
+			if (!(off & 0x80000000)) {
+				offset = off;
+			} else {
+				uint32_t off64[2];
+				if ((off & 0x7fffffff) != off64_nr)
+					die("inconsistent 64b offset index");
+				if (fread(off64, 8, 1, stdin) != 1)
+					die("unable to read 64b offset %u", off64_nr);
+				offset = (((uint64_t)ntohl(off64[0])) << 32) |
+						     ntohl(off64[1]);
+				off64_nr++;
+			}
+			printf("%llu %s (%08x)\n", (unsigned long long) offset,
+			       sha1_to_hex(entries[i].sha1),
+			       ntohl(entries[i].crc));
+		}
+		free(entries);
+	}
 	return 0;
 }

t/t5301-sliding-window.sh

@@ -12,7 +12,7 @@ test_expect_success \
      for i in a b c
      do
          echo $i >$i &&
-         dd if=/dev/urandom bs=32k count=1 >>$i &&
+         test-genrandom "$i" 32768 >>$i &&
          git-update-index --add $i || return 1
      done &&
      echo d >d && cat c >>d && git-update-index --add d &&

t/t5302-pack-index.sh

@@ -0,0 +1,146 @@
+#!/bin/sh
+#
+# Copyright (c) 2007 Nicolas Pitre
+#
+
+test_description='pack index with 64-bit offsets and object CRC'
+. ./test-lib.sh
+
+test_expect_success \
+    'setup' \
+    'rm -rf .git
+     git-init &&
+     for i in `seq -w 100`
+     do
+         echo $i >file_$i &&
+         test-genrandom "$i" 8192 >>file_$i &&
+         git-update-index --add file_$i || return 1
+     done &&
+     echo 101 >file_101 && tail -c 8192 file_100 >>file_101 &&
+     git-update-index --add file_101 &&
+     tree=`git-write-tree` &&
+     commit=`git-commit-tree $tree </dev/null` && {
+	 echo $tree &&
+	 git-ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\)	.*/\\1/"
+     } >obj-list &&
+     git-update-ref HEAD $commit'
+
+test_expect_success \
+    'pack-objects with index version 1' \
+    'pack1=$(git-pack-objects --index-version=1 test-1 <obj-list) &&
+     git-verify-pack -v "test-1-${pack1}.pack"'
+
+test_expect_success \
+    'pack-objects with index version 2' \
+    'pack2=$(git-pack-objects --index-version=2 test-2 <obj-list) &&
+     git-verify-pack -v "test-2-${pack2}.pack"'
+
+test_expect_success \
+    'both packs should be identical' \
+    'cmp "test-1-${pack1}.pack" "test-2-${pack2}.pack"'
+
+test_expect_failure \
+    'index v1 and index v2 should be different' \
+    'cmp "test-1-${pack1}.idx" "test-2-${pack2}.idx"'
+
+test_expect_success \
+    'index-pack with index version 1' \
+    'git-index-pack --index-version=1 -o 1.idx "test-1-${pack1}.pack"'
+
+test_expect_success \
+    'index-pack with index version 2' \
+    'git-index-pack --index-version=2 -o 2.idx "test-1-${pack1}.pack"'
+
+test_expect_success \
+    'index-pack results should match pack-objects ones' \
+    'cmp "test-1-${pack1}.idx" "1.idx" &&
+     cmp "test-2-${pack2}.idx" "2.idx"'
+
+test_expect_success \
+    'index v2: force some 64-bit offsets with pack-objects' \
+    'pack3=$(git-pack-objects --index-version=2,0x40000 test-3 <obj-list) &&
+     git-verify-pack -v "test-3-${pack3}.pack"'
+
+test_expect_failure \
+    '64-bit offsets: should be different from previous index v2 results' \
+    'cmp "test-2-${pack2}.idx" "test-3-${pack3}.idx"'
+
+test_expect_success \
+    'index v2: force some 64-bit offsets with index-pack' \
+    'git-index-pack --index-version=2,0x40000 -o 3.idx "test-1-${pack1}.pack"'
+
+test_expect_success \
+    '64-bit offsets: index-pack result should match pack-objects one' \
+    'cmp "test-3-${pack3}.idx" "3.idx"'
+
+test_expect_success \
+    '[index v1] 1) stream pack to repository' \
+    'git-index-pack --index-version=1 --stdin < "test-1-${pack1}.pack" &&
+     git-prune-packed &&
+     git-count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
+     cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
+     cmp "test-1-${pack1}.idx"  ".git/objects/pack/pack-${pack1}.idx"'
+
+test_expect_success \
+    '[index v1] 2) create a stealth corruption in a delta base reference' \
+    '# this test assumes a delta smaller than 16 bytes at the end of the pack
+     git-show-index <1.idx | sort -n | tail -n 1 | (
+       read delta_offs delta_sha1 &&
+       git-cat-file blob "$delta_sha1" > blob_1 &&
+       chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
+       dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($delta_offs + 1)) \
+	  if=".git/objects/pack/pack-${pack1}.idx" skip=$((256 * 4 + 4)) \
+	  bs=1 count=20 conv=notrunc &&
+       git-cat-file blob "$delta_sha1" > blob_2 )'
+
+test_expect_failure \
+    '[index v1] 3) corrupted delta happily returned wrong data' \
+    'cmp blob_1 blob_2'
+
+test_expect_failure \
+    '[index v1] 4) confirm that the pack is actually corrupted' \
+    'git-fsck --full $commit'
+
+test_expect_success \
+    '[index v1] 5) pack-objects happily reuses corrupted data' \
+    'pack4=$(git-pack-objects test-4 <obj-list) &&
+     test -f "test-4-${pack1}.pack"'
+
+test_expect_failure \
+    '[index v1] 6) newly created pack is BAD !' \
+    'git-verify-pack -v "test-4-${pack1}.pack"'
+
+test_expect_success \
+    '[index v2] 1) stream pack to repository' \
+    'rm -f .git/objects/pack/* &&
+     git-index-pack --index-version=2,0x40000 --stdin < "test-1-${pack1}.pack" &&
+     git-prune-packed &&
+     git-count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
+     cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
+     cmp "test-3-${pack1}.idx"  ".git/objects/pack/pack-${pack1}.idx"'
+
+test_expect_success \
+    '[index v2] 2) create a stealth corruption in a delta base reference' \
+    '# this test assumes a delta smaller than 16 bytes at the end of the pack
+     git-show-index <1.idx | sort -n | tail -n 1 | (
+       read delta_offs delta_sha1 delta_crc &&
+       git-cat-file blob "$delta_sha1" > blob_3 &&
+       chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
+       dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($delta_offs + 1)) \
+	  if=".git/objects/pack/pack-${pack1}.idx" skip=$((8 + 256 * 4)) \
+	  bs=1 count=20 conv=notrunc &&
+       git-cat-file blob "$delta_sha1" > blob_4 )'
+
+test_expect_failure \
+    '[index v2] 3) corrupted delta happily returned wrong data' \
+    'cmp blob_3 blob_4'
+
+test_expect_failure \
+    '[index v2] 4) confirm that the pack is actually corrupted' \
+    'git-fsck --full $commit'
+
+test_expect_failure \
+    '[index v2] 5) pack-objects refuses to reuse corrupted data' \
+    'git-pack-objects test-5 <obj-list'
+
+test_done

test-genrandom.c

@@ -0,0 +1,34 @@
+/*
+ * Simple random data generator used to create reproducible test files.
+ * This is inspired from POSIX.1-2001 implementation example for rand().
+ * Copyright (C) 2007 by Nicolas Pitre, licensed under the GPL version 2.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char *argv[])
+{
+	unsigned long count, next = 0;
+	unsigned char *c;
+
+	if (argc < 2 || argc > 3) {
+		fprintf( stderr, "Usage: %s <seed_string> [<size>]", argv[0]);
+		return 1;
+	}
+
+	c = (unsigned char *) argv[1];
+	do {
+		next = next * 11 + *c;
+	} while (*c++);
+
+	count = (argc == 3) ? strtoul(argv[2], NULL, 0) : -1L;
+
+	while (count--) {
+		next = next * 1103515245 + 12345;
+		if (putchar((next >> 16) & 0xff) == EOF)
+			return -1;
+	}
+
+	return 0;
+}