git-pack-objects: write the pack files with a SHA1 csum

We want to be able to check their integrity later, and putting the
sha1-sum of the contents at the end is a good thing.  The writing
routines are generic, so we could try to re-use them for the index file,
instead of having the same logic duplicated.

Update unpack-objects to know about the extra 20 bytes at the end
of the index.

Makefile

@@ -45,9 +45,9 @@ install: $(PROG) $(SCRIPTS)
 
 LIB_OBJS=read-cache.o sha1_file.o usage.o object.o commit.o tree.o blob.o \
 	 tag.o delta.o date.o index.o diff-delta.o patch-delta.o entry.o \
-	 epoch.o refs.o
+	 epoch.o refs.o csum-file.o
 LIB_FILE=libgit.a
-LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h
+LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h csum-file.h
 
 LIB_H += strbuf.h
 LIB_OBJS += strbuf.o

csum-file.c

@@ -0,0 +1,120 @@
+/*
+ * csum-file.c
+ *
+ * Copyright (C) 2005 Linus Torvalds
+ *
+ * Simple file write infrastructure for writing SHA1-summed
+ * files. Useful when you write a file that you want to be
+ * able to verify hasn't been messed with afterwards.
+ */
+#include "cache.h"
+#include "csum-file.h"
+
+static int sha1flush(struct sha1file *f, unsigned int count)
+{
+	void *buf = f->buffer;
+
+	for (;;) {
+		int ret = write(f->fd, buf, count);
+		if (ret > 0) {
+			buf += ret;
+			count -= ret;
+			if (count)
+				continue;
+			return 0;
+		}
+		if (!ret)
+			die("sha1 file write error. Out of diskspace");
+		if (errno == EAGAIN || errno == EINTR)
+			continue;
+		die("sha1 file write error (%s)", strerror(errno));
+	}
+}
+
+int sha1close(struct sha1file *f)
+{
+	unsigned offset = f->offset;
+	if (offset) {
+		SHA1_Update(&f->ctx, f->buffer, offset);
+		sha1flush(f, offset);
+	}
+	SHA1_Final(f->buffer, &f->ctx);
+	sha1flush(f, 20);
+	return 0;
+}
+
+int sha1write(struct sha1file *f, void *buf, unsigned int count)
+{
+	while (count) {
+		unsigned offset = f->offset;
+		unsigned left = sizeof(f->buffer) - offset;
+		unsigned nr = count > left ? left : count;
+
+		memcpy(f->buffer + offset, buf, nr);
+		count -= nr;
+		offset += nr;
+		left -= nr;
+		if (!left) {
+			SHA1_Update(&f->ctx, f->buffer, offset);
+			sha1flush(f, offset);
+			offset = 0;
+		}
+		f->offset = offset;
+	}
+	return 0;
+}
+
+struct sha1file *sha1create(const char *fmt, ...)
+{
+	static char filename[PATH_MAX];
+	struct sha1file *f;
+	unsigned len;
+	va_list arg;
+	int fd;
+
+	va_start(arg, fmt);
+	len = vsnprintf(filename, PATH_MAX, fmt, arg);
+	va_end(arg);
+
+	if (len >= PATH_MAX)
+		die("you wascally wabbit, you");
+	fd = open(filename, O_CREAT | O_EXCL | O_WRONLY, 0644);
+	if (fd < 0)
+		die("unable to open %s (%s)", filename, strerror(errno));
+	f = xmalloc(sizeof(*f));
+	f->fd = fd;
+	f->error = 0;
+	f->offset = 0;
+	SHA1_Init(&f->ctx);
+	return f;
+}
+
+int sha1write_compressed(struct sha1file *f, void *in, unsigned int size)
+{
+	z_stream stream;
+	unsigned long maxsize;
+	void *out;
+
+	memset(&stream, 0, sizeof(stream));
+	deflateInit(&stream, Z_DEFAULT_COMPRESSION);
+	maxsize = deflateBound(&stream, size);
+	out = xmalloc(maxsize);
+
+	/* Compress it */
+	stream.next_in = in;
+	stream.avail_in = size;
+
+	stream.next_out = out;
+	stream.avail_out = maxsize;
+
+	while (deflate(&stream, Z_FINISH) == Z_OK)
+		/* nothing */;
+	deflateEnd(&stream);
+
+	size = stream.total_out;
+	sha1write(f, out, size);
+	free(out);
+	return size;
+}
+
+

csum-file.h

@@ -0,0 +1,17 @@
+#ifndef CSUM_FILE_H
+#define CSUM_FILE_H
+
+/* A SHA1-protected file */
+struct sha1file {
+	int fd, error;
+	unsigned long offset;
+	SHA_CTX ctx;
+	unsigned char buffer[8192];
+};
+
+extern struct sha1file *sha1create(const char *fmt, ...);
+extern int sha1close(struct sha1file *);
+extern int sha1write(struct sha1file *, void *, unsigned int);
+extern int sha1write_compressed(struct sha1file *, void *, unsigned int);
+
+#endif

pack-objects.c

@@ -2,6 +2,7 @@
 #include "cache.h"
 #include "object.h"
 #include "delta.h"
+#include "csum-file.h"
 
 static const char pack_usage[] = "git-pack-objects [--window=N] [--depth=N] base-name < object-list";
 
@@ -29,51 +30,6 @@ static struct object_entry *objects = NULL;
 static int nr_objects = 0, nr_alloc = 0;
 static const char *base_name;
 
-struct myfile {
-	int fd;
-	unsigned long chars;
-	unsigned char buffer[8192];
-};
-
-static FILE *create_file(const char *suffix)
-{
-	static char filename[PATH_MAX];
-	unsigned len;
-
-	len = snprintf(filename, PATH_MAX, "%s.%s", base_name, suffix);
-	if (len >= PATH_MAX)
-		die("you wascally wabbit, you");
-	return fopen(filename, "w");
-}
-
-static unsigned long fwrite_compressed(void *in, unsigned long size, FILE *f)
-{
-	z_stream stream;
-	unsigned long maxsize;
-	void *out;
-
-	memset(&stream, 0, sizeof(stream));
-	deflateInit(&stream, Z_DEFAULT_COMPRESSION);
-	maxsize = deflateBound(&stream, size);
-	out = xmalloc(maxsize);
-
-	/* Compress it */
-	stream.next_in = in;
-	stream.avail_in = size;
-
-	stream.next_out = out;
-	stream.avail_out = maxsize;
-
-	while (deflate(&stream, Z_FINISH) == Z_OK)
-		/* nothing */;
-	deflateEnd(&stream);
-
-	size = stream.total_out;
-	fwrite(out, size, 1, f);
-	free(out);
-	return size;
-}
-
 static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
 {
 	unsigned long othersize, delta_size;
@@ -92,7 +48,7 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
 	return delta_buf;
 }
 
-static unsigned long write_object(FILE *f, struct object_entry *entry)
+static unsigned long write_object(struct sha1file *f, struct object_entry *entry)
 {
 	unsigned long size;
 	char type[10];
@@ -121,8 +77,8 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
 	}
 	datalen = htonl(size);
 	memcpy(header+1, &datalen, 4);
-	fwrite(header, hdrlen, 1, f);
+	sha1write(f, header, hdrlen);
-	datalen = fwrite_compressed(buf, size, f);
+	datalen = sha1write_compressed(f, buf, size);
 	free(buf);
 	return hdrlen + datalen;
 }
@@ -130,7 +86,7 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
 static void write_pack_file(void)
 {
 	int i;
-	FILE *f = create_file("pack");
+	struct sha1file *f = sha1create("%s.%s", base_name, "pack");
 	unsigned long offset = 0;
 	unsigned long mb;
 
@@ -139,7 +95,7 @@ static void write_pack_file(void)
 		entry->offset = offset;
 		offset += write_object(f, entry);
 	}
-	fclose(f);
+	sha1close(f);
 	mb = offset >> 20;
 	offset &= 0xfffff;
 }
@@ -147,7 +103,7 @@ static void write_pack_file(void)
 static void write_index_file(void)
 {
 	int i;
-	FILE *f = create_file("idx");
+	struct sha1file *f = sha1create("%s.%s", base_name, "idx");
 	struct object_entry **list = sorted_by_sha;
 	struct object_entry **last = list + nr_objects;
 	unsigned int array[256];
@@ -168,7 +124,7 @@ static void write_index_file(void)
 		array[i] = htonl(next - sorted_by_sha);
 		list = next;
 	}
-	fwrite(array, 256, sizeof(int), f);
+	sha1write(f, array, 256 * sizeof(int));
 
 	/*
 	 * Write the actual SHA1 entries..
@@ -177,10 +133,10 @@ static void write_index_file(void)
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *entry = *list++;
 		unsigned int offset = htonl(entry->offset);
-		fwrite(&offset, 4, 1, f);
+		sha1write(f, &offset, 4);
-		fwrite(entry->sha1, 20, 1, f);
+		sha1write(f, entry->sha1, 20);
 	}
-	fclose(f);
+	sha1close(f);
 }
 
 static void add_object_entry(unsigned char *sha1, unsigned int hash)

unpack-objects.c

@@ -61,7 +61,7 @@ static int check_index(void)
 	unsigned int nr;
 	int i;
 
-	if (index_size < 4*256)
+	if (index_size < 4*256 + 20)
 		return error("index file too small");
 	nr = 0;
 	for (i = 0; i < 256; i++) {
@@ -70,11 +70,14 @@ static int check_index(void)
 			return error("non-monotonic index");
 		nr = n;
 	}
-	if (index_size != 4*256 + nr * 24) {
+	/*
-		printf("index_size=%lu, expected %u (%u)\n",
+	 * Total size:
-		       index_size, 4*256 + nr * 24, nr);
+	 *  - 256 index entries 4 bytes each
+	 *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
+	 *  - 20-byte SHA1 file checksum
+	 */
+	if (index_size != 4*256 + nr * 24 + 20)
 		return error("wrong index file size");
-	}
 
 	nr_entries = nr;
 	pack_list = xmalloc(nr * sizeof(struct pack_entry *));