undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'np/index-pack-memsave'

Browse Source 
* np/index-pack-memsave:
  index-pack: smarter memory usage when appending objects
  index-pack: rationalize unpack_entry_data()
  index-pack: smarter memory usage when resolving deltas
This commit is contained in:
Junio C Hamano 2010-05-21 04:02:19 -07:00
commit e43e48cfb6
1 changed files with 45 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
builtin/index-pack.c
View File

@ -266,26 +266,23 @@ static void unlink_base_data(struct base_data *c)
static void *unpack_entry_data(unsigned long offset, unsigned long size) static void *unpack_entry_data(unsigned long offset, unsigned long size)
{ {
int status;
z_stream stream; z_stream stream;
void *buf = xmalloc(size); void *buf = xmalloc(size);
memset(&stream, 0, sizeof(stream)); memset(&stream, 0, sizeof(stream));
git_inflate_init(&stream);
stream.next_out = buf; stream.next_out = buf;
stream.avail_out = size; stream.avail_out = size;
stream.next_in = fill(1);
stream.avail_in = input_len;
git_inflate_init(&stream);
for (;;) { do {
int ret = git_inflate(&stream, 0);
use(input_len - stream.avail_in);
if (stream.total_out == size && ret == Z_STREAM_END)
break;
if (ret != Z_OK)
bad_object(offset, "inflate returned %d", ret);
stream.next_in = fill(1); stream.next_in = fill(1);
stream.avail_in = input_len; stream.avail_in = input_len;
} status = git_inflate(&stream, 0);
use(input_len - stream.avail_in);
} while (status == Z_OK);
if (stream.total_out != size || status != Z_STREAM_END)
bad_object(offset, "inflate returned %d", status);
git_inflate_end(&stream); git_inflate_end(&stream);
return buf; return buf;
} }
@ -359,34 +356,38 @@ static void *get_data_from_pack(struct object_entry *obj)
{ {
off_t from = obj[0].idx.offset + obj[0].hdr_size; off_t from = obj[0].idx.offset + obj[0].hdr_size;
unsigned long len = obj[1].idx.offset - from; unsigned long len = obj[1].idx.offset - from;
unsigned long rdy = 0; unsigned char *data, *inbuf;
unsigned char *src, *data;
z_stream stream; z_stream stream;
int st; int status;
data = xmalloc(obj->size);
inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
memset(&stream, 0, sizeof(stream));
git_inflate_init(&stream);
stream.next_out = data;
stream.avail_out = obj->size;
src = xmalloc(len);
data = src;
do { do {
ssize_t n = pread(pack_fd, data + rdy, len - rdy, from + rdy); ssize_t n = (len < 64*1024) ? len : 64*1024;
n = pread(pack_fd, inbuf, n, from);
if (n < 0) if (n < 0)
die_errno("cannot pread pack file"); die_errno("cannot pread pack file");
if (!n) if (!n)
die("premature end of pack file, %lu bytes missing", die("premature end of pack file, %lu bytes missing", len);
len - rdy); from += n;
rdy += n; len -= n;
} while (rdy < len); stream.next_in = inbuf;
data = xmalloc(obj->size); stream.avail_in = n;
memset(&stream, 0, sizeof(stream)); status = git_inflate(&stream, 0);
stream.next_out = data; } while (len && status == Z_OK && !stream.avail_in);
stream.avail_out = obj->size;
stream.next_in = src; /* This has been inflated OK when first encountered, so... */
stream.avail_in = len; if (status != Z_STREAM_END || stream.total_out != obj->size)
git_inflate_init(&stream);
while ((st = git_inflate(&stream, Z_FINISH)) == Z_OK);
git_inflate_end(&stream);
if (st != Z_STREAM_END || stream.total_out != obj->size)
die("serious inflate inconsistency"); die("serious inflate inconsistency");
free(src);
git_inflate_end(&stream);
free(inbuf);
return data; return data;
} }
@ -668,25 +669,25 @@ static void parse_pack_objects(unsigned char *sha1)
static int write_compressed(struct sha1file *f, void *in, unsigned int size) static int write_compressed(struct sha1file *f, void *in, unsigned int size)
{ {
z_stream stream; z_stream stream;
unsigned long maxsize; int status;
void *out; unsigned char outbuf[4096];
memset(&stream, 0, sizeof(stream)); memset(&stream, 0, sizeof(stream));
deflateInit(&stream, zlib_compression_level); deflateInit(&stream, zlib_compression_level);
maxsize = deflateBound(&stream, size);
out = xmalloc(maxsize);
/* Compress it */
stream.next_in = in; stream.next_in = in;
stream.avail_in = size; stream.avail_in = size;
stream.next_out = out;
stream.avail_out = maxsize;
while (deflate(&stream, Z_FINISH) == Z_OK);
deflateEnd(&stream);
do {
stream.next_out = outbuf;
stream.avail_out = sizeof(outbuf);
status = deflate(&stream, Z_FINISH);
sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out);
} while (status == Z_OK);
if (status != Z_STREAM_END)
die("unable to deflate appended object (%d)", status);
size = stream.total_out; size = stream.total_out;
sha1write(f, out, size); deflateEnd(&stream);
free(out);
return size; return size;
} }