undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

git-pack-objects: cache small deltas between big objects

Browse Source 
Creating deltas between big blobs is a CPU and memory intensive task.
In the writing phase, all (not reused) deltas are redone.

This patch adds support for caching deltas from the deltifing phase, so
that that the writing phase is faster.

The caching is limited to small deltas to avoid increasing memory usage very much.
The implemented limit is (memory needed to create the delta)/1024.

Signed-off-by: Martin Koegler <mkoegler@auto.tuwien.ac.at>
Signed-off-by: Junio C Hamano <junkio@cox.net>
This commit is contained in:
Martin Koegler 2007-05-28 23:20:58 +02:00 committed by Junio C Hamano
parent a588d88aaf
commit 074b2eea29
2 changed files with 59 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Documentation/config.txt
View File

@ -567,6 +567,11 @@ pack.compression::
slowest. If not set, defaults to core.compression. If that is
not set, defaults to -1.
pack.deltaCacheSize::
The maxium memory in bytes used for caching deltas in
gitlink:git-pack-objects[1].
A value of 0 means no limit. Defaults to 0.
pull.octopus::
The default merge strategy to use when pulling multiple branches
at once.

39
builtin-pack-objects.c
View File

@ -36,6 +36,7 @@ struct object_entry {
struct object_entry *delta_sibling; /* other deltified objects who
* uses the same base as me
*/
void *delta_data; /* cached delta (uncompressed) */
unsigned long delta_size; /* delta data size (uncompressed) */
enum object_type type;
enum object_type in_pack_type; /* could be delta */
@ -76,6 +77,9 @@ static struct progress progress_state;
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
static int pack_compression_seen;
static unsigned long delta_cache_size = 0;
static unsigned long max_delta_cache_size = 0;
/*
* The object names in objects array are hashed with this hashtable,
* to help looking up the entry by object name.
@ -405,6 +409,12 @@ static unsigned long write_object(struct sha1file *f,
z_stream stream;
unsigned long maxsize;
void *out;
if (entry->delta_data && usable_delta) {
buf = entry->delta_data;
size = entry->delta_size;
obj_type = (allow_ofs_delta && entry->delta->offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
} else {
buf = read_sha1_file(entry->sha1, &type, &size);
if (!buf)
die("unable to read %s", sha1_to_hex(entry->sha1));
@ -424,6 +434,7 @@ static unsigned long write_object(struct sha1file *f,
*/
obj_type = type;
}
}
/* compress the data to store and put compressed length in datalen */
memset(&stream, 0, sizeof(stream));
deflateInit(&stream, pack_compression_level);
@ -1385,6 +1396,20 @@ struct unpacked {
struct delta_index *index;
};
static int delta_cacheable(struct unpacked *trg, struct unpacked *src,
unsigned long src_size, unsigned long trg_size,
unsigned long delta_size)
{
if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)
return 0;
/* cache delta, if objects are large enough compared to delta size */
if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))
return 1;
return 0;
}
/*
* We search for deltas _backwards_ in a list sorted by type and
* by size, so that we see progressively smaller and smaller files.
@ -1466,9 +1491,19 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
if (!delta_buf)
return 0;
if (trg_entry->delta_data) {
delta_cache_size -= trg_entry->delta_size;
free(trg_entry->delta_data);
}
trg_entry->delta_data = 0;
trg_entry->delta = src_entry;
trg_entry->delta_size = delta_size;
trg_entry->depth = src_entry->depth + 1;
if (delta_cacheable(src, trg, src_size, trg_size, delta_size)) {
trg_entry->delta_data = xrealloc(delta_buf, delta_size);
delta_cache_size += trg_entry->delta_size;
} else
free(delta_buf);
return 1;
}
@ -1615,6 +1650,10 @@ static int git_pack_config(const char *k, const char *v)
pack_compression_seen = 1;
return 0;
}
if (!strcmp(k, "pack.deltacachesize")) {
max_delta_cache_size = git_config_int(k, v);
return 0;
}
return git_default_config(k, v);
}