undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[PATCH] mkdelta enhancements (take 2)

Browse Source 
Although it was described as such, git-mkdelta didn't really attempt to
find the best delta against any previous object in the list, but was
only able to create a delta against the preceeding object.  This patch
reworks the code to fix that limitation and hopefully makes it a bit
clearer than before, including fixing the delta loop detection which was
broken.

This means that

	git-mkdelta sha1 sha2 sha3 sha4 sha5 sha6

will now create a sha2 delta against sha1, a sha3 delta against either
sha2 or sha1 and keep the best one, a sha4 delta against either sha3,
sha2 or sha1, etc.  The --max-behind argument limits that search for the
best delta to the specified number of previous objects in the list.  If
no limit is specified it is unlimited (note: it might run out of
memory with long object lists).

Also added a -q (quiet) switch so it is possible to have 3 levels of
output: -q for nothing, -v for verbose, and if none of -q nor -v is
specified then only actual changes on the object database are shown.

Finally the git-deltafy-script has been updated accordingly, and some
bugs fixed (thanks to Stephen C. Tweedie for spotting them).

This version has been toroughly tested and I think it is ready
for public consumption.

Signed-off-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
This commit is contained in:
Nicolas Pitre 2005-05-29 21:52:19 -04:00 committed by Linus Torvalds
parent a3e870f2e2
commit 53d4b46085
2 changed files with 245 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
git-deltafy-script Normal file → Executable file
View File

@ -1,40 +1,67 @@
#!/bin/bash #!/bin/bash
# Script to deltafy an entire GIT repository based on the commit list. # Example script to deltafy an entire GIT repository based on the commit list.
# The most recent version of a file is the reference and previous versions # The most recent version of a file is the reference and previous versions
# are made delta against the best earlier version available. And so on for # are made delta against the best earlier version available. And so on for
# successive versions going back in time. This way the delta overhead is # successive versions going back in time. This way the increasing delta
# pushed towards older version of any given file. # overhead is pushed towards older versions of any given file.
#
# NOTE: the "best earlier version" is not implemented in mkdelta yet
# and therefore only the next eariler version is used at this time.
#
# TODO: deltafy tree objects as well.
# #
# The -d argument allows to provide a limit on the delta chain depth. # The -d argument allows to provide a limit on the delta chain depth.
# If 0 is passed then everything is undeltafied. # If 0 is passed then everything is undeltafied. Limiting the delta
# depth is meaningful for subsequent access performance to old revisions.
# A value of 16 might be a good compromize between performance and good
# space saving. Current default is unbounded.
#
# The --max-behind=30 argument is passed to git-mkdelta so to keep
# combinations and memory usage bounded a bit. If you have lots of memory
# and CPU power you may remove it (or set to 0) to let git-mkdelta find the
# best delta match regardless of the number of revisions for a given file.
# You can also make the value smaller to make it faster and less
# memory hungry. A value of 5 ought to still give pretty good results.
# When set to 0 or ommitted then look behind is unbounded. Note that
# git-mkdelta might die with a segmentation fault in that case if it
# runs out of memory. Note that the GIT repository will still be consistent
# even if git-mkdelta dies unexpectedly.
set -e set -e
depth= depth=
[ "$1" == "-d" ] && depth="--max-depth=$2" && shift 2 [ "$1" == "-d" ] && depth="--max-depth=$2" && shift 2
function process_list() {
if [ "$list" ]; then
echo "Processing $curr_file"
echo "$head $list" | xargs git-mkdelta $depth --max-behind=30 -v
fi
}
curr_file="" curr_file=""
git-rev-list HEAD | git-rev-list HEAD |
git-diff-tree -r --stdin | git-diff-tree -r -t --stdin |
awk '/^:/ { if ($5 == "M" || $5 == "N") print $4, $6 }' | awk '/^:/ { if ($5 == "M" || $5 == "N") print $4, $6;
if ($5 == "M") print $3, $6 }' |
LC_ALL=C sort -s -k 2 | uniq | LC_ALL=C sort -s -k 2 | uniq |
while read sha1 file; do while read sha1 file; do
if [ "$file" == "$curr_file" ]; then if [ "$file" == "$curr_file" ]; then
list="$list $sha1" list="$list $sha1"
else else
if [ "$list" ]; then process_list
echo "Processing $curr_file"
echo "$head $list" | xargs git-mkdelta $depth -v
fi
curr_file="$file" curr_file="$file"
list="" list=""
head="$sha1" head="$sha1"
fi fi
done done
process_list
curr_file="root directory"
head=""
list="$(
git-rev-list HEAD |
while read commit; do
git-cat-file commit $commit |
sed -n 's/tree //p;Q'
done
)"
process_list

360
mkdelta.c
View File

@ -98,21 +98,16 @@ static void *create_delta_object(char *buf, unsigned long len,
return create_object(buf, len, hdr, hdrlen, size); return create_object(buf, len, hdr, hdrlen, size);
} }
static unsigned long get_object_size(unsigned char *sha1) static void *get_buffer(unsigned char *sha1, char *type,
{ unsigned long *size, unsigned long *compsize)
struct stat st;
if (stat(sha1_file_name(sha1), &st))
die("%s: %s", sha1_to_hex(sha1), strerror(errno));
return st.st_size;
}
static void *get_buffer(unsigned char *sha1, char *type, unsigned long *size)
{ {
unsigned long mapsize; unsigned long mapsize;
void *map = map_sha1_file(sha1, &mapsize); void *map = map_sha1_file(sha1, &mapsize);
if (map) { if (map) {
void *buffer = unpack_sha1_file(map, mapsize, type, size); void *buffer = unpack_sha1_file(map, mapsize, type, size);
munmap(map, mapsize); munmap(map, mapsize);
if (compsize)
*compsize = mapsize;
if (buffer) if (buffer)
return buffer; return buffer;
} }
@ -120,198 +115,249 @@ static void *get_buffer(unsigned char *sha1, char *type, unsigned long *size)
return NULL; return NULL;
} }
static void *expand_delta(void *delta, unsigned long delta_size, char *type, static void *expand_delta(void *delta, unsigned long *size, char *type,
unsigned long *size, unsigned int *depth, char *head) unsigned int *depth, unsigned char **links)
{ {
void *buf = NULL; void *buf = NULL;
*depth++; unsigned int level = (*depth)++;
if (delta_size < 20) { if (*size < 20) {
error("delta object is bad"); error("delta object is bad");
free(delta); free(delta);
} else { } else {
unsigned long ref_size; unsigned long ref_size;
void *ref = get_buffer(delta, type, &ref_size); void *ref = get_buffer(delta, type, &ref_size, NULL);
if (ref && !strcmp(type, "delta")) if (ref && !strcmp(type, "delta"))
ref = expand_delta(ref, ref_size, type, &ref_size, ref = expand_delta(ref, &ref_size, type, depth, links);
depth, head); else if (ref)
else {
memcpy(head, delta, 20); *links = xmalloc(*depth * 20);
if (ref) }
buf = patch_delta(ref, ref_size, delta+20, if (ref) {
delta_size-20, size); buf = patch_delta(ref, ref_size, delta+20, *size-20, size);
free(ref); free(ref);
if (buf)
memcpy(*links + level*20, delta, 20);
else
free(*links);
}
free(delta); free(delta);
} }
return buf; return buf;
} }
static char *mkdelta_usage = static char *mkdelta_usage =
"mkdelta [ --max-depth=N ] <reference_sha1> <target_sha1> [ <next_sha1> ... ]"; "mkdelta [--max-depth=N] [--max-behind=N] <reference_sha1> <target_sha1> [<next_sha1> ...]";
struct delta {
unsigned char sha1[20]; /* object sha1 */
unsigned long size; /* object size */
void *buf; /* object content */
unsigned char *links; /* delta reference links */
unsigned int depth; /* delta depth */
};
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
unsigned char sha1_ref[20], sha1_trg[20], head_ref[20], head_trg[20]; struct delta *ref, trg;
char type_ref[20], type_trg[20]; char ref_type[20], trg_type[20], *skip_reason;
void *buf_ref, *buf_trg, *buf_delta; void *best_buf;
unsigned long size_ref, size_trg, size_orig, size_delta; unsigned long best_size, orig_size, orig_compsize;
unsigned int depth_ref, depth_trg, depth_max = -1; unsigned int r, orig_ref, best_ref, nb_refs, next_ref, max_refs = 0;
int i, verbose = 0; unsigned int i, duplicate, skip_lvl, verbose = 0, quiet = 0;
unsigned int max_depth = -1;
for (i = 1; i < argc; i++) { for (i = 1; i < argc; i++) {
if (!strcmp(argv[i], "-v")) { if (!strcmp(argv[i], "-v")) {
verbose = 1; verbose = 1;
quiet = 0;
} else if (!strcmp(argv[i], "-q")) {
quiet = 1;
verbose = 0;
} else if (!strcmp(argv[i], "-d") && i+1 < argc) { } else if (!strcmp(argv[i], "-d") && i+1 < argc) {
depth_max = atoi(argv[++i]); max_depth = atoi(argv[++i]);
} else if (!strncmp(argv[i], "--max-depth=", 12)) { } else if (!strncmp(argv[i], "--max-depth=", 12)) {
depth_max = atoi(argv[i]+12); max_depth = atoi(argv[i]+12);
} else if (!strcmp(argv[i], "-b") && i+1 < argc) {
max_refs = atoi(argv[++i]);
} else if (!strncmp(argv[i], "--max-behind=", 13)) {
max_refs = atoi(argv[i]+13);
} else } else
break; break;
} }
if (i + (depth_max != 0) >= argc) if (i + (max_depth != 0) >= argc)
usage(mkdelta_usage); usage(mkdelta_usage);
if (get_sha1(argv[i], sha1_ref)) if (!max_refs || max_refs > argc - i)
die("bad sha1 %s", argv[i]); max_refs = argc - i;
depth_ref = 0; ref = xmalloc(max_refs * sizeof(*ref));
buf_ref = get_buffer(sha1_ref, type_ref, &size_ref); for (r = 0; r < max_refs; r++)
if (buf_ref && !strcmp(type_ref, "delta")) ref[r].buf = ref[r].links = NULL;
buf_ref = expand_delta(buf_ref, size_ref, type_ref, next_ref = nb_refs = 0;
&size_ref, &depth_ref, head_ref);
else
memcpy(head_ref, sha1_ref, 20);
if (!buf_ref)
die("unable to obtain initial object %s", argv[i]);
if (depth_ref > depth_max) { do {
if (restore_original_object(buf_ref, size_ref, type_ref, sha1_ref)) if (get_sha1(argv[i], trg.sha1))
die("unable to restore %s", argv[i]);
if (verbose)
printf("undelta %s (depth was %d)\n", argv[i], depth_ref);
depth_ref = 0;
}
/*
* TODO: deltafication should be tried against any early object
* in the object list and not only the previous object.
*/
while (++i < argc) {
if (get_sha1(argv[i], sha1_trg))
die("bad sha1 %s", argv[i]); die("bad sha1 %s", argv[i]);
depth_trg = 0; trg.buf = get_buffer(trg.sha1, trg_type, &trg.size, &orig_compsize);
buf_trg = get_buffer(sha1_trg, type_trg, &size_trg); if (trg.buf && !trg.size) {
if (buf_trg && !size_trg) {
if (verbose) if (verbose)
printf("skip %s (object is empty)\n", argv[i]); printf("skip %s (object is empty)\n", argv[i]);
continue; continue;
} }
size_orig = size_trg; orig_size = trg.size;
if (buf_trg && !strcmp(type_trg, "delta")) { orig_ref = -1;
if (!memcmp(buf_trg, sha1_ref, 20)) { trg.depth = 0;
/* delta already in place */ trg.links = NULL;
depth_ref++; if (trg.buf && !strcmp(trg_type, "delta")) {
memcpy(sha1_ref, sha1_trg, 20); for (r = 0; r < nb_refs; r++)
buf_ref = patch_delta(buf_ref, size_ref, if (!memcmp(trg.buf, ref[r].sha1, 20))
buf_trg+20, size_trg-20, break;
&size_ref); if (r < nb_refs) {
if (!buf_ref) /* no need to reload the reference object */
die("unable to apply delta %s", argv[i]); trg.depth = ref[r].depth + 1;
if (depth_ref > depth_max) { trg.links = xmalloc(trg.depth*20);
if (restore_original_object(buf_ref, size_ref, memcpy(trg.links, trg.buf, 20);
type_ref, sha1_ref)) memcpy(trg.links+20, ref[r].links, ref[r].depth*20);
die("unable to restore %s", argv[i]); trg.buf = patch_delta(ref[r].buf, ref[r].size,
if (verbose) trg.buf+20, trg.size-20,
printf("undelta %s (depth was %d)\n", argv[i], depth_ref); &trg.size);
depth_ref = 0; strcpy(trg_type, ref_type);
continue; orig_ref = r;
} } else {
if (verbose) trg.buf = expand_delta(trg.buf, &trg.size, trg_type,
printf("skip %s (delta already in place)\n", argv[i]); &trg.depth, &trg.links);
continue;
} }
buf_trg = expand_delta(buf_trg, size_trg, type_trg, }
&size_trg, &depth_trg, head_trg); if (!trg.buf)
} else
memcpy(head_trg, sha1_trg, 20);
if (!buf_trg)
die("unable to read target object %s", argv[i]); die("unable to read target object %s", argv[i]);
if (depth_trg > depth_max) { if (!nb_refs) {
if (restore_original_object(buf_trg, size_trg, type_trg, sha1_trg)) strcpy(ref_type, trg_type);
die("unable to restore %s", argv[i]); } else if (max_depth && strcmp(ref_type, trg_type)) {
if (verbose)
printf("undelta %s (depth was %d)\n", argv[i], depth_trg);
depth_trg = 0;
size_orig = size_trg;
}
if (depth_max == 0)
goto skip;
if (strcmp(type_ref, type_trg))
die("type mismatch for object %s", argv[i]); die("type mismatch for object %s", argv[i]);
if (!size_ref) {
if (verbose)
printf("skip %s (initial object is empty)\n", argv[i]);
goto skip;
} }
if (depth_ref + 1 > depth_max) { duplicate = 0;
if (verbose) best_buf = NULL;
printf("skip %s (exceeding max link depth)\n", argv[i]); best_size = -1;
goto skip; best_ref = -1;
} skip_lvl = 0;
skip_reason = NULL;
for (r = 0; max_depth && r < nb_refs; r++) {
void *delta_buf, *comp_buf;
unsigned long delta_size, comp_size;
unsigned int l;
if (!memcmp(head_ref, sha1_trg, 20)) { duplicate = !memcmp(trg.sha1, ref[r].sha1, 20);
if (verbose) if (duplicate) {
printf("skip %s (would create a loop)\n", argv[i]); skip_reason = "already seen";
goto skip; break;
} }
if (ref[r].depth >= max_depth) {
buf_delta = diff_delta(buf_ref, size_ref, buf_trg, size_trg, &size_delta); if (skip_lvl < 1) {
if (!buf_delta) skip_reason = "exceeding max link depth";
die("out of memory"); skip_lvl = 1;
}
/* no need to even try to compress if original continue;
uncompressed is already smaller */ }
if (size_delta+20 < size_orig) { for (l = 0; l < ref[r].depth; l++)
void *buf_obj; if (!memcmp(trg.sha1, ref[r].links + l*20, 20))
unsigned long size_obj; break;
buf_obj = create_delta_object(buf_delta, size_delta, if (l != ref[r].depth) {
sha1_ref, &size_obj); if (skip_lvl < 2) {
free(buf_delta); skip_reason = "would create a loop";
size_orig = get_object_size(sha1_trg); skip_lvl = 2;
if (size_obj >= size_orig) { }
free(buf_obj); continue;
if (verbose) }
printf("skip %s (original is smaller)\n", argv[i]); if (trg.depth < max_depth && r == orig_ref) {
goto skip; if (skip_lvl < 3) {
skip_reason = "delta already in place";
skip_lvl = 3;
}
continue;
}
delta_buf = diff_delta(ref[r].buf, ref[r].size,
trg.buf, trg.size, &delta_size);
if (!delta_buf)
die("out of memory");
if (trg.depth < max_depth &&
delta_size+20 >= orig_size) {
/* no need to even try to compress if original
object is smaller than this delta */
free(delta_buf);
if (skip_lvl < 4) {
skip_reason = "no size reduction";
skip_lvl = 4;
}
continue;
}
comp_buf = create_delta_object(delta_buf, delta_size,
ref[r].sha1, &comp_size);
if (!comp_buf)
die("out of memory");
free(delta_buf);
if (trg.depth < max_depth &&
comp_size >= orig_compsize) {
free(comp_buf);
if (skip_lvl < 5) {
skip_reason = "no size reduction";
skip_lvl = 5;
}
continue;
}
if ((comp_size < best_size) ||
(comp_size == best_size &&
ref[r].depth < ref[best_ref].depth)) {
free(best_buf);
best_buf = comp_buf;
best_size = comp_size;
best_ref = r;
} }
if (replace_object(buf_obj, size_obj, sha1_trg))
die("unable to write delta for %s", argv[i]);
free(buf_obj);
depth_ref++;
if (verbose)
printf("delta %s (size=%ld.%02ld%%, depth=%d)\n",
argv[i], size_obj*100 / size_orig,
(size_obj*10000 / size_orig)%100,
depth_ref);
} else {
free(buf_delta);
if (verbose)
printf("skip %s (original is smaller)\n", argv[i]);
skip:
depth_ref = depth_trg;
memcpy(head_ref, head_trg, 20);
} }
free(buf_ref); if (best_buf) {
buf_ref = buf_trg; if (replace_object(best_buf, best_size, trg.sha1))
size_ref = size_trg; die("unable to write delta for %s", argv[i]);
memcpy(sha1_ref, sha1_trg, 20); free(best_buf);
} free(trg.links);
trg.depth = ref[best_ref].depth + 1;
trg.links = xmalloc(trg.depth*20);
memcpy(trg.links, ref[best_ref].sha1, 20);
memcpy(trg.links+20, ref[best_ref].links, ref[best_ref].depth*20);
if (!quiet)
printf("delta %s (size=%ld.%02ld%% depth=%d dist=%d)\n",
argv[i], best_size*100 / orig_compsize,
(best_size*10000 / orig_compsize)%100,
trg.depth,
(next_ref - best_ref + max_refs)
% (max_refs + 1) + 1);
} else if (trg.depth > max_depth) {
if (restore_original_object(trg.buf, trg.size, trg_type, trg.sha1))
die("unable to restore %s", argv[i]);
if (!quiet)
printf("undelta %s (depth was %d)\n",
argv[i], trg.depth);
trg.depth = 0;
free(trg.links);
trg.links = NULL;
} else if (skip_reason && verbose) {
printf("skip %s (%s)\n", argv[i], skip_reason);
}
if (!duplicate) {
free(ref[next_ref].buf);
free(ref[next_ref].links);
ref[next_ref] = trg;
if (++next_ref > nb_refs)
nb_refs = next_ref;
if (next_ref == max_refs)
next_ref = 0;
} else {
free(trg.buf);
free(trg.links);
}
} while (++i < argc);
return 0; return 0;
} }