From 1385bb7ba39128de0a5bc4ff6e8a5ad03fc49205 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Fri, 27 Mar 2015 07:32:41 -0400 Subject: [PATCH 1/3] reachable: only mark local objects as recent When pruning and repacking a repository that has an alternate object store configured, we may traverse a large number of objects in the alternate. This serves no purpose, and may be expensive to do. A longer explanation is below. Commits d3038d2 and abcb865 taught prune and pack-objects (respectively) to treat "recent" objects as tips for reachability, so that we keep whole chunks of history. They built on the object traversal in 660c889 (sha1_file: add for_each iterators for loose and packed objects, 2014-10-15), which covers both local and alternate objects. In both cases, covering alternate objects is unnecessary, as both commands can only drop objects from the local repository. In the case of prune, we traverse only the local object directory. And in the case of repacking, while we may or may not include local objects in our pack, we will never reach into the alternate with "repack -d". The "-l" option is only a question of whether we are migrating objects from the alternate into our repository, or leaving them untouched. It is possible that we may drop an object that is depended upon by another object in the alternate. For example, imagine two repositories, A and B, with A pointing to B as an alternate. Now imagine a commit that is in B which references a tree that is only in A. Traversing from recent objects in B might prevent A from dropping that tree. But this case isn't worth covering. Repo B should take responsibility for its own objects. It would never have had the commit in the first place if it did not also have the tree, and assuming it is using the same "keep recent chunks of history" scheme, then it would itself keep the tree, as well. So checking the alternate objects is not worth doing, and come with a significant performance impact. In both cases, we skip any recent objects that have already been marked SEEN (i.e., that we know are already reachable for prune, or included in the pack for a repack). So there is a slight waste of time in opening the alternate packs at all, only to notice that we have already considered each object. But much worse, the alternate repository may have a large number of objects that are not reachable from the local repository at all, and we end up adding them to the traversal. We can fix this by considering only local unseen objects. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- cache.h | 8 +++++--- reachable.c | 6 ++++-- sha1_file.c | 9 +++++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cache.h b/cache.h index 4743f7ed9f..c7565f6acb 100644 --- a/cache.h +++ b/cache.h @@ -1283,14 +1283,16 @@ int for_each_loose_file_in_objdir_buf(struct strbuf *path, /* * Iterate over loose and packed objects in both the local - * repository and any alternates repositories. + * repository and any alternates repositories (unless the + * LOCAL_ONLY flag is set). */ +#define FOR_EACH_OBJECT_LOCAL_ONLY 0x1 typedef int each_packed_object_fn(const unsigned char *sha1, struct packed_git *pack, uint32_t pos, void *data); -extern int for_each_loose_object(each_loose_object_fn, void *); -extern int for_each_packed_object(each_packed_object_fn, void *); +extern int for_each_loose_object(each_loose_object_fn, void *, unsigned flags); +extern int for_each_packed_object(each_packed_object_fn, void *, unsigned flags); struct object_info { /* Request */ diff --git a/reachable.c b/reachable.c index a647267ae9..69fa6851da 100644 --- a/reachable.c +++ b/reachable.c @@ -142,10 +142,12 @@ int add_unseen_recent_objects_to_traversal(struct rev_info *revs, data.revs = revs; data.timestamp = timestamp; - r = for_each_loose_object(add_recent_loose, &data); + r = for_each_loose_object(add_recent_loose, &data, + FOR_EACH_OBJECT_LOCAL_ONLY); if (r) return r; - return for_each_packed_object(add_recent_packed, &data); + return for_each_packed_object(add_recent_packed, &data, + FOR_EACH_OBJECT_LOCAL_ONLY); } void mark_reachable_objects(struct rev_info *revs, int mark_reflog, diff --git a/sha1_file.c b/sha1_file.c index a41cc4f65f..9711ef817d 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -3418,7 +3418,7 @@ static int loose_from_alt_odb(struct alternate_object_database *alt, return r; } -int for_each_loose_object(each_loose_object_fn cb, void *data) +int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags) { struct loose_alt_odb_data alt; int r; @@ -3428,6 +3428,9 @@ int for_each_loose_object(each_loose_object_fn cb, void *data) if (r) return r; + if (flags & FOR_EACH_OBJECT_LOCAL_ONLY) + return 0; + alt.cb = cb; alt.data = data; return foreach_alt_odb(loose_from_alt_odb, &alt); @@ -3452,13 +3455,15 @@ static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn c return r; } -int for_each_packed_object(each_packed_object_fn cb, void *data) +int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags) { struct packed_git *p; int r = 0; prepare_packed_git(); for (p = packed_git; p; p = p->next) { + if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) + continue; r = for_each_object_in_pack(p, cb, data); if (r) break; From b5f52f372e85c6e461b6123cd7eebd544b439020 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Mon, 20 Apr 2015 15:54:03 -0400 Subject: [PATCH 2/3] sha1_file: freshen pack objects before loose When writing out an object file, we first check whether it already exists and if so optimize out the write. Prior to 33d4221, we did this by calling has_sha1_file(), which will check for packed objects followed by loose. Since that commit, we check loose objects first. For the common case of a repository whose objects are mostly packed, this means we will make a lot of extra access() system calls checking for loose objects. We should follow the same packed-then-loose order that all of our other lookups use. Reported-by: Stefan Saasen Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- sha1_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sha1_file.c b/sha1_file.c index 9711ef817d..cd6c102ccf 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -3014,7 +3014,7 @@ int write_sha1_file(const void *buf, unsigned long len, const char *type, unsign write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen); if (returnsha1) hashcpy(returnsha1, sha1); - if (freshen_loose_object(sha1) || freshen_packed_object(sha1)) + if (freshen_packed_object(sha1) || freshen_loose_object(sha1)) return 0; return write_loose_object(sha1, hdr, hdrlen, buf, len, 0); } From ee1c6c34ac64c1e10b17a50710ea7002b7e7241f Mon Sep 17 00:00:00 2001 From: Jeff King Date: Mon, 20 Apr 2015 15:55:00 -0400 Subject: [PATCH 3/3] sha1_file: only freshen packs once per run Since 33d4221 (write_sha1_file: freshen existing objects, 2014-10-15), we update the mtime of existing objects that we would have written out (had they not existed). For the common case in which many objects are packed, we may update the mtime on a single packfile repeatedly. This can result in a noticeable performance problem if calling utime() is expensive (e.g., because your storage is on NFS). We can fix this by keeping a per-pack flag that lets us freshen only once per program invocation. An alternative would be to keep the packed_git.mtime flag up to date as we freshen, and freshen only once every N seconds. In practice, it's not worth the complexity. We are racing against prune expiration times here, which inherently must be set to accomodate reasonable program running times (because they really care about the time between an object being written and it becoming referenced, and the latter is typically the last step a program takes). Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- cache.h | 1 + sha1_file.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cache.h b/cache.h index c7565f6acb..e917833391 100644 --- a/cache.h +++ b/cache.h @@ -1168,6 +1168,7 @@ extern struct packed_git { int pack_fd; unsigned pack_local:1, pack_keep:1, + freshened:1, do_not_close:1; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ diff --git a/sha1_file.c b/sha1_file.c index cd6c102ccf..bf1bdbcdf5 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -2999,7 +2999,14 @@ static int freshen_loose_object(const unsigned char *sha1) static int freshen_packed_object(const unsigned char *sha1) { struct pack_entry e; - return find_pack_entry(sha1, &e) && freshen_file(e.p->pack_name); + if (!find_pack_entry(sha1, &e)) + return 0; + if (e.p->freshened) + return 1; + if (!freshen_file(e.p->pack_name)) + return 0; + e.p->freshened = 1; + return 1; } int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)