From 44da6f69ecc4200a488b0647be9f5cb75cae6c4d Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 4 Oct 2012 03:58:15 -0400 Subject: [PATCH 1/4] peel_ref: use faster deref_tag_noverify When we are asked to peel a ref to a sha1, we internally call deref_tag, which will recursively parse each tagged object until we reach a non-tag. This has the benefit that we will verify our ability to load and parse the pointed-to object. However, there is a performance downside: we may not need to load that object at all (e.g., if we are listing peeled simply listing peeled refs), or it may be a large object that should follow a streaming code path (e.g., an annotated tag of a large blob). It makes more sense for peel_ref to choose the fast thing rather than performing the extra check, for two reasons: 1. We will already sometimes short-circuit the tag parsing in favor of a peeled entry from a packed-refs file. So we are already favoring speed in some cases, and it is not wise for a caller to rely on peel_ref to detect corruption. 2. We already silently ignore much larger corruptions, like a ref that points to a non-existent object, or a tag object that exists but is corrupted. 2. peel_ref is not the right place to check for such a database corruption. It is returning only the sha1 anyway, not the actual object. Any callers which use that sha1 to load an object will soon discover the corruption anyway, so we are really just pushing back the discovery to later in the program. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- refs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refs.c b/refs.c index da74a2b29a..0a916a0c2a 100644 --- a/refs.c +++ b/refs.c @@ -1225,7 +1225,7 @@ int peel_ref(const char *refname, unsigned char *sha1) fallback: o = parse_object(base); if (o && o->type == OBJ_TAG) { - o = deref_tag(o, refname, 0); + o = deref_tag_noverify(o); if (o) { hashcpy(sha1, o->sha1); return 0; From e6dbffa67b8e4c463a8fe18e8599b8623d7f0485 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 4 Oct 2012 04:00:19 -0400 Subject: [PATCH 2/4] peel_ref: do not return a null sha1 The idea of the peel_ref function is to dereference tag objects recursively until we hit a non-tag, and return the sha1. Conceptually, it should return 0 if it is successful (and fill in the sha1), or -1 if there was nothing to peel. However, the current behavior is much more confusing. For a regular loose ref, the behavior is as described above. But there is an optimization to reuse the peeled-ref value for a ref that came from a packed-refs file. If we have such a ref, we return its peeled value, even if that peeled value is null (indicating that we know the ref definitely does _not_ peel). It might seem like such information is useful to the caller, who would then know not to bother loading and trying to peel the object. Except that they should not bother loading and trying to peel the object _anyway_, because that fallback is already handled by peel_ref. In other words, the whole point of calling this function is that it handles those details internally, and you either get a sha1, or you know that it is not peel-able. This patch catches the null sha1 case internally and converts it into a -1 return value (i.e., there is nothing to peel). This simplifies callers, which do not need to bother checking themselves. Two callers are worth noting: - in pack-objects, a comment indicates that there is a difference between non-peelable tags and unannotated tags. But that is not the case (before or after this patch). Whether you get a null sha1 has to do with internal details of how peel_ref operated. - in show-ref, if peel_ref returns a failure, the caller tries to decide whether to try peeling manually based on whether the REF_ISPACKED flag is set. But this doesn't make any sense. If the flag is set, that does not necessarily mean the ref came from a packed-refs file with the "peeled" extension. But it doesn't matter, because even if it didn't, there's no point in trying to peel it ourselves, as peel_ref would already have done so. In other words, the fallback peeling is guaranteed to fail. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/describe.c | 2 +- builtin/pack-objects.c | 1 - builtin/show-ref.c | 23 +++-------------------- refs.c | 2 ++ 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/builtin/describe.c b/builtin/describe.c index 9f63067f50..94b06069a0 100644 --- a/builtin/describe.c +++ b/builtin/describe.c @@ -144,7 +144,7 @@ static int get_name(const char *path, const unsigned char *sha1, int flag, void if (!all && !might_be_tag) return 0; - if (!peel_ref(path, peeled) && !is_null_sha1(peeled)) { + if (!peel_ref(path, peeled)) { is_tag = !!hashcmp(sha1, peeled); } else { hashcpy(peeled, sha1); diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 782e7d0c38..035ed3b2d0 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -2033,7 +2033,6 @@ static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, vo if (!prefixcmp(path, "refs/tags/") && /* is a tag? */ !peel_ref(path, peeled) && /* peelable? */ - !is_null_sha1(peeled) && /* annotated tag? */ locate_object_entry(peeled)) /* object packed? */ add_object_entry(sha1, OBJ_TAG, NULL, 0); return 0; diff --git a/builtin/show-ref.c b/builtin/show-ref.c index 3911661900..aaac2b22f9 100644 --- a/builtin/show-ref.c +++ b/builtin/show-ref.c @@ -28,7 +28,6 @@ static void show_one(const char *refname, const unsigned char *sha1) static int show_ref(const char *refname, const unsigned char *sha1, int flag, void *cbdata) { - struct object *obj; const char *hex; unsigned char peeled[20]; @@ -79,25 +78,9 @@ match: if (!deref_tags) return 0; - if ((flag & REF_ISPACKED) && !peel_ref(refname, peeled)) { - if (!is_null_sha1(peeled)) { - hex = find_unique_abbrev(peeled, abbrev); - printf("%s %s^{}\n", hex, refname); - } - } - else { - obj = parse_object(sha1); - if (!obj) - die("git show-ref: bad ref %s (%s)", refname, - sha1_to_hex(sha1)); - if (obj->type == OBJ_TAG) { - obj = deref_tag(obj, refname, 0); - if (!obj) - die("git show-ref: bad tag at ref %s (%s)", refname, - sha1_to_hex(sha1)); - hex = find_unique_abbrev(obj->sha1, abbrev); - printf("%s %s^{}\n", hex, refname); - } + if (!peel_ref(refname, peeled)) { + hex = find_unique_abbrev(peeled, abbrev); + printf("%s %s^{}\n", hex, refname); } return 0; } diff --git a/refs.c b/refs.c index 0a916a0c2a..f672ad93a6 100644 --- a/refs.c +++ b/refs.c @@ -1202,6 +1202,8 @@ int peel_ref(const char *refname, unsigned char *sha1) if (current_ref && (current_ref->name == refname || !strcmp(current_ref->name, refname))) { if (current_ref->flag & REF_KNOWS_PEELED) { + if (is_null_sha1(current_ref->u.value.peeled)) + return -1; hashcpy(sha1, current_ref->u.value.peeled); return 0; } From 6c4a060d7d895c90610eb442cb4f910fe43ef13a Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 4 Oct 2012 04:02:53 -0400 Subject: [PATCH 3/4] peel_ref: check object type before loading The point of peel_ref is to dereference tags; if the base object is not a tag, then we can return early without even loading the object into memory. This patch accomplishes that by checking sha1_object_info for the type. For a packed object, we can get away with just looking in the pack index. For a loose object, we only need to inflate the first couple of header bytes. This is a bit of a gamble; if we do find a tag object, then we will end up loading the content anyway, and the extra lookup will have been wasteful. However, if it is not a tag object, then we save loading the object entirely. Depending on the ratio of non-tags to tags in the input, this can be a minor win or minor loss. However, it does give us one potential major win: if a ref points to a large blob (e.g., via an unannotated tag), then we can avoid looking at it entirely. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- refs.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/refs.c b/refs.c index f672ad93a6..02e47b1157 100644 --- a/refs.c +++ b/refs.c @@ -1225,8 +1225,15 @@ int peel_ref(const char *refname, unsigned char *sha1) } fallback: - o = parse_object(base); - if (o && o->type == OBJ_TAG) { + o = lookup_unknown_object(base); + if (o->type == OBJ_NONE) { + int type = sha1_object_info(base, NULL); + if (type < 0) + return -1; + o->type = type; + } + + if (o->type == OBJ_TAG) { o = deref_tag_noverify(o); if (o) { hashcpy(sha1, o->sha1); From 435c83323713066723e4194cd89c31bbec7011a2 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 4 Oct 2012 04:03:33 -0400 Subject: [PATCH 4/4] upload-pack: use peel_ref for ref advertisements When upload-pack advertises refs, we attempt to peel tags and advertise the peeled version. We currently hand-roll the tag dereferencing, and use as many optimizations as we can to avoid loading non-tag objects into memory. Not only has peel_ref recently learned these optimizations, too, but it also contains an even more important one: it has access to the "peeled" data from the pack-refs file. That means we can avoid not only loading annotated tags entirely, but also avoid doing any kind of object lookup at all. This cut the CPU time to advertise refs by 50% in the linux-2.6 repo, as measured by: echo 0000 | git-upload-pack . >/dev/null best-of-five, warm cache, objects and refs fully packed: [before] [after] real 0m0.026s real 0m0.013s user 0m0.024s user 0m0.008s sys 0m0.000s sys 0m0.000s Those numbers are irrelevantly small compared to an actual fetch. Here's a larger repo (400K refs, of which 12K are unique, and of which only 107 are unique annotated tags): [before] [after] real 0m0.704s real 0m0.596s user 0m0.600s user 0m0.496s sys 0m0.096s sys 0m0.092s This shows only a 15% speedup (mostly because it has fewer actual tags to parse), but a larger absolute value (100ms, which isn't a lot compared to a real fetch, but this advertisement happens on every fetch, even if the client is just finding out they are completely up to date). In truly pathological cases, where you have a large number of unique annotated tags, it can make an even bigger difference. Here are the numbers for a linux-2.6 repository that has had every seventh commit tagged (so about 50K tags): [before] [after] real 0m0.443s real 0m0.097s user 0m0.416s user 0m0.080s sys 0m0.024s sys 0m0.012s Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- upload-pack.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/upload-pack.c b/upload-pack.c index 2e90ccb74f..6142421ea1 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -727,12 +727,7 @@ static int send_ref(const char *refname, const unsigned char *sha1, int flag, vo " include-tag multi_ack_detailed"; struct object *o = lookup_unknown_object(sha1); const char *refname_nons = strip_namespace(refname); - - if (o->type == OBJ_NONE) { - o->type = sha1_object_info(sha1, NULL); - if (o->type < 0) - die("git upload-pack: cannot find object %s:", sha1_to_hex(sha1)); - } + unsigned char peeled[20]; if (capabilities) packet_write(1, "%s %s%c%s%s agent=%s\n", @@ -747,11 +742,8 @@ static int send_ref(const char *refname, const unsigned char *sha1, int flag, vo o->flags |= OUR_REF; nr_our_refs++; } - if (o->type == OBJ_TAG) { - o = deref_tag_noverify(o); - if (o) - packet_write(1, "%s %s^{}\n", sha1_to_hex(o->sha1), refname_nons); - } + if (!peel_ref(refname, peeled)) + packet_write(1, "%s %s^{}\n", sha1_to_hex(peeled), refname_nons); return 0; }