git-commit-vandalism/connected.c
Patrick Steinhardt f45022dc2f connected: do not sort input revisions
In order to compute whether objects reachable from a set of tips are all
connected, we do a revision walk with these tips as positive references
and `--not --all`. `--not --all` will cause the revision walk to load
all preexisting references as uninteresting, which can be very expensive
in repositories with many references.

Benchmarking the git-rev-list(1) command highlights that by far the most
expensive single phase is initial sorting of the input revisions: after
all references have been loaded, we first sort commits by author date.
In a real-world repository with about 2.2 million references, it makes
up about 40% of the total runtime of git-rev-list(1).

Ultimately, the connectivity check shouldn't really bother about the
order of input revisions at all. We only care whether we can actually
walk all objects until we hit the cut-off point. So sorting the input is
a complete waste of time.

Introduce a new "--unsorted-input" flag to git-rev-list(1) which will
cause it to not sort the commits and adjust the connectivity check to
always pass the flag. This results in the following speedups, executed
in a clone of gitlab-org/gitlab [1]:

    Benchmark #1: git rev-list  --objects --quiet --not --all --not $(cat newrev)
      Time (mean ± σ):      7.639 s ±  0.065 s    [User: 7.304 s, System: 0.335 s]
      Range (min … max):    7.543 s …  7.742 s    10 runs

    Benchmark #2: git rev-list --unsorted-input --objects --quiet --not --all --not $newrev
      Time (mean ± σ):      4.995 s ±  0.044 s    [User: 4.657 s, System: 0.337 s]
      Range (min … max):    4.909 s …  5.048 s    10 runs

    Summary
      'git rev-list --unsorted-input --objects --quiet --not --all --not $(cat newrev)' ran
        1.53 ± 0.02 times faster than 'git rev-list  --objects --quiet --not --all --not $newrev'

[1]: https://gitlab.com/gitlab-org/gitlab.git. Note that not all refs
     are visible to clients.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-08-09 09:51:12 -07:00

155 lines
4.3 KiB
C

#include "cache.h"
#include "object-store.h"
#include "run-command.h"
#include "sigchain.h"
#include "connected.h"
#include "transport.h"
#include "packfile.h"
#include "promisor-remote.h"
/*
* If we feed all the commits we want to verify to this command
*
* $ git rev-list --objects --stdin --not --all
*
* and if it does not error out, that means everything reachable from
* these commits locally exists and is connected to our existing refs.
* Note that this does _not_ validate the individual objects.
*
* Returns 0 if everything is connected, non-zero otherwise.
*/
int check_connected(oid_iterate_fn fn, void *cb_data,
struct check_connected_options *opt)
{
struct child_process rev_list = CHILD_PROCESS_INIT;
FILE *rev_list_in;
struct check_connected_options defaults = CHECK_CONNECTED_INIT;
struct object_id oid;
int err = 0;
struct packed_git *new_pack = NULL;
struct transport *transport;
size_t base_len;
if (!opt)
opt = &defaults;
transport = opt->transport;
if (fn(cb_data, &oid)) {
if (opt->err_fd)
close(opt->err_fd);
return err;
}
if (transport && transport->smart_options &&
transport->smart_options->self_contained_and_connected &&
transport->pack_lockfiles.nr == 1 &&
strip_suffix(transport->pack_lockfiles.items[0].string,
".keep", &base_len)) {
struct strbuf idx_file = STRBUF_INIT;
strbuf_add(&idx_file, transport->pack_lockfiles.items[0].string,
base_len);
strbuf_addstr(&idx_file, ".idx");
new_pack = add_packed_git(idx_file.buf, idx_file.len, 1);
strbuf_release(&idx_file);
}
if (has_promisor_remote()) {
/*
* For partial clones, we don't want to have to do a regular
* connectivity check because we have to enumerate and exclude
* all promisor objects (slow), and then the connectivity check
* itself becomes a no-op because in a partial clone every
* object is a promisor object. Instead, just make sure we
* received, in a promisor packfile, the objects pointed to by
* each wanted ref.
*
* Before checking for promisor packs, be sure we have the
* latest pack-files loaded into memory.
*/
reprepare_packed_git(the_repository);
do {
struct packed_git *p;
for (p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_promisor)
continue;
if (find_pack_entry_one(oid.hash, p))
goto promisor_pack_found;
}
/*
* Fallback to rev-list with oid and the rest of the
* object IDs provided by fn.
*/
goto no_promisor_pack_found;
promisor_pack_found:
;
} while (!fn(cb_data, &oid));
return 0;
}
no_promisor_pack_found:
if (opt->shallow_file) {
strvec_push(&rev_list.args, "--shallow-file");
strvec_push(&rev_list.args, opt->shallow_file);
}
strvec_push(&rev_list.args,"rev-list");
strvec_push(&rev_list.args, "--objects");
strvec_push(&rev_list.args, "--stdin");
if (has_promisor_remote())
strvec_push(&rev_list.args, "--exclude-promisor-objects");
if (!opt->is_deepening_fetch) {
strvec_push(&rev_list.args, "--not");
strvec_push(&rev_list.args, "--all");
}
strvec_push(&rev_list.args, "--quiet");
strvec_push(&rev_list.args, "--alternate-refs");
if (opt->progress)
strvec_pushf(&rev_list.args, "--progress=%s",
_("Checking connectivity"));
strvec_push(&rev_list.args, "--unsorted-input");
rev_list.git_cmd = 1;
rev_list.env = opt->env;
rev_list.in = -1;
rev_list.no_stdout = 1;
if (opt->err_fd)
rev_list.err = opt->err_fd;
else
rev_list.no_stderr = opt->quiet;
if (start_command(&rev_list))
return error(_("Could not run 'git rev-list'"));
sigchain_push(SIGPIPE, SIG_IGN);
rev_list_in = xfdopen(rev_list.in, "w");
do {
/*
* If index-pack already checked that:
* - there are no dangling pointers in the new pack
* - the pack is self contained
* Then if the updated ref is in the new pack, then we
* are sure the ref is good and not sending it to
* rev-list for verification.
*/
if (new_pack && find_pack_entry_one(oid.hash, new_pack))
continue;
if (fprintf(rev_list_in, "%s\n", oid_to_hex(&oid)) < 0)
break;
} while (!fn(cb_data, &oid));
if (ferror(rev_list_in) || fflush(rev_list_in)) {
if (errno != EPIPE && errno != EINVAL)
error_errno(_("failed write to rev-list"));
err = -1;
}
if (fclose(rev_list_in))
err = error_errno(_("failed to close rev-list's stdin"));
sigchain_pop(SIGPIPE);
return finish_command(&rev_list) || err;
}