Merge branch 'jt/lazy-fetch'
Updates to on-demand fetching code in lazily cloned repositories. * jt/lazy-fetch: fetch: no FETCH_HEAD display if --no-write-fetch-head fetch-pack: remove no_dependents code promisor-remote: lazy-fetch objects in subprocess fetch-pack: do not lazy-fetch during ref iteration fetch: only populate existing_refs if needed fetch: avoid reading submodule config until needed fetch: allow refspecs specified through stdin negotiator/noop: add noop fetch negotiator
This commit is contained in:
Junio C Hamano
commit
b4100f366c
@ -60,7 +60,10 @@ fetch.negotiationAlgorithm::
|
||||
sent when negotiating the contents of the packfile to be sent by the
|
||||
server. Set to "skipping" to use an algorithm that skips commits in an
|
||||
effort to converge faster, but may result in a larger-than-necessary
|
||||
packfile; The default is "default" which instructs Git to use the default algorithm
|
||||
packfile; or set to "noop" to not send any information at all, which
|
||||
will almost certainly result in a larger-than-necessary packfile, but
|
||||
will skip the negotiation step.
|
||||
The default is "default" which instructs Git to use the default algorithm
|
||||
that never skips commits (unless the server has acknowledged it or one
|
||||
of its descendants). If `feature.experimental` is enabled, then this
|
||||
setting defaults to "skipping".
|
||||
|
||||
@ -48,6 +48,10 @@ include::fetch-options.txt[]
|
||||
|
||||
include::pull-fetch-param.txt[]
|
||||
|
||||
--stdin::
|
||||
Read refspecs, one per line, from stdin in addition to those provided
|
||||
as arguments. The "tag <name>" format is not supported.
|
||||
|
||||
include::urls-remotes.txt[]
|
||||
|
||||
|
||||
|
||||
@ -171,20 +171,13 @@ additional flag.
|
||||
Fetching Missing Objects
|
||||
------------------------
|
||||
|
||||
- Fetching of objects is done using the existing transport mechanism using
|
||||
transport_fetch_refs(), setting a new transport option
|
||||
TRANS_OPT_NO_DEPENDENTS to indicate that only the objects themselves are
|
||||
desired, not any object that they refer to.
|
||||
+
|
||||
Because some transports invoke fetch_pack() in the same process, fetch_pack()
|
||||
has been updated to not use any object flags when the corresponding argument
|
||||
(no_dependents) is set.
|
||||
- Fetching of objects is done by invoking a "git fetch" subprocess.
|
||||
|
||||
- The local repository sends a request with the hashes of all requested
|
||||
objects as "want" lines, and does not perform any packfile negotiation.
|
||||
objects, and does not perform any packfile negotiation.
|
||||
It then receives a packfile.
|
||||
|
||||
- Because we are reusing the existing fetch-pack mechanism, fetching
|
||||
- Because we are reusing the existing fetch mechanism, fetching
|
||||
currently fetches all objects referred to by the requested objects, even
|
||||
though they are not necessary.
|
||||
|
||||
|
||||
1
Makefile
1
Makefile
@ -908,6 +908,7 @@ LIB_OBJS += mergesort.o
|
||||
LIB_OBJS += midx.o
|
||||
LIB_OBJS += name-hash.o
|
||||
LIB_OBJS += negotiator/default.o
|
||||
LIB_OBJS += negotiator/noop.o
|
||||
LIB_OBJS += negotiator/skipping.o
|
||||
LIB_OBJS += notes-cache.o
|
||||
LIB_OBJS += notes-merge.o
|
||||
|
||||
@ -153,10 +153,6 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
|
||||
args.from_promisor = 1;
|
||||
continue;
|
||||
}
|
||||
if (!strcmp("--no-dependents", arg)) {
|
||||
args.no_dependents = 1;
|
||||
continue;
|
||||
}
|
||||
if (skip_prefix(arg, ("--" CL_ARG__FILTER "="), &arg)) {
|
||||
parse_list_objects_filter(&args.filter_options, arg);
|
||||
continue;
|
||||
|
||||
@ -80,6 +80,7 @@ static struct list_objects_filter_options filter_options;
|
||||
static struct string_list server_options = STRING_LIST_INIT_DUP;
|
||||
static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP;
|
||||
static int fetch_write_commit_graph = -1;
|
||||
static int stdin_refspecs = 0;
|
||||
|
||||
static int git_fetch_config(const char *k, const char *v, void *cb)
|
||||
{
|
||||
@ -205,6 +206,8 @@ static struct option builtin_fetch_options[] = {
|
||||
N_("check for forced-updates on all updated branches")),
|
||||
OPT_BOOL(0, "write-commit-graph", &fetch_write_commit_graph,
|
||||
N_("write the commit-graph after fetching")),
|
||||
OPT_BOOL(0, "stdin", &stdin_refspecs,
|
||||
N_("accept refspecs from stdin")),
|
||||
OPT_END()
|
||||
};
|
||||
|
||||
@ -442,6 +445,7 @@ static struct ref *get_ref_map(struct remote *remote,
|
||||
struct ref *orefs = NULL, **oref_tail = &orefs;
|
||||
|
||||
struct hashmap existing_refs;
|
||||
int existing_refs_populated = 0;
|
||||
|
||||
if (rs->nr) {
|
||||
struct refspec *fetch_refspec;
|
||||
@ -535,15 +539,18 @@ static struct ref *get_ref_map(struct remote *remote,
|
||||
|
||||
ref_map = ref_remove_duplicates(ref_map);
|
||||
|
||||
refname_hash_init(&existing_refs);
|
||||
for_each_ref(add_one_refname, &existing_refs);
|
||||
|
||||
for (rm = ref_map; rm; rm = rm->next) {
|
||||
if (rm->peer_ref) {
|
||||
const char *refname = rm->peer_ref->name;
|
||||
struct refname_hash_entry *peer_item;
|
||||
unsigned int hash = strhash(refname);
|
||||
|
||||
if (!existing_refs_populated) {
|
||||
refname_hash_init(&existing_refs);
|
||||
for_each_ref(add_one_refname, &existing_refs);
|
||||
existing_refs_populated = 1;
|
||||
}
|
||||
|
||||
peer_item = hashmap_get_entry_from_hash(&existing_refs,
|
||||
hash, refname,
|
||||
struct refname_hash_entry, ent);
|
||||
@ -553,7 +560,8 @@ static struct ref *get_ref_map(struct remote *remote,
|
||||
}
|
||||
}
|
||||
}
|
||||
hashmap_free_entries(&existing_refs, struct refname_hash_entry, ent);
|
||||
if (existing_refs_populated)
|
||||
hashmap_free_entries(&existing_refs, struct refname_hash_entry, ent);
|
||||
|
||||
return ref_map;
|
||||
}
|
||||
@ -1015,11 +1023,17 @@ static int store_updated_refs(const char *raw_url, const char *remote_name,
|
||||
rc |= update_local_ref(ref, what, rm, ¬e,
|
||||
summary_width);
|
||||
free(ref);
|
||||
} else
|
||||
} else if (write_fetch_head || dry_run) {
|
||||
/*
|
||||
* Display fetches written to FETCH_HEAD (or
|
||||
* would be written to FETCH_HEAD, if --dry-run
|
||||
* is set).
|
||||
*/
|
||||
format_display(¬e, '*',
|
||||
*kind ? kind : "branch", NULL,
|
||||
*what ? what : "HEAD",
|
||||
"FETCH_HEAD", summary_width);
|
||||
}
|
||||
if (note.len) {
|
||||
if (verbosity >= 0 && !shown_url) {
|
||||
fprintf(stderr, _("From %.*s\n"),
|
||||
@ -1680,7 +1694,8 @@ static inline void fetch_one_setup_partial(struct remote *remote)
|
||||
return;
|
||||
}
|
||||
|
||||
static int fetch_one(struct remote *remote, int argc, const char **argv, int prune_tags_ok)
|
||||
static int fetch_one(struct remote *remote, int argc, const char **argv,
|
||||
int prune_tags_ok, int use_stdin_refspecs)
|
||||
{
|
||||
struct refspec rs = REFSPEC_INIT_FETCH;
|
||||
int i;
|
||||
@ -1737,6 +1752,13 @@ static int fetch_one(struct remote *remote, int argc, const char **argv, int pru
|
||||
}
|
||||
}
|
||||
|
||||
if (use_stdin_refspecs) {
|
||||
struct strbuf line = STRBUF_INIT;
|
||||
while (strbuf_getline_lf(&line, stdin) != EOF)
|
||||
refspec_append(&rs, line.buf);
|
||||
strbuf_release(&line);
|
||||
}
|
||||
|
||||
if (server_options.nr)
|
||||
gtransport->server_options = &server_options;
|
||||
|
||||
@ -1771,12 +1793,18 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
|
||||
free(anon);
|
||||
}
|
||||
|
||||
fetch_config_from_gitmodules(&submodule_fetch_jobs_config,
|
||||
&recurse_submodules);
|
||||
git_config(git_fetch_config, NULL);
|
||||
|
||||
argc = parse_options(argc, argv, prefix,
|
||||
builtin_fetch_options, builtin_fetch_usage, 0);
|
||||
if (recurse_submodules != RECURSE_SUBMODULES_OFF) {
|
||||
int *sfjc = submodule_fetch_jobs_config == -1
|
||||
? &submodule_fetch_jobs_config : NULL;
|
||||
int *rs = recurse_submodules == RECURSE_SUBMODULES_DEFAULT
|
||||
? &recurse_submodules : NULL;
|
||||
|
||||
fetch_config_from_gitmodules(sfjc, rs);
|
||||
}
|
||||
|
||||
if (deepen_relative) {
|
||||
if (deepen_relative < 0)
|
||||
@ -1837,7 +1865,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
|
||||
if (remote) {
|
||||
if (filter_options.choice || has_promisor_remote())
|
||||
fetch_one_setup_partial(remote);
|
||||
result = fetch_one(remote, argc, argv, prune_tags_ok);
|
||||
result = fetch_one(remote, argc, argv, prune_tags_ok, stdin_refspecs);
|
||||
} else {
|
||||
int max_children = max_jobs;
|
||||
|
||||
@ -1845,6 +1873,10 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
|
||||
die(_("--filter can only be used with the remote "
|
||||
"configured in extensions.partialclone"));
|
||||
|
||||
if (stdin_refspecs)
|
||||
die(_("--stdin can only be used when fetching "
|
||||
"from one remote"));
|
||||
|
||||
if (max_children < 0)
|
||||
max_children = fetch_parallel_config;
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include "fetch-negotiator.h"
|
||||
#include "negotiator/default.h"
|
||||
#include "negotiator/skipping.h"
|
||||
#include "negotiator/noop.h"
|
||||
#include "repository.h"
|
||||
|
||||
void fetch_negotiator_init(struct repository *r,
|
||||
@ -13,6 +14,10 @@ void fetch_negotiator_init(struct repository *r,
|
||||
skipping_negotiator_init(negotiator);
|
||||
return;
|
||||
|
||||
case FETCH_NEGOTIATION_NOOP:
|
||||
noop_negotiator_init(negotiator);
|
||||
return;
|
||||
|
||||
case FETCH_NEGOTIATION_DEFAULT:
|
||||
default:
|
||||
default_negotiator_init(negotiator);
|
||||
|
||||
187
fetch-pack.c
187
fetch-pack.c
@ -108,24 +108,48 @@ static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
|
||||
cb(negotiator, cache.items[i]);
|
||||
}
|
||||
|
||||
static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
|
||||
int mark_tags_complete)
|
||||
{
|
||||
enum object_type type;
|
||||
struct object_info info = { .typep = &type };
|
||||
|
||||
while (1) {
|
||||
if (oid_object_info_extended(the_repository, oid, &info,
|
||||
OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK))
|
||||
return NULL;
|
||||
if (type == OBJ_TAG) {
|
||||
struct tag *tag = (struct tag *)
|
||||
parse_object(the_repository, oid);
|
||||
|
||||
if (!tag->tagged)
|
||||
return NULL;
|
||||
if (mark_tags_complete)
|
||||
tag->object.flags |= COMPLETE;
|
||||
oid = &tag->tagged->oid;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (type == OBJ_COMMIT)
|
||||
return (struct commit *) parse_object(the_repository, oid);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
|
||||
const char *refname,
|
||||
const struct object_id *oid)
|
||||
{
|
||||
struct object *o = deref_tag(the_repository,
|
||||
parse_object(the_repository, oid),
|
||||
refname, 0);
|
||||
|
||||
if (o && o->type == OBJ_COMMIT)
|
||||
negotiator->add_tip(negotiator, (struct commit *)o);
|
||||
struct commit *c = deref_without_lazy_fetch(oid, 0);
|
||||
|
||||
if (c)
|
||||
negotiator->add_tip(negotiator, c);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
|
||||
int flag, void *cb_data)
|
||||
{
|
||||
return rev_list_insert_ref(cb_data, refname, oid);
|
||||
return rev_list_insert_ref(cb_data, oid);
|
||||
}
|
||||
|
||||
enum ack_type {
|
||||
@ -201,7 +225,7 @@ static void send_request(struct fetch_pack_args *args,
|
||||
static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
|
||||
struct object *obj)
|
||||
{
|
||||
rev_list_insert_ref(negotiator, NULL, &obj->oid);
|
||||
rev_list_insert_ref(negotiator, &obj->oid);
|
||||
}
|
||||
|
||||
#define INITIAL_FLUSH 16
|
||||
@ -230,13 +254,12 @@ static void mark_tips(struct fetch_negotiator *negotiator,
|
||||
int i;
|
||||
|
||||
if (!negotiation_tips) {
|
||||
for_each_ref(rev_list_insert_ref_oid, negotiator);
|
||||
for_each_rawref(rev_list_insert_ref_oid, negotiator);
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < negotiation_tips->nr; i++)
|
||||
rev_list_insert_ref(negotiator, NULL,
|
||||
&negotiation_tips->oid[i]);
|
||||
rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -262,10 +285,8 @@ static int find_common(struct fetch_negotiator *negotiator,
|
||||
PACKET_READ_CHOMP_NEWLINE |
|
||||
PACKET_READ_DIE_ON_ERR_PACKET);
|
||||
|
||||
if (!args->no_dependents) {
|
||||
mark_tips(negotiator, args->negotiation_tips);
|
||||
for_each_cached_alternate(negotiator, insert_one_alternate_object);
|
||||
}
|
||||
mark_tips(negotiator, args->negotiation_tips);
|
||||
for_each_cached_alternate(negotiator, insert_one_alternate_object);
|
||||
|
||||
fetching = 0;
|
||||
for ( ; refs ; refs = refs->next) {
|
||||
@ -282,12 +303,8 @@ static int find_common(struct fetch_negotiator *negotiator,
|
||||
* We use lookup_object here because we are only
|
||||
* interested in the case we *know* the object is
|
||||
* reachable and we have already scanned it.
|
||||
*
|
||||
* Do this only if args->no_dependents is false (if it is true,
|
||||
* we cannot trust the object flags).
|
||||
*/
|
||||
if (!args->no_dependents &&
|
||||
((o = lookup_object(the_repository, remote)) != NULL) &&
|
||||
if (((o = lookup_object(the_repository, remote)) != NULL) &&
|
||||
(o->flags & COMPLETE)) {
|
||||
continue;
|
||||
}
|
||||
@ -387,8 +404,6 @@ static int find_common(struct fetch_negotiator *negotiator,
|
||||
trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
|
||||
flushes = 0;
|
||||
retval = -1;
|
||||
if (args->no_dependents)
|
||||
goto done;
|
||||
while ((oid = negotiator->next(negotiator))) {
|
||||
packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
|
||||
print_verbose(args, "have %s", oid_to_hex(oid));
|
||||
@ -503,21 +518,11 @@ static struct commit_list *complete;
|
||||
|
||||
static int mark_complete(const struct object_id *oid)
|
||||
{
|
||||
struct object *o = parse_object(the_repository, oid);
|
||||
struct commit *commit = deref_without_lazy_fetch(oid, 1);
|
||||
|
||||
while (o && o->type == OBJ_TAG) {
|
||||
struct tag *t = (struct tag *) o;
|
||||
if (!t->tagged)
|
||||
break; /* broken repository */
|
||||
o->flags |= COMPLETE;
|
||||
o = parse_object(the_repository, &t->tagged->oid);
|
||||
}
|
||||
if (o && o->type == OBJ_COMMIT) {
|
||||
struct commit *commit = (struct commit *)o;
|
||||
if (!(commit->object.flags & COMPLETE)) {
|
||||
commit->object.flags |= COMPLETE;
|
||||
commit_list_insert(commit, &complete);
|
||||
}
|
||||
if (commit && !(commit->object.flags & COMPLETE)) {
|
||||
commit->object.flags |= COMPLETE;
|
||||
commit_list_insert(commit, &complete);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -653,9 +658,7 @@ struct loose_object_iter {
|
||||
|
||||
/*
|
||||
* Mark recent commits available locally and reachable from a local ref as
|
||||
* COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
|
||||
* COMMON_REF (otherwise, we are not planning to participate in negotiation, and
|
||||
* thus do not need COMMON_REF marks).
|
||||
* COMPLETE.
|
||||
*
|
||||
* The cutoff time for recency is determined by this heuristic: it is the
|
||||
* earliest commit time of the objects in refs that are commits and that we know
|
||||
@ -702,7 +705,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
|
||||
*/
|
||||
trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
|
||||
if (!args->deepen) {
|
||||
for_each_ref(mark_complete_oid, NULL);
|
||||
for_each_rawref(mark_complete_oid, NULL);
|
||||
for_each_cached_alternate(NULL, mark_alternate_complete);
|
||||
commit_list_sort_by_date(&complete);
|
||||
if (cutoff)
|
||||
@ -716,16 +719,12 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
|
||||
*/
|
||||
trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
|
||||
for (ref = *refs; ref; ref = ref->next) {
|
||||
struct object *o = deref_tag(the_repository,
|
||||
lookup_object(the_repository,
|
||||
&ref->old_oid),
|
||||
NULL, 0);
|
||||
struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0);
|
||||
|
||||
if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
|
||||
if (!c || !(c->object.flags & COMPLETE))
|
||||
continue;
|
||||
|
||||
negotiator->known_common(negotiator,
|
||||
(struct commit *)o);
|
||||
negotiator->known_common(negotiator, c);
|
||||
}
|
||||
trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
|
||||
|
||||
@ -967,12 +966,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
|
||||
struct fetch_negotiator negotiator_alloc;
|
||||
struct fetch_negotiator *negotiator;
|
||||
|
||||
if (args->no_dependents) {
|
||||
negotiator = NULL;
|
||||
} else {
|
||||
negotiator = &negotiator_alloc;
|
||||
fetch_negotiator_init(r, negotiator);
|
||||
}
|
||||
negotiator = &negotiator_alloc;
|
||||
fetch_negotiator_init(r, negotiator);
|
||||
|
||||
sort_ref_list(&ref, ref_compare_name);
|
||||
QSORT(sought, nr_sought, cmp_ref_by_name);
|
||||
@ -1060,15 +1055,11 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
|
||||
if (!server_supports_hash(the_hash_algo->name, NULL))
|
||||
die(_("Server does not support this repository's object format"));
|
||||
|
||||
if (!args->no_dependents) {
|
||||
mark_complete_and_common_ref(negotiator, args, &ref);
|
||||
filter_refs(args, &ref, sought, nr_sought);
|
||||
if (everything_local(args, &ref)) {
|
||||
packet_flush(fd[1]);
|
||||
goto all_done;
|
||||
}
|
||||
} else {
|
||||
filter_refs(args, &ref, sought, nr_sought);
|
||||
mark_complete_and_common_ref(negotiator, args, &ref);
|
||||
filter_refs(args, &ref, sought, nr_sought);
|
||||
if (everything_local(args, &ref)) {
|
||||
packet_flush(fd[1]);
|
||||
goto all_done;
|
||||
}
|
||||
if (find_common(negotiator, args, fd, &oid, ref) < 0)
|
||||
if (!args->keep_pack)
|
||||
@ -1117,7 +1108,7 @@ static void add_shallow_requests(struct strbuf *req_buf,
|
||||
packet_buf_write(req_buf, "deepen-relative\n");
|
||||
}
|
||||
|
||||
static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf)
|
||||
static void add_wants(const struct ref *wants, struct strbuf *req_buf)
|
||||
{
|
||||
int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
|
||||
|
||||
@ -1134,12 +1125,8 @@ static void add_wants(int no_dependents, const struct ref *wants, struct strbuf
|
||||
* We use lookup_object here because we are only
|
||||
* interested in the case we *know* the object is
|
||||
* reachable and we have already scanned it.
|
||||
*
|
||||
* Do this only if args->no_dependents is false (if it is true,
|
||||
* we cannot trust the object flags).
|
||||
*/
|
||||
if (!no_dependents &&
|
||||
((o = lookup_object(the_repository, remote)) != NULL) &&
|
||||
if (((o = lookup_object(the_repository, remote)) != NULL) &&
|
||||
(o->flags & COMPLETE)) {
|
||||
continue;
|
||||
}
|
||||
@ -1273,19 +1260,14 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
|
||||
}
|
||||
|
||||
/* add wants */
|
||||
add_wants(args->no_dependents, wants, &req_buf);
|
||||
add_wants(wants, &req_buf);
|
||||
|
||||
if (args->no_dependents) {
|
||||
packet_buf_write(&req_buf, "done");
|
||||
ret = 1;
|
||||
} else {
|
||||
/* Add all of the common commits we've found in previous rounds */
|
||||
add_common(&req_buf, common);
|
||||
/* Add all of the common commits we've found in previous rounds */
|
||||
add_common(&req_buf, common);
|
||||
|
||||
/* Add initial haves */
|
||||
ret = add_haves(negotiator, seen_ack, &req_buf,
|
||||
haves_to_send, in_vain);
|
||||
}
|
||||
/* Add initial haves */
|
||||
ret = add_haves(negotiator, seen_ack, &req_buf,
|
||||
haves_to_send, in_vain);
|
||||
|
||||
/* Send request */
|
||||
packet_buf_flush(&req_buf);
|
||||
@ -1545,12 +1527,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
|
||||
struct string_list packfile_uris = STRING_LIST_INIT_DUP;
|
||||
int i;
|
||||
|
||||
if (args->no_dependents) {
|
||||
negotiator = NULL;
|
||||
} else {
|
||||
negotiator = &negotiator_alloc;
|
||||
fetch_negotiator_init(r, negotiator);
|
||||
}
|
||||
negotiator = &negotiator_alloc;
|
||||
fetch_negotiator_init(r, negotiator);
|
||||
|
||||
packet_reader_init(&reader, fd[0], NULL, 0,
|
||||
PACKET_READ_CHOMP_NEWLINE |
|
||||
@ -1574,21 +1552,16 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
|
||||
args->deepen = 1;
|
||||
|
||||
/* Filter 'ref' by 'sought' and those that aren't local */
|
||||
if (!args->no_dependents) {
|
||||
mark_complete_and_common_ref(negotiator, args, &ref);
|
||||
filter_refs(args, &ref, sought, nr_sought);
|
||||
if (everything_local(args, &ref))
|
||||
state = FETCH_DONE;
|
||||
else
|
||||
state = FETCH_SEND_REQUEST;
|
||||
|
||||
mark_tips(negotiator, args->negotiation_tips);
|
||||
for_each_cached_alternate(negotiator,
|
||||
insert_one_alternate_object);
|
||||
} else {
|
||||
filter_refs(args, &ref, sought, nr_sought);
|
||||
mark_complete_and_common_ref(negotiator, args, &ref);
|
||||
filter_refs(args, &ref, sought, nr_sought);
|
||||
if (everything_local(args, &ref))
|
||||
state = FETCH_DONE;
|
||||
else
|
||||
state = FETCH_SEND_REQUEST;
|
||||
}
|
||||
|
||||
mark_tips(negotiator, args->negotiation_tips);
|
||||
for_each_cached_alternate(negotiator,
|
||||
insert_one_alternate_object);
|
||||
break;
|
||||
case FETCH_SEND_REQUEST:
|
||||
if (!negotiation_started) {
|
||||
@ -1909,20 +1882,6 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
|
||||
if (nr_sought)
|
||||
nr_sought = remove_duplicates_in_refs(sought, nr_sought);
|
||||
|
||||
if (args->no_dependents && !args->filter_options.choice) {
|
||||
/*
|
||||
* The protocol does not support requesting that only the
|
||||
* wanted objects be sent, so approximate this by setting a
|
||||
* "blob:none" filter if no filter is already set. This works
|
||||
* for all object types: note that wanted blobs will still be
|
||||
* sent because they are directly specified as a "want".
|
||||
*
|
||||
* NEEDSWORK: Add an option in the protocol to request that
|
||||
* only the wanted objects be sent, and implement it.
|
||||
*/
|
||||
parse_list_objects_filter(&args->filter_options, "blob:none");
|
||||
}
|
||||
|
||||
if (version != protocol_v2 && !ref) {
|
||||
packet_flush(fd[1]);
|
||||
die(_("no matching remote head"));
|
||||
|
||||
14
fetch-pack.h
14
fetch-pack.h
@ -50,20 +50,6 @@ struct fetch_pack_args {
|
||||
*/
|
||||
unsigned from_promisor:1;
|
||||
|
||||
/*
|
||||
* Attempt to fetch only the wanted objects, and not any objects
|
||||
* referred to by them. Due to protocol limitations, extraneous
|
||||
* objects may still be included. (When fetching non-blob
|
||||
* objects, only blobs are excluded; when fetching a blob, the
|
||||
* blob itself will still be sent. The client does not need to
|
||||
* know whether a wanted object is a blob or not.)
|
||||
*
|
||||
* If 1, fetch_pack() will also not modify any object flags.
|
||||
* This allows fetch_pack() to safely be called by any function,
|
||||
* regardless of which object flags it uses (if any).
|
||||
*/
|
||||
unsigned no_dependents:1;
|
||||
|
||||
/*
|
||||
* Because fetch_pack() overwrites the shallow file upon a
|
||||
* successful deepening non-clone fetch, if this struct
|
||||
|
||||
44
negotiator/noop.c
Normal file
44
negotiator/noop.c
Normal file
@ -0,0 +1,44 @@
|
||||
#include "cache.h"
|
||||
#include "noop.h"
|
||||
#include "../commit.h"
|
||||
#include "../fetch-negotiator.h"
|
||||
|
||||
static void known_common(struct fetch_negotiator *n, struct commit *c)
|
||||
{
|
||||
/* do nothing */
|
||||
}
|
||||
|
||||
static void add_tip(struct fetch_negotiator *n, struct commit *c)
|
||||
{
|
||||
/* do nothing */
|
||||
}
|
||||
|
||||
static const struct object_id *next(struct fetch_negotiator *n)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int ack(struct fetch_negotiator *n, struct commit *c)
|
||||
{
|
||||
/*
|
||||
* This negotiator does not emit any commits, so there is no commit to
|
||||
* be acknowledged. If there is any ack, there is a bug.
|
||||
*/
|
||||
BUG("ack with noop negotiator, which does not emit any commits");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void release(struct fetch_negotiator *n)
|
||||
{
|
||||
/* nothing to release */
|
||||
}
|
||||
|
||||
void noop_negotiator_init(struct fetch_negotiator *negotiator)
|
||||
{
|
||||
negotiator->known_common = known_common;
|
||||
negotiator->add_tip = add_tip;
|
||||
negotiator->next = next;
|
||||
negotiator->ack = ack;
|
||||
negotiator->release = release;
|
||||
negotiator->data = NULL;
|
||||
}
|
||||
8
negotiator/noop.h
Normal file
8
negotiator/noop.h
Normal file
@ -0,0 +1,8 @@
|
||||
#ifndef NEGOTIATOR_NOOP_H
|
||||
#define NEGOTIATOR_NOOP_H
|
||||
|
||||
struct fetch_negotiator;
|
||||
|
||||
void noop_negotiator_init(struct fetch_negotiator *negotiator);
|
||||
|
||||
#endif
|
||||
@ -3,6 +3,7 @@
|
||||
#include "promisor-remote.h"
|
||||
#include "config.h"
|
||||
#include "transport.h"
|
||||
#include "strvec.h"
|
||||
|
||||
static char *repository_format_partial_clone;
|
||||
static const char *core_partial_clone_filter_default;
|
||||
@ -12,39 +13,34 @@ void set_repository_format_partial_clone(char *partial_clone)
|
||||
repository_format_partial_clone = xstrdup_or_null(partial_clone);
|
||||
}
|
||||
|
||||
static int fetch_refs(const char *remote_name, struct ref *ref)
|
||||
{
|
||||
struct remote *remote;
|
||||
struct transport *transport;
|
||||
int res;
|
||||
|
||||
remote = remote_get(remote_name);
|
||||
if (!remote->url[0])
|
||||
die(_("Remote with no URL"));
|
||||
transport = transport_get(remote, remote->url[0]);
|
||||
|
||||
transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
|
||||
transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");
|
||||
res = transport_fetch_refs(transport, ref);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static int fetch_objects(const char *remote_name,
|
||||
const struct object_id *oids,
|
||||
int oid_nr)
|
||||
{
|
||||
struct ref *ref = NULL;
|
||||
struct child_process child = CHILD_PROCESS_INIT;
|
||||
int i;
|
||||
FILE *child_in;
|
||||
|
||||
child.git_cmd = 1;
|
||||
child.in = -1;
|
||||
strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
|
||||
"fetch", remote_name, "--no-tags",
|
||||
"--no-write-fetch-head", "--recurse-submodules=no",
|
||||
"--filter=blob:none", "--stdin", NULL);
|
||||
if (start_command(&child))
|
||||
die(_("promisor-remote: unable to fork off fetch subprocess"));
|
||||
child_in = xfdopen(child.in, "w");
|
||||
|
||||
for (i = 0; i < oid_nr; i++) {
|
||||
struct ref *new_ref = alloc_ref(oid_to_hex(&oids[i]));
|
||||
oidcpy(&new_ref->old_oid, &oids[i]);
|
||||
new_ref->exact_oid = 1;
|
||||
new_ref->next = ref;
|
||||
ref = new_ref;
|
||||
if (fputs(oid_to_hex(&oids[i]), child_in) < 0)
|
||||
die_errno(_("promisor-remote: could not write to fetch subprocess"));
|
||||
if (fputc('\n', child_in) < 0)
|
||||
die_errno(_("promisor-remote: could not write to fetch subprocess"));
|
||||
}
|
||||
return fetch_refs(remote_name, ref);
|
||||
|
||||
if (fclose(child_in) < 0)
|
||||
die_errno(_("promisor-remote: could not close stdin to fetch subprocess"));
|
||||
return finish_command(&child) ? -1 : 0;
|
||||
}
|
||||
|
||||
static struct promisor_remote *promisors;
|
||||
|
||||
@ -43,7 +43,6 @@ struct options {
|
||||
/* see documentation of corresponding flag in fetch-pack.h */
|
||||
from_promisor : 1,
|
||||
|
||||
no_dependents : 1,
|
||||
atomic : 1,
|
||||
object_format : 1;
|
||||
const struct git_hash_algo *hash_algo;
|
||||
@ -193,9 +192,6 @@ static int set_option(const char *name, const char *value)
|
||||
} else if (!strcmp(name, "from-promisor")) {
|
||||
options.from_promisor = 1;
|
||||
return 0;
|
||||
} else if (!strcmp(name, "no-dependents")) {
|
||||
options.no_dependents = 1;
|
||||
return 0;
|
||||
} else if (!strcmp(name, "filter")) {
|
||||
options.filter = xstrdup(value);
|
||||
return 0;
|
||||
@ -1178,8 +1174,6 @@ static int fetch_git(struct discovery *heads,
|
||||
strvec_push(&args, "--deepen-relative");
|
||||
if (options.from_promisor)
|
||||
strvec_push(&args, "--from-promisor");
|
||||
if (options.no_dependents)
|
||||
strvec_push(&args, "--no-dependents");
|
||||
if (options.filter)
|
||||
strvec_pushf(&args, "--filter=%s", options.filter);
|
||||
strvec_push(&args, url.buf);
|
||||
|
||||
@ -39,6 +39,8 @@ void prepare_repo_settings(struct repository *r)
|
||||
if (!repo_config_get_string(r, "fetch.negotiationalgorithm", &strval)) {
|
||||
if (!strcasecmp(strval, "skipping"))
|
||||
r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_SKIPPING;
|
||||
else if (!strcasecmp(strval, "noop"))
|
||||
r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_NOOP;
|
||||
else
|
||||
r->settings.fetch_negotiation_algorithm = FETCH_NEGOTIATION_DEFAULT;
|
||||
}
|
||||
|
||||
@ -23,6 +23,7 @@ enum fetch_negotiation_setting {
|
||||
FETCH_NEGOTIATION_NONE = 0,
|
||||
FETCH_NEGOTIATION_DEFAULT = 1,
|
||||
FETCH_NEGOTIATION_SKIPPING = 2,
|
||||
FETCH_NEGOTIATION_NOOP = 3,
|
||||
};
|
||||
|
||||
struct repo_settings {
|
||||
|
||||
@ -777,10 +777,14 @@ static int gitmodules_fetch_config(const char *var, const char *value, void *cb)
|
||||
{
|
||||
struct fetch_config *config = cb;
|
||||
if (!strcmp(var, "submodule.fetchjobs")) {
|
||||
*(config->max_children) = parse_submodule_fetchjobs(var, value);
|
||||
if (config->max_children)
|
||||
*(config->max_children) =
|
||||
parse_submodule_fetchjobs(var, value);
|
||||
return 0;
|
||||
} else if (!strcmp(var, "fetch.recursesubmodules")) {
|
||||
*(config->recurse_submodules) = parse_fetch_recurse_submodules_arg(var, value);
|
||||
if (config->recurse_submodules)
|
||||
*(config->recurse_submodules) =
|
||||
parse_fetch_recurse_submodules_arg(var, value);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -183,7 +183,7 @@ test_expect_success 'missing CLI object, but promised, passes fsck' '
|
||||
'
|
||||
|
||||
test_expect_success 'fetching of missing objects' '
|
||||
rm -rf repo &&
|
||||
rm -rf repo err &&
|
||||
test_create_repo server &&
|
||||
test_commit -C server foo &&
|
||||
git -C server repack -a -d --write-bitmap-index &&
|
||||
@ -194,7 +194,10 @@ test_expect_success 'fetching of missing objects' '
|
||||
|
||||
git -C repo config core.repositoryformatversion 1 &&
|
||||
git -C repo config extensions.partialclone "origin" &&
|
||||
git -C repo cat-file -p "$HASH" &&
|
||||
git -C repo cat-file -p "$HASH" 2>err &&
|
||||
|
||||
# Ensure that no spurious FETCH_HEAD messages are written
|
||||
! grep FETCH_HEAD err &&
|
||||
|
||||
# Ensure that the .promisor file is written, and check that its
|
||||
# associated packfile contains the object
|
||||
@ -214,7 +217,7 @@ test_expect_success 'fetching of missing objects works with ref-in-want enabled'
|
||||
rm -rf repo/.git/objects/* &&
|
||||
rm -f trace &&
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C repo cat-file -p "$HASH" &&
|
||||
grep "git< fetch=.*ref-in-want" trace
|
||||
grep "fetch< fetch=.*ref-in-want" trace
|
||||
'
|
||||
|
||||
test_expect_success 'fetching of missing objects from another promisor remote' '
|
||||
|
||||
@ -20,7 +20,7 @@ test_expect_success 'git show batches blobs' '
|
||||
# Ensure that there is exactly 1 negotiation by checking that there is
|
||||
# only 1 "done" line sent. ("done" marks the end of negotiation.)
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client show HEAD &&
|
||||
grep "git> done" trace >done_lines &&
|
||||
grep "fetch> done" trace >done_lines &&
|
||||
test_line_count = 1 done_lines
|
||||
'
|
||||
|
||||
@ -44,7 +44,7 @@ test_expect_success 'diff batches blobs' '
|
||||
# Ensure that there is exactly 1 negotiation by checking that there is
|
||||
# only 1 "done" line sent. ("done" marks the end of negotiation.)
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff HEAD^ HEAD &&
|
||||
grep "git> done" trace >done_lines &&
|
||||
grep "fetch> done" trace >done_lines &&
|
||||
test_line_count = 1 done_lines
|
||||
'
|
||||
|
||||
@ -127,7 +127,7 @@ test_expect_success 'diff with rename detection batches blobs' '
|
||||
# only 1 "done" line sent. ("done" marks the end of negotiation.)
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --raw -M HEAD^ HEAD >out &&
|
||||
grep ":100644 100644.*R[0-9][0-9][0-9].*b.*c" out &&
|
||||
grep "git> done" trace >done_lines &&
|
||||
grep "fetch> done" trace >done_lines &&
|
||||
test_line_count = 1 done_lines
|
||||
'
|
||||
|
||||
@ -175,7 +175,7 @@ test_expect_success 'diff --break-rewrites fetches only if necessary, and batche
|
||||
# by checking that there is only 1 "done" line sent. ("done" marks the
|
||||
# end of negotiation.)
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client diff --break-rewrites --raw -M HEAD^ HEAD &&
|
||||
grep "git> done" trace >done_lines &&
|
||||
grep "fetch> done" trace >done_lines &&
|
||||
test_line_count = 1 done_lines
|
||||
'
|
||||
|
||||
|
||||
@ -528,7 +528,7 @@ test_expect_success 'prefetch objects' '
|
||||
TWO=$(git -C server rev-parse three_branch^) &&
|
||||
git -C client fetch --filter=blob:none origin "$TWO" &&
|
||||
GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch &&
|
||||
grep "git> done" trace >donelines &&
|
||||
grep "fetch> done" trace >donelines &&
|
||||
test_line_count = 1 donelines
|
||||
'
|
||||
|
||||
|
||||
@ -543,16 +543,18 @@ test_expect_success 'fetch into the current branch with --update-head-ok' '
|
||||
|
||||
'
|
||||
|
||||
test_expect_success 'fetch --dry-run does not touch FETCH_HEAD' '
|
||||
rm -f .git/FETCH_HEAD &&
|
||||
git fetch --dry-run . &&
|
||||
! test -f .git/FETCH_HEAD
|
||||
test_expect_success 'fetch --dry-run does not touch FETCH_HEAD, but still prints what would be written' '
|
||||
rm -f .git/FETCH_HEAD err &&
|
||||
git fetch --dry-run . 2>err &&
|
||||
! test -f .git/FETCH_HEAD &&
|
||||
grep FETCH_HEAD err
|
||||
'
|
||||
|
||||
test_expect_success '--no-write-fetch-head does not touch FETCH_HEAD' '
|
||||
rm -f .git/FETCH_HEAD &&
|
||||
git fetch --no-write-fetch-head . &&
|
||||
! test -f .git/FETCH_HEAD
|
||||
test_expect_success '--no-write-fetch-head does not touch FETCH_HEAD, and does not print what would be written' '
|
||||
rm -f .git/FETCH_HEAD err &&
|
||||
git fetch --no-write-fetch-head . 2>err &&
|
||||
! test -f .git/FETCH_HEAD &&
|
||||
! grep FETCH_HEAD err
|
||||
'
|
||||
|
||||
test_expect_success '--write-fetch-head gets defeated by --dry-run' '
|
||||
|
||||
22
t/t5554-noop-fetch-negotiator.sh
Executable file
22
t/t5554-noop-fetch-negotiator.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
|
||||
test_description='test noop fetch negotiator'
|
||||
. ./test-lib.sh
|
||||
|
||||
test_expect_success 'noop negotiator does not emit any "have"' '
|
||||
rm -f trace &&
|
||||
|
||||
test_create_repo server &&
|
||||
test_commit -C server to_fetch &&
|
||||
|
||||
test_create_repo client &&
|
||||
test_commit -C client we_have &&
|
||||
|
||||
test_config -C client fetch.negotiationalgorithm noop &&
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch "$(pwd)/server" &&
|
||||
|
||||
! grep "fetch> have" trace &&
|
||||
grep "fetch> done" trace
|
||||
'
|
||||
|
||||
test_done
|
||||
@ -704,7 +704,7 @@ test_expect_success 'batch missing blob request during checkout' '
|
||||
# Ensure that there is only one negotiation by checking that there is
|
||||
# only "done" line sent. ("done" marks the end of negotiation.)
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ &&
|
||||
grep "git> done" trace >done_lines &&
|
||||
grep "fetch> done" trace >done_lines &&
|
||||
test_line_count = 1 done_lines
|
||||
'
|
||||
|
||||
|
||||
@ -433,6 +433,26 @@ test_expect_success 'fetch lazy-fetches only to resolve deltas, protocol v2' '
|
||||
grep "want $(cat hash)" trace
|
||||
'
|
||||
|
||||
test_expect_success 'fetch does not lazy-fetch missing targets of its refs' '
|
||||
rm -rf server client trace &&
|
||||
|
||||
test_create_repo server &&
|
||||
test_config -C server uploadpack.allowfilter 1 &&
|
||||
test_config -C server uploadpack.allowanysha1inwant 1 &&
|
||||
test_commit -C server foo &&
|
||||
|
||||
git clone --filter=blob:none "file://$(pwd)/server" client &&
|
||||
# Make all refs point to nothing by deleting all objects.
|
||||
rm client/.git/objects/pack/* &&
|
||||
|
||||
test_commit -C server bar &&
|
||||
GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
|
||||
--no-tags --recurse-submodules=no \
|
||||
origin refs/tags/bar &&
|
||||
FOO_HASH=$(git -C server rev-parse foo) &&
|
||||
! grep "want $FOO_HASH" trace
|
||||
'
|
||||
|
||||
# The following two tests must be in this order. It is important that
|
||||
# the srv.bare repository did not have tags during clone, but has tags
|
||||
# in the fetch.
|
||||
|
||||
@ -232,9 +232,6 @@ static int set_git_option(struct git_transport_options *opts,
|
||||
} else if (!strcmp(name, TRANS_OPT_FROM_PROMISOR)) {
|
||||
opts->from_promisor = !!value;
|
||||
return 0;
|
||||
} else if (!strcmp(name, TRANS_OPT_NO_DEPENDENTS)) {
|
||||
opts->no_dependents = !!value;
|
||||
return 0;
|
||||
} else if (!strcmp(name, TRANS_OPT_LIST_OBJECTS_FILTER)) {
|
||||
list_objects_filter_die_if_populated(&opts->filter_options);
|
||||
parse_list_objects_filter(&opts->filter_options, value);
|
||||
@ -359,7 +356,6 @@ static int fetch_refs_via_pack(struct transport *transport,
|
||||
args.cloning = transport->cloning;
|
||||
args.update_shallow = data->options.update_shallow;
|
||||
args.from_promisor = data->options.from_promisor;
|
||||
args.no_dependents = data->options.no_dependents;
|
||||
args.filter_options = data->options.filter_options;
|
||||
args.stateless_rpc = transport->stateless_rpc;
|
||||
args.server_options = transport->server_options;
|
||||
|
||||
@ -19,8 +19,6 @@ struct git_transport_options {
|
||||
/* see documentation of corresponding flag in fetch-pack.h */
|
||||
unsigned from_promisor : 1;
|
||||
|
||||
unsigned no_dependents : 1;
|
||||
|
||||
/*
|
||||
* If this transport supports connect or stateless-connect,
|
||||
* the corresponding field in struct fetch_pack_args is copied
|
||||
@ -204,12 +202,6 @@ void transport_check_allowed(const char *type);
|
||||
/* Indicate that these objects are being fetched by a promisor */
|
||||
#define TRANS_OPT_FROM_PROMISOR "from-promisor"
|
||||
|
||||
/*
|
||||
* Indicate that only the objects wanted need to be fetched, not their
|
||||
* dependents
|
||||
*/
|
||||
#define TRANS_OPT_NO_DEPENDENTS "no-dependents"
|
||||
|
||||
/* Filter objects for partial clone and fetch */
|
||||
#define TRANS_OPT_LIST_OBJECTS_FILTER "filter"
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user