Merge branch 'nd/clone-connectivity-shortcut'

Special case "git clone" and use lighter-weight implementation to
check the completeness of the history behind refs.

* nd/clone-connectivity-shortcut:
  clone: open a shortcut for connectivity check
  index-pack: remove dead code (it should never happen)
  fetch-pack: prepare updated shallow file before fetching the pack
  clone: let the user know when check_everything_connected is run
This commit is contained in:
Junio C Hamano 2013-06-06 12:17:55 -07:00
commit 72e719292d
13 changed files with 192 additions and 55 deletions

View File

@ -74,6 +74,9 @@ OPTIONS
--strict::
Die, if the pack contains broken objects or links.
--check-self-contained-and-connected::
Die if the pack contains broken links. For internal use only.
--threads=<n>::
Specifies the number of threads to spawn when resolving
deltas. This requires that index-pack be compiled with

View File

@ -541,12 +541,18 @@ static void update_remote_refs(const struct ref *refs,
const struct ref *mapped_refs,
const struct ref *remote_head_points_at,
const char *branch_top,
const char *msg)
const char *msg,
struct transport *transport)
{
const struct ref *rm = mapped_refs;
if (check_everything_connected(iterate_ref_map, 0, &rm))
if (0 <= option_verbosity)
printf(_("Checking connectivity... "));
if (check_everything_connected_with_transport(iterate_ref_map,
0, &rm, transport))
die(_("remote did not send all necessary objects"));
if (0 <= option_verbosity)
printf(_("done\n"));
if (refs) {
write_remote_refs(mapped_refs);
@ -890,6 +896,9 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
if (option_upload_pack)
transport_set_option(transport, TRANS_OPT_UPLOADPACK,
option_upload_pack);
if (transport->smart_options && !option_depth)
transport->smart_options->check_self_contained_and_connected = 1;
}
refs = transport_get_remote_refs(transport);
@ -951,7 +960,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
transport_fetch_refs(transport, mapped_refs);
update_remote_refs(refs, mapped_refs, remote_head_points_at,
branch_top.buf, reflog_msg.buf);
branch_top.buf, reflog_msg.buf, transport);
update_head(our_head_points_at, remote_head, reflog_msg.buf);

View File

@ -77,8 +77,10 @@ static int nr_threads;
static int from_stdin;
static int strict;
static int do_fsck_object;
static int verbose;
static int show_stat;
static int check_self_contained_and_connected;
static struct progress *progress;
@ -187,13 +189,13 @@ static int mark_link(struct object *obj, int type, void *data)
/* The content of each linked object must have been checked
or it must be already present in the object database */
static void check_object(struct object *obj)
static unsigned check_object(struct object *obj)
{
if (!obj)
return;
return 0;
if (!(obj->flags & FLAG_LINK))
return;
return 0;
if (!(obj->flags & FLAG_CHECKED)) {
unsigned long size;
@ -201,17 +203,20 @@ static void check_object(struct object *obj)
if (type != obj->type || type <= 0)
die(_("object of unexpected type"));
obj->flags |= FLAG_CHECKED;
return;
return 1;
}
return 0;
}
static void check_objects(void)
static unsigned check_objects(void)
{
unsigned i, max;
unsigned i, max, foreign_nr = 0;
max = get_max_object_index();
for (i = 0; i < max; i++)
check_object(get_indexed_object(i));
foreign_nr += check_object(get_indexed_object(i));
return foreign_nr;
}
@ -747,8 +752,7 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
int eaten;
void *buf = (void *) data;
if (!buf)
buf = new_data = get_data_from_pack(obj_entry);
assert(data && "data can only be NULL for large _blobs_");
/*
* we do not need to free the memory here, as the
@ -757,7 +761,8 @@ static void sha1_object(const void *data, struct object_entry *obj_entry,
obj = parse_object_buffer(sha1, type, size, buf, &eaten);
if (!obj)
die(_("invalid %s"), typename(type));
if (fsck_object(obj, 1, fsck_error_function))
if (do_fsck_object &&
fsck_object(obj, 1, fsck_error_function))
die(_("Error in object"));
if (fsck_walk(obj, mark_link, NULL))
die(_("Not all child objects of %s are reachable"), sha1_to_hex(obj->sha1));
@ -1491,6 +1496,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
struct pack_idx_entry **idx_objects;
struct pack_idx_option opts;
unsigned char pack_sha1[20];
unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */
if (argc == 2 && !strcmp(argv[1], "-h"))
usage(index_pack_usage);
@ -1512,6 +1518,10 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
fix_thin_pack = 1;
} else if (!strcmp(arg, "--strict")) {
strict = 1;
do_fsck_object = 1;
} else if (!strcmp(arg, "--check-self-contained-and-connected")) {
strict = 1;
check_self_contained_and_connected = 1;
} else if (!strcmp(arg, "--verify")) {
verify = 1;
} else if (!strcmp(arg, "--verify-stat")) {
@ -1625,7 +1635,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
conclude_pack(fix_thin_pack, curr_pack, pack_sha1);
free(deltas);
if (strict)
check_objects();
foreign_nr = check_objects();
if (show_stat)
show_pack_info(stat_only);
@ -1651,5 +1661,11 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
if (index_name == NULL)
free((void *) curr_index);
/*
* Let the caller know this pack is not self contained
*/
if (check_self_contained_and_connected && foreign_nr)
return 1;
return 0;
}

View File

@ -176,6 +176,8 @@ extern int for_each_commit_graft(each_commit_graft_fn, void *);
extern int is_repository_shallow(void);
extern struct commit_list *get_shallow_commits(struct object_array *heads,
int depth, int shallow_flag, int not_shallow_flag);
extern void check_shallow_file_for_update(void);
extern void set_alternate_shallow_file(const char *path);
int is_descendant_of(struct commit *, struct commit_list *);
int in_merge_bases(struct commit *, struct commit *);

View File

@ -2,7 +2,12 @@
#include "run-command.h"
#include "sigchain.h"
#include "connected.h"
#include "transport.h"
int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data)
{
return check_everything_connected_with_transport(fn, quiet, cb_data, NULL);
}
/*
* If we feed all the commits we want to verify to this command
*
@ -14,7 +19,10 @@
*
* Returns 0 if everything is connected, non-zero otherwise.
*/
int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data)
int check_everything_connected_with_transport(sha1_iterate_fn fn,
int quiet,
void *cb_data,
struct transport *transport)
{
struct child_process rev_list;
const char *argv[] = {"rev-list", "--objects",
@ -22,10 +30,23 @@ int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data)
char commit[41];
unsigned char sha1[20];
int err = 0;
struct packed_git *new_pack = NULL;
if (fn(cb_data, sha1))
return err;
if (transport && transport->smart_options &&
transport->smart_options->self_contained_and_connected &&
transport->pack_lockfile &&
!suffixcmp(transport->pack_lockfile, ".keep")) {
struct strbuf idx_file = STRBUF_INIT;
strbuf_addstr(&idx_file, transport->pack_lockfile);
strbuf_setlen(&idx_file, idx_file.len - 5); /* ".keep" */
strbuf_addstr(&idx_file, ".idx");
new_pack = add_packed_git(idx_file.buf, idx_file.len, 1);
strbuf_release(&idx_file);
}
if (quiet)
argv[5] = "--quiet";
@ -42,6 +63,17 @@ int check_everything_connected(sha1_iterate_fn fn, int quiet, void *cb_data)
commit[40] = '\n';
do {
/*
* If index-pack already checked that:
* - there are no dangling pointers in the new pack
* - the pack is self contained
* Then if the updated ref is in the new pack, then we
* are sure the ref is good and not sending it to
* rev-list for verification.
*/
if (new_pack && find_pack_entry_one(sha1, new_pack))
continue;
memcpy(commit, sha1_to_hex(sha1), 40);
if (write_in_full(rev_list.in, commit, 41) < 0) {
if (errno != EPIPE && errno != EINVAL)

View File

@ -1,6 +1,8 @@
#ifndef CONNECTED_H
#define CONNECTED_H
struct transport;
/*
* Take callback data, and return next object name in the buffer.
* When called after returning the name for the last object, return -1
@ -16,5 +18,8 @@ typedef int (*sha1_iterate_fn)(void *, unsigned char [20]);
* Return 0 if Ok, non zero otherwise (i.e. some missing objects)
*/
extern int check_everything_connected(sha1_iterate_fn, int quiet, void *cb_data);
extern int check_everything_connected_with_transport(sha1_iterate_fn, int quiet,
void *cb_data,
struct transport *transport);
#endif /* CONNECTED_H */

View File

@ -20,6 +20,8 @@ static int no_done;
static int fetch_fsck_objects = -1;
static int transfer_fsck_objects = -1;
static int agent_supported;
static struct lock_file shallow_lock;
static const char *alternate_shallow_file;
#define COMPLETE (1U << 0)
#define COMMON (1U << 1)
@ -683,12 +685,13 @@ static int get_pack(struct fetch_pack_args *args,
int xd[2], char **pack_lockfile)
{
struct async demux;
const char *argv[20];
const char *argv[22];
char keep_arg[256];
char hdr_arg[256];
const char **av;
int do_keep = args->keep_pack;
struct child_process cmd;
int ret;
memset(&demux, 0, sizeof(demux));
if (use_sideband) {
@ -724,6 +727,11 @@ static int get_pack(struct fetch_pack_args *args,
do_keep = 1;
}
if (alternate_shallow_file) {
*av++ = "--shallow-file";
*av++ = alternate_shallow_file;
}
if (do_keep) {
if (pack_lockfile)
cmd.out = -1;
@ -740,11 +748,14 @@ static int get_pack(struct fetch_pack_args *args,
strcpy(keep_arg + s, "localhost");
*av++ = keep_arg;
}
if (args->check_self_contained_and_connected)
*av++ = "--check-self-contained-and-connected";
}
else {
*av++ = "unpack-objects";
if (args->quiet || args->no_progress)
*av++ = "-q";
args->check_self_contained_and_connected = 0;
}
if (*hdr_arg)
*av++ = hdr_arg;
@ -765,7 +776,12 @@ static int get_pack(struct fetch_pack_args *args,
close(cmd.out);
}
if (finish_command(&cmd))
ret = finish_command(&cmd);
if (!ret || (args->check_self_contained_and_connected && ret == 1))
args->self_contained_and_connected =
args->check_self_contained_and_connected &&
ret == 0;
else
die("%s failed", argv[0]);
if (use_sideband && finish_async(&demux))
die("error in sideband demultiplexer");
@ -779,6 +795,27 @@ static int cmp_ref_by_name(const void *a_, const void *b_)
return strcmp(a->name, b->name);
}
static void setup_alternate_shallow(void)
{
struct strbuf sb = STRBUF_INIT;
int fd;
check_shallow_file_for_update();
fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
LOCK_DIE_ON_ERROR);
if (write_shallow_commits(&sb, 0)) {
if (write_in_full(fd, sb.buf, sb.len) != sb.len)
die_errno("failed to write to %s", shallow_lock.filename);
alternate_shallow_file = shallow_lock.filename;
} else
/*
* is_repository_shallow() sees empty string as "no
* shallow file".
*/
alternate_shallow_file = "";
strbuf_release(&sb);
}
static struct ref *do_fetch_pack(struct fetch_pack_args *args,
int fd[2],
const struct ref *orig_ref,
@ -858,6 +895,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
if (args->stateless_rpc)
packet_flush(fd[1]);
if (args->depth > 0)
setup_alternate_shallow();
if (get_pack(args, fd, pack_lockfile))
die("git fetch-pack: fetch failed.");
@ -936,15 +975,9 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
struct ref **sought, int nr_sought,
char **pack_lockfile)
{
struct stat st;
struct ref *ref_cpy;
fetch_pack_setup();
if (args->depth > 0) {
if (stat(git_path("shallow"), &st))
st.st_mtime = 0;
}
if (nr_sought)
nr_sought = remove_duplicates_in_refs(sought, nr_sought);
@ -954,35 +987,12 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
}
ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
if (args->depth > 0) {
static struct lock_file lock;
struct cache_time mtime;
struct strbuf sb = STRBUF_INIT;
char *shallow = git_path("shallow");
int fd;
mtime.sec = st.st_mtime;
mtime.nsec = ST_MTIME_NSEC(st);
if (stat(shallow, &st)) {
if (mtime.sec)
die("shallow file was removed during fetch");
} else if (st.st_mtime != mtime.sec
#ifdef USE_NSEC
|| ST_MTIME_NSEC(st) != mtime.nsec
#endif
)
die("shallow file was changed during fetch");
fd = hold_lock_file_for_update(&lock, shallow,
LOCK_DIE_ON_ERROR);
if (!write_shallow_commits(&sb, 0)
|| write_in_full(fd, sb.buf, sb.len) != sb.len) {
unlink_or_warn(shallow);
rollback_lock_file(&lock);
} else {
commit_lock_file(&lock);
}
strbuf_release(&sb);
if (alternate_shallow_file) {
if (*alternate_shallow_file == '\0') { /* --unshallow */
unlink_or_warn(git_path("shallow"));
rollback_lock_file(&shallow_lock);
} else
commit_lock_file(&shallow_lock);
}
reprepare_packed_git();

View File

@ -16,7 +16,9 @@ struct fetch_pack_args {
verbose:1,
no_progress:1,
include_tag:1,
stateless_rpc:1;
stateless_rpc:1,
check_self_contained_and_connected:1,
self_contained_and_connected:1;
};
/*

7
git.c
View File

@ -4,6 +4,7 @@
#include "help.h"
#include "quote.h"
#include "run-command.h"
#include "commit.h"
const char git_usage_string[] =
"git [--version] [--help] [-c name=value]\n"
@ -146,6 +147,12 @@ static int handle_options(const char ***argv, int *argc, int *envchanged)
setenv(GIT_LITERAL_PATHSPECS_ENVIRONMENT, "0", 1);
if (envchanged)
*envchanged = 1;
} else if (!strcmp(cmd, "--shallow-file")) {
(*argv)++;
(*argc)--;
set_alternate_shallow_file((*argv)[0]);
if (envchanged)
*envchanged = 1;
} else {
fprintf(stderr, "Unknown option: %s\n", cmd);
usage(git_usage_string);

View File

@ -3,6 +3,16 @@
#include "tag.h"
static int is_shallow = -1;
static struct stat shallow_stat;
static char *alternate_shallow_file;
void set_alternate_shallow_file(const char *path)
{
if (is_shallow != -1)
die("BUG: is_repository_shallow must not be called before set_alternate_shallow_file");
free(alternate_shallow_file);
alternate_shallow_file = path ? xstrdup(path) : NULL;
}
int register_shallow(const unsigned char *sha1)
{
@ -21,12 +31,21 @@ int is_repository_shallow(void)
{
FILE *fp;
char buf[1024];
const char *path = alternate_shallow_file;
if (is_shallow >= 0)
return is_shallow;
fp = fopen(git_path("shallow"), "r");
if (!fp) {
if (!path)
path = git_path("shallow");
/*
* fetch-pack sets '--shallow-file ""' as an indicator that no
* shallow file should be used. We could just open it and it
* will likely fail. But let's do an explicit check instead.
*/
if (!*path ||
stat(path, &shallow_stat) ||
(fp = fopen(path, "r")) == NULL) {
is_shallow = 0;
return is_shallow;
}
@ -108,3 +127,22 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth,
return result;
}
void check_shallow_file_for_update(void)
{
struct stat st;
if (!is_shallow)
return;
else if (is_shallow == -1)
die("BUG: shallow must be initialized by now");
if (stat(git_path("shallow"), &st))
die("shallow file was removed during fetch");
else if (st.st_mtime != shallow_stat.st_mtime
#ifdef USE_NSEC
|| ST_MTIME_NSEC(st) != ST_MTIME_NSEC(shallow_stat)
#endif
)
die("shallow file was changed during fetch");
}

View File

@ -135,6 +135,13 @@ test_expect_success 'clone shallow depth 1' '
test "`git --git-dir=shallow0/.git rev-list --count HEAD`" = 1
'
test_expect_success 'clone shallow depth 1 with fsck' '
git config --global fetch.fsckobjects true &&
git clone --no-single-branch --depth 1 "file://$(pwd)/." shallow0fsck &&
test "`git --git-dir=shallow0fsck/.git rev-list --count HEAD`" = 1 &&
git config --global --unset fetch.fsckobjects
'
test_expect_success 'clone shallow' '
git clone --no-single-branch --depth 2 "file://$(pwd)/." shallow
'

View File

@ -534,6 +534,8 @@ static int fetch_refs_via_pack(struct transport *transport,
args.quiet = (transport->verbose < 0);
args.no_progress = !transport->progress;
args.depth = data->options.depth;
args.check_self_contained_and_connected =
data->options.check_self_contained_and_connected;
if (!data->got_remote_heads) {
connect_setup(transport, 0, 0);
@ -551,6 +553,8 @@ static int fetch_refs_via_pack(struct transport *transport,
refs = NULL;
data->conn = NULL;
data->got_remote_heads = 0;
data->options.self_contained_and_connected =
args.self_contained_and_connected;
free_refs(refs_tmp);

View File

@ -8,6 +8,8 @@ struct git_transport_options {
unsigned thin : 1;
unsigned keep : 1;
unsigned followtags : 1;
unsigned check_self_contained_and_connected : 1;
unsigned self_contained_and_connected : 1;
int depth;
const char *uploadpack;
const char *receivepack;