From 9b8dc263e1b0d470cc67a824837d8884ae3e7136 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:08:43 +0100 Subject: [PATCH 01/15] upload-pack: no longer call rev-list It is trivial to do now, and it is needed for the upcoming shallow clone stuff. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- upload-pack.c | 95 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/upload-pack.c b/upload-pack.c index 4572fff07c..7f7df2afd3 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -9,6 +9,9 @@ #include "object.h" #include "commit.h" #include "exec_cmd.h" +#include "diff.h" +#include "revision.h" +#include "list-objects.h" static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=nn] "; @@ -57,6 +60,40 @@ static ssize_t send_client_data(int fd, const char *data, ssize_t sz) return safe_write(fd, data, sz); } +FILE *pack_pipe = NULL; +static void show_commit(struct commit *commit) +{ + if (commit->object.flags & BOUNDARY) + fputc('-', pack_pipe); + if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0) + die("broken output pipe"); + fputc('\n', pack_pipe); + fflush(pack_pipe); + free(commit->buffer); + commit->buffer = NULL; +} + +static void show_object(struct object_array_entry *p) +{ + /* An object with name "foo\n0000000..." can be used to + * confuse downstream git-pack-objects very badly. + */ + const char *ep = strchr(p->name, '\n'); + if (ep) { + fprintf(pack_pipe, "%s %.*s\n", sha1_to_hex(p->item->sha1), + (int) (ep - p->name), + p->name); + } + else + fprintf(pack_pipe, "%s %s\n", + sha1_to_hex(p->item->sha1), p->name); +} + +static void show_edge(struct commit *commit) +{ + fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1)); +} + static void create_pack_file(void) { /* Pipes between rev-list to pack-objects, pack-objects to us @@ -78,48 +115,38 @@ static void create_pack_file(void) if (!pid_rev_list) { int i; - int args; - const char **argv; - const char **p; - char *buf; + struct rev_info revs; + + pack_pipe = fdopen(lp_pipe[1], "w"); + + if (create_full_pack) + use_thin_pack = 0; /* no point doing it */ + init_revisions(&revs, NULL); + revs.tag_objects = 1; + revs.tree_objects = 1; + revs.blob_objects = 1; + if (use_thin_pack) + revs.edge_hint = 1; if (create_full_pack) { - args = 10; - use_thin_pack = 0; /* no point doing it */ - } - else - args = have_obj.nr + want_obj.nr + 5; - p = xmalloc(args * sizeof(char *)); - argv = (const char **) p; - buf = xmalloc(args * 45); - - dup2(lp_pipe[1], 1); - close(0); - close(lp_pipe[0]); - close(lp_pipe[1]); - *p++ = "rev-list"; - *p++ = use_thin_pack ? "--objects-edge" : "--objects"; - if (create_full_pack) - *p++ = "--all"; - else { + const char *args[] = {"rev-list", "--all", NULL}; + setup_revisions(2, args, &revs, NULL); + } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; - *p++ = buf; - memcpy(buf, sha1_to_hex(o->sha1), 41); - buf += 41; + add_pending_object(&revs, o, NULL); } - } - if (!create_full_pack) for (i = 0; i < have_obj.nr; i++) { struct object *o = have_obj.objects[i].item; - *p++ = buf; - *buf++ = '^'; - memcpy(buf, sha1_to_hex(o->sha1), 41); - buf += 41; + o->flags |= UNINTERESTING; + add_pending_object(&revs, o, NULL); } - *p++ = NULL; - execv_git_cmd(argv); - die("git-upload-pack: unable to exec git-rev-list"); + setup_revisions(0, NULL, &revs, NULL); + } + prepare_revision_walk(&revs); + mark_edges_uninteresting(revs.commits, &revs, show_edge); + traverse_commit_list(&revs, show_commit, show_object); + exit(0); } if (pipe(pu_pipe) < 0) From ed09aef06fda2ba06a7412e3fa43ab1c3449f723 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:09:06 +0100 Subject: [PATCH 02/15] support fetching into a shallow repository A shallow commit is a commit which has parents, which in turn are "grafted away", i.e. the commit appears as if it were a root. Since these shallow commits should not be edited by the user, but only by core git, they are recorded in the file $GIT_DIR/shallow. A repository containing shallow commits is called shallow. The advantage of a shallow repository is that even if the upstream contains lots of history, your local (shallow) repository needs not occupy much disk space. The disadvantage is that you might miss a merge base when pulling some remote branch. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- Makefile | 2 +- commit.c | 21 +++++++++++ commit.h | 8 ++++- fetch-pack.c | 4 +++ shallow.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++ upload-pack.c | 22 ++++++++++-- 6 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 shallow.c diff --git a/Makefile b/Makefile index 36ce8cd606..ea5d2cf723 100644 --- a/Makefile +++ b/Makefile @@ -260,7 +260,7 @@ LIB_OBJS = \ revision.o pager.o tree-walk.o xdiff-interface.o \ write_or_die.o trace.o list-objects.o grep.o \ alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \ - color.o wt-status.o archive-zip.o archive-tar.o + color.o wt-status.o archive-zip.o archive-tar.o shallow.o BUILTIN_OBJS = \ builtin-add.o \ diff --git a/commit.c b/commit.c index a6d543eee7..bffa278868 100644 --- a/commit.c +++ b/commit.c @@ -1,6 +1,7 @@ #include "cache.h" #include "tag.h" #include "commit.h" +#include "pkt-line.h" int save_commit_buffer = 1; @@ -221,6 +222,8 @@ static void prepare_commit_graft(void) return; graft_file = get_graft_file(); read_graft_file(graft_file); + /* make sure shallows are read */ + is_repository_shallow(); commit_graft_prepared = 1; } @@ -234,6 +237,24 @@ static struct commit_graft *lookup_commit_graft(const unsigned char *sha1) return commit_graft[pos]; } +int write_shallow_commits(int fd, int use_pack_protocol) +{ + int i, count = 0; + for (i = 0; i < commit_graft_nr; i++) + if (commit_graft[i]->nr_parent < 0) { + const char *hex = + sha1_to_hex(commit_graft[i]->sha1); + count++; + if (use_pack_protocol) + packet_write(fd, "shallow %s", hex); + else { + write(fd, hex, 40); + write(fd, "\n", 1); + } + } + return count; +} + int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size) { char *tail = buffer; diff --git a/commit.h b/commit.h index fc13de9780..c559510b46 100644 --- a/commit.h +++ b/commit.h @@ -97,7 +97,7 @@ void sort_in_topological_order_fn(struct commit_list ** list, int lifo, struct commit_graft { unsigned char sha1[20]; - int nr_parent; + int nr_parent; /* < 0 if shallow commit */ unsigned char parent[FLEX_ARRAY][20]; /* more */ }; @@ -107,4 +107,10 @@ int read_graft_file(const char *graft_file); extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2, int cleanup); +extern int register_shallow(const unsigned char *sha1); +extern int write_shallow_commits(int fd, int use_pack_protocol); +extern int is_repository_shallow(); +extern struct commit_list *get_shallow_commits(struct object_array *heads, + int depth); + #endif /* COMMIT_H */ diff --git a/fetch-pack.c b/fetch-pack.c index 0a169dce85..bc5e725055 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -180,6 +180,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, packet_write(fd[1], "want %s\n", sha1_to_hex(remote)); fetching++; } + if (is_repository_shallow()) + write_shallow_commits(fd[1], 1); packet_flush(fd[1]); if (!fetching) return 1; @@ -523,6 +525,8 @@ static int fetch_pack(int fd[2], int nr_match, char **match) int status; get_remote_heads(fd[0], &ref, 0, NULL, 0); + if (is_repository_shallow() && !server_supports("shallow")) + die("Server does not support shallow clients"); if (server_supports("multi_ack")) { if (verbose) fprintf(stderr, "Server supports multi_ack\n"); diff --git a/shallow.c b/shallow.c new file mode 100644 index 0000000000..3cf2127134 --- /dev/null +++ b/shallow.c @@ -0,0 +1,97 @@ +#include "cache.h" +#include "commit.h" + +static int is_shallow = -1; + +int register_shallow(const unsigned char *sha1) +{ + struct commit_graft *graft = + xmalloc(sizeof(struct commit_graft)); + struct commit *commit = lookup_commit(sha1); + + hashcpy(graft->sha1, sha1); + graft->nr_parent = -1; + if (commit && commit->object.parsed) + commit->parents = NULL; + return register_commit_graft(graft, 0); +} + +int is_repository_shallow() +{ + FILE *fp; + char buf[1024]; + + if (is_shallow >= 0) + return is_shallow; + + fp = fopen(git_path("shallow"), "r"); + if (!fp) { + is_shallow = 0; + return is_shallow; + } + is_shallow = 1; + + while (fgets(buf, sizeof(buf), fp)) { + unsigned char sha1[20]; + if (get_sha1_hex(buf, sha1)) + die("bad shallow line: %s", buf); + register_shallow(sha1); + } + fclose(fp); + return is_shallow; +} + +struct commit_list *get_shallow_commits(struct object_array *heads, int depth) +{ + int i = 0, cur_depth = 0; + struct commit_list *result = NULL; + struct object_array stack = {0, 0, NULL}; + struct commit *commit = NULL; + + while (commit || i < heads->nr || stack.nr) { + struct commit_list *p; + if (!commit) { + if (i < heads->nr) { + commit = (struct commit *) + heads->objects[i++].item; + if (commit->object.type != OBJ_COMMIT) { + commit = NULL; + continue; + } + commit->util = xcalloc(1, sizeof(int)); + cur_depth = 0; + } else { + commit = (struct commit *) + stack.objects[--stack.nr].item; + cur_depth = *(int *)commit->util; + } + } + parse_commit(commit); + cur_depth++; + for (p = commit->parents, commit = NULL; p; p = p->next) { + if (!p->item->util) { + int *pointer = xmalloc(sizeof(int)); + p->item->util = pointer; + *pointer = cur_depth; + } else { + int *pointer = p->item->util; + if (cur_depth >= *pointer) + continue; + *pointer = cur_depth; + } + if (cur_depth < depth) { + if (p->next) + add_object_array(&p->item->object, + NULL, &stack); + else { + commit = p->item; + cur_depth = *(int *)commit->util; + } + } else + commit_list_insert(p->item, &result); + } + } + + return result; +} + diff --git a/upload-pack.c b/upload-pack.c index 7f7df2afd3..8dd6121dab 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -486,6 +486,7 @@ static int get_common_commits(void) static void receive_needs(void) { + struct object_array shallows = {0, 0, NULL}; static char line[1000]; int len; @@ -495,8 +496,19 @@ static void receive_needs(void) len = packet_read_line(0, line, sizeof(line)); reset_timeout(); if (!len) - return; + break; + if (!strncmp("shallow ", line, 8)) { + unsigned char sha1[20]; + struct object *object; + if (get_sha1(line + 8, sha1)) + die("invalid shallow line: %s", line); + object = parse_object(sha1); + if (!object) + die("did not find object for %s", line); + add_object_array(object, NULL, &shallows); + continue; + } if (strncmp("want ", line, 5) || get_sha1_hex(line+5, sha1_buf)) die("git-upload-pack: protocol error, " @@ -528,11 +540,17 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (shallows.nr > 0) { + int i; + for (i = 0; i < shallows.nr; i++) + register_shallow(shallows.objects[i].item->sha1); + } } static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data) { - static const char *capabilities = "multi_ack thin-pack side-band side-band-64k ofs-delta"; + static const char *capabilities = "multi_ack thin-pack side-band" + " side-band-64k ofs-delta shallow"; struct object *o = parse_object(sha1); if (!o) From 016e6ccbe03438454777e43dd73d67844296a3fd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:09:29 +0100 Subject: [PATCH 03/15] allow cloning a repository "shallowly" By specifying a depth, you can now clone a repository such that all fetched ancestor-chains' length is at most "depth". For example, if the upstream repository has only 2 branches ("A" and "B"), which are linear, and you specify depth 3, you will get A, A~1, A~2, A~3, B, B~1, B~2, and B~3. The ends are automatically made shallow commits. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- fetch-pack.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++- git-clone.sh | 19 +++++++++++++--- upload-pack.c | 21 +++++++++++++++++- 3 files changed, 96 insertions(+), 5 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index bc5e725055..f335bd42b7 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -11,8 +11,9 @@ static int keep_pack; static int quiet; static int verbose; static int fetch_all; +static int depth; static const char fetch_pack_usage[] = -"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory ..."; +"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=] [host:]directory ..."; static const char *exec = "git-upload-pack"; #define COMPLETE (1U << 0) @@ -182,10 +183,29 @@ static int find_common(int fd[2], unsigned char *result_sha1, } if (is_repository_shallow()) write_shallow_commits(fd[1], 1); + if (depth > 0) + packet_write(fd[1], "deepen %d", depth); packet_flush(fd[1]); if (!fetching) return 1; + if (depth > 0) { + char line[1024]; + unsigned char sha1[20]; + int len; + + while ((len = packet_read_line(fd[0], line, sizeof(line)))) { + if (!strncmp("shallow ", line, 8)) { + if (get_sha1_hex(line + 8, sha1)) + die("invalid shallow line: %s", line); + /* no need making it shallow if we have it already */ + if (lookup_object(sha1)) + continue; + register_shallow(sha1); + } + } + } + flushes = 0; retval = -1; while ((sha1 = get_rev())) { @@ -576,6 +596,8 @@ int main(int argc, char **argv) char *dest = NULL, **heads; int fd[2]; pid_t pid; + struct stat st; + struct lock_file lock; setup_git_directory(); @@ -609,6 +631,12 @@ int main(int argc, char **argv) verbose = 1; continue; } + if (!strncmp("--depth=", arg, 8)) { + depth = strtol(arg + 8, NULL, 0); + if (stat(git_path("shallow"), &st)) + st.st_mtime = 0; + continue; + } usage(fetch_pack_usage); } dest = arg; @@ -618,6 +646,8 @@ int main(int argc, char **argv) } if (!dest) usage(fetch_pack_usage); + if (is_repository_shallow() && depth > 0) + die("Deepening of a shallow repository not yet supported!"); pid = git_connect(fd, dest, exec); if (pid < 0) return 1; @@ -639,5 +669,34 @@ int main(int argc, char **argv) } } + if (!ret && depth > 0) { + struct cache_time mtime; + char *shallow = git_path("shallow"); + int fd; + + mtime.sec = st.st_mtime; +#ifdef USE_NSEC + mtime.usec = st.st_mtim.usec; +#endif + if (stat(shallow, &st)) { + if (mtime.sec) + die("shallow file was removed during fetch"); + } else if (st.st_mtime != mtime.sec +#ifdef USE_NSEC + || st.st_mtim.usec != mtime.usec +#endif + ) + die("shallow file was changed during fetch"); + + fd = hold_lock_file_for_update(&lock, shallow, 1); + if (!write_shallow_commits(fd, 0)) { + unlink(lock.filename); + rollback_lock_file(&lock); + } else { + close(fd); + commit_lock_file(&lock); + } + } + return !!ret; } diff --git a/git-clone.sh b/git-clone.sh index 9ed4135544..8c0a93ebd1 100755 --- a/git-clone.sh +++ b/git-clone.sh @@ -14,7 +14,7 @@ die() { } usage() { - die "Usage: $0 [--template=] [--use-immingled-remote] [--reference ] [--bare] [-l [-s]] [-q] [-u ] [--origin ] [-n] []" + die "Usage: $0 [--template=] [--use-immingled-remote] [--reference ] [--bare] [-l [-s]] [-q] [-u ] [--origin ] [--depth ] [-n] []" } get_repo_base() { @@ -116,6 +116,7 @@ reference= origin= origin_override= use_separate_remote=t +depth= while case "$#,$1" in 0,*) break ;; @@ -161,6 +162,10 @@ while *,-u|*,--upload-pack) shift upload_pack="--exec=$1" ;; + 1,--depth) usage;; + *,--depth) + shift + depth="--depth=$1";; *,-*) usage ;; *) break ;; esac @@ -265,6 +270,10 @@ yes,yes) *) case "$repo" in rsync://*) + case "$depth" in + "") ;; + *) die "shallow over rsync not supported" ;; + esac rsync $quiet -av --ignore-existing \ --exclude info "$repo/objects/" "$GIT_DIR/objects/" || exit @@ -293,6 +302,10 @@ yes,yes) git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1 ;; https://*|http://*|ftp://*) + case "$depth" in + "") ;; + *) die "shallow over http or ftp not supported" ;; + esac if test -z "@@NO_CURL@@" then clone_dumb_http "$repo" "$D" @@ -302,8 +315,8 @@ yes,yes) ;; *) case "$upload_pack" in - '') git-fetch-pack --all -k $quiet "$repo" ;; - *) git-fetch-pack --all -k $quiet "$upload_pack" "$repo" ;; + '') git-fetch-pack --all -k $quiet $depth "$repo" ;; + *) git-fetch-pack --all -k $quiet "$upload_pack" $depth "$repo" ;; esac >"$GIT_DIR/CLONE_HEAD" || die "fetch-pack from '$repo' failed." ;; diff --git a/upload-pack.c b/upload-pack.c index 8dd6121dab..ebe1e5ae4d 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -488,7 +488,7 @@ static void receive_needs(void) { struct object_array shallows = {0, 0, NULL}; static char line[1000]; - int len; + int len, depth = 0; for (;;) { struct object *o; @@ -509,6 +509,13 @@ static void receive_needs(void) add_object_array(object, NULL, &shallows); continue; } + if (!strncmp("deepen ", line, 7)) { + char *end; + depth = strtol(line + 7, &end, 0); + if (end == line + 7 || depth <= 0) + die("Invalid deepen: %s", line); + continue; + } if (strncmp("want ", line, 5) || get_sha1_hex(line+5, sha1_buf)) die("git-upload-pack: protocol error, " @@ -540,6 +547,18 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (depth > 0) { + struct commit_list *result, *backup; + if (shallows.nr > 0) + die("Deepening a shallow repository not yet supported"); + backup = result = get_shallow_commits(&want_obj, depth); + while (result) { + packet_write(1, "shallow %s", + sha1_to_hex(result->item->object.sha1)); + result = result->next; + } + free_commit_list(backup); + } if (shallows.nr > 0) { int i; for (i = 0; i < shallows.nr; i++) From f53514bc2d82f2f5cc7b447575e74aa266ed46f0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:09:53 +0100 Subject: [PATCH 04/15] allow deepening of a shallow repository Now, by saying "git fetch -depth " you can deepen a shallow repository. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- commit.c | 13 ++++++++++++ commit.h | 3 ++- fetch-pack.c | 22 +++++++++++++------ git-fetch.sh | 14 ++++++++++++- shallow.c | 8 +++++-- upload-pack.c | 58 +++++++++++++++++++++++++++++++++++++++++---------- 6 files changed, 97 insertions(+), 21 deletions(-) diff --git a/commit.c b/commit.c index bffa278868..d5103cd3c6 100644 --- a/commit.c +++ b/commit.c @@ -255,6 +255,19 @@ int write_shallow_commits(int fd, int use_pack_protocol) return count; } +int unregister_shallow(const unsigned char *sha1) +{ + int pos = commit_graft_pos(sha1); + if (pos < 0) + return -1; + if (pos + 1 < commit_graft_nr) + memcpy(commit_graft + pos, commit_graft + pos + 1, + sizeof(struct commit_graft *) + * (commit_graft_nr - pos - 1)); + commit_graft_nr--; + return 0; +} + int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size) { char *tail = buffer; diff --git a/commit.h b/commit.h index c559510b46..e9e158f4e1 100644 --- a/commit.h +++ b/commit.h @@ -108,9 +108,10 @@ int read_graft_file(const char *graft_file); extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2, int cleanup); extern int register_shallow(const unsigned char *sha1); +extern int unregister_shallow(const unsigned char *sha1); extern int write_shallow_commits(int fd, int use_pack_protocol); extern int is_repository_shallow(); extern struct commit_list *get_shallow_commits(struct object_array *heads, - int depth); + int depth, int shallow_flag, int not_shallow_flag); #endif /* COMMIT_H */ diff --git a/fetch-pack.c b/fetch-pack.c index f335bd42b7..c3064b94ad 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -202,7 +202,17 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (lookup_object(sha1)) continue; register_shallow(sha1); - } + } else if (!strncmp("unshallow ", line, 10)) { + if (get_sha1_hex(line + 10, sha1)) + die("invalid unshallow line: %s", line); + if (!lookup_object(sha1)) + die("object not found: %s", line); + /* make sure that it is parsed as shallow */ + parse_object(sha1); + if (unregister_shallow(sha1)) + die("no shallow found: %s", line); + } else + die("expected shallow/unshallow, got %s", line); } } @@ -391,9 +401,11 @@ static int everything_local(struct ref **refs, int nr_match, char **match) } } - for_each_ref(mark_complete, NULL); - if (cutoff) - mark_recent_complete_commits(cutoff); + if (!depth) { + for_each_ref(mark_complete, NULL); + if (cutoff) + mark_recent_complete_commits(cutoff); + } /* * Mark all complete remote refs as common refs. @@ -646,8 +658,6 @@ int main(int argc, char **argv) } if (!dest) usage(fetch_pack_usage); - if (is_repository_shallow() && depth > 0) - die("Deepening of a shallow repository not yet supported!"); pid = git_connect(fd, dest, exec); if (pid < 0) return 1; diff --git a/git-fetch.sh b/git-fetch.sh index eb32476bbd..0b1e6d1071 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -21,6 +21,7 @@ update_head_ok= exec= upload_pack= keep= +shallow_depth= while case "$#" in 0) break ;; esac do case "$1" in @@ -56,6 +57,13 @@ do --reflog-action=*) rloga=`expr "z$1" : 'z-[^=]*=\(.*\)'` ;; + --depth=*) + shallow_depth="--depth=`expr "z$1" : 'z-[^=]*=\(.*\)'`" + ;; + --depth) + shift + shallow_depth="--depth=$1" + ;; -*) usage ;; @@ -296,6 +304,8 @@ fetch_main () { # There are transports that can fetch only one head at a time... case "$remote" in http://* | https://* | ftp://*) + test -n "$shallow_depth" && + die "shallow clone with http not supported" proto=`expr "$remote" : '\([^:]*\):'` if [ -n "$GIT_SSL_NO_VERIFY" ]; then curl_extra_args="-k" @@ -324,6 +334,8 @@ fetch_main () { git-http-fetch -v -a "$head" "$remote/" || exit ;; rsync://*) + test -n "$shallow_depth" && + die "shallow clone with rsync not supported" TMP_HEAD="$GIT_DIR/TMP_HEAD" rsync -L -q "$remote/$remote_name" "$TMP_HEAD" || exit 1 head=$(git-rev-parse --verify TMP_HEAD) @@ -371,7 +383,7 @@ fetch_main () { pack_lockfile= IFS=" $LF" ( - git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote" + git-fetch-pack --thin $exec $keep $shallow_depth "$remote" $rref || echo failed "$remote" ) | while read sha1 remote_name do diff --git a/shallow.c b/shallow.c index 3cf2127134..58a7b20d79 100644 --- a/shallow.c +++ b/shallow.c @@ -41,7 +41,8 @@ int is_repository_shallow() return is_shallow; } -struct commit_list *get_shallow_commits(struct object_array *heads, int depth) +struct commit_list *get_shallow_commits(struct object_array *heads, int depth, + int shallow_flag, int not_shallow_flag) { int i = 0, cur_depth = 0; struct commit_list *result = NULL; @@ -67,6 +68,7 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth) } } parse_commit(commit); + commit->object.flags |= not_shallow_flag; cur_depth++; for (p = commit->parents, commit = NULL; p; p = p->next) { if (!p->item->util) { @@ -87,8 +89,10 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth) commit = p->item; cur_depth = *(int *)commit->util; } - } else + } else { commit_list_insert(p->item, &result); + p->item->object.flags |= shallow_flag; + } } } diff --git a/upload-pack.c b/upload-pack.c index ebe1e5ae4d..4a9d6720a5 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -22,6 +22,10 @@ static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=n #define COMMON_KNOWN (1u << 14) #define REACHABLE (1u << 15) +#define SHALLOW (1u << 16) +#define NOT_SHALLOW (1u << 17) +#define CLIENT_SHALLOW (1u << 18) + static unsigned long oldest_have; static int multi_ack, nr_our_refs; @@ -134,6 +138,7 @@ static void create_pack_file(void) } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; + o->flags &= ~UNINTERESTING; add_pending_object(&revs, o, NULL); } for (i = 0; i < have_obj.nr; i++) { @@ -501,16 +506,19 @@ static void receive_needs(void) if (!strncmp("shallow ", line, 8)) { unsigned char sha1[20]; struct object *object; + use_thin_pack = 0; if (get_sha1(line + 8, sha1)) die("invalid shallow line: %s", line); object = parse_object(sha1); if (!object) die("did not find object for %s", line); + object->flags |= CLIENT_SHALLOW; add_object_array(object, NULL, &shallows); continue; } if (!strncmp("deepen ", line, 7)) { char *end; + use_thin_pack = 0; depth = strtol(line + 7, &end, 0); if (end == line + 7 || depth <= 0) die("Invalid deepen: %s", line); @@ -547,23 +555,51 @@ static void receive_needs(void) add_object_array(o, NULL, &want_obj); } } + if (depth == 0 && shallows.nr == 0) + return; if (depth > 0) { struct commit_list *result, *backup; - if (shallows.nr > 0) - die("Deepening a shallow repository not yet supported"); - backup = result = get_shallow_commits(&want_obj, depth); + int i; + backup = result = get_shallow_commits(&want_obj, depth, + SHALLOW, NOT_SHALLOW); while (result) { - packet_write(1, "shallow %s", - sha1_to_hex(result->item->object.sha1)); + struct object *object = &result->item->object; + if (!(object->flags & CLIENT_SHALLOW)) { + packet_write(1, "shallow %s", + sha1_to_hex(object->sha1)); + register_shallow(object->sha1); + } result = result->next; } free_commit_list(backup); - } - if (shallows.nr > 0) { - int i; - for (i = 0; i < shallows.nr; i++) - register_shallow(shallows.objects[i].item->sha1); - } + for (i = 0; i < shallows.nr; i++) { + struct object *object = shallows.objects[i].item; + if (object->flags & NOT_SHALLOW) { + struct commit_list *parents; + packet_write(1, "unshallow %s", + sha1_to_hex(object->sha1)); + object->flags &= ~CLIENT_SHALLOW; + /* make sure the real parents are parsed */ + unregister_shallow(object->sha1); + parse_commit((struct commit *)object); + parents = ((struct commit *)object)->parents; + while (parents) { + add_object_array(&parents->item->object, + NULL, &want_obj); + parents = parents->next; + } + } + /* make sure commit traversal conforms to client */ + register_shallow(object->sha1); + } + packet_flush(1); + } else + if (shallows.nr > 0) { + int i; + for (i = 0; i < shallows.nr; i++) + register_shallow(shallows.objects[i].item->sha1); + } + free(shallows.objects); } static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data) From abef3a1625b7aff168a5fa062432dc6ef2006963 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Sat, 11 Nov 2006 14:57:23 +0100 Subject: [PATCH 05/15] Shallow clone: do not ignore shallowness when following tags Tags should be considered when truncating the commit list. The patch below fixes it, and fetches the right number of commits for each tag. However the correct fix is probably to not fetch historical tags at all. Signed-off-by: Junio C Hamano --- shallow.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shallow.c b/shallow.c index 58a7b20d79..2db1dc428f 100644 --- a/shallow.c +++ b/shallow.c @@ -1,5 +1,6 @@ #include "cache.h" #include "commit.h" +#include "tag.h" static int is_shallow = -1; @@ -54,7 +55,7 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, if (!commit) { if (i < heads->nr) { commit = (struct commit *) - heads->objects[i++].item; + deref_tag(heads->objects[i++].item, NULL, 0); if (commit->object.type != OBJ_COMMIT) { commit = NULL; continue; From 16ad35791099e5da118a0b6cfa0c3db4e2feff81 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 30 Oct 2006 20:10:13 +0100 Subject: [PATCH 06/15] add tests for shallow stuff Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- t/t5500-fetch-pack.sh | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh index f7625a6f46..fa10840303 100755 --- a/t/t5500-fetch-pack.sh +++ b/t/t5500-fetch-pack.sh @@ -128,4 +128,49 @@ pull_to_client 2nd "B" $((64*3)) pull_to_client 3rd "A" $((1*3)) # old fails +test_expect_success "clone shallow" "git-clone --depth 2 . shallow" + +(cd shallow; git-count-objects -v) > count.shallow + +test_expect_success "clone shallow object count" \ + "test \"in-pack: 18\" = \"$(grep in-pack count.shallow)\"" + +test_expect_success "clone shallow object count (part 2)" \ + "test -z \"$(grep -v in-pack count.shallow | sed "s/^.*: 0//")\"" + +test_expect_success "fsck in shallow repo" \ + "(cd shallow; git-fsck-objects --full)" + +#test_done; exit + +add B66 $B65 +add B67 $B66 + +test_expect_success "pull in shallow repo" \ + "(cd shallow; git pull .. B)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 6\" = \"$(grep count count.shallow)\"" + +add B68 $B67 +add B69 $B68 + +test_expect_success "deepening pull in shallow repo" \ + "(cd shallow; git pull --depth 4 .. B)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 12\" = \"$(grep count count.shallow)\"" + +test_expect_success "deepening fetch in shallow repo" \ + "(cd shallow; git fetch --depth 4 .. A:A)" + +(cd shallow; git-count-objects -v) > count.shallow +test_expect_success "clone shallow object count" \ + "test \"count: 18\" = \"$(grep count count.shallow)\"" + +test_expect_failure "pull in shallow repo with missing merge base" \ + "(cd shallow; git pull --depth 4 .. A)" + test_done From cf01bd52efeb21084652b2b06b778ed458004e8f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:04:56 -0800 Subject: [PATCH 07/15] We should make sure that the protocol is still extensible. This just reformats if .. else if .. else chain to make it clear we are handling extended response from the other end. --- fetch-pack.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index c3064b94ad..6fd457035e 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -189,7 +189,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (!fetching) return 1; - if (depth > 0) { + if (depth > 0) { char line[1024]; unsigned char sha1[20]; int len; @@ -202,7 +202,9 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (lookup_object(sha1)) continue; register_shallow(sha1); - } else if (!strncmp("unshallow ", line, 10)) { + continue; + } + if (!strncmp("unshallow ", line, 10)) { if (get_sha1_hex(line + 10, sha1)) die("invalid unshallow line: %s", line); if (!lookup_object(sha1)) @@ -211,8 +213,9 @@ static int find_common(int fd[2], unsigned char *result_sha1, parse_object(sha1); if (unregister_shallow(sha1)) die("no shallow found: %s", line); - } else - die("expected shallow/unshallow, got %s", line); + continue; + } + die("expected shallow/unshallow, got %s", line); } } From fcd1e3190612fc59c2d820537280b5164d290d9d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:04:56 -0800 Subject: [PATCH 08/15] Why does it mean we do not have to register shallow if we have one? --- fetch-pack.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index 6fd457035e..d00573d022 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -198,9 +198,6 @@ static int find_common(int fd[2], unsigned char *result_sha1, if (!strncmp("shallow ", line, 8)) { if (get_sha1_hex(line + 8, sha1)) die("invalid shallow line: %s", line); - /* no need making it shallow if we have it already */ - if (lookup_object(sha1)) - continue; register_shallow(sha1); continue; } From c6702f4b95699c6bd47337516c32e3408dda0595 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:47:45 -0800 Subject: [PATCH 09/15] Why didn't we mark want_obj as ~UNINTERESTING in the old code? Is this something we would want to do regardless of shallow clone? --- upload-pack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/upload-pack.c b/upload-pack.c index 4a9d6720a5..7e3c437d8e 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -138,6 +138,7 @@ static void create_pack_file(void) } else { for (i = 0; i < want_obj.nr; i++) { struct object *o = want_obj.objects[i].item; + /* why??? */ o->flags &= ~UNINTERESTING; add_pending_object(&revs, o, NULL); } From 176d45cb20eb4ffc661d73383aec01943e056e10 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 13 Nov 2006 22:47:46 -0800 Subject: [PATCH 10/15] shallow clone: unparse and reparse an unshallowed commit Otherwise we would not read the real parents from the commit object. --- upload-pack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/upload-pack.c b/upload-pack.c index 7e3c437d8e..d5b47507db 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -582,6 +582,7 @@ static void receive_needs(void) object->flags &= ~CLIENT_SHALLOW; /* make sure the real parents are parsed */ unregister_shallow(object->sha1); + object->parsed = 0; parse_commit((struct commit *)object); parents = ((struct commit *)object)->parents; while (parents) { From d6491e3a214541aeb016b0688d0920845b03932d Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:58:04 +0100 Subject: [PATCH 11/15] fetch-pack: Properly remove the shallow file when it becomes empty. The code was unlinking the lock file instead. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano --- fetch-pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fetch-pack.c b/fetch-pack.c index d00573d022..bb310b644f 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -700,7 +700,7 @@ int main(int argc, char **argv) fd = hold_lock_file_for_update(&lock, shallow, 1); if (!write_shallow_commits(fd, 0)) { - unlink(lock.filename); + unlink(shallow); rollback_lock_file(&lock); } else { close(fd); From 1f2de769812f0adea598269d788c9c75d23e80b8 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:58:25 +0100 Subject: [PATCH 12/15] upload-pack: Check for NOT_SHALLOW flag before sending a shallow to the client. A commit may have been put on the shallow list, and then reached from another branch and marked NOT_SHALLOW without being removed from the list. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano --- upload-pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upload-pack.c b/upload-pack.c index d5b47507db..d4a7b625f4 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -565,7 +565,7 @@ static void receive_needs(void) SHALLOW, NOT_SHALLOW); while (result) { struct object *object = &result->item->object; - if (!(object->flags & CLIENT_SHALLOW)) { + if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) { packet_write(1, "shallow %s", sha1_to_hex(object->sha1)); register_shallow(object->sha1); From d158631549b87e11f2415622e421546539b09b67 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:59:12 +0100 Subject: [PATCH 13/15] git-fetch: Reset shallow_depth before auto-following tags. Otherwise fetching the tags could also fetch commits up to the specified depth, which isn't the expected behavior. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano --- git-fetch.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-fetch.sh b/git-fetch.sh index 0b1e6d1071..f0645d97c9 100755 --- a/git-fetch.sh +++ b/git-fetch.sh @@ -461,6 +461,8 @@ case "$no_tags$tags" in case "$taglist" in '') ;; ?*) + # do not deepen a shallow tree when following tags + shallow_depth= fetch_main "$taglist" ;; esac esac From d64d6c9fc712cf6fad9a3ec7f659cf843ee2e18d Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 15:58:50 +0100 Subject: [PATCH 14/15] get_shallow_commits: Avoid memory leak if a commit has been reached already. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano --- shallow.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/shallow.c b/shallow.c index 2db1dc428f..3d53d17423 100644 --- a/shallow.c +++ b/shallow.c @@ -60,7 +60,9 @@ struct commit_list *get_shallow_commits(struct object_array *heads, int depth, commit = NULL; continue; } - commit->util = xcalloc(1, sizeof(int)); + if (!commit->util) + commit->util = xmalloc(sizeof(int)); + *(int *)commit->util = 0; cur_depth = 0; } else { commit = (struct commit *) From 4bcb310c2539b66d535e87508d1b7a90fe29c083 Mon Sep 17 00:00:00 2001 From: Alexandre Julliard Date: Fri, 24 Nov 2006 16:00:13 +0100 Subject: [PATCH 15/15] fetch-pack: Do not fetch tags for shallow clones. A better fix may be to only fetch tags that point to commits that we are downloading, but git-clone doesn't have support for following tags. This will happen automatically on the next git-fetch though. Signed-off-by: Alexandre Julliard Signed-off-by: Junio C Hamano --- fetch-pack.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fetch-pack.c b/fetch-pack.c index bb310b644f..80979b83be 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -342,7 +342,8 @@ static void filter_refs(struct ref **refs, int nr_match, char **match) if (!memcmp(ref->name, "refs/", 5) && check_ref_format(ref->name + 5)) ; /* trash */ - else if (fetch_all) { + else if (fetch_all && + (!depth || strncmp(ref->name, "refs/tags/", 10) )) { *newtail = ref; ref->next = NULL; newtail = &ref->next;