fetch-pack: -k option to keep downloaded pack.

Split out the functions that deal with the socketpair after
finishing git protocol handshake to receive the packed data into
a separate file, and use it in fetch-pack to keep/explode the
received pack data.  We earlier had something like that on
clone-pack side once, but the list discussion resulted in the
decision that it makes sense to always keep the pack for
clone-pack, so unpacking option is not enabled on the clone-pack
side, but we later still could do so easily if we wanted to with
this change.

Signed-off-by: Junio C Hamano <junkio@cox.net>

Documentation/git-fetch-pack.txt

@@ -8,7 +8,7 @@ git-fetch-pack - Receive missing objects from another repository.
 
 SYNOPSIS
 --------
-git-fetch-pack [-q] [--exec=<git-upload-pack>] [<host>:]<directory> [<refs>...]
+git-fetch-pack [-q] [-k] [--exec=<git-upload-pack>] [<host>:]<directory> [<refs>...]
 
 DESCRIPTION
 -----------
@@ -29,6 +29,11 @@ OPTIONS
 	Pass '-q' flag to 'git-unpack-objects'; this makes the
 	cloning process less verbose.
 
+-k::
+	Do not invoke 'git-unpack-objects' on received data, but
+	create a single packfile out of it instead, and store it
+	in the object database.
+
 --exec=<git-upload-pack>::
 	Use this to specify the path to 'git-upload-pack' on the
 	remote side, if is not found on your $PATH.

Makefile

@@ -175,6 +175,7 @@ LIB_OBJS = \
 	quote.o read-cache.o refs.o run-command.o \
 	server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \
 	tag.o tree.o usage.o config.o environment.o ctype.o copy.o \
+	fetch-clone.o \
 	$(DIFF_OBJS)
 
 LIBS = $(LIB_FILE)

cache.h

@@ -338,4 +338,9 @@ extern char git_default_name[MAX_GITNAME];
 extern char git_commit_encoding[MAX_ENCODING_LENGTH];
 
 extern int copy_fd(int ifd, int ofd);
+
+/* Finish off pack transfer receiving end */
+extern int receive_unpack_pack(int fd[2], const char *me, int quiet);
+extern int receive_keep_pack(int fd[2], const char *me);
+
 #endif /* CACHE_H */

clone-pack.c

@@ -1,7 +1,6 @@
 #include "cache.h"
 #include "refs.h"
 #include "pkt-line.h"
-#include <sys/wait.h>
 
 static const char clone_pack_usage[] =
 "git-clone-pack [--exec=<git-upload-pack>] [<host>:]<directory> [<heads>]*";
@@ -112,139 +111,6 @@ static void write_refs(struct ref *ref)
 	free(head_path);
 }
 
-static int finish_pack(const char *pack_tmp_name)
-{
-	int pipe_fd[2];
-	pid_t pid;
-	char idx[PATH_MAX];
-	char final[PATH_MAX];
-	char hash[41];
-	unsigned char sha1[20];
-	char *cp;
-	int err = 0;
-
-	if (pipe(pipe_fd) < 0)
-		die("git-clone-pack: unable to set up pipe");
-
-	strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */
-	cp = strrchr(idx, '/');
-	memcpy(cp, "/pidx", 5);
-
-	pid = fork();
-	if (pid < 0)
-		die("git-clone-pack: unable to fork off git-index-pack");
-	if (!pid) {
-		close(0);
-		dup2(pipe_fd[1], 1);
-		close(pipe_fd[0]);
-		close(pipe_fd[1]);
-		execlp("git-index-pack","git-index-pack",
-		       "-o", idx, pack_tmp_name, NULL);
-		error("cannot exec git-index-pack <%s> <%s>",
-		      idx, pack_tmp_name);
-		exit(1);
-	}
-	close(pipe_fd[1]);
-	if (read(pipe_fd[0], hash, 40) != 40) {
-		error("git-clone-pack: unable to read from git-index-pack");
-		err = 1;
-	}
-	close(pipe_fd[0]);
-
-	for (;;) {
-		int status, code;
-		int retval = waitpid(pid, &status, 0);
-
-		if (retval < 0) {
-			if (errno == EINTR)
-				continue;
-			error("waitpid failed (%s)", strerror(retval));
-			goto error_die;
-		}
-		if (WIFSIGNALED(status)) {
-			int sig = WTERMSIG(status);
-			error("git-index-pack died of signal %d", sig);
-			goto error_die;
-		}
-		if (!WIFEXITED(status)) {
-			error("git-index-pack died of unnatural causes %d",
-			      status);
-			goto error_die;
-		}
-		code = WEXITSTATUS(status);
-		if (code) {
-			error("git-index-pack died with error code %d", code);
-			goto error_die;
-		}
-		if (err)
-			goto error_die;
-		break;
-	}
-	hash[40] = 0;
-	if (get_sha1_hex(hash, sha1)) {
-		error("git-index-pack reported nonsense '%s'", hash);
-		goto error_die;
-	}
-	/* Now we have pack in pack_tmp_name[], and
-	 * idx in idx[]; rename them to their final names.
-	 */
-	snprintf(final, sizeof(final),
-		 "%s/pack/pack-%s.pack", get_object_directory(), hash);
-	move_temp_to_file(pack_tmp_name, final);
-	chmod(final, 0444);
-	snprintf(final, sizeof(final),
-		 "%s/pack/pack-%s.idx", get_object_directory(), hash);
-	move_temp_to_file(idx, final);
-	chmod(final, 0444);
-	return 0;
-
- error_die:
-	unlink(idx);
-	unlink(pack_tmp_name);
-	exit(1);
-}
-
-static int clone_without_unpack(int fd[2])
-{
-	char tmpfile[PATH_MAX];
-	int ofd, ifd;
-
-	ifd = fd[0];
-	snprintf(tmpfile, sizeof(tmpfile),
-		 "%s/pack/tmp-XXXXXX", get_object_directory());
-	ofd = mkstemp(tmpfile);
-	if (ofd < 0)
-		return error("unable to create temporary file %s", tmpfile);
-
-	while (1) {
-		char buf[8192];
-		ssize_t sz, wsz, pos;
-		sz = read(ifd, buf, sizeof(buf));
-		if (sz == 0)
-			break;
-		if (sz < 0) {
-			error("error reading pack (%s)", strerror(errno));
-			close(ofd);
-			unlink(tmpfile);
-			return -1;
-		}
-		pos = 0;
-		while (pos < sz) {
-			wsz = write(ofd, buf + pos, sz - pos);
-			if (wsz < 0) {
-				error("error writing pack (%s)",
-				      strerror(errno));
-				close(ofd);
-				unlink(tmpfile);
-				return -1;
-			}
-			pos += wsz;
-		}
-	}
-	close(ofd);
-	return finish_pack(tmpfile);
-}
-
 static int clone_pack(int fd[2], int nr_match, char **match)
 {
 	struct ref *refs;
@@ -257,7 +123,7 @@ static int clone_pack(int fd[2], int nr_match, char **match)
 	}
 	clone_handshake(fd, refs);
 
-	status = clone_without_unpack(fd);
+	status = receive_keep_pack(fd, "git-clone-pack");
 
 	if (!status) {
 		if (nr_match == 0)

fetch-clone.c

@@ -0,0 +1,172 @@
+#include "cache.h"
+#include <sys/wait.h>
+
+static int finish_pack(const char *pack_tmp_name, const char *me)
+{
+	int pipe_fd[2];
+	pid_t pid;
+	char idx[PATH_MAX];
+	char final[PATH_MAX];
+	char hash[41];
+	unsigned char sha1[20];
+	char *cp;
+	int err = 0;
+
+	if (pipe(pipe_fd) < 0)
+		die("%s: unable to set up pipe", me);
+
+	strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */
+	cp = strrchr(idx, '/');
+	memcpy(cp, "/pidx", 5);
+
+	pid = fork();
+	if (pid < 0)
+		die("git-clone-pack: unable to fork off git-index-pack");
+	if (!pid) {
+		close(0);
+		dup2(pipe_fd[1], 1);
+		close(pipe_fd[0]);
+		close(pipe_fd[1]);
+		execlp("git-index-pack","git-index-pack",
+		       "-o", idx, pack_tmp_name, NULL);
+		error("cannot exec git-index-pack <%s> <%s>",
+		      idx, pack_tmp_name);
+		exit(1);
+	}
+	close(pipe_fd[1]);
+	if (read(pipe_fd[0], hash, 40) != 40) {
+		error("%s: unable to read from git-index-pack", me);
+		err = 1;
+	}
+	close(pipe_fd[0]);
+
+	for (;;) {
+		int status, code;
+		int retval = waitpid(pid, &status, 0);
+
+		if (retval < 0) {
+			if (errno == EINTR)
+				continue;
+			error("waitpid failed (%s)", strerror(retval));
+			goto error_die;
+		}
+		if (WIFSIGNALED(status)) {
+			int sig = WTERMSIG(status);
+			error("git-index-pack died of signal %d", sig);
+			goto error_die;
+		}
+		if (!WIFEXITED(status)) {
+			error("git-index-pack died of unnatural causes %d",
+			      status);
+			goto error_die;
+		}
+		code = WEXITSTATUS(status);
+		if (code) {
+			error("git-index-pack died with error code %d", code);
+			goto error_die;
+		}
+		if (err)
+			goto error_die;
+		break;
+	}
+	hash[40] = 0;
+	if (get_sha1_hex(hash, sha1)) {
+		error("git-index-pack reported nonsense '%s'", hash);
+		goto error_die;
+	}
+	/* Now we have pack in pack_tmp_name[], and
+	 * idx in idx[]; rename them to their final names.
+	 */
+	snprintf(final, sizeof(final),
+		 "%s/pack/pack-%s.pack", get_object_directory(), hash);
+	move_temp_to_file(pack_tmp_name, final);
+	chmod(final, 0444);
+	snprintf(final, sizeof(final),
+		 "%s/pack/pack-%s.idx", get_object_directory(), hash);
+	move_temp_to_file(idx, final);
+	chmod(final, 0444);
+	return 0;
+
+ error_die:
+	unlink(idx);
+	unlink(pack_tmp_name);
+	exit(1);
+}
+
+int receive_unpack_pack(int fd[2], const char *me, int quiet)
+{
+	int status;
+	pid_t pid;
+
+	pid = fork();
+	if (pid < 0)
+		die("%s: unable to fork off git-unpack-objects", me);
+	if (!pid) {
+		dup2(fd[0], 0);
+		close(fd[0]);
+		close(fd[1]);
+		execlp("git-unpack-objects", "git-unpack-objects",
+		       quiet ? "-q" : NULL, NULL);
+		die("git-unpack-objects exec failed");
+	}
+	close(fd[0]);
+	close(fd[1]);
+	while (waitpid(pid, &status, 0) < 0) {
+		if (errno != EINTR)
+			die("waiting for git-unpack-objects: %s",
+			    strerror(errno));
+	}
+	if (WIFEXITED(status)) {
+		int code = WEXITSTATUS(status);
+		if (code)
+			die("git-unpack-objects died with error code %d",
+			    code);
+		return 0;
+	}
+	if (WIFSIGNALED(status)) {
+		int sig = WTERMSIG(status);
+		die("git-unpack-objects died of signal %d", sig);
+	}
+	die("git-unpack-objects died of unnatural causes %d", status);
+}
+
+int receive_keep_pack(int fd[2], const char *me)
+{
+	char tmpfile[PATH_MAX];
+	int ofd, ifd;
+
+	ifd = fd[0];
+	snprintf(tmpfile, sizeof(tmpfile),
+		 "%s/pack/tmp-XXXXXX", get_object_directory());
+	ofd = mkstemp(tmpfile);
+	if (ofd < 0)
+		return error("unable to create temporary file %s", tmpfile);
+
+	while (1) {
+		char buf[8192];
+		ssize_t sz, wsz, pos;
+		sz = read(ifd, buf, sizeof(buf));
+		if (sz == 0)
+			break;
+		if (sz < 0) {
+			error("error reading pack (%s)", strerror(errno));
+			close(ofd);
+			unlink(tmpfile);
+			return -1;
+		}
+		pos = 0;
+		while (pos < sz) {
+			wsz = write(ofd, buf + pos, sz - pos);
+			if (wsz < 0) {
+				error("error writing pack (%s)",
+				      strerror(errno));
+				close(ofd);
+				unlink(tmpfile);
+				return -1;
+			}
+			pos += wsz;
+		}
+	}
+	close(ofd);
+	return finish_pack(tmpfile, me);
+}

fetch-pack.c

@@ -3,13 +3,12 @@
 #include "pkt-line.h"
 #include "commit.h"
 #include "tag.h"
-#include <time.h>
-#include <sys/wait.h>
 
+static int keep_pack;
 static int quiet;
 static int verbose;
 static const char fetch_pack_usage[] =
-"git-fetch-pack [-q] [-v] [--exec=upload-pack] [host:]directory <refs>...";
+"git-fetch-pack [-q] [-v] [-k] [--exec=upload-pack] [host:]directory <refs>...";
 static const char *exec = "git-upload-pack";
 
 #define COMPLETE	(1U << 0)
@@ -363,7 +362,6 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
 	struct ref *ref;
 	unsigned char sha1[20];
 	int status;
-	pid_t pid;
 
 	get_remote_heads(fd[0], &ref, 0, NULL, 0);
 	if (server_supports("multi_ack")) {
@@ -381,40 +379,22 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
 	}
 	if (find_common(fd, sha1, ref) < 0)
 		fprintf(stderr, "warning: no common commits\n");
-	pid = fork();
+
-	if (pid < 0)
+	if (keep_pack)
-		die("git-fetch-pack: unable to fork off git-unpack-objects");
+		status = receive_keep_pack(fd, "git-fetch-pack");
-	if (!pid) {
+	else
-		dup2(fd[0], 0);
+		status = receive_unpack_pack(fd, "git-fetch-pack", quiet);
-		close(fd[0]);
+
-		close(fd[1]);
+	if (status)
-		execlp("git-unpack-objects", "git-unpack-objects",
+		die("git-fetch-pack: fetch failed.");
-		       quiet ? "-q" : NULL, NULL);
+
-		die("git-unpack-objects exec failed");
+ all_done:
+	while (ref) {
+		printf("%s %s\n",
+		       sha1_to_hex(ref->old_sha1), ref->name);
+		ref = ref->next;
 	}
-	close(fd[0]);
+	return 0;
-	close(fd[1]);
-	while (waitpid(pid, &status, 0) < 0) {
-		if (errno != EINTR)
-			die("waiting for git-unpack-objects: %s", strerror(errno));
-	}
-	if (WIFEXITED(status)) {
-		int code = WEXITSTATUS(status);
-		if (code)
-			die("git-unpack-objects died with error code %d", code);
-all_done:
-		while (ref) {
-			printf("%s %s\n",
-			       sha1_to_hex(ref->old_sha1), ref->name);
-			ref = ref->next;
-		}
-		return 0;
-	}
-	if (WIFSIGNALED(status)) {
-		int sig = WTERMSIG(status);
-		die("git-unpack-objects died of signal %d", sig);
-	}
-	die("Sherlock Holmes! git-unpack-objects died of unnatural causes %d!", status);
 }
 
 int main(int argc, char **argv)
@@ -440,6 +420,10 @@ int main(int argc, char **argv)
 				quiet = 1;
 				continue;
 			}
+			if (!strcmp("-k", arg)) {
+				keep_pack = 1;
+				continue;
+			}
 			if (!strcmp("-v", arg)) {
 				verbose = 1;
 				continue;