Merge branch 'hv/submodule-recurse-push'

"git push --recurse-submodules" learns to optionally look into the
histories of submodules bound to the superproject and push them out.

By Heiko Voigt
* hv/submodule-recurse-push:
  push: teach --recurse-submodules the on-demand option
  Refactor submodule push check to use string list instead of integer
  Teach revision walking machinery to walk multiple times sequencially

.gitignore

@@ -185,6 +185,7 @@
 /test-mktemp
 /test-parse-options
 /test-path-utils
+/test-revision-walking
 /test-run-command
 /test-sha1
 /test-sigchain

Documentation/git-push.txt

@@ -170,10 +170,16 @@ useful if you write an alias or script around 'git push'.
 	is specified. This flag forces progress status even if the
 	standard error stream is not directed to a terminal.
 
---recurse-submodules=check::
+--recurse-submodules=check|on-demand::
-	Check whether all submodule commits used by the revisions to be
+	Make sure all submodule commits used by the revisions to be
-	pushed are available on a remote tracking branch. Otherwise the
+	pushed are available on a remote tracking branch. If 'check' is
-	push will be aborted and the command will exit with non-zero status.
+	used git will verify that all submodule commits that changed in
+	the revisions to be pushed are available on at least one remote
+	of the submodule. If any commits are missing the push will be
+	aborted and exit with non-zero status. If 'on-demand' is used
+	all submodules that changed in the revisions to be pushed will
+	be pushed. If on-demand was not able to push all necessary
+	revisions it will also be aborted and exit with non-zero status.
 
 
 include::urls-remotes.txt[]

Documentation/technical/api-revision-walking.txt

@@ -56,6 +56,11 @@ function.
 	returning a `struct commit *` each time you call it. The end of the
 	revision list is indicated by returning a NULL pointer.
 
+`reset_revision_walk`::
+
+	Reset the flags used by the revision walking api. You can use
+	this to do multiple sequencial revision walks.
+
 Data structures
 ---------------
 

Makefile

@@ -485,6 +485,7 @@ TEST_PROGRAMS_NEED_X += test-mergesort
 TEST_PROGRAMS_NEED_X += test-mktemp
 TEST_PROGRAMS_NEED_X += test-parse-options
 TEST_PROGRAMS_NEED_X += test-path-utils
+TEST_PROGRAMS_NEED_X += test-revision-walking
 TEST_PROGRAMS_NEED_X += test-run-command
 TEST_PROGRAMS_NEED_X += test-sha1
 TEST_PROGRAMS_NEED_X += test-sigchain

builtin/push.c

@@ -284,13 +284,21 @@ static int option_parse_recurse_submodules(const struct option *opt,
 				   const char *arg, int unset)
 {
 	int *flags = opt->value;
+
+	if (*flags & (TRANSPORT_RECURSE_SUBMODULES_CHECK |
+		      TRANSPORT_RECURSE_SUBMODULES_ON_DEMAND))
+		die("%s can only be used once.", opt->long_name);
+
 	if (arg) {
 		if (!strcmp(arg, "check"))
 			*flags |= TRANSPORT_RECURSE_SUBMODULES_CHECK;
+		else if (!strcmp(arg, "on-demand"))
+			*flags |= TRANSPORT_RECURSE_SUBMODULES_ON_DEMAND;
 		else
 			die("bad %s argument: %s", opt->long_name, arg);
 	} else
-		die("option %s needs an argument (check)", opt->long_name);
+		die("option %s needs an argument (check|on-demand)",
+				opt->long_name);
 
 	return 0;
 }

object.c

@@ -286,3 +286,14 @@ void object_array_remove_duplicates(struct object_array *array)
 		array->nr = dst;
 	}
 }
+
+void clear_object_flags(unsigned flags)
+{
+	int i;
+
+	for (i=0; i < obj_hash_size; i++) {
+		struct object *obj = obj_hash[i];
+		if (obj)
+			obj->flags &= ~flags;
+	}
+}

object.h

@@ -76,4 +76,6 @@ void add_object_array(struct object *obj, const char *name, struct object_array
 void add_object_array_with_mode(struct object *obj, const char *name, struct object_array *array, unsigned mode);
 void object_array_remove_duplicates(struct object_array *);
 
+void clear_object_flags(unsigned flags);
+
 #endif /* OBJECT_H */

revision.c

@@ -2062,6 +2062,11 @@ static void set_children(struct rev_info *revs)
 	}
 }
 
+void reset_revision_walk(void)
+{
+	clear_object_flags(SEEN | ADDED | SHOWN);
+}
+
 int prepare_revision_walk(struct rev_info *revs)
 {
 	int nr = revs->pending.nr;

revision.h

@@ -192,6 +192,7 @@ extern void parse_revision_opt(struct rev_info *revs, struct parse_opt_ctx_t *ct
 				 const char * const usagestr[]);
 extern int handle_revision_arg(const char *arg, struct rev_info *revs,int flags,int cant_be_filename);
 
+extern void reset_revision_walk(void);
 extern int prepare_revision_walk(struct rev_info *revs);
 extern struct commit *get_revision(struct rev_info *revs);
 extern char *get_revision_mark(const struct rev_info *revs, const struct commit *commit);

submodule.c

@@ -357,21 +357,19 @@ static void collect_submodules_from_diff(struct diff_queue_struct *q,
 					 void *data)
 {
 	int i;
-	int *needs_pushing = data;
+	struct string_list *needs_pushing = data;
 
 	for (i = 0; i < q->nr; i++) {
 		struct diff_filepair *p = q->queue[i];
 		if (!S_ISGITLINK(p->two->mode))
 			continue;
-		if (submodule_needs_pushing(p->two->path, p->two->sha1)) {
+		if (submodule_needs_pushing(p->two->path, p->two->sha1))
-			*needs_pushing = 1;
+			string_list_insert(needs_pushing, p->two->path);
-			break;
-		}
 	}
 }
 
-
+static void find_unpushed_submodule_commits(struct commit *commit,
-static void commit_need_pushing(struct commit *commit, int *needs_pushing)
+		struct string_list *needs_pushing)
 {
 	struct rev_info rev;
 
@@ -382,14 +380,15 @@ static void commit_need_pushing(struct commit *commit, int *needs_pushing)
 	diff_tree_combined_merge(commit, 1, &rev);
 }
 
-int check_submodule_needs_pushing(unsigned char new_sha1[20], const char *remotes_name)
+int find_unpushed_submodules(unsigned char new_sha1[20],
+		const char *remotes_name, struct string_list *needs_pushing)
 {
 	struct rev_info rev;
 	struct commit *commit;
 	const char *argv[] = {NULL, NULL, "--not", "NULL", NULL};
 	int argc = ARRAY_SIZE(argv) - 1;
 	char *sha1_copy;
-	int needs_pushing = 0;
+
 	struct strbuf remotes_arg = STRBUF_INIT;
 
 	strbuf_addf(&remotes_arg, "--remotes=%s", remotes_name);
@@ -401,13 +400,62 @@ int check_submodule_needs_pushing(unsigned char new_sha1[20], const char *remote
 	if (prepare_revision_walk(&rev))
 		die("revision walk setup failed");
 
-	while ((commit = get_revision(&rev)) && !needs_pushing)
+	while ((commit = get_revision(&rev)) != NULL)
-		commit_need_pushing(commit, &needs_pushing);
+		find_unpushed_submodule_commits(commit, needs_pushing);
 
+	reset_revision_walk();
 	free(sha1_copy);
 	strbuf_release(&remotes_arg);
 
-	return needs_pushing;
+	return needs_pushing->nr;
+}
+
+static int push_submodule(const char *path)
+{
+	if (add_submodule_odb(path))
+		return 1;
+
+	if (for_each_remote_ref_submodule(path, has_remote, NULL) > 0) {
+		struct child_process cp;
+		const char *argv[] = {"push", NULL};
+
+		memset(&cp, 0, sizeof(cp));
+		cp.argv = argv;
+		cp.env = local_repo_env;
+		cp.git_cmd = 1;
+		cp.no_stdin = 1;
+		cp.dir = path;
+		if (run_command(&cp))
+			return 0;
+		close(cp.out);
+	}
+
+	return 1;
+}
+
+int push_unpushed_submodules(unsigned char new_sha1[20], const char *remotes_name)
+{
+	int i, ret = 1;
+	struct string_list needs_pushing;
+
+	memset(&needs_pushing, 0, sizeof(struct string_list));
+	needs_pushing.strdup_strings = 1;
+
+	if (!find_unpushed_submodules(new_sha1, remotes_name, &needs_pushing))
+		return 1;
+
+	for (i = 0; i < needs_pushing.nr; i++) {
+		const char *path = needs_pushing.items[i].string;
+		fprintf(stderr, "Pushing submodule '%s'\n", path);
+		if (!push_submodule(path)) {
+			fprintf(stderr, "Unable to push submodule '%s'\n", path);
+			ret = 0;
+		}
+	}
+
+	string_list_clear(&needs_pushing, 0);
+
+	return ret;
 }
 
 static int is_submodule_commit_present(const char *path, unsigned char sha1[20])
@@ -741,6 +789,7 @@ static int find_first_merges(struct object_array *result, const char *path,
 		if (in_merge_bases(b, &commit, 1))
 			add_object_array(o, NULL, &merges);
 	}
+	reset_revision_walk();
 
 	/* Now we've got all merges that contain a and b. Prune all
 	 * merges that contain another found merge and save them in

submodule.h

@@ -29,6 +29,8 @@ int fetch_populated_submodules(int num_options, const char **options,
 unsigned is_submodule_modified(const char *path, int ignore_untracked);
 int merge_submodule(unsigned char result[20], const char *path, const unsigned char base[20],
 		    const unsigned char a[20], const unsigned char b[20], int search);
-int check_submodule_needs_pushing(unsigned char new_sha1[20], const char *remotes_name);
+int find_unpushed_submodules(unsigned char new_sha1[20], const char *remotes_name,
+		struct string_list *needs_pushing);
+int push_unpushed_submodules(unsigned char new_sha1[20], const char *remotes_name);
 
 #endif

t/t0062-revision-walking.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Copyright (c) 2012 Heiko Voigt
+#
+
+test_description='Test revision walking api'
+
+. ./test-lib.sh
+
+cat >run_twice_expected <<-EOF
+1st
+ > add b
+ > add a
+2nd
+ > add b
+ > add a
+EOF
+
+test_expect_success 'setup' '
+	echo a > a &&
+	git add a &&
+	git commit -m "add a" &&
+	echo b > b &&
+	git add b &&
+	git commit -m "add b"
+'
+
+test_expect_success 'revision walking can be done twice' '
+	test-revision-walking run-twice > run_twice_actual
+	test_cmp run_twice_expected run_twice_actual
+'
+
+test_done

t/t5531-deep-submodule-push.sh

@@ -119,4 +119,98 @@ test_expect_success 'push succeeds if submodule has no remote and is on the firs
 	)
 '
 
+test_expect_success 'push unpushed submodules when not needed' '
+	(
+		cd work &&
+		(
+			cd gar/bage &&
+			git checkout master &&
+			>junk5 &&
+			git add junk5 &&
+			git commit -m "Fifth junk" &&
+			git push &&
+			git rev-parse origin/master >../../../expected
+		) &&
+		git checkout master &&
+		git add gar/bage &&
+		git commit -m "Fifth commit for gar/bage" &&
+		git push --recurse-submodules=on-demand ../pub.git master
+	) &&
+	(
+		cd submodule.git &&
+		git rev-parse master >../actual
+	) &&
+	test_cmp expected actual
+'
+
+test_expect_success 'push unpushed submodules when not needed 2' '
+	(
+		cd submodule.git &&
+		git rev-parse master >../expected
+	) &&
+	(
+		cd work &&
+		(
+			cd gar/bage &&
+			>junk6 &&
+			git add junk6 &&
+			git commit -m "Sixth junk"
+		) &&
+		>junk2 &&
+		git add junk2 &&
+		git commit -m "Second junk for work" &&
+		git push --recurse-submodules=on-demand ../pub.git master
+	) &&
+	(
+		cd submodule.git &&
+		git rev-parse master >../actual
+	) &&
+	test_cmp expected actual
+'
+
+test_expect_success 'push unpushed submodules recursively' '
+	(
+		cd work &&
+		(
+			cd gar/bage &&
+			git checkout master &&
+			> junk7 &&
+			git add junk7 &&
+			git commit -m "Seventh junk" &&
+			git rev-parse master >../../../expected
+		) &&
+		git checkout master &&
+		git add gar/bage &&
+		git commit -m "Seventh commit for gar/bage" &&
+		git push --recurse-submodules=on-demand ../pub.git master
+	) &&
+	(
+		cd submodule.git &&
+		git rev-parse master >../actual
+	) &&
+	test_cmp expected actual
+'
+
+test_expect_success 'push unpushable submodule recursively fails' '
+	(
+		cd work &&
+		(
+			cd gar/bage &&
+			git rev-parse origin/master >../../../expected &&
+			git checkout master~0 &&
+			> junk8 &&
+			git add junk8 &&
+			git commit -m "Eighth junk"
+		) &&
+		git add gar/bage &&
+		git commit -m "Eighth commit for gar/bage" &&
+		test_must_fail git push --recurse-submodules=on-demand ../pub.git master
+	) &&
+	(
+		cd submodule.git &&
+		git rev-parse master >../actual
+	) &&
+	test_cmp expected actual
+'
+
 test_done

test-revision-walking.c

@@ -0,0 +1,66 @@
+/*
+ * test-revision-walking.c: test revision walking API.
+ *
+ * (C) 2012 Heiko Voigt <hvoigt@hvoigt.net>
+ *
+ * This code is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "cache.h"
+#include "commit.h"
+#include "diff.h"
+#include "revision.h"
+
+static void print_commit(struct commit *commit)
+{
+	struct strbuf sb = STRBUF_INIT;
+	struct pretty_print_context ctx = {0};
+	ctx.date_mode = DATE_NORMAL;
+	format_commit_message(commit, " %m %s", &sb, &ctx);
+	printf("%s\n", sb.buf);
+	strbuf_release(&sb);
+}
+
+static int run_revision_walk(void)
+{
+	struct rev_info rev;
+	struct commit *commit;
+	const char *argv[] = {NULL, "--all", NULL};
+	int argc = ARRAY_SIZE(argv) - 1;
+	int got_revision = 0;
+
+	init_revisions(&rev, NULL);
+	setup_revisions(argc, argv, &rev, NULL);
+	if (prepare_revision_walk(&rev))
+		die("revision walk setup failed");
+
+	while ((commit = get_revision(&rev)) != NULL) {
+		print_commit(commit);
+		got_revision = 1;
+	}
+
+	reset_revision_walk();
+	return got_revision;
+}
+
+int main(int argc, char **argv)
+{
+	if (argc < 2)
+		return 1;
+
+	if (!strcmp(argv[1], "run-twice")) {
+		printf("1st\n");
+		if (!run_revision_walk())
+			return 1;
+		printf("2nd\n");
+		if (!run_revision_walk())
+			return 1;
+
+		return 0;
+	}
+
+	fprintf(stderr, "check usage\n");
+	return 1;
+}

transport.c

@@ -11,6 +11,7 @@
 #include "branch.h"
 #include "url.h"
 #include "submodule.h"
+#include "string-list.h"
 
 /* rsync support */
 
@@ -1013,6 +1014,25 @@ void transport_set_verbosity(struct transport *transport, int verbosity,
 		transport->progress = verbosity >= 0 && isatty(2);
 }
 
+static void die_with_unpushed_submodules(struct string_list *needs_pushing)
+{
+	int i;
+
+	fprintf(stderr, "The following submodule paths contain changes that can\n"
+			"not be found on any remote:\n");
+	for (i = 0; i < needs_pushing->nr; i++)
+		printf("  %s\n", needs_pushing->items[i].string);
+	fprintf(stderr, "\nPlease try\n\n"
+			"	git push --recurse-submodules=on-demand\n\n"
+			"or cd to the path and use\n\n"
+			"	git push\n\n"
+			"to push them to a remote.\n\n");
+
+	string_list_clear(needs_pushing, 0);
+
+	die("Aborting.");
+}
+
 int transport_push(struct transport *transport,
 		   int refspec_nr, const char **refspec, int flags,
 		   int *nonfastforward)
@@ -1053,12 +1073,27 @@ int transport_push(struct transport *transport,
 			flags & TRANSPORT_PUSH_MIRROR,
 			flags & TRANSPORT_PUSH_FORCE);
 
-		if ((flags & TRANSPORT_RECURSE_SUBMODULES_CHECK) && !is_bare_repository()) {
+		if ((flags & TRANSPORT_RECURSE_SUBMODULES_ON_DEMAND) && !is_bare_repository()) {
 			struct ref *ref = remote_refs;
 			for (; ref; ref = ref->next)
 				if (!is_null_sha1(ref->new_sha1) &&
-				    check_submodule_needs_pushing(ref->new_sha1,transport->remote->name))
+				    !push_unpushed_submodules(ref->new_sha1,
-					die("There are unpushed submodules, aborting.");
+					    transport->remote->name))
+				    die ("Failed to push all needed submodules!");
+		}
+
+		if ((flags & (TRANSPORT_RECURSE_SUBMODULES_ON_DEMAND |
+			      TRANSPORT_RECURSE_SUBMODULES_CHECK)) && !is_bare_repository()) {
+			struct ref *ref = remote_refs;
+			struct string_list needs_pushing;
+
+			memset(&needs_pushing, 0, sizeof(struct string_list));
+			needs_pushing.strdup_strings = 1;
+			for (; ref; ref = ref->next)
+				if (!is_null_sha1(ref->new_sha1) &&
+				    find_unpushed_submodules(ref->new_sha1,
+					    transport->remote->name, &needs_pushing))
+					die_with_unpushed_submodules(&needs_pushing);
 		}
 
 		push_ret = transport->push_refs(transport, remote_refs, flags);

transport.h

@@ -103,6 +103,7 @@ struct transport {
 #define TRANSPORT_PUSH_SET_UPSTREAM 32
 #define TRANSPORT_RECURSE_SUBMODULES_CHECK 64
 #define TRANSPORT_PUSH_PRUNE 128
+#define TRANSPORT_RECURSE_SUBMODULES_ON_DEMAND 256
 
 #define TRANSPORT_SUMMARY_WIDTH (2 * DEFAULT_ABBREV + 3)