Merge branch 'fc/fast-export-fixes'

Various updates to fast-export used in the context of the remote
helper interface.

* fc/fast-export-fixes:
  fast-export: make sure updated refs get updated
  fast-export: don't handle uninteresting refs
  fast-export: fix comparison in tests
  fast-export: trivial cleanup
  remote-testgit: implement the "done" feature manually
  remote-testgit: report success after an import
  remote-testgit: exercise more features
  remote-testgit: cleanup tests
  remote-testgit: remove irrelevant test
  remote-testgit: remove non-local functionality
  Add new simplified git-remote-testgit
  Rename git-remote-testgit to git-remote-testpy
  remote-helpers: fix failure message
  remote-testgit: fix direction of marks
  fast-export: avoid importing blob marks

.gitignore

@@ -124,7 +124,7 @@
 /git-remote-ftps
 /git-remote-fd
 /git-remote-ext
-/git-remote-testgit
+/git-remote-testpy
 /git-remote-testsvn
 /git-repack
 /git-replace

Documentation/git-remote-testgit.txt

@@ -19,7 +19,7 @@ testcase for the remote-helper functionality, and as an example to
 show remote-helper authors one possible implementation.
 
 The best way to learn more is to read the comments and source code in
-'git-remote-testgit.py'.
+'git-remote-testgit'.
 
 SEE ALSO
 --------

Makefile

@@ -478,7 +478,7 @@ SCRIPT_PERL += git-relink.perl
 SCRIPT_PERL += git-send-email.perl
 SCRIPT_PERL += git-svn.perl
 
-SCRIPT_PYTHON += git-remote-testgit.py
+SCRIPT_PYTHON += git-remote-testpy.py
 SCRIPT_PYTHON += git-p4.py
 
 SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) \

builtin/fast-export.c

@@ -474,18 +474,21 @@ static void handle_tag(const char *name, struct tag *tag)
 	       (int)message_size, (int)message_size, message ? message : "");
 }
 
-static void get_tags_and_duplicates(struct object_array *pending,
+static void get_tags_and_duplicates(struct rev_cmdline_info *info,
 				    struct string_list *extra_refs)
 {
 	struct tag *tag;
 	int i;
 
-	for (i = 0; i < pending->nr; i++) {
+	for (i = 0; i < info->nr; i++) {
-		struct object_array_entry *e = pending->objects + i;
+		struct rev_cmdline_entry *e = info->rev + i;
 		unsigned char sha1[20];
-		struct commit *commit = commit;
+		struct commit *commit;
 		char *full_name;
 
+		if (e->flags & UNINTERESTING)
+			continue;
+
 		if (dwim_ref(e->name, strlen(e->name), sha1, &full_name) != 1)
 			continue;
 
@@ -523,10 +526,14 @@ static void get_tags_and_duplicates(struct object_array *pending,
 				typename(e->item->type));
 			continue;
 		}
-		if (commit->util)
+
-			/* more than one name for the same object */
+		/*
+		 * This ref will not be updated through a commit, lets make
+		 * sure it gets properly updated eventually.
+		 */
+		if (commit->util || commit->object.flags & SHOWN)
 			string_list_append(extra_refs, full_name)->util = commit;
-		else
+		if (!commit->util)
 			commit->util = full_name;
 	}
 }
@@ -614,6 +621,10 @@ static void import_marks(char *input_file)
 		if (object->flags & SHOWN)
 			error("Object %s already has a mark", sha1_to_hex(sha1));
 
+		if (object->type != OBJ_COMMIT)
+			/* only commits */
+			continue;
+
 		mark_object(object, mark);
 		if (last_idnum < mark)
 			last_idnum = mark;
@@ -677,7 +688,7 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix)
 	if (import_filename && revs.prune_data.nr)
 		full_tree = 1;
 
-	get_tags_and_duplicates(&revs.pending, &extra_refs);
+	get_tags_and_duplicates(&revs.cmdline, &extra_refs);
 
 	if (prepare_revision_walk(&revs))
 		die("revision walk setup failed");

git-remote-testgit

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Copyright (c) 2012 Felipe Contreras
+
+alias=$1
+url=$2
+
+dir="$GIT_DIR/testgit/$alias"
+prefix="refs/testgit/$alias"
+
+default_refspec="refs/heads/*:${prefix}/heads/*"
+
+refspec="${GIT_REMOTE_TESTGIT_REFSPEC-$default_refspec}"
+
+test -z "$refspec" && prefix="refs"
+
+export GIT_DIR="$url/.git"
+
+mkdir -p "$dir"
+
+if test -z "$GIT_REMOTE_TESTGIT_NO_MARKS"
+then
+	gitmarks="$dir/git.marks"
+	testgitmarks="$dir/testgit.marks"
+	test -e "$gitmarks" || >"$gitmarks"
+	test -e "$testgitmarks" || >"$testgitmarks"
+	testgitmarks_args=( "--"{import,export}"-marks=$testgitmarks" )
+fi
+
+while read line
+do
+	case $line in
+	capabilities)
+		echo 'import'
+		echo 'export'
+		test -n "$refspec" && echo "refspec $refspec"
+		if test -n "$gitmarks"
+		then
+			echo "*import-marks $gitmarks"
+			echo "*export-marks $gitmarks"
+		fi
+		echo
+		;;
+	list)
+		git for-each-ref --format='? %(refname)' 'refs/heads/'
+		head=$(git symbolic-ref HEAD)
+		echo "@$head HEAD"
+		echo
+		;;
+	import*)
+		# read all import lines
+		while true
+		do
+			ref="${line#* }"
+			refs="$refs $ref"
+			read line
+			test "${line%% *}" != "import" && break
+		done
+
+		if test -n "$gitmarks"
+		then
+			echo "feature import-marks=$gitmarks"
+			echo "feature export-marks=$gitmarks"
+		fi
+		echo "feature done"
+		git fast-export "${testgitmarks_args[@]}" $refs |
+		sed -e "s#refs/heads/#${prefix}/heads/#g"
+		echo "done"
+		;;
+	export)
+		before=$(git for-each-ref --format='%(refname) %(objectname)')
+
+		git fast-import "${testgitmarks_args[@]}" --quiet
+
+		after=$(git for-each-ref --format='%(refname) %(objectname)')
+
+		# figure out which refs were updated
+		join -e 0 -o '0 1.2 2.2' -a 2 <(echo "$before") <(echo "$after") |
+		while read ref a b
+		do
+			test $a == $b && continue
+			echo "ok $ref"
+		done
+
+		echo
+		;;
+	'')
+		exit
+		;;
+	esac
+done

git-remote-testpy.py

@@ -91,7 +91,7 @@ def do_capabilities(repo, args):
     if not os.path.exists(dirname):
         os.makedirs(dirname)
 
-    path = os.path.join(dirname, 'testgit.marks')
+    path = os.path.join(dirname, 'git.marks')
 
     print "*export-marks %s" % path
     if os.path.exists(path):

git_remote_helpers/git/importer.py

@@ -39,7 +39,7 @@ class GitImporter(object):
             gitdir = self.repo.gitpath
         else:
             gitdir = os.path.abspath(os.path.join(dirname, '.git'))
-        path = os.path.abspath(os.path.join(dirname, 'git.marks'))
+        path = os.path.abspath(os.path.join(dirname, 'testgit.marks'))
 
         if not os.path.exists(dirname):
             os.makedirs(dirname)

t/t5800-remote-testpy.sh

@@ -3,12 +3,12 @@
 # Copyright (c) 2010 Sverre Rabbelier
 #
 
-test_description='Test remote-helper import and export commands'
+test_description='Test python remote-helper framework'
 
 . ./test-lib.sh
 
 if ! test_have_prereq PYTHON ; then
-	skip_all='skipping git-remote-hg tests, python not available'
+	skip_all='skipping python remote-helper tests, python not available'
 	test_done
 fi
 
@@ -17,7 +17,7 @@ import sys
 if sys.hexversion < 0x02040000:
     sys.exit(1)
 ' || {
-	skip_all='skipping git-remote-hg tests, python version < 2.4'
+	skip_all='skipping python remote-helper tests, python version < 2.4'
 	test_done
 }
 
@@ -38,12 +38,12 @@ test_expect_success 'setup repository' '
 '
 
 test_expect_success 'cloning from local repo' '
-	git clone "testgit::${PWD}/server" localclone &&
+	git clone "testpy::${PWD}/server" localclone &&
 	test_cmp public/file localclone/file
 '
 
 test_expect_success 'cloning from remote repo' '
-	git clone "testgit::file://${PWD}/server" clone &&
+	git clone "testpy::file://${PWD}/server" clone &&
 	test_cmp public/file clone/file
 '
 
@@ -73,11 +73,11 @@ test_expect_success 'pushing to local repo' '
 '
 
 # Generally, skip this test.  It demonstrates a now-fixed race in
-# git-remote-testgit, but is too slow to leave in for general use.
+# git-remote-testpy, but is too slow to leave in for general use.
 : test_expect_success 'racily pushing to local repo' '
 	test_when_finished "rm -rf server2 localclone2" &&
 	cp -R server server2 &&
-	git clone "testgit::${PWD}/server2" localclone2 &&
+	git clone "testpy::${PWD}/server2" localclone2 &&
 	(cd localclone2 &&
 	echo content >>file &&
 	git commit -a -m three &&

t/t5801-remote-helpers.sh

@@ -0,0 +1,169 @@
+#!/bin/sh
+#
+# Copyright (c) 2010 Sverre Rabbelier
+#
+
+test_description='Test remote-helper import and export commands'
+
+. ./test-lib.sh
+
+if ! type "${BASH-bash}" >/dev/null 2>&1; then
+	skip_all='skipping remote-testgit tests, bash not available'
+	test_done
+fi
+
+compare_refs() {
+	git --git-dir="$1/.git" rev-parse --verify $2 >expect &&
+	git --git-dir="$3/.git" rev-parse --verify $4 >actual &&
+	test_cmp expect actual
+}
+
+test_expect_success 'setup repository' '
+	git init server &&
+	(cd server &&
+	 echo content >file &&
+	 git add file &&
+	 git commit -m one)
+'
+
+test_expect_success 'cloning from local repo' '
+	git clone "testgit::${PWD}/server" local &&
+	test_cmp server/file local/file
+'
+
+test_expect_success 'create new commit on remote' '
+	(cd server &&
+	 echo content >>file &&
+	 git commit -a -m two)
+'
+
+test_expect_success 'pulling from local repo' '
+	(cd local && git pull) &&
+	test_cmp server/file local/file
+'
+
+test_expect_success 'pushing to local repo' '
+	(cd local &&
+	echo content >>file &&
+	git commit -a -m three &&
+	git push) &&
+	compare_refs local HEAD server HEAD
+'
+
+test_expect_success 'fetch new branch' '
+	(cd server &&
+	 git reset --hard &&
+	 git checkout -b new &&
+	 echo content >>file &&
+	 git commit -a -m five
+	) &&
+	(cd local &&
+	 git fetch origin new
+	) &&
+	compare_refs server HEAD local FETCH_HEAD
+'
+
+test_expect_success 'fetch multiple branches' '
+	(cd local &&
+	 git fetch
+	) &&
+	compare_refs server master local refs/remotes/origin/master &&
+	compare_refs server new local refs/remotes/origin/new
+'
+
+test_expect_success 'push when remote has extra refs' '
+	(cd local &&
+	 git reset --hard origin/master &&
+	 echo content >>file &&
+	 git commit -a -m six &&
+	 git push
+	) &&
+	compare_refs local master server master
+'
+
+test_expect_success 'push new branch by name' '
+	(cd local &&
+	 git checkout -b new-name  &&
+	 echo content >>file &&
+	 git commit -a -m seven &&
+	 git push origin new-name
+	) &&
+	compare_refs local HEAD server refs/heads/new-name
+'
+
+test_expect_failure 'push new branch with old:new refspec' '
+	(cd local &&
+	 git push origin new-name:new-refspec
+	) &&
+	compare_refs local HEAD server refs/heads/new-refspec
+'
+
+test_expect_success 'cloning without refspec' '
+	GIT_REMOTE_TESTGIT_REFSPEC="" \
+	git clone "testgit::${PWD}/server" local2 &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_success 'pulling without refspecs' '
+	(cd local2 &&
+	git reset --hard &&
+	GIT_REMOTE_TESTGIT_REFSPEC="" git pull) &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_failure 'pushing without refspecs' '
+	test_when_finished "(cd local2 && git reset --hard origin)" &&
+	(cd local2 &&
+	echo content >>file &&
+	git commit -a -m ten &&
+	GIT_REMOTE_TESTGIT_REFSPEC="" git push) &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_success 'pulling with straight refspec' '
+	(cd local2 &&
+	GIT_REMOTE_TESTGIT_REFSPEC="*:*" git pull) &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_failure 'pushing with straight refspec' '
+	test_when_finished "(cd local2 && git reset --hard origin)" &&
+	(cd local2 &&
+	echo content >>file &&
+	git commit -a -m eleven &&
+	GIT_REMOTE_TESTGIT_REFSPEC="*:*" git push) &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_success 'pulling without marks' '
+	(cd local2 &&
+	GIT_REMOTE_TESTGIT_NO_MARKS=1 git pull) &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_failure 'pushing without marks' '
+	test_when_finished "(cd local2 && git reset --hard origin)" &&
+	(cd local2 &&
+	echo content >>file &&
+	git commit -a -m twelve &&
+	GIT_REMOTE_TESTGIT_NO_MARKS=1 git push) &&
+	compare_refs local2 HEAD server HEAD
+'
+
+test_expect_success 'push all with existing object' '
+	(cd local &&
+	git branch dup2 master &&
+	git push origin --all
+	) &&
+	compare_refs local dup2 server dup2
+'
+
+test_expect_success 'push ref with existing object' '
+	(cd local &&
+	git branch dup master &&
+	git push origin dup
+	) &&
+	compare_refs local dup server dup
+'
+
+test_done

t/t9350-fast-export.sh

@@ -303,7 +303,7 @@ test_expect_success 'dropping tag of filtered out object' '
 (
 	cd limit-by-paths &&
 	git fast-export --tag-of-filtered-object=drop mytag -- there > output &&
-	test_cmp output expected
+	test_cmp expected output
 )
 '
 
@@ -320,7 +320,7 @@ test_expect_success 'rewriting tag of filtered out object' '
 (
 	cd limit-by-paths &&
 	git fast-export --tag-of-filtered-object=rewrite mytag -- there > output &&
-	test_cmp output expected
+	test_cmp expected output
 )
 '
 
@@ -351,7 +351,7 @@ test_expect_failure 'no exact-ref revisions included' '
 	(
 		cd limit-by-paths &&
 		git fast-export master~2..master~1 > output &&
-		test_cmp output expected
+		test_cmp expected output
 	)
 '
 
@@ -440,4 +440,63 @@ test_expect_success 'fast-export quotes pathnames' '
 	)
 '
 
+test_expect_success 'test bidirectionality' '
+	>marks-cur &&
+	>marks-new &&
+	git init marks-test &&
+	git fast-export --export-marks=marks-cur --import-marks=marks-cur --branches | \
+	git --git-dir=marks-test/.git fast-import --export-marks=marks-new --import-marks=marks-new &&
+	(cd marks-test &&
+	git reset --hard &&
+	echo Wohlauf > file &&
+	git commit -a -m "back in time") &&
+	git --git-dir=marks-test/.git fast-export --export-marks=marks-new --import-marks=marks-new --branches | \
+	git fast-import --export-marks=marks-cur --import-marks=marks-cur
+'
+
+cat > expected << EOF
+blob
+mark :13
+data 5
+bump
+
+commit refs/heads/master
+mark :14
+author A U Thor <author@example.com> 1112912773 -0700
+committer C O Mitter <committer@example.com> 1112912773 -0700
+data 5
+bump
+from :12
+M 100644 :13 file
+
+EOF
+
+test_expect_success 'avoid uninteresting refs' '
+	> tmp-marks &&
+	git fast-export --import-marks=tmp-marks \
+		--export-marks=tmp-marks master > /dev/null &&
+	git tag v1.0 &&
+	git branch uninteresting &&
+	echo bump > file &&
+	git commit -a -m bump &&
+	git fast-export --import-marks=tmp-marks \
+		--export-marks=tmp-marks ^uninteresting ^v1.0 master > actual &&
+	test_cmp expected actual
+'
+
+cat > expected << EOF
+reset refs/heads/master
+from :14
+
+EOF
+
+test_expect_success 'refs are updated even if no commits need to be exported' '
+	> tmp-marks &&
+	git fast-export --import-marks=tmp-marks \
+		--export-marks=tmp-marks master > /dev/null &&
+	git fast-export --import-marks=tmp-marks \
+		--export-marks=tmp-marks master > actual &&
+	test_cmp expected actual
+'
+
 test_done