From 0a9feffc47b19b81aba7ee5a93473b4c91f44ea9 Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Mon, 22 Aug 2011 09:33:05 +0100 Subject: [PATCH 1/9] git-p4: Allow setting rename/copy detection threshold Copy and rename detection arguments (-C and -M) allow setting a threshold value for the similarity ratio. If the similarity is below this threshold the rename or copy is ignored and the file is added as new. This patch allows setting git-p4.detectRenames and git-p4.detectCopies options to an integer value to set the respective threshold. Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6b9de9e7e0..29a5390fbd 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -774,15 +774,20 @@ class P4Submit(Command, P4UserMap): if not self.detectRenames: # If not explicitly set check the config variable - self.detectRenames = gitConfig("git-p4.detectRenames").lower() == "true" + self.detectRenames = gitConfig("git-p4.detectRenames") - if self.detectRenames: + if self.detectRenames.lower() == "false" or self.detectRenames == "": + diffOpts = "" + elif self.detectRenames.lower() == "true": diffOpts = "-M" else: - diffOpts = "" + diffOpts = "-M%s" % self.detectRenames - if gitConfig("git-p4.detectCopies").lower() == "true": + detectCopies = gitConfig("git-p4.detectCopies") + if detectCopies.lower() == "true": diffOpts += " -C" + elif detectCopies != "" and detectCopies.lower() != "false": + diffOpts += " -C%s" % detectCopies if gitConfig("git-p4.detectCopiesHarder").lower() == "true": diffOpts += " --find-copies-harder" From 807371a92654634d482b31b1d5dd3dff86193990 Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Mon, 22 Aug 2011 09:33:06 +0100 Subject: [PATCH 2/9] git-p4: Add description of rename/copy detection options Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index caa4bb3e30..2ffbccc9bd 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -232,6 +232,31 @@ git-p4.skipUserNameCheck When submitting, git-p4 checks that the git commits are authored by the current p4 user, and warns if they are not. This disables the check. +git-p4.detectRenames + +Detect renames when submitting changes to Perforce server. Will enable -M git +argument. Can be optionally set to a number representing the threshold +percentage value of the rename detection. + + git config [--global] git-p4.detectRenames true + git config [--global] git-p4.detectRenames 50 + +git-p4.detectCopies + +Detect copies when submitting changes to Perforce server. Will enable -C git +argument. Can be optionally set to a number representing the threshold +percentage value of the copy detection. + + git config [--global] git-p4.detectCopies true + git config [--global] git-p4.detectCopies 80 + +git-p4.detectCopiesHarder + +Detect copies even between files that did not change when submitting changes to +Perforce server. Will enable --find-copies-harder git argument. + + git config [--global] git-p4.detectCopies true + Implementation Details... ========================= From 52dced8a568a03d62ceff8e2cd9631074d08dca0 Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Mon, 22 Aug 2011 09:33:07 +0100 Subject: [PATCH 3/9] git-p4: Add test case for rename detection Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- t/t9800-git-p4.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/t/t9800-git-p4.sh b/t/t9800-git-p4.sh index 97ec9753b4..5a5fd0f35c 100755 --- a/t/t9800-git-p4.sh +++ b/t/t9800-git-p4.sh @@ -269,6 +269,59 @@ test_expect_success 'initial import time from top change time' ' test $p4time = $gittime ' +# Rename a file and confirm that rename is not detected in P4. +# Rename the new file again with detectRenames option enabled and confirm that +# this is detected in P4. +# Rename the new file again adding an extra line, configure a big threshold in +# detectRenames and confirm that rename is not detected in P4. +# Repeat, this time with a smaller threshold and confirm that the rename is +# detected in P4. +test_expect_success 'detect renames' ' + "$GITP4" clone --dest="$git" //depot@all && + test_when_finished cleanup_git && + cd "$git" && + git config git-p4.skipSubmitEditCheck true && + + git mv file1 file4 && + git commit -a -m "Rename file1 to file4" && + git diff-tree -r -M HEAD && + "$GITP4" submit && + p4 filelog //depot/file4 && + ! p4 filelog //depot/file4 | grep -q "branch from" && + + git mv file4 file5 && + git commit -a -m "Rename file4 to file5" && + git diff-tree -r -M HEAD && + git config git-p4.detectRenames true && + "$GITP4" submit && + p4 filelog //depot/file5 && + p4 filelog //depot/file5 | grep -q "branch from //depot/file4" && + + git mv file5 file6 && + echo update >>file6 && + git add file6 && + git commit -a -m "Rename file5 to file6 with changes" && + git diff-tree -r -M HEAD && + level=$(git diff-tree -r -M HEAD | sed 1d | cut -f1 | cut -d" " -f5 | sed "s/R0*//") && + test -n "$level" && test "$level" -gt 0 && test "$level" -lt 98 && + git config git-p4.detectRenames $((level + 2)) && + "$GITP4" submit && + p4 filelog //depot/file6 && + ! p4 filelog //depot/file6 | grep -q "branch from" && + + git mv file6 file7 && + echo update >>file7 && + git add file7 && + git commit -a -m "Rename file6 to file7 with changes" && + git diff-tree -r -M HEAD && + level=$(git diff-tree -r -M HEAD | sed 1d | cut -f1 | cut -d" " -f5 | sed "s/R0*//") && + test -n "$level" && test "$level" -gt 2 && test "$level" -lt 100 && + git config git-p4.detectRenames $((level - 2)) && + "$GITP4" submit && + p4 filelog //depot/file7 && + p4 filelog //depot/file7 | grep -q "branch from //depot/file6" +' + test_expect_success 'shutdown' ' pid=`pgrep -f p4d` && test -n "$pid" && From c5cd4ef0fd0838b3a7c97e53f361ba15db736138 Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Mon, 22 Aug 2011 09:33:08 +0100 Subject: [PATCH 4/9] git-p4: Add test case for copy detection Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- t/t9800-git-p4.sh | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/t/t9800-git-p4.sh b/t/t9800-git-p4.sh index 5a5fd0f35c..9d4d4bfd90 100755 --- a/t/t9800-git-p4.sh +++ b/t/t9800-git-p4.sh @@ -322,6 +322,89 @@ test_expect_success 'detect renames' ' p4 filelog //depot/file7 | grep -q "branch from //depot/file6" ' +# Copy a file and confirm that copy is not detected in P4. +# Copy a file with detectCopies option enabled and confirm that copy is not +# detected in P4. +# Modify and copy a file with detectCopies option enabled and confirm that copy +# is detected in P4. +# Copy a file with detectCopies and detectCopiesHarder options enabled and +# confirm that copy is detected in P4. +# Modify and copy a file, configure a bigger threshold in detectCopies and +# confirm that copy is not detected in P4. +# Modify and copy a file, configure a smaller threshold in detectCopies and +# confirm that copy is detected in P4. +test_expect_success 'detect copies' ' + "$GITP4" clone --dest="$git" //depot@all && + test_when_finished cleanup_git && + cd "$git" && + git config git-p4.skipSubmitEditCheck true && + + cp file2 file8 && + git add file8 && + git commit -a -m "Copy file2 to file8" && + git diff-tree -r -C HEAD && + "$GITP4" submit && + p4 filelog //depot/file8 && + ! p4 filelog //depot/file8 | grep -q "branch from" && + + cp file2 file9 && + git add file9 && + git commit -a -m "Copy file2 to file9" && + git diff-tree -r -C HEAD && + git config git-p4.detectCopies true && + "$GITP4" submit && + p4 filelog //depot/file9 && + ! p4 filelog //depot/file9 | grep -q "branch from" && + + echo "file2" >>file2 && + cp file2 file10 && + git add file2 file10 && + git commit -a -m "Modify and copy file2 to file10" && + git diff-tree -r -C HEAD && + "$GITP4" submit && + p4 filelog //depot/file10 && + p4 filelog //depot/file10 | grep -q "branch from //depot/file" && + + cp file2 file11 && + git add file11 && + git commit -a -m "Copy file2 to file11" && + git diff-tree -r -C --find-copies-harder HEAD && + src=$(git diff-tree -r -C --find-copies-harder HEAD | sed 1d | cut -f2) && + test "$src" = file10 && + git config git-p4.detectCopiesHarder true && + "$GITP4" submit && + p4 filelog //depot/file11 && + p4 filelog //depot/file11 | grep -q "branch from //depot/file" && + + cp file2 file12 && + echo "some text" >>file12 && + git add file12 && + git commit -a -m "Copy file2 to file12 with changes" && + git diff-tree -r -C --find-copies-harder HEAD && + level=$(git diff-tree -r -C --find-copies-harder HEAD | sed 1d | cut -f1 | cut -d" " -f5 | sed "s/C0*//") && + test -n "$level" && test "$level" -gt 0 && test "$level" -lt 98 && + src=$(git diff-tree -r -C --find-copies-harder HEAD | sed 1d | cut -f2) && + test "$src" = file10 && + git config git-p4.detectCopies $((level + 2)) && + "$GITP4" submit && + p4 filelog //depot/file12 && + ! p4 filelog //depot/file12 | grep -q "branch from" && + + cp file2 file13 && + echo "different text" >>file13 && + git add file13 && + git commit -a -m "Copy file2 to file13 with changes" && + git diff-tree -r -C --find-copies-harder HEAD && + level=$(git diff-tree -r -C --find-copies-harder HEAD | sed 1d | cut -f1 | cut -d" " -f5 | sed "s/C0*//") && + test -n "$level" && test "$level" -gt 2 && test "$level" -lt 100 && + src=$(git diff-tree -r -C --find-copies-harder HEAD | sed 1d | cut -f2) && + test "$src" = file10 && + git config git-p4.detectCopies $((level - 2)) && + "$GITP4" submit && + p4 filelog //depot/file13 && + p4 filelog //depot/file13 | grep -q "branch from //depot/file" +' + test_expect_success 'shutdown' ' pid=`pgrep -f p4d` && test -n "$pid" && From 68cbcf1b2575ca151ed3b11698916bcbbd1b890c Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Mon, 22 Aug 2011 09:33:09 +0100 Subject: [PATCH 5/9] git-p4: Process detectCopiesHarder with --bool Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 29a5390fbd..0db3e72665 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -789,7 +789,7 @@ class P4Submit(Command, P4UserMap): elif detectCopies != "" and detectCopies.lower() != "false": diffOpts += " -C%s" % detectCopies - if gitConfig("git-p4.detectCopiesHarder").lower() == "true": + if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true": diffOpts += " --find-copies-harder" diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id)) From 04d277b39e3e40f939fd7935c30beb48452015ec Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Fri, 19 Aug 2011 00:44:03 +0100 Subject: [PATCH 6/9] git-p4: Correct branch base depot path detection When branch detection is enabled each branch is named in git after their relative depot path in Perforce. To do this the depot paths are compared against each other to find their common base path. The current algorithm makes this comparison on a character by character basis. Assuming we have the following branches: //depot/branches/featureA //depot/branches/featureB Then the base depot path would be //depot/branches/feature, which is an invalid depot path. The current patch fixes this by splitting the path into a list and comparing the list entries, making it choose correctly //depot/branches as the base path. Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0db3e72665..72a5b6c183 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1829,12 +1829,14 @@ class P4Sync(Command, P4UserMap): else: paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, min(len(cur), len(prev))): - if cur[i] <> prev[i]: + prev_list = prev.split("/") + cur_list = cur.split("/") + for i in range(0, min(len(cur_list), len(prev_list))): + if cur_list[i] <> prev_list[i]: i = i - 1 break - paths.append (cur[:i + 1]) + paths.append ("/".join(cur_list[:i + 1])) self.previousDepotPaths = paths From 8ace74c00e87cbb16d7303761ff4f6b91505ee20 Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Fri, 19 Aug 2011 00:44:04 +0100 Subject: [PATCH 7/9] git-p4: Allow filtering Perforce branches by user All branches in the Perforce server are downloaded to allow branch detection. If you have a centralized server on a remote location and there is a big number of branches this operation can take some time. This patch adds the configuration option git-p4.branchUser to allow filtering the branch list by user. Although this limits the branch maintenance in Perforce to be done by a single user, it might be an advantage when the number of branches being used in a specific depot is very small when compared with the branches available in the server. Signed-off-by: Vitor Antunes Acked-by: Pete Wyckoff Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4 | 8 +++++++- contrib/fast-import/git-p4.txt | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 72a5b6c183..6314c20ac1 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1455,7 +1455,13 @@ class P4Sync(Command, P4UserMap): def getBranchMapping(self): lostAndFoundBranches = set() - for info in p4CmdList("branches"): + user = gitConfig("git-p4.branchUser") + if len(user) > 0: + command = "branches -u %s" % user + else: + command = "branches" + + for info in p4CmdList(command): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 while details.has_key("View%s" % viewIdx): diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 2ffbccc9bd..97b66b9297 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -257,6 +257,12 @@ Perforce server. Will enable --find-copies-harder git argument. git config [--global] git-p4.detectCopies true +git-p4.branchUser + +Only use branch specifications defined by the selected username. + + git config [--global] git-p4.branchUser username + Implementation Details... ========================= From 7199cf131c5420b631f8402498f17ec42cdb3b3a Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Fri, 19 Aug 2011 00:44:05 +0100 Subject: [PATCH 8/9] git-p4: Allow branch definition with git config Perforce does not strictly require the usage of branch specifications to create branches. In these cases the branch detection code of git-p4 will not be able to import them. This patch adds support for git-p4.branchList configuration option, allowing branches to be defined in git config. Signed-off-by: Vitor Antunes Signed-off-by: Junio C Hamano --- contrib/fast-import/git-p4 | 24 ++++++++++++++++++++++++ contrib/fast-import/git-p4.txt | 7 +++++++ 2 files changed, 31 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6314c20ac1..2f7b270566 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -342,6 +342,11 @@ def gitConfig(key, args = None): # set args to "--bool", for instance _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip() return _gitConfig[key] +def gitConfigList(key): + if not _gitConfig.has_key(key): + _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep) + return _gitConfig[key] + def p4BranchesInGit(branchesAreInRemotes = True): branches = {} @@ -1490,6 +1495,25 @@ class P4Sync(Command, P4UserMap): if source not in self.knownBranches: lostAndFoundBranches.add(source) + # Perforce does not strictly require branches to be defined, so we also + # check git config for a branch list. + # + # Example of branch definition in git config file: + # [git-p4] + # branchList=main:branchA + # branchList=main:branchB + # branchList=branchA:branchC + configBranches = gitConfigList("git-p4.branchList") + for branch in configBranches: + if branch: + (source, destination) = branch.split(":") + self.knownBranches[destination] = source + + lostAndFoundBranches.discard(destination) + + if source not in self.knownBranches: + lostAndFoundBranches.add(source) + for branch in lostAndFoundBranches: self.knownBranches[branch] = branch diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 97b66b9297..52003ae904 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -263,6 +263,13 @@ Only use branch specifications defined by the selected username. git config [--global] git-p4.branchUser username +git-p4.branchList + +List of branches to be imported when branch detection is enabled. + + git config [--global] git-p4.branchList main:branchA + git config [--global] --add git-p4.branchList main:branchB + Implementation Details... ========================= From 7ca97f6ecc66f1d70ca126edd33b6a3065c914f7 Mon Sep 17 00:00:00 2001 From: Vitor Antunes Date: Fri, 19 Aug 2011 00:44:06 +0100 Subject: [PATCH 9/9] git-p4: Add simple test case for branch import Create a basic branch structure in P4 and clone it with git-p4. Also, make an update on P4 side and check if git-p4 imports it correctly. The branch structure is created in such a way that git-p4 will fail to import updates if patch "git-p4: Correct branch base depot path detection" is not applied. Signed-off-by: Vitor Antunes Signed-off-by: Junio C Hamano --- t/t9800-git-p4.sh | 64 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/t/t9800-git-p4.sh b/t/t9800-git-p4.sh index 9d4d4bfd90..39e7f78e28 100755 --- a/t/t9800-git-p4.sh +++ b/t/t9800-git-p4.sh @@ -405,6 +405,70 @@ test_expect_success 'detect copies' ' p4 filelog //depot/file13 | grep -q "branch from //depot/file" ' +# Create a simple branch structure in P4 depot to check if it is correctly +# cloned. +test_expect_success 'add simple p4 branches' ' + cd "$cli" && + mkdir branch1 && + cd branch1 && + echo file1 >file1 && + echo file2 >file2 && + p4 add file* && + p4 submit -d "branch1" && + p4 integrate //depot/branch1/... //depot/branch2/... && + p4 submit -d "branch2" && + echo file3 >file3 && + p4 add file3 && + p4 submit -d "add file3 in branch1" && + p4 open file2 && + echo update >>file2 && + p4 submit -d "update file2 in branch1" && + p4 integrate //depot/branch1/... //depot/branch3/... && + p4 submit -d "branch3" && + cd "$TRASH_DIRECTORY" +' + +# Configure branches through git-config and clone them. +# All files are tested to make sure branches were cloned correctly. +# Finally, make an update to branch1 on P4 side to check if it is imported +# correctly by git-p4. +test_expect_success 'git-p4 clone simple branches' ' + git init "$git" && + cd "$git" && + git config git-p4.branchList branch1:branch2 && + git config --add git-p4.branchList branch1:branch3 && + cd "$TRASH_DIRECTORY" && + "$GITP4" clone --dest="$git" --detect-branches //depot@all && + cd "$git" && + git log --all --graph --decorate --stat && + git reset --hard p4/depot/branch1 && + test -f file1 && + test -f file2 && + test -f file3 && + grep -q update file2 && + git reset --hard p4/depot/branch2 && + test -f file1 && + test -f file2 && + test \! -z file3 && + ! grep -q update file2 && + git reset --hard p4/depot/branch3 && + test -f file1 && + test -f file2 && + test -f file3 && + grep -q update file2 && + cd "$cli" && + cd branch1 && + p4 edit file2 && + echo file2_ >> file2 && + p4 submit -d "update file2 in branch3" && + cd "$git" && + git reset --hard p4/depot/branch1 && + "$GITP4" rebase && + grep -q file2_ file2 && + cd "$TRASH_DIRECTORY" && + rm -rf "$git" && mkdir "$git" +' + test_expect_success 'shutdown' ' pid=`pgrep -f p4d` && test -n "$pid" &&