From 16d6b8ab6fd7f68bfd9f4d312965cb99e8ad911b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 00:16:59 +0100 Subject: [PATCH 001/260] Initial import of a python script to import changesets from Perforce into git. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 150 ++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 contrib/fast-import/p4-fast-export.py diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py new file mode 100644 index 0000000000..238f6e3524 --- /dev/null +++ b/contrib/fast-import/p4-fast-export.py @@ -0,0 +1,150 @@ +#!/usr/bin/python +# +# p4-fast-export.py +# +# Author: Simon Hausmann +# License: MIT +# +# TODO: - fix date parsing (how hard can it be?) +# - support integrations (at least p4i) +# - support incremental imports +# - create tags +# - instead of reading all files into a variable try to pipe from +# - p4 print directly to stdout. need to figure out file size somehow +# though. +# - support p4 submit (hah!) +# - don't hardcode the import to master +# +import os, string, sys + +# yep, that's hardcoded right. will fix to a commandline option rsn :) +prefix = "//depot/qt/main/" +# that's in revision range syntax, for example @2342,523634 +changeRange = "" + +def describe(change): + output = os.popen("p4 describe %s" % change).readlines() + + firstLine = output[0] + + author = firstLine.split(" ")[3] + author = author[:author.find("@")] + + filesSection = 0 + try: + filesSection = output.index("Affected files ...\n") + except ValueError: + sys.stderr.write("Change %s doesn't seem to affect any files. Weird.\n" % change) + return [], [], [], [] + + differencesSection = 0 + try: + differencesSection = output.index("Differences ...\n") + except ValueError: + sys.stderr.write("Change %s doesn't seem to have a differences section. Weird.\n" % change) + return [], [], [], [] + + log = output[2:filesSection - 1] + + lines = output[filesSection + 2:differencesSection - 1] + + changed = [] + removed = [] + + for line in lines: + # chop off "... " and trailing newline + line = line[4:len(line) - 1] + + lastSpace = line.rfind(" ") + if lastSpace == -1: + sys.stderr.write("trouble parsing line %s, skipping!\n" % line) + continue + + operation = line[lastSpace + 1:] + path = line[:lastSpace] + + if operation == "delete": + removed.append(path) + else: + changed.append(path) + + return author, log, changed, removed + +def p4cat(path): + return os.popen("p4 print -q \"%s\"" % path).read() + +def stripRevision(path): + hashPos = path.rindex("#") + return path[:hashPos] + +def getUserMap(): + users = {} + output = os.popen("p4 users") + for line in output: + firstSpace = line.index(" ") + secondSpace = line.index(" ", firstSpace + 1) + key = line[:firstSpace] + email = line[firstSpace + 1:secondSpace] + openParenPos = line.index("(", secondSpace) + closedParenPos = line.index(")", openParenPos) + name = line[openParenPos + 1:closedParenPos] + + users[key] = name + " " + email + + return users + + +users = getUserMap() + +output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() + +changes = [] +for line in output: + changeNum = line.split(" ")[1] + changes.append(changeNum) + +changes.reverse() + +sys.stderr.write("\n") + +cnt = 0 +for change in changes: + [ author, log, changedFiles, removedFiles ] = describe(change) + sys.stderr.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + cnt = cnt + 1 +# sys.stderr.write("%s\n" % log) +# sys.stderr.write("%s\n" % changedFiles) +# sys.stderr.write("%s\n" % removedFiles) + + print "commit refs/heads/master" + if author in users: + print "committer %s 1 2" % users[author] + else: + print "committer %s 1 2" % author + print "data < Date: Wed, 31 Jan 2007 09:39:20 +0100 Subject: [PATCH 002/260] Added basic support for specifying the depot path to import from as well as the range of perforce changes. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 238f6e3524..abd6da4668 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -17,10 +17,22 @@ # import os, string, sys -# yep, that's hardcoded right. will fix to a commandline option rsn :) -prefix = "//depot/qt/main/" -# that's in revision range syntax, for example @2342,523634 +if len(sys.argv) != 2: + sys.stderr.write("usage: %s //depot/path[@revRange]\n" % sys.argv[0]); + sys.stderr.write("\n example:\n"); + sys.stderr.write(" %s //depot/my/project -- to import everything\n"); + sys.stderr.write(" %s //depot/my/project@1,6 -- to import only from revision 1 to 6\n"); + sys.stderr.write("\n"); + sys.exit(1) + +prefix = sys.argv[1] changeRange = "" +try: + atIdx = prefix.index("@") + changeRange = prefix[atIdx:] + prefix = prefix[0:atIdx] +except ValueError: + changeRange = "" def describe(change): output = os.popen("p4 describe %s" % change).readlines() From 06bb04454fb91fbc144ed2b3abb3492c923f98f5 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 09:49:41 +0100 Subject: [PATCH 003/260] Slightly improved help usage output and made specifying the trailing slash for the depot path optional. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index abd6da4668..8df3d73392 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -20,8 +20,10 @@ import os, string, sys if len(sys.argv) != 2: sys.stderr.write("usage: %s //depot/path[@revRange]\n" % sys.argv[0]); sys.stderr.write("\n example:\n"); - sys.stderr.write(" %s //depot/my/project -- to import everything\n"); - sys.stderr.write(" %s //depot/my/project@1,6 -- to import only from revision 1 to 6\n"); + sys.stderr.write(" %s //depot/my/project/ -- to import everything\n"); + sys.stderr.write(" %s //depot/my/project/@1,6 -- to import only from revision 1 to 6\n"); + sys.stderr.write("\n"); + sys.stderr.write(" (a ... is not needed in the path p4 specification, it's added implicitly)\n"); sys.stderr.write("\n"); sys.exit(1) @@ -34,6 +36,9 @@ try: except ValueError: changeRange = "" +if not prefix.endswith("/"): + prefix += "/" + def describe(change): output = os.popen("p4 describe %s" % change).readlines() From 72b2f0ada30adbb2880d551bd92d9519396897dd Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 16:39:46 +0100 Subject: [PATCH 004/260] Implemented basic support for converting the date of the perforce change to the git format. The timezone isn't correctly set up yet though. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 8df3d73392..689349d551 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -5,8 +5,7 @@ # Author: Simon Hausmann # License: MIT # -# TODO: - fix date parsing (how hard can it be?) -# - support integrations (at least p4i) +# TODO: - support integrations (at least p4i) # - support incremental imports # - create tags # - instead of reading all files into a variable try to pipe from @@ -15,7 +14,7 @@ # - support p4 submit (hah!) # - don't hardcode the import to master # -import os, string, sys +import os, string, sys, time if len(sys.argv) != 2: sys.stderr.write("usage: %s //depot/path[@revRange]\n" % sys.argv[0]); @@ -44,22 +43,25 @@ def describe(change): firstLine = output[0] - author = firstLine.split(" ")[3] + splitted = firstLine.split(" ") + author = splitted[3] author = author[:author.find("@")] + tm = time.strptime(splitted[5] + " " + splitted[6] + time.tzname[0], "%Y/%m/%d %H:%M:%S %Z") + epoch = int(time.mktime(tm)) filesSection = 0 try: filesSection = output.index("Affected files ...\n") except ValueError: sys.stderr.write("Change %s doesn't seem to affect any files. Weird.\n" % change) - return [], [], [], [] + return [], [], [], [], [] differencesSection = 0 try: differencesSection = output.index("Differences ...\n") except ValueError: sys.stderr.write("Change %s doesn't seem to have a differences section. Weird.\n" % change) - return [], [], [], [] + return [], [], [], [], [] log = output[2:filesSection - 1] @@ -85,7 +87,7 @@ def describe(change): else: changed.append(path) - return author, log, changed, removed + return author, log, epoch, changed, removed def p4cat(path): return os.popen("p4 print -q \"%s\"" % path).read() @@ -124,9 +126,9 @@ changes.reverse() sys.stderr.write("\n") -cnt = 0 +cnt = 1 for change in changes: - [ author, log, changedFiles, removedFiles ] = describe(change) + [ author, log, epoch, changedFiles, removedFiles ] = describe(change) sys.stderr.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) cnt = cnt + 1 # sys.stderr.write("%s\n" % log) @@ -135,9 +137,9 @@ for change in changes: print "commit refs/heads/master" if author in users: - print "committer %s 1 2" % users[author] + print "committer %s %s +0000" % (users[author], epoch) else: - print "committer %s 1 2" % author + print "committer %s %s +0000" % (author, epoch) print "data < Date: Wed, 31 Jan 2007 19:43:16 +0100 Subject: [PATCH 005/260] Some fixes to the timezone conversion between the date of a perforce change and the git commit. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 689349d551..f3b5f35cb3 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -46,7 +46,7 @@ def describe(change): splitted = firstLine.split(" ") author = splitted[3] author = author[:author.find("@")] - tm = time.strptime(splitted[5] + " " + splitted[6] + time.tzname[0], "%Y/%m/%d %H:%M:%S %Z") + tm = time.strptime(splitted[5] + " " + splitted[6], "%Y/%m/%d %H:%M:%S ") epoch = int(time.mktime(tm)) filesSection = 0 @@ -126,6 +126,8 @@ changes.reverse() sys.stderr.write("\n") +tz = - time.timezone / 36 + cnt = 1 for change in changes: [ author, log, epoch, changedFiles, removedFiles ] = describe(change) @@ -137,9 +139,9 @@ for change in changes: print "commit refs/heads/master" if author in users: - print "committer %s %s +0000" % (users[author], epoch) + print "committer %s %s %s" % (users[author], epoch, tz) else: - print "committer %s %s +0000" % (author, epoch) + print "committer %s %s %s" % (author, epoch, tz) print "data < Date: Wed, 31 Jan 2007 20:16:26 +0100 Subject: [PATCH 006/260] Speed up the import of individual files from Perforce into git by passing the output of "p4 print" directly to git fast-import. Also try to set the mode of the file in git correctly based on file type heuristics. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 31 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index f3b5f35cb3..72a4fd70a5 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -5,12 +5,11 @@ # Author: Simon Hausmann # License: MIT # -# TODO: - support integrations (at least p4i) +# TODO: +# - support integrations (at least p4i) # - support incremental imports # - create tags # - instead of reading all files into a variable try to pipe from -# - p4 print directly to stdout. need to figure out file size somehow -# though. # - support p4 submit (hah!) # - don't hardcode the import to master # @@ -92,6 +91,17 @@ def describe(change): def p4cat(path): return os.popen("p4 print -q \"%s\"" % path).read() +def p4Stat(path): + output = os.popen("p4 fstat -Ol \"%s\"" % path).readlines() + fileSize = 0 + mode = 644 + for line in output: + if line.startswith("... headType x"): + mode = 755 + elif line.startswith("... fileSize "): + fileSize = long(line[12:]) + return mode, fileSize + def stripRevision(path): hashPos = path.rindex("#") return path[:hashPos] @@ -112,7 +122,6 @@ def getUserMap(): return users - users = getUserMap() output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() @@ -133,9 +142,6 @@ for change in changes: [ author, log, epoch, changedFiles, removedFiles ] = describe(change) sys.stderr.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) cnt = cnt + 1 -# sys.stderr.write("%s\n" % log) -# sys.stderr.write("%s\n" % changedFiles) -# sys.stderr.write("%s\n" % removedFiles) print "commit refs/heads/master" if author in users: @@ -154,10 +160,13 @@ for change in changes: sys.stderr.write("\nchanged files: ignoring path %s outside of %s in change %s\n" % (f, prefix, change)) continue relpath = f[len(prefix):] - print "M 644 inline %s" % stripRevision(relpath) - data = p4cat(f) - print "data %s" % len(data) - sys.stdout.write(data) + + [mode, fileSize] = p4Stat(f) + + print "M %s inline %s" % (mode, stripRevision(relpath)) + print "data %s" % fileSize + sys.stdout.flush(); + os.system("p4 print -q \"%s\"" % f) print "" for f in removedFiles: From 3f2ddd47c7f311516dc7092b0a89b46291e40271 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 20:48:39 +0100 Subject: [PATCH 007/260] Removed unused p4cat function and added helper function for the perforce python interface (p4Cmd). Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 72a4fd70a5..e284b19502 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -14,6 +14,7 @@ # - don't hardcode the import to master # import os, string, sys, time +import marshal if len(sys.argv) != 2: sys.stderr.write("usage: %s //depot/path[@revRange]\n" % sys.argv[0]); @@ -37,6 +38,18 @@ except ValueError: if not prefix.endswith("/"): prefix += "/" +def p4Cmd(cmd): + pipe = os.popen("p4 -G %s" % cmd, "rb") + result = {} + try: + while True: + entry = marshal.load(pipe) + result.update(entry) + except EOFError: + pass + pipe.close() + return result + def describe(change): output = os.popen("p4 describe %s" % change).readlines() @@ -88,9 +101,6 @@ def describe(change): return author, log, epoch, changed, removed -def p4cat(path): - return os.popen("p4 print -q \"%s\"" % path).read() - def p4Stat(path): output = os.popen("p4 fstat -Ol \"%s\"" % path).readlines() fileSize = 0 From 701ce876331eca10031c104f42cc6fccc425ac94 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 21:54:56 +0100 Subject: [PATCH 008/260] Changed the import mechanism to write to git fast-import through a pipe instead of having p4-fast-export write to stdout and let the caller connect it to git fast-import. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 52 ++++++++++++++------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index e284b19502..662dd01d8d 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -14,16 +14,16 @@ # - don't hardcode the import to master # import os, string, sys, time -import marshal +import marshal, popen2 if len(sys.argv) != 2: - sys.stderr.write("usage: %s //depot/path[@revRange]\n" % sys.argv[0]); - sys.stderr.write("\n example:\n"); - sys.stderr.write(" %s //depot/my/project/ -- to import everything\n"); - sys.stderr.write(" %s //depot/my/project/@1,6 -- to import only from revision 1 to 6\n"); - sys.stderr.write("\n"); - sys.stderr.write(" (a ... is not needed in the path p4 specification, it's added implicitly)\n"); - sys.stderr.write("\n"); + print "usage: %s //depot/path[@revRange]" % sys.argv[0] + print "\n example:" + print " %s //depot/my/project/ -- to import everything" + print " %s //depot/my/project/@1,6 -- to import only from revision 1 to 6" + print "" + print " (a ... is not needed in the path p4 specification, it's added implicitly)" + print "" sys.exit(1) prefix = sys.argv[1] @@ -147,23 +147,23 @@ sys.stderr.write("\n") tz = - time.timezone / 36 +gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") + cnt = 1 for change in changes: [ author, log, epoch, changedFiles, removedFiles ] = describe(change) - sys.stderr.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) cnt = cnt + 1 - print "commit refs/heads/master" + gitStream.write("commit refs/heads/master\n") if author in users: - print "committer %s %s %s" % (users[author], epoch, tz) + gitStream.write("committer %s %s %s\n" % (users[author], epoch, tz)) else: - print "committer %s %s %s" % (author, epoch, tz) - print "data < %s %s\n" % (author, epoch, tz)) + gitStream.write("data < Date: Wed, 31 Jan 2007 22:13:17 +0100 Subject: [PATCH 009/260] Minor code cleanups and ported some p4 interfacing code over to the p4 python mode. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 53 +++++++-------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 662dd01d8d..3ccef526eb 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -51,54 +51,30 @@ def p4Cmd(cmd): return result def describe(change): - output = os.popen("p4 describe %s" % change).readlines() + describeOutput = p4Cmd("describe %s" % change) - firstLine = output[0] + author = describeOutput["user"] + epoch = describeOutput["time"] - splitted = firstLine.split(" ") - author = splitted[3] - author = author[:author.find("@")] - tm = time.strptime(splitted[5] + " " + splitted[6], "%Y/%m/%d %H:%M:%S ") - epoch = int(time.mktime(tm)) - - filesSection = 0 - try: - filesSection = output.index("Affected files ...\n") - except ValueError: - sys.stderr.write("Change %s doesn't seem to affect any files. Weird.\n" % change) - return [], [], [], [], [] - - differencesSection = 0 - try: - differencesSection = output.index("Differences ...\n") - except ValueError: - sys.stderr.write("Change %s doesn't seem to have a differences section. Weird.\n" % change) - return [], [], [], [], [] - - log = output[2:filesSection - 1] - - lines = output[filesSection + 2:differencesSection - 1] + log = describeOutput["desc"] changed = [] removed = [] - for line in lines: - # chop off "... " and trailing newline - line = line[4:len(line) - 1] + i = 0 + while describeOutput.has_key("depotFile%s" % i): + path = describeOutput["depotFile%s" % i] + rev = describeOutput["rev%s" % i] + action = describeOutput["action%s" % i] + path = path + "#" + rev - lastSpace = line.rfind(" ") - if lastSpace == -1: - sys.stderr.write("trouble parsing line %s, skipping!\n" % line) - continue - - operation = line[lastSpace + 1:] - path = line[:lastSpace] - - if operation == "delete": + if action == "delete": removed.append(path) else: changed.append(path) + i = i + 1 + return author, log, epoch, changed, removed def p4Stat(path): @@ -161,8 +137,7 @@ for change in changes: else: gitStream.write("committer %s %s %s\n" % (author, epoch, tz)) gitStream.write("data < Date: Wed, 31 Jan 2007 22:19:18 +0100 Subject: [PATCH 010/260] Instead of parsing the output of "p4 users" use the python objects of "p4 -G users". Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 28 +++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 3ccef526eb..d3e65f0f39 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -38,18 +38,25 @@ except ValueError: if not prefix.endswith("/"): prefix += "/" -def p4Cmd(cmd): +def p4CmdList(cmd): pipe = os.popen("p4 -G %s" % cmd, "rb") - result = {} + result = [] try: while True: entry = marshal.load(pipe) - result.update(entry) + result.append(entry) except EOFError: pass pipe.close() return result +def p4Cmd(cmd): + list = p4CmdList(cmd) + result = {} + for entry in list: + result.update(entry) + return result; + def describe(change): describeOutput = p4Cmd("describe %s" % change) @@ -94,18 +101,11 @@ def stripRevision(path): def getUserMap(): users = {} - output = os.popen("p4 users") - for line in output: - firstSpace = line.index(" ") - secondSpace = line.index(" ", firstSpace + 1) - key = line[:firstSpace] - email = line[firstSpace + 1:secondSpace] - openParenPos = line.index("(", secondSpace) - closedParenPos = line.index(")", openParenPos) - name = line[openParenPos + 1:closedParenPos] - - users[key] = name + " " + email + for output in p4CmdList("users"): + if not output.has_key("User"): + continue + users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" return users users = getUserMap() From 0dd0b9d01116d57c612a5fa90de803bdd836ce11 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 22:31:28 +0100 Subject: [PATCH 011/260] Ported the remaining functions that parsed p4 shell output over to the p4 python interface. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 89 ++++++++++----------------- 1 file changed, 31 insertions(+), 58 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index d3e65f0f39..45d5157961 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -57,47 +57,8 @@ def p4Cmd(cmd): result.update(entry) return result; -def describe(change): - describeOutput = p4Cmd("describe %s" % change) - - author = describeOutput["user"] - epoch = describeOutput["time"] - - log = describeOutput["desc"] - - changed = [] - removed = [] - - i = 0 - while describeOutput.has_key("depotFile%s" % i): - path = describeOutput["depotFile%s" % i] - rev = describeOutput["rev%s" % i] - action = describeOutput["action%s" % i] - path = path + "#" + rev - - if action == "delete": - removed.append(path) - else: - changed.append(path) - - i = i + 1 - - return author, log, epoch, changed, removed - -def p4Stat(path): - output = os.popen("p4 fstat -Ol \"%s\"" % path).readlines() - fileSize = 0 - mode = 644 - for line in output: - if line.startswith("... headType x"): - mode = 755 - elif line.startswith("... fileSize "): - fileSize = long(line[12:]) - return mode, fileSize - -def stripRevision(path): - hashPos = path.rindex("#") - return path[:hashPos] +def p4FileSize(path): + return int(p4Cmd("fstat -Ol \"%s\"" % path)["fileSize"]) def getUserMap(): users = {} @@ -127,38 +88,50 @@ gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") cnt = 1 for change in changes: - [ author, log, epoch, changedFiles, removedFiles ] = describe(change) + description = p4Cmd("describe %s" % change) + sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) cnt = cnt + 1 + epoch = description["time"] + author = description["user"] + gitStream.write("commit refs/heads/master\n") if author in users: gitStream.write("committer %s %s %s\n" % (users[author], epoch, tz)) else: gitStream.write("committer %s %s %s\n" % (author, epoch, tz)) gitStream.write("data < Date: Wed, 31 Jan 2007 22:38:07 +0100 Subject: [PATCH 012/260] Avoid calling fstat for every imported file (slow!) and instead read the file data first into the python process and use the length of the bytes read for the size field of git fast-import. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 45d5157961..133447c4e7 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -57,9 +57,6 @@ def p4Cmd(cmd): result.update(entry) return result; -def p4FileSize(path): - return int(p4Cmd("fstat -Ol \"%s\"" % path)["fileSize"]) - def getUserMap(): users = {} @@ -121,14 +118,15 @@ for change in changes: if action == "delete": gitStream.write("D %s\n" % relPath) else: - fileSize = p4FileSize(depotPath) mode = 644 if description["type%s" % fnum].startswith("x"): mode = 755 + data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() + gitStream.write("M %s inline %s\n" % (mode, relPath)) - gitStream.write("data %s\n" % fileSize) - gitStream.write(os.popen("p4 print -q \"%s\"" % depotPath).read()) + gitStream.write("data %s\n" % len(data)) + gitStream.write(data) gitStream.write("\n") fnum = fnum + 1 From f26037dce365b3e525596c6f0236ab203d87a872 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 22:41:08 +0100 Subject: [PATCH 013/260] Permit calling p4-fast-export with a depot path that has the typical ... wildcard at the end. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 133447c4e7..72e01224bf 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -35,6 +35,9 @@ try: except ValueError: changeRange = "" +if prefix.endswith("..."): + prefix = prefix[:-3] + if not prefix.endswith("/"): prefix += "/" From 214bed82390479bdba31337ac420bb09e9b366e9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 22:47:53 +0100 Subject: [PATCH 014/260] Fixed displaying import progress by calling flush on stdout. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 72e01224bf..a1dc54013e 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -91,6 +91,7 @@ for change in changes: description = p4Cmd("describe %s" % change) sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.flush() cnt = cnt + 1 epoch = description["time"] From 71f7c0d0bb20936fc831e49bbfc9355f1e5ca211 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 31 Jan 2007 23:03:01 +0100 Subject: [PATCH 015/260] Create a git tag for every changeset imported from perforce. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index a1dc54013e..588554d672 100644 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -98,12 +98,17 @@ for change in changes: author = description["user"] gitStream.write("commit refs/heads/master\n") + committer = "" if author in users: - gitStream.write("committer %s %s %s\n" % (users[author], epoch, tz)) + committer = "%s %s %s" % (users[author], epoch, tz) else: - gitStream.write("committer %s %s %s\n" % (author, epoch, tz)) + committer = "%s %s %s" % (author, epoch, tz) + + gitStream.write("committer %s\n" % committer) + gitStream.write("data < Date: Wed, 31 Jan 2007 23:09:24 +0100 Subject: [PATCH 016/260] Fix file permissions of p4-fast-export.py to be executable. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 contrib/fast-import/p4-fast-export.py diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py old mode 100644 new mode 100755 From 61b3cf7c471d2f94eefcf6616c1204f32e641542 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 1 Feb 2007 00:08:51 +0100 Subject: [PATCH 017/260] Started working on incremental imports from Perforce. Try to find the last imported p4 change number from the git tags and try to pass the right parent for commits to git fast-import. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 37 +++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 588554d672..da3eb35841 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -26,6 +26,8 @@ if len(sys.argv) != 2: print "" sys.exit(1) +master = "refs/heads/p4" +branch = "refs/heads/p4-import" prefix = sys.argv[1] changeRange = "" try: @@ -70,6 +72,25 @@ def getUserMap(): return users users = getUserMap() +topMerge = "" + +incremental = 0 +# try incremental import +if len(changeRange) == 0: + try: + sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % master) + output = sout.read() + tagIdx = output.index(" tags/p4/") + caretIdx = output.index("^") + revision = int(output[tagIdx + 9 : caretIdx]) + 1 + changeRange = "@%s,#head" % revision + topMerge = os.popen("git-rev-parse %s" % master).read()[:-1] + incremental = 1 + except: + pass + +if incremental == 0: + branch = master output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() @@ -80,6 +101,10 @@ for line in output: changes.reverse() +if len(changes) == 0: + print "no changes to import!" + sys.exit(1) + sys.stderr.write("\n") tz = - time.timezone / 36 @@ -97,7 +122,7 @@ for change in changes: epoch = description["time"] author = description["user"] - gitStream.write("commit refs/heads/master\n") + gitStream.write("commit %s\n" % branch) committer = "" if author in users: committer = "%s %s %s" % (users[author], epoch, tz) @@ -111,6 +136,10 @@ for change in changes: gitStream.write("\n[ imported from %s; change %s ]\n" % (prefix, change)) gitStream.write("EOT\n\n") + if len(topMerge) > 0: + gitStream.write("merge %s\n" % topMerge) + topMerge = "" + fnum = 0 while description.has_key("depotFile%s" % fnum): path = description["depotFile%s" % fnum] @@ -143,7 +172,7 @@ for change in changes: gitStream.write("\n") gitStream.write("tag p4/%s\n" % change) - gitStream.write("from refs/heads/master\n"); + gitStream.write("from %s\n" % branch); gitStream.write("tagger %s\n" % committer); gitStream.write("data 0\n\n") @@ -152,4 +181,8 @@ gitStream.close() gitOutput.close() gitError.close() +if incremental == 1: + os.popen("git rebase p4-import p4") + os.popen("git branch -d p4-import") + print "" From f16255f559c96b8c5fb096b00c94a69f5fcc498f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 1 Feb 2007 08:23:39 +0100 Subject: [PATCH 018/260] Simplify the incremental import by elimination the need for a temporary import branch. It turns out that git fast-import can "resume" from an existing branch just fine. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index da3eb35841..c5b15206b1 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -7,11 +7,7 @@ # # TODO: # - support integrations (at least p4i) -# - support incremental imports -# - create tags -# - instead of reading all files into a variable try to pipe from # - support p4 submit (hah!) -# - don't hardcode the import to master # import os, string, sys, time import marshal, popen2 @@ -26,8 +22,7 @@ if len(sys.argv) != 2: print "" sys.exit(1) -master = "refs/heads/p4" -branch = "refs/heads/p4-import" +branch = "refs/heads/p4" prefix = sys.argv[1] changeRange = "" try: @@ -74,24 +69,18 @@ def getUserMap(): users = getUserMap() topMerge = "" -incremental = 0 -# try incremental import if len(changeRange) == 0: try: - sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % master) + sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) output = sout.read() tagIdx = output.index(" tags/p4/") caretIdx = output.index("^") revision = int(output[tagIdx + 9 : caretIdx]) + 1 changeRange = "@%s,#head" % revision - topMerge = os.popen("git-rev-parse %s" % master).read()[:-1] - incremental = 1 + topMerge = os.popen("git-rev-parse %s" % branch).read()[:-1] except: pass -if incremental == 0: - branch = master - output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() changes = [] @@ -181,8 +170,4 @@ gitStream.close() gitOutput.close() gitError.close() -if incremental == 1: - os.popen("git rebase p4-import p4") - os.popen("git branch -d p4-import") - print "" From 68f1336fe370ca2153d691631876b0f85a3fb17f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 1 Feb 2007 17:42:23 +0100 Subject: [PATCH 019/260] Code cleanups, move the code to create a commit with fast-import into a separate function out of the main loop. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 126 ++++++++++++++------------ 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index c5b15206b1..8455c1f41c 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -25,6 +25,9 @@ if len(sys.argv) != 2: branch = "refs/heads/p4" prefix = sys.argv[1] changeRange = "" +users = {} +initialParent = "" + try: atIdx = prefix.index("@") changeRange = prefix[atIdx:] @@ -57,6 +60,68 @@ def p4Cmd(cmd): result.update(entry) return result; +def commit(details): + global initialParent + global users + + epoch = details["time"] + author = details["user"] + + gitStream.write("commit %s\n" % branch) + committer = "" + if author in users: + committer = "%s %s %s" % (users[author], epoch, tz) + else: + committer = "%s %s %s" % (author, epoch, tz) + + gitStream.write("committer %s\n" % committer) + + gitStream.write("data < 0: + gitStream.write("merge %s\n" % initialParent) + initialParent = "" + + fnum = 0 + while details.has_key("depotFile%s" % fnum): + path = details["depotFile%s" % fnum] + if not path.startswith(prefix): + print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, prefix, change) + fnum = fnum + 1 + continue + + rev = details["rev%s" % fnum] + depotPath = path + "#" + rev + relPath = path[len(prefix):] + action = details["action%s" % fnum] + + if action == "delete": + gitStream.write("D %s\n" % relPath) + else: + mode = 644 + if details["type%s" % fnum].startswith("x"): + mode = 755 + + data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() + + gitStream.write("M %s inline %s\n" % (mode, relPath)) + gitStream.write("data %s\n" % len(data)) + gitStream.write(data) + gitStream.write("\n") + + fnum = fnum + 1 + + gitStream.write("\n") + + gitStream.write("tag p4/%s\n" % change) + gitStream.write("from %s\n" % branch); + gitStream.write("tagger %s\n" % committer); + gitStream.write("data 0\n\n") + + def getUserMap(): users = {} @@ -67,7 +132,6 @@ def getUserMap(): return users users = getUserMap() -topMerge = "" if len(changeRange) == 0: try: @@ -77,7 +141,7 @@ if len(changeRange) == 0: caretIdx = output.index("^") revision = int(output[tagIdx + 9 : caretIdx]) + 1 changeRange = "@%s,#head" % revision - topMerge = os.popen("git-rev-parse %s" % branch).read()[:-1] + initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1] except: pass @@ -108,63 +172,7 @@ for change in changes: sys.stdout.flush() cnt = cnt + 1 - epoch = description["time"] - author = description["user"] - - gitStream.write("commit %s\n" % branch) - committer = "" - if author in users: - committer = "%s %s %s" % (users[author], epoch, tz) - else: - committer = "%s %s %s" % (author, epoch, tz) - - gitStream.write("committer %s\n" % committer) - - gitStream.write("data < 0: - gitStream.write("merge %s\n" % topMerge) - topMerge = "" - - fnum = 0 - while description.has_key("depotFile%s" % fnum): - path = description["depotFile%s" % fnum] - if not path.startswith(prefix): - print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, prefix, change) - fnum = fnum + 1 - continue - - rev = description["rev%s" % fnum] - depotPath = path + "#" + rev - relPath = path[len(prefix):] - action = description["action%s" % fnum] - - if action == "delete": - gitStream.write("D %s\n" % relPath) - else: - mode = 644 - if description["type%s" % fnum].startswith("x"): - mode = 755 - - data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() - - gitStream.write("M %s inline %s\n" % (mode, relPath)) - gitStream.write("data %s\n" % len(data)) - gitStream.write(data) - gitStream.write("\n") - - fnum = fnum + 1 - - gitStream.write("\n") - - gitStream.write("tag p4/%s\n" % change) - gitStream.write("from %s\n" % branch); - gitStream.write("tagger %s\n" % committer); - gitStream.write("data 0\n\n") - + commit(description) gitStream.close() gitOutput.close() From 6d48d12f5d34a2acb6d8b56bfe3c26d4db92b1a5 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 1 Feb 2007 18:19:55 +0100 Subject: [PATCH 020/260] Initial support for importing a directory from Perforce at a specified revision. Use p4 files //depot/path/...@revision to determine the state of the project and create a "fake" git commit from it. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 98 ++++++++++++++++++--------- 1 file changed, 67 insertions(+), 31 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 8455c1f41c..19f5f03476 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -25,15 +25,21 @@ if len(sys.argv) != 2: branch = "refs/heads/p4" prefix = sys.argv[1] changeRange = "" +revision = "" users = {} initialParent = "" -try: +if prefix.find("@") != -1: atIdx = prefix.index("@") changeRange = prefix[atIdx:] + if changeRange.find(",") == -1: + revision = changeRange + changeRange = "" prefix = prefix[0:atIdx] -except ValueError: - changeRange = "" +elif prefix.find("#") != -1: + hashIdx = prefix.index("#") + revision = prefix[hashIdx:] + prefix = prefix[0:hashIdx] if prefix.endswith("..."): prefix = prefix[:-3] @@ -78,7 +84,7 @@ def commit(details): gitStream.write("data < 0: @@ -116,7 +122,7 @@ def commit(details): gitStream.write("\n") - gitStream.write("tag p4/%s\n" % change) + gitStream.write("tag p4/%s\n" % details["change"]) gitStream.write("from %s\n" % branch); gitStream.write("tagger %s\n" % committer); gitStream.write("data 0\n\n") @@ -139,43 +145,73 @@ if len(changeRange) == 0: output = sout.read() tagIdx = output.index(" tags/p4/") caretIdx = output.index("^") - revision = int(output[tagIdx + 9 : caretIdx]) + 1 - changeRange = "@%s,#head" % revision + rev = int(output[tagIdx + 9 : caretIdx]) + 1 + changeRange = "@%s,#head" % rev initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1] except: pass -output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() - -changes = [] -for line in output: - changeNum = line.split(" ")[1] - changes.append(changeNum) - -changes.reverse() - -if len(changes) == 0: - print "no changes to import!" - sys.exit(1) - sys.stderr.write("\n") tz = - time.timezone / 36 -gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") +if len(revision) > 0: + print "Doing initial import of %s from revision %s" % (prefix, revision) -cnt = 1 -for change in changes: - description = p4Cmd("describe %s" % change) + details = { "user" : "git perforce import user", "time" : int(time.time()) } + details["desc"] = "Initial import of %s from the state at revision %s" % (prefix, revision) + details["change"] = revision + newestRevision = 0 - sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) - sys.stdout.flush() - cnt = cnt + 1 + fileCnt = 0 + for info in p4CmdList("files %s...%s" % (prefix, revision)): + if info["action"] == "delete": + continue + for prop in [ "depotFile", "rev", "action", "type" ]: + details["%s%s" % (prop, fileCnt)] = info[prop] - commit(description) + change = info["change"] + if change > newestRevision: + newestRevision = change -gitStream.close() -gitOutput.close() -gitError.close() + fileCnt = fileCnt + 1 + + details["change"] = newestRevision + + gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") + commit(details) + + gitStream.close() + gitOutput.close() + gitError.close() +else: + output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() + + changes = [] + for line in output: + changeNum = line.split(" ")[1] + changes.append(changeNum) + + changes.reverse() + + if len(changes) == 0: + print "no changes to import!" + sys.exit(1) + + gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") + + cnt = 1 + for change in changes: + description = p4Cmd("describe %s" % change) + + sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.flush() + cnt = cnt + 1 + + commit(description) + + gitStream.close() + gitOutput.close() + gitError.close() print "" From c4cf2d4f8793ea8f889052391d72dd4066e88155 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 7 Feb 2007 22:57:01 +0100 Subject: [PATCH 021/260] Minor cleanups and print an error message of git fast-import if it fails. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 19f5f03476..06de0eae6d 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -154,6 +154,9 @@ if len(changeRange) == 0: sys.stderr.write("\n") tz = - time.timezone / 36 +tzsign = ("%s" % tz)[0] +if tzsign != '+' and tzsign != '-': + tz = "+" + ("%s" % tz) if len(revision) > 0: print "Doing initial import of %s from revision %s" % (prefix, revision) @@ -165,21 +168,25 @@ if len(revision) > 0: fileCnt = 0 for info in p4CmdList("files %s...%s" % (prefix, revision)): - if info["action"] == "delete": - continue - for prop in [ "depotFile", "rev", "action", "type" ]: - details["%s%s" % (prop, fileCnt)] = info[prop] - change = info["change"] if change > newestRevision: newestRevision = change + if info["action"] == "delete": + continue + + for prop in [ "depotFile", "rev", "action", "type" ]: + details["%s%s" % (prop, fileCnt)] = info[prop] + fileCnt = fileCnt + 1 details["change"] = newestRevision gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") - commit(details) + try: + commit(details) + except: + print gitError.read() gitStream.close() gitOutput.close() From e3d37cf0980646b3dbc19e01684c665482a3129d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 7 Feb 2007 23:09:49 +0100 Subject: [PATCH 022/260] Fixed incremental imports by using the correct "from" command instead of "merge" with git fast-import. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 06de0eae6d..3e573cd786 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -88,7 +88,7 @@ def commit(details): gitStream.write("EOT\n\n") if len(initialParent) > 0: - gitStream.write("merge %s\n" % initialParent) + gitStream.write("from %s\n" % initialParent) initialParent = "" fnum = 0 From 1cd573866a76d62866b65c458e723983ba58117e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 7 Feb 2007 23:25:56 +0100 Subject: [PATCH 023/260] Make incremental imports easier to use by storing the p4 depot path after an import in .git/config and re-using it when we're invoked again later. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 3e573cd786..be4fd36c36 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -12,7 +12,14 @@ import os, string, sys, time import marshal, popen2 -if len(sys.argv) != 2: +branch = "refs/heads/p4" +prefix = os.popen("git-repo-config --get p4.depotpath").read() +if len(prefix) != 0: + prefix = prefix[:-1] + +if len(sys.argv) == 1 and len(prefix) != 0: + print "[using previously specified depot path %s]" % prefix +elif len(sys.argv) != 2: print "usage: %s //depot/path[@revRange]" % sys.argv[0] print "\n example:" print " %s //depot/my/project/ -- to import everything" @@ -21,9 +28,12 @@ if len(sys.argv) != 2: print " (a ... is not needed in the path p4 specification, it's added implicitly)" print "" sys.exit(1) +else: + if len(prefix) != 0 and prefix != sys.argv[1]: + print "previous import used depot path %s and now %s was specified. this doesn't work!" % (prefix, sys.argv[1]) + sys.exit(1) + prefix = sys.argv[1] -branch = "refs/heads/p4" -prefix = sys.argv[1] changeRange = "" revision = "" users = {} @@ -222,3 +232,6 @@ else: gitError.close() print "" + +os.popen("git-repo-config p4.depotpath %s" % prefix).read() + From 23efd2545b292208740b0f2fb7446d99ce744000 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 7 Feb 2007 23:37:54 +0100 Subject: [PATCH 024/260] Make specifying the revision ranges more convenient. Added support for @all as revision range specifier to import all changes to a given depot path. Also default to an import of #head if no revrange is specified. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index be4fd36c36..16e3d8d42a 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -13,7 +13,7 @@ import os, string, sys, time import marshal, popen2 branch = "refs/heads/p4" -prefix = os.popen("git-repo-config --get p4.depotpath").read() +prefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() if len(prefix) != 0: prefix = prefix[:-1] @@ -22,7 +22,8 @@ if len(sys.argv) == 1 and len(prefix) != 0: elif len(sys.argv) != 2: print "usage: %s //depot/path[@revRange]" % sys.argv[0] print "\n example:" - print " %s //depot/my/project/ -- to import everything" + print " %s //depot/my/project/ -- to import the current head" + print " %s //depot/my/project/@all -- to import everything" print " %s //depot/my/project/@1,6 -- to import only from revision 1 to 6" print "" print " (a ... is not needed in the path p4 specification, it's added implicitly)" @@ -42,7 +43,9 @@ initialParent = "" if prefix.find("@") != -1: atIdx = prefix.index("@") changeRange = prefix[atIdx:] - if changeRange.find(",") == -1: + if changeRange == "@all": + changeRange = "" + elif changeRange.find(",") == -1: revision = changeRange changeRange = "" prefix = prefix[0:atIdx] @@ -50,6 +53,8 @@ elif prefix.find("#") != -1: hashIdx = prefix.index("#") revision = prefix[hashIdx:] prefix = prefix[0:hashIdx] +elif len(previousDepotPath) == 0: + revision = "#head" if prefix.endswith("..."): prefix = prefix[:-3] From 1e30c07dfcd0ca74ff4aec2004b5722e69b5a639 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 7 Feb 2007 23:51:51 +0100 Subject: [PATCH 025/260] Fix calculation of the newest imported revision for #head imports. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 16e3d8d42a..36381fbecd 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -183,7 +183,7 @@ if len(revision) > 0: fileCnt = 0 for info in p4CmdList("files %s...%s" % (prefix, revision)): - change = info["change"] + change = int(info["change"]) if change > newestRevision: newestRevision = change From 7315866824607e0a7da807dbee84c60cfe5c5719 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 8 Feb 2007 15:45:16 +0100 Subject: [PATCH 026/260] Catch io exceptions from git fast-import again and print the error message. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 36381fbecd..9f4b16a974 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -230,7 +230,11 @@ else: sys.stdout.flush() cnt = cnt + 1 - commit(description) + try: + commit(description) + except: + print gitError.read() + sys.exit(1) gitStream.close() gitOutput.close() @@ -240,3 +244,4 @@ print "" os.popen("git-repo-config p4.depotpath %s" % prefix).read() +sys.exit(0) From c9c527d7b60f406e143c1f7947a8c6c5ea9d9a8a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 8 Feb 2007 15:53:11 +0100 Subject: [PATCH 027/260] Made the name of the git branch used for the perforce import configurable through a new --branch= commandline option. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 9f4b16a974..c44292473a 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -10,16 +10,26 @@ # - support p4 submit (hah!) # import os, string, sys, time -import marshal, popen2 +import marshal, popen2, getopt branch = "refs/heads/p4" prefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() if len(prefix) != 0: prefix = prefix[:-1] -if len(sys.argv) == 1 and len(prefix) != 0: +try: + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=" ]) +except getopt.GetoptError: + print "fixme, syntax error" + sys.exit(1) + +for o, a in opts: + if o == "--branch": + branch = "refs/heads/" + a + +if len(args) == 0 and len(prefix) != 0: print "[using previously specified depot path %s]" % prefix -elif len(sys.argv) != 2: +elif len(args) != 1: print "usage: %s //depot/path[@revRange]" % sys.argv[0] print "\n example:" print " %s //depot/my/project/ -- to import the current head" @@ -30,10 +40,10 @@ elif len(sys.argv) != 2: print "" sys.exit(1) else: - if len(prefix) != 0 and prefix != sys.argv[1]: - print "previous import used depot path %s and now %s was specified. this doesn't work!" % (prefix, sys.argv[1]) + if len(prefix) != 0 and prefix != args[0]: + print "previous import used depot path %s and now %s was specified. this doesn't work!" % (prefix, args[0]) sys.exit(1) - prefix = sys.argv[1] + prefix = args[0] changeRange = "" revision = "" From 20c7bc76b90084da6b99a1627427c000c06ef53a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 8 Feb 2007 23:00:19 +0100 Subject: [PATCH 028/260] Added a little helper script to debug the output of the p4 python interface. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-debug.p4 | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 contrib/fast-import/p4-debug.p4 diff --git a/contrib/fast-import/p4-debug.p4 b/contrib/fast-import/p4-debug.p4 new file mode 100755 index 0000000000..8fb159fd6a --- /dev/null +++ b/contrib/fast-import/p4-debug.p4 @@ -0,0 +1,25 @@ +#!/usr/bin/python +# +# p4-debug.py +# +# Author: Simon Hausmann +# License: MIT +# +# executes a p4 command with -G and prints the resulting python dicts +# +import os, string, sys +import marshal, popen2 + +cmd = "" +for arg in sys.argv[1:]: + cmd += arg + " " + +pipe = os.popen("p4 -G %s" % cmd, "rb") +try: + while True: + entry = marshal.load(pipe) + print entry +except EOFError: + pass +pipe.close() + From b41507a42769c9e40d024593c6924769d9cb961b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Feb 2007 09:25:22 +0100 Subject: [PATCH 029/260] Minor code cleanups. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index c44292473a..d1faa7c290 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -183,6 +183,8 @@ tzsign = ("%s" % tz)[0] if tzsign != '+' and tzsign != '-': tz = "+" + ("%s" % tz) +gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") + if len(revision) > 0: print "Doing initial import of %s from revision %s" % (prefix, revision) @@ -207,15 +209,11 @@ if len(revision) > 0: details["change"] = newestRevision - gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") try: commit(details) except: print gitError.read() - gitStream.close() - gitOutput.close() - gitError.close() else: output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() @@ -230,8 +228,6 @@ else: print "no changes to import!" sys.exit(1) - gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") - cnt = 1 for change in changes: description = p4Cmd("describe %s" % change) @@ -246,12 +242,12 @@ else: print gitError.read() sys.exit(1) - gitStream.close() - gitOutput.close() - gitError.close() - print "" +gitStream.close() +gitOutput.close() +gitError.close() + os.popen("git-repo-config p4.depotpath %s" % prefix).read() sys.exit(0) From 8718f3ec9a550c8c2419576a3765c20189029c99 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Feb 2007 10:05:29 +0100 Subject: [PATCH 030/260] Avoid the excessive use of git tags for every perforce change and instead just create one git tag for the last imported change. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index d1faa7c290..907a56dc8a 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -49,6 +49,9 @@ changeRange = "" revision = "" users = {} initialParent = "" +lastChange = "" +lastCommitter = "" +initialTag = "" if prefix.find("@") != -1: atIdx = prefix.index("@") @@ -94,6 +97,8 @@ def p4Cmd(cmd): def commit(details): global initialParent global users + global lastChange + global lastCommitter epoch = details["time"] author = details["user"] @@ -147,11 +152,8 @@ def commit(details): gitStream.write("\n") - gitStream.write("tag p4/%s\n" % details["change"]) - gitStream.write("from %s\n" % branch); - gitStream.write("tagger %s\n" % committer); - gitStream.write("data 0\n\n") - + lastChange = details["change"] + lastCommitter = committer def getUserMap(): users = {} @@ -173,6 +175,7 @@ if len(changeRange) == 0: rev = int(output[tagIdx + 9 : caretIdx]) + 1 changeRange = "@%s,#head" % rev initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1] + initialTag = "p4/%s" % (int(rev) - 1) except: pass @@ -244,10 +247,17 @@ else: print "" +gitStream.write("tag p4/%s\n" % lastChange) +gitStream.write("from %s\n" % branch); +gitStream.write("tagger %s\n" % lastCommitter); +gitStream.write("data 0\n\n") + gitStream.close() gitOutput.close() gitError.close() os.popen("git-repo-config p4.depotpath %s" % prefix).read() +if len(initialTag) > 0: + os.popen("git tag -d %s" % initialTag).read() sys.exit(0) From fe2193183add84d185c0691166fde4460a333412 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Feb 2007 10:05:51 +0100 Subject: [PATCH 031/260] Changed the default git import branch from "p4" to "master". Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 907a56dc8a..1f19cbc560 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -12,7 +12,7 @@ import os, string, sys, time import marshal, popen2, getopt -branch = "refs/heads/p4" +branch = "refs/heads/master" prefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() if len(prefix) != 0: prefix = prefix[:-1] From f7d63b0c99945f4b358df06e4d09837f66db6a88 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Feb 2007 10:26:03 +0100 Subject: [PATCH 032/260] Added a little helper script to remove unused tags from the perforce import. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-clean-tags.py | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 contrib/fast-import/p4-clean-tags.py diff --git a/contrib/fast-import/p4-clean-tags.py b/contrib/fast-import/p4-clean-tags.py new file mode 100755 index 0000000000..0be51f6405 --- /dev/null +++ b/contrib/fast-import/p4-clean-tags.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +# +# p4-debug.py +# +# Author: Simon Hausmann +# License: MIT +# +# removes unused p4 import tags +# +import os, string, sys +import popen2, getopt + +branch = "refs/heads/master" + +try: + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=" ]) +except getopt.GetoptError: + print "fixme, syntax error" + sys.exit(1) + +for o, a in opts: + if o == "--branch": + branch = "refs/heads/" + a + +sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) +output = sout.read() +tagIdx = output.index(" tags/p4/") +caretIdx = output.index("^") +rev = int(output[tagIdx + 9 : caretIdx]) + +allTags = os.popen("git tag -l p4/").readlines() +for i in range(len(allTags)): + allTags[i] = int(allTags[i][3:-1]) + +allTags.sort() + +allTags.remove(rev) + +for rev in allTags: + print os.popen("git tag -d p4/%s" % rev).read() From fc21f8a1dab090ceb63c479705104cbb95585a2f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 11 Feb 2007 18:04:39 +0100 Subject: [PATCH 033/260] Create lightweight git tags (using the "reset" trick) for the incremental import instead of full-blown ones. Also fix parsing the output of git name-rev for figuring out the last imported p4 change number. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 1f19cbc560..989513888a 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -170,9 +170,14 @@ if len(changeRange) == 0: try: sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) output = sout.read() + if output.endswith("\n"): + output = output[:-1] tagIdx = output.index(" tags/p4/") - caretIdx = output.index("^") - rev = int(output[tagIdx + 9 : caretIdx]) + 1 + caretIdx = output.find("^") + endPos = len(output) + if caretIdx != -1: + endPos = caretIdx + rev = int(output[tagIdx + 9 : endPos]) + 1 changeRange = "@%s,#head" % rev initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1] initialTag = "p4/%s" % (int(rev) - 1) @@ -247,10 +252,9 @@ else: print "" -gitStream.write("tag p4/%s\n" % lastChange) -gitStream.write("from %s\n" % branch); -gitStream.write("tagger %s\n" % lastCommitter); -gitStream.write("data 0\n\n") +gitStream.write("reset refs/tags/p4/%s\n" % lastChange) +gitStream.write("from %s\n\n" % branch); + gitStream.close() gitOutput.close() From 12d04ca7da3bcfec123fb56d3d143ccda729a7f4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 11 Feb 2007 21:35:34 +0100 Subject: [PATCH 034/260] Cleanups, remove unused variable. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 989513888a..61b9e67e54 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -50,7 +50,6 @@ revision = "" users = {} initialParent = "" lastChange = "" -lastCommitter = "" initialTag = "" if prefix.find("@") != -1: @@ -98,7 +97,6 @@ def commit(details): global initialParent global users global lastChange - global lastCommitter epoch = details["time"] author = details["user"] @@ -153,7 +151,6 @@ def commit(details): gitStream.write("\n") lastChange = details["change"] - lastCommitter = committer def getUserMap(): users = {} From 44b3add6519d6a85fb75b90173ecd6e7ec1ba650 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Feb 2007 20:28:58 +0100 Subject: [PATCH 035/260] Code cleanups. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 33 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 61b9e67e54..07d6e53852 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -93,7 +93,20 @@ def p4Cmd(cmd): result.update(entry) return result; -def commit(details): +def extractFilesFromCommit(commit): + files = [] + fnum = 0 + while commit.has_key("depotFile%s" % fnum): + file = {} + file["path"] = commit["depotFile%s" % fnum] + file["rev"] = commit["rev%s" % fnum] + file["action"] = commit["action%s" % fnum] + file["type"] = commit["type%s" % fnum] + files.append(file) + fnum = fnum + 1 + return files + +def commit(details, files, branch): global initialParent global users global lastChange @@ -119,24 +132,22 @@ def commit(details): gitStream.write("from %s\n" % initialParent) initialParent = "" - fnum = 0 - while details.has_key("depotFile%s" % fnum): - path = details["depotFile%s" % fnum] + for file in files: + path = file["path"] if not path.startswith(prefix): print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, prefix, change) - fnum = fnum + 1 continue - rev = details["rev%s" % fnum] + rev = file["rev"] depotPath = path + "#" + rev relPath = path[len(prefix):] - action = details["action%s" % fnum] + action = file["action"] if action == "delete": gitStream.write("D %s\n" % relPath) else: mode = 644 - if details["type%s" % fnum].startswith("x"): + if file["type"].startswith("x"): mode = 755 data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -146,8 +157,6 @@ def commit(details): gitStream.write(data) gitStream.write("\n") - fnum = fnum + 1 - gitStream.write("\n") lastChange = details["change"] @@ -215,7 +224,7 @@ if len(revision) > 0: details["change"] = newestRevision try: - commit(details) + commit(details, extractFilesFromCommit(details), branch) except: print gitError.read() @@ -242,7 +251,7 @@ else: cnt = cnt + 1 try: - commit(description) + commit(description, extractFilesFromCommit(description), branch) except: print gitError.read() sys.exit(1) From 766887e110898d4ec6f08d19061db323a1ac2b27 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Feb 2007 21:04:59 +0100 Subject: [PATCH 036/260] Started work on p4 branch detection (experimental!). Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 59 +++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 07d6e53852..01bf5baf9a 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -14,11 +14,12 @@ import marshal, popen2, getopt branch = "refs/heads/master" prefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() +detectBranches = False if len(prefix) != 0: prefix = prefix[:-1] try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=" ]) + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -26,6 +27,8 @@ except getopt.GetoptError: for o, a in opts: if o == "--branch": branch = "refs/heads/" + a + elif o == "--detect-branches": + detectBranches = True if len(args) == 0 and len(prefix) != 0: print "[using previously specified depot path %s]" % prefix @@ -97,8 +100,13 @@ def extractFilesFromCommit(commit): files = [] fnum = 0 while commit.has_key("depotFile%s" % fnum): + path = commit["depotFile%s" % fnum] + if not path.startswith(prefix): + print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, prefix, change) + continue + file = {} - file["path"] = commit["depotFile%s" % fnum] + file["path"] = path file["rev"] = commit["rev%s" % fnum] file["action"] = commit["action%s" % fnum] file["type"] = commit["type%s" % fnum] @@ -106,7 +114,37 @@ def extractFilesFromCommit(commit): fnum = fnum + 1 return files -def commit(details, files, branch): +def branchesForCommit(files): + branches = set() + + for file in files: + relativePath = file["path"][len(prefix):] + # strip off the filename + relativePath = relativePath[0:relativePath.rfind("/")] + + if len(branches) == 0: + branches.add(relativePath) + continue + + ###### this needs more testing :) + knownBranch = False + for branch in branches: + if relativePath == branch: + knownBranch = True + break + if relativePath.startswith(branch): + knownBranch = True + break + if branch.startswith(relativePath): + branches.remove(branch) + break + + if not knownBranch: + branches.add(relativePath) + + return branches + +def commit(details, files, branch, prefix): global initialParent global users global lastChange @@ -134,10 +172,6 @@ def commit(details, files, branch): for file in files: path = file["path"] - if not path.startswith(prefix): - print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, prefix, change) - continue - rev = file["rev"] depotPath = path + "#" + rev relPath = path[len(prefix):] @@ -224,7 +258,7 @@ if len(revision) > 0: details["change"] = newestRevision try: - commit(details, extractFilesFromCommit(details), branch) + commit(details, extractFilesFromCommit(details), branch, prefix) except: print gitError.read() @@ -251,7 +285,14 @@ else: cnt = cnt + 1 try: - commit(description, extractFilesFromCommit(description), branch) + files = extractFilesFromCommit(description) + if detectBranches: + for branch in branchesForCommit(files): + branchPrefix = prefix + branch + "/" + branch = "refs/heads/" + branch + commit(description, files, branch, branchPrefix) + else: + commit(description, files, branch, prefix) except: print gitError.read() sys.exit(1) From dcacf8b447be74efaa5335f42c502ee1c397c436 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Feb 2007 21:41:45 +0100 Subject: [PATCH 037/260] More fixes in heuristic p4 branch detection based on common path components. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 108 ++++++++++++++++++-------- 1 file changed, 74 insertions(+), 34 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 01bf5baf9a..f9653f1344 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -12,11 +12,12 @@ import os, string, sys, time import marshal, popen2, getopt +knownBranches = set() branch = "refs/heads/master" -prefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() +globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() detectBranches = False -if len(prefix) != 0: - prefix = prefix[:-1] +if len(globalPrefix) != 0: + globalPrefix = globalPrefix[:-1] try: opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches" ]) @@ -30,8 +31,8 @@ for o, a in opts: elif o == "--detect-branches": detectBranches = True -if len(args) == 0 and len(prefix) != 0: - print "[using previously specified depot path %s]" % prefix +if len(args) == 0 and len(globalPrefix) != 0: + print "[using previously specified depot path %s]" % globalPrefix elif len(args) != 1: print "usage: %s //depot/path[@revRange]" % sys.argv[0] print "\n example:" @@ -43,10 +44,10 @@ elif len(args) != 1: print "" sys.exit(1) else: - if len(prefix) != 0 and prefix != args[0]: - print "previous import used depot path %s and now %s was specified. this doesn't work!" % (prefix, args[0]) + if len(globalPrefix) != 0 and globalPrefix != args[0]: + print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0]) sys.exit(1) - prefix = args[0] + globalPrefix = args[0] changeRange = "" revision = "" @@ -55,27 +56,27 @@ initialParent = "" lastChange = "" initialTag = "" -if prefix.find("@") != -1: - atIdx = prefix.index("@") - changeRange = prefix[atIdx:] +if globalPrefix.find("@") != -1: + atIdx = globalPrefix.index("@") + changeRange = globalPrefix[atIdx:] if changeRange == "@all": changeRange = "" elif changeRange.find(",") == -1: revision = changeRange changeRange = "" - prefix = prefix[0:atIdx] -elif prefix.find("#") != -1: - hashIdx = prefix.index("#") - revision = prefix[hashIdx:] - prefix = prefix[0:hashIdx] + globalPrefix = globalPrefix[0:atIdx] +elif globalPrefix.find("#") != -1: + hashIdx = globalPrefix.index("#") + revision = globalPrefix[hashIdx:] + globalPrefix = globalPrefix[0:hashIdx] elif len(previousDepotPath) == 0: revision = "#head" -if prefix.endswith("..."): - prefix = prefix[:-3] +if globalPrefix.endswith("..."): + globalPrefix = globalPrefix[:-3] -if not prefix.endswith("/"): - prefix += "/" +if not globalPrefix.endswith("/"): + globalPrefix += "/" def p4CmdList(cmd): pipe = os.popen("p4 -G %s" % cmd, "rb") @@ -101,8 +102,8 @@ def extractFilesFromCommit(commit): fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] - if not path.startswith(prefix): - print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, prefix, change) + if not path.startswith(globalPrefix): + print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) continue file = {} @@ -118,7 +119,7 @@ def branchesForCommit(files): branches = set() for file in files: - relativePath = file["path"][len(prefix):] + relativePath = file["path"][len(globalPrefix):] # strip off the filename relativePath = relativePath[0:relativePath.rfind("/")] @@ -144,7 +145,7 @@ def branchesForCommit(files): return branches -def commit(details, files, branch, prefix): +def commit(details, files, branch, branchPrefix): global initialParent global users global lastChange @@ -163,7 +164,7 @@ def commit(details, files, branch, prefix): gitStream.write("data < 0: @@ -172,9 +173,47 @@ def commit(details, files, branch, prefix): for file in files: path = file["path"] + if not path.startswith(branchPrefix): + continue + action = file["action"] + if action != "integrate" and action != "branch": + continue rev = file["rev"] depotPath = path + "#" + rev - relPath = path[len(prefix):] + + log = p4CmdList("filelog \"%s\"" % depotPath) + if len(log) != 1: + print "eek! I got confused by the filelog of %s" % depotPath + sys.exit(1); + + log = log[0] + if log["action0"] != action: + print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) + sys.exit(1); + + if not log["how0,0"].endswith(" from"): + print "eek! file %s was not branched but instead: %s" % (depotPath, log["how0,0"]) + sys.exit(1); + + source = log["file0,0"] + if source.startswith(branchPrefix): + continue + + relPath = source[len(globalPrefix):] + + for branch in knownBranches: + if relPath.startswith(branch): + gitStream.write("merge refs/heads/%s\n" % branch) + break + + for file in files: + path = file["path"] + if not path.startswith(branchPrefix): + print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, change) + continue + rev = file["rev"] + depotPath = path + "#" + rev + relPath = path[len(branchPrefix):] action = file["action"] if action == "delete": @@ -234,15 +273,15 @@ if tzsign != '+' and tzsign != '-': gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") if len(revision) > 0: - print "Doing initial import of %s from revision %s" % (prefix, revision) + print "Doing initial import of %s from revision %s" % (globalPrefix, revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = "Initial import of %s from the state at revision %s" % (prefix, revision) + details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision) details["change"] = revision newestRevision = 0 fileCnt = 0 - for info in p4CmdList("files %s...%s" % (prefix, revision)): + for info in p4CmdList("files %s...%s" % (globalPrefix, revision)): change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -258,12 +297,12 @@ if len(revision) > 0: details["change"] = newestRevision try: - commit(details, extractFilesFromCommit(details), branch, prefix) + commit(details, extractFilesFromCommit(details), branch, globalPrefix) except: print gitError.read() else: - output = os.popen("p4 changes %s...%s" % (prefix, changeRange)).readlines() + output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines() changes = [] for line in output: @@ -288,11 +327,12 @@ else: files = extractFilesFromCommit(description) if detectBranches: for branch in branchesForCommit(files): - branchPrefix = prefix + branch + "/" + knownBranches.add(branch) + branchPrefix = globalPrefix + branch + "/" branch = "refs/heads/" + branch commit(description, files, branch, branchPrefix) else: - commit(description, files, branch, prefix) + commit(description, files, branch, globalPrefix) except: print gitError.read() sys.exit(1) @@ -307,7 +347,7 @@ gitStream.close() gitOutput.close() gitError.close() -os.popen("git-repo-config p4.depotpath %s" % prefix).read() +os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read() if len(initialTag) > 0: os.popen("git tag -d %s" % initialTag).read() From 53b03239aa899677bcadf69f757354b72797e457 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Feb 2007 21:44:02 +0100 Subject: [PATCH 038/260] After marking a p4 branch as merged don't ever merge it in git again. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index f9653f1344..5838ca3c68 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -171,6 +171,8 @@ def commit(details, files, branch, branchPrefix): gitStream.write("from %s\n" % initialParent) initialParent = "" + mergedBranches = set() + for file in files: path = file["path"] if not path.startswith(branchPrefix): @@ -202,8 +204,9 @@ def commit(details, files, branch, branchPrefix): relPath = source[len(globalPrefix):] for branch in knownBranches: - if relPath.startswith(branch): + if relPath.startswith(branch) and branch not in mergedBranches: gitStream.write("merge refs/heads/%s\n" % branch) + mergedBranches.add(branch) break for file in files: From 77083daac72612ab8ded6cba82a72924f3a867b4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Feb 2007 21:56:46 +0100 Subject: [PATCH 039/260] Set git fast-import marks for every imported change for future use. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 5838ca3c68..488533b07e 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -154,6 +154,7 @@ def commit(details, files, branch, branchPrefix): author = details["user"] gitStream.write("commit %s\n" % branch) + gitStream.write("mark :%s\n" % details["change"]) committer = "" if author in users: committer = "%s %s %s" % (users[author], epoch, tz) From 930d3cc94e98ccecfa8762ef79f7c29af4a72769 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Feb 2007 22:05:21 +0100 Subject: [PATCH 040/260] When trying to map p4 integrations to git merges just record it as a single merge with the newest p4 change as secondary parent. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 488533b07e..0a3e5c9934 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -172,7 +172,8 @@ def commit(details, files, branch, branchPrefix): gitStream.write("from %s\n" % initialParent) initialParent = "" - mergedBranches = set() + #mergedBranches = set() + merge = 0 for file in files: path = file["path"] @@ -202,13 +203,28 @@ def commit(details, files, branch, branchPrefix): if source.startswith(branchPrefix): continue - relPath = source[len(globalPrefix):] + lastSourceRev = log["erev0,0"] - for branch in knownBranches: - if relPath.startswith(branch) and branch not in mergedBranches: - gitStream.write("merge refs/heads/%s\n" % branch) - mergedBranches.add(branch) - break + sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) + if len(sourceLog) != 1: + print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) + sys.exit(1); + sourceLog = sourceLog[0] + + change = int(sourceLog["change0"]) + if change > merge: + merge = change + +# relPath = source[len(globalPrefix):] +# +# for branch in knownBranches: +# if relPath.startswith(branch) and branch not in mergedBranches: +# gitStream.write("merge refs/heads/%s\n" % branch) +# mergedBranches.add(branch) +# break + + if merge != 0: + gitStream.write("merge :%s\n" % merge) for file in files: path = file["path"] From 0563a4538a33c7ae9da41628d6f8849c38a2f935 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 14 Feb 2007 17:13:17 +0100 Subject: [PATCH 041/260] Make it possible to specify the p4 changes to import through a text file (for debugging) and made various improvements to the branch/merge heuristic detection. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 128 ++++++++++++++++++-------- 1 file changed, 90 insertions(+), 38 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 0a3e5c9934..b5dc6f6767 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -13,14 +13,16 @@ import os, string, sys, time import marshal, popen2, getopt knownBranches = set() +committedChanges = set() branch = "refs/heads/master" globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() detectBranches = False +changesFile = "" if len(globalPrefix) != 0: globalPrefix = globalPrefix[:-1] try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches" ]) + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -30,6 +32,8 @@ for o, a in opts: branch = "refs/heads/" + a elif o == "--detect-branches": detectBranches = True + elif o == "--changesfile": + changesFile = a if len(args) == 0 and len(globalPrefix) != 0: print "[using previously specified depot path %s]" % globalPrefix @@ -53,7 +57,7 @@ changeRange = "" revision = "" users = {} initialParent = "" -lastChange = "" +lastChange = 0 initialTag = "" if globalPrefix.find("@") != -1: @@ -104,6 +108,7 @@ def extractFilesFromCommit(commit): path = commit["depotFile%s" % fnum] if not path.startswith(globalPrefix): print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) + fnum = fnum + 1 continue file = {} @@ -115,7 +120,15 @@ def extractFilesFromCommit(commit): fnum = fnum + 1 return files +def isSubPathOf(first, second): + if not first.startswith(second): + return False + if first == second: + return True + return first[len(second)] == "/" + def branchesForCommit(files): + global knownBranches branches = set() for file in files: @@ -123,9 +136,10 @@ def branchesForCommit(files): # strip off the filename relativePath = relativePath[0:relativePath.rfind("/")] - if len(branches) == 0: - branches.add(relativePath) - continue +# if len(branches) == 0: +# branches.add(relativePath) +# knownBranches.add(relativePath) +# continue ###### this needs more testing :) knownBranch = False @@ -133,15 +147,32 @@ def branchesForCommit(files): if relativePath == branch: knownBranch = True break - if relativePath.startswith(branch): +# if relativePath.startswith(branch): + if isSubPathOf(relativePath, branch): knownBranch = True break - if branch.startswith(relativePath): +# if branch.startswith(relativePath): + if isSubPathOf(branch, relativePath): branches.remove(branch) break - if not knownBranch: - branches.add(relativePath) + if knownBranch: + continue + + for branch in knownBranches: + #if relativePath.startswith(branch): + if isSubPathOf(relativePath, branch): + if len(branches) == 0: + relativePath = branch + else: + knownBranch = True + break + + if knownBranch: + continue + + branches.add(relativePath) + knownBranches.add(relativePath) return branches @@ -149,12 +180,14 @@ def commit(details, files, branch, branchPrefix): global initialParent global users global lastChange + global committedChanges epoch = details["time"] author = details["user"] gitStream.write("commit %s\n" % branch) gitStream.write("mark :%s\n" % details["change"]) + committedChanges.add(int(details["change"])) committer = "" if author in users: committer = "%s %s %s" % (users[author], epoch, tz) @@ -173,9 +206,11 @@ def commit(details, files, branch, branchPrefix): initialParent = "" #mergedBranches = set() - merge = 0 + merges = set() for file in files: + if lastChange == 0: + continue path = file["path"] if not path.startswith(branchPrefix): continue @@ -195,9 +230,14 @@ def commit(details, files, branch, branchPrefix): print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) sys.exit(1); - if not log["how0,0"].endswith(" from"): - print "eek! file %s was not branched but instead: %s" % (depotPath, log["how0,0"]) - sys.exit(1); + branchAction = log["how0,0"] +# if branchAction == "branch into" or branchAction == "ignored": +# continue # ignore for branching + + if not branchAction.endswith(" from"): + continue # ignore for branching +# print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) +# sys.exit(1); source = log["file0,0"] if source.startswith(branchPrefix): @@ -212,8 +252,7 @@ def commit(details, files, branch, branchPrefix): sourceLog = sourceLog[0] change = int(sourceLog["change0"]) - if change > merge: - merge = change + merges.add(change) # relPath = source[len(globalPrefix):] # @@ -223,13 +262,14 @@ def commit(details, files, branch, branchPrefix): # mergedBranches.add(branch) # break - if merge != 0: - gitStream.write("merge :%s\n" % merge) + for merge in merges: + if merge in committedChanges: + gitStream.write("merge :%s\n" % merge) for file in files: path = file["path"] if not path.startswith(branchPrefix): - print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, change) + print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) continue rev = file["rev"] depotPath = path + "#" + rev @@ -252,7 +292,7 @@ def commit(details, files, branch, branchPrefix): gitStream.write("\n") - lastChange = details["change"] + lastChange = int(details["change"]) def getUserMap(): users = {} @@ -322,14 +362,26 @@ if len(revision) > 0: print gitError.read() else: - output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines() - changes = [] - for line in output: - changeNum = line.split(" ")[1] - changes.append(changeNum) - changes.reverse() + if len(changesFile) > 0: + output = open(changesFile).readlines() + changeSet = set() + for line in output: + changeSet.add(int(line)) + + for change in changeSet: + changes.append(change) + + changes.sort() + else: + output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines() + + for line in output: + changeNum = line.split(" ")[1] + changes.append(changeNum) + + changes.reverse() if len(changes) == 0: print "no changes to import!" @@ -343,19 +395,19 @@ else: sys.stdout.flush() cnt = cnt + 1 - try: - files = extractFilesFromCommit(description) - if detectBranches: - for branch in branchesForCommit(files): - knownBranches.add(branch) - branchPrefix = globalPrefix + branch + "/" - branch = "refs/heads/" + branch - commit(description, files, branch, branchPrefix) - else: - commit(description, files, branch, globalPrefix) - except: - print gitError.read() - sys.exit(1) +# try: + files = extractFilesFromCommit(description) + if detectBranches: + for branch in branchesForCommit(files): + knownBranches.add(branch) + branchPrefix = globalPrefix + branch + "/" + branch = "refs/heads/" + branch + commit(description, files, branch, branchPrefix) + else: + commit(description, files, branch, globalPrefix) +# except: +# print gitError.read() +# sys.exit(1) print "" From f1e9b5345efade26961289860ced9e5c428360a4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 15 Feb 2007 02:16:14 +1000 Subject: [PATCH 042/260] Use sets.Set() instead of set() to run also with older versions of Python. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index b5dc6f6767..76c4b9d323 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -11,9 +11,10 @@ # import os, string, sys, time import marshal, popen2, getopt +from sets import Set; -knownBranches = set() -committedChanges = set() +knownBranches = Set() +committedChanges = Set() branch = "refs/heads/master" globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() detectBranches = False @@ -129,7 +130,7 @@ def isSubPathOf(first, second): def branchesForCommit(files): global knownBranches - branches = set() + branches = Set() for file in files: relativePath = file["path"][len(globalPrefix):] @@ -205,8 +206,8 @@ def commit(details, files, branch, branchPrefix): gitStream.write("from %s\n" % initialParent) initialParent = "" - #mergedBranches = set() - merges = set() + #mergedBranches = Set() + merges = Set() for file in files: if lastChange == 0: @@ -366,7 +367,7 @@ else: if len(changesFile) > 0: output = open(changesFile).readlines() - changeSet = set() + changeSet = Set() for line in output: changeSet.add(int(line)) From 90dc3dfdc8ab903f5eb12f7081dd43b67c8c112a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 18 Feb 2007 01:18:22 +1000 Subject: [PATCH 043/260] Fix single-branch imports by skipping the branch/merge detection correctly. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 76c4b9d323..f37fdcf96e 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -210,7 +210,7 @@ def commit(details, files, branch, branchPrefix): merges = Set() for file in files: - if lastChange == 0: + if lastChange == 0 or not detectBranches: continue path = file["path"] if not path.startswith(branchPrefix): From 9c9fec3d28a77240fc7dbf44ba2f0805fed5f012 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Feb 2007 17:51:07 +0100 Subject: [PATCH 044/260] Added p4 delete behavioural emulation as todo item. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index f37fdcf96e..6fd86cf735 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -8,6 +8,8 @@ # TODO: # - support integrations (at least p4i) # - support p4 submit (hah!) +# - emulate p4's delete behavior: if a directory becomes empty delete it. continue +# with parent dir until non-empty dir is found. # import os, string, sys, time import marshal, popen2, getopt From 161958cc2f71eefbf514b2265741f7d2eb1a5f0b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Feb 2007 09:03:39 +0100 Subject: [PATCH 045/260] Added support for --silent so that p4-fast-export can be called from cronjobs. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 6fd86cf735..b39548917a 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -15,6 +15,7 @@ import os, string, sys, time import marshal, popen2, getopt from sets import Set; +silent = False knownBranches = Set() committedChanges = Set() branch = "refs/heads/master" @@ -25,7 +26,7 @@ if len(globalPrefix) != 0: globalPrefix = globalPrefix[:-1] try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=" ]) + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -37,6 +38,8 @@ for o, a in opts: detectBranches = True elif o == "--changesfile": changesFile = a + elif o == "--silent": + silent= True if len(args) == 0 and len(globalPrefix) != 0: print "[using previously specified depot path %s]" % globalPrefix @@ -110,7 +113,8 @@ def extractFilesFromCommit(commit): while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] if not path.startswith(globalPrefix): - print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) + if not silent: + print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) fnum = fnum + 1 continue @@ -272,7 +276,8 @@ def commit(details, files, branch, branchPrefix): for file in files: path = file["path"] if not path.startswith(branchPrefix): - print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) + if not silent: + print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) continue rev = file["rev"] depotPath = path + "#" + rev @@ -387,15 +392,17 @@ else: changes.reverse() if len(changes) == 0: - print "no changes to import!" + if not silent: + print "no changes to import!" sys.exit(1) cnt = 1 for change in changes: description = p4Cmd("describe %s" % change) - sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) - sys.stdout.flush() + if not silent: + sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.flush() cnt = cnt + 1 # try: @@ -412,7 +419,8 @@ else: # print gitError.read() # sys.exit(1) -print "" +if not silent: + print "" gitStream.write("reset refs/tags/p4/%s\n" % lastChange) gitStream.write("from %s\n\n" % branch); From 47e33ec082a4ee5d2925760cb1150b25f4ef392e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Feb 2007 09:22:36 +0100 Subject: [PATCH 046/260] More work in --silent support. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index b39548917a..0541f84188 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -42,7 +42,8 @@ for o, a in opts: silent= True if len(args) == 0 and len(globalPrefix) != 0: - print "[using previously specified depot path %s]" % globalPrefix + if not silent: + print "[using previously specified depot path %s]" % globalPrefix elif len(args) != 1: print "usage: %s //depot/path[@revRange]" % sys.argv[0] print "\n example:" From 5ea919de226177e52f62d1c92208ad9687fd76d7 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Feb 2007 10:20:53 +0100 Subject: [PATCH 047/260] Don't print a plain newline at the end of the execution (avoids bogus cron error mails). Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 0541f84188..8f5b8fe314 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -332,8 +332,6 @@ if len(changeRange) == 0: except: pass -sys.stderr.write("\n") - tz = - time.timezone / 36 tzsign = ("%s" % tz)[0] if tzsign != '+' and tzsign != '-': From 6392a40e5ec1cc1190f5870f6d7c9cc3710dfd46 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 7 Mar 2007 19:58:54 +0100 Subject: [PATCH 048/260] Adjust the output parsing of git name-rev to handle the output of the latest git version. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-clean-tags.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-clean-tags.py b/contrib/fast-import/p4-clean-tags.py index 0be51f6405..924ff89cc6 100755 --- a/contrib/fast-import/p4-clean-tags.py +++ b/contrib/fast-import/p4-clean-tags.py @@ -25,7 +25,10 @@ for o, a in opts: sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) output = sout.read() tagIdx = output.index(" tags/p4/") -caretIdx = output.index("^") +try: + caretIdx = output.index("^") +except: + caretIdx = len(output) - 1 rev = int(output[tagIdx + 9 : caretIdx]) allTags = os.popen("git tag -l p4/").readlines() From 3ef674bd4b2f651d769cd5b464ba26313eab290a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 8 Mar 2007 21:27:59 +0100 Subject: [PATCH 049/260] Work in progress on detecting branches. Added a disk-cache p4 output so debugging imports is faster. Added --known-branches commandline option for pre-defining branches. Various other fixes... Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 160 +++++++++++++++++--------- 1 file changed, 103 insertions(+), 57 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 8f5b8fe314..a2cca31173 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -11,12 +11,15 @@ # - emulate p4's delete behavior: if a directory becomes empty delete it. continue # with parent dir until non-empty dir is found. # -import os, string, sys, time -import marshal, popen2, getopt +import os, string, sys, time, os.path +import marshal, popen2, getopt, sha from sets import Set; +cacheDebug = False + silent = False knownBranches = Set() +createdBranches = Set() committedChanges = Set() branch = "refs/heads/master" globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() @@ -26,7 +29,7 @@ if len(globalPrefix) != 0: globalPrefix = globalPrefix[:-1] try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent" ]) + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -40,6 +43,9 @@ for o, a in opts: changesFile = a elif o == "--silent": silent= True + elif o == "--known-branches": + for branch in o.split(","): + knownBranches.add(branch) if len(args) == 0 and len(globalPrefix) != 0: if not silent: @@ -89,8 +95,37 @@ if globalPrefix.endswith("..."): if not globalPrefix.endswith("/"): globalPrefix += "/" +def p4File(depotPath): + cacheKey = "/tmp/p4cache/data-" + sha.new(depotPath).hexdigest() + + data = 0 + try: + if not cacheDebug: + raise + data = open(cacheKey, "rb").read() + except: + data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() + if cacheDebug: + open(cacheKey, "wb").write(data) + + return data + def p4CmdList(cmd): - pipe = os.popen("p4 -G %s" % cmd, "rb") + fullCmd = "p4 -G %s" % cmd; + + cacheKey = sha.new(fullCmd).hexdigest() + cacheKey = "/tmp/p4cache/cmd-" + cacheKey + + cached = True + pipe = 0 + try: + if not cacheDebug: + raise + pipe = open(cacheKey, "rb") + except: + cached = False + pipe = os.popen(fullCmd, "rb") + result = [] try: while True: @@ -99,6 +134,13 @@ def p4CmdList(cmd): except EOFError: pass pipe.close() + + if not cached and cacheDebug: + pipe = open(cacheKey, "wb") + for r in result: + marshal.dump(r, pipe) + pipe.close() + return result def p4Cmd(cmd): @@ -114,8 +156,8 @@ def extractFilesFromCommit(commit): while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] if not path.startswith(globalPrefix): - if not silent: - print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) +# if not silent: +# print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) fnum = fnum + 1 continue @@ -184,41 +226,8 @@ def branchesForCommit(files): return branches -def commit(details, files, branch, branchPrefix): - global initialParent - global users - global lastChange - global committedChanges - - epoch = details["time"] - author = details["user"] - - gitStream.write("commit %s\n" % branch) - gitStream.write("mark :%s\n" % details["change"]) - committedChanges.add(int(details["change"])) - committer = "" - if author in users: - committer = "%s %s %s" % (users[author], epoch, tz) - else: - committer = "%s %s %s" % (author, epoch, tz) - - gitStream.write("committer %s\n" % committer) - - gitStream.write("data < 0: - gitStream.write("from %s\n" % initialParent) - initialParent = "" - - #mergedBranches = Set() - merges = Set() - +def findBranchParent(branchPrefix, files): for file in files: - if lastChange == 0 or not detectBranches: - continue path = file["path"] if not path.startswith(branchPrefix): continue @@ -259,26 +268,51 @@ def commit(details, files, branch, branchPrefix): sys.exit(1); sourceLog = sourceLog[0] - change = int(sourceLog["change0"]) - merges.add(change) + relPath = source[len(globalPrefix):] + # strip off the filename + relPath = relPath[0:relPath.rfind("/")] -# relPath = source[len(globalPrefix):] -# -# for branch in knownBranches: -# if relPath.startswith(branch) and branch not in mergedBranches: -# gitStream.write("merge refs/heads/%s\n" % branch) -# mergedBranches.add(branch) -# break + for branch in knownBranches: + if isSubPathOf(relPath, branch): +# print "determined parent branch branch %s due to change in file %s" % (branch, source) + return "refs/heads/%s" % branch +# else: +# print "%s is not a subpath of branch %s" % (relPath, branch) - for merge in merges: - if merge in committedChanges: - gitStream.write("merge :%s\n" % merge) + return "" + +def commit(details, files, branch, branchPrefix, parent): + global users + global lastChange + global committedChanges + + epoch = details["time"] + author = details["user"] + + gitStream.write("commit %s\n" % branch) + gitStream.write("mark :%s\n" % details["change"]) + committedChanges.add(int(details["change"])) + committer = "" + if author in users: + committer = "%s %s %s" % (users[author], epoch, tz) + else: + committer = "%s %s %s" % (author, epoch, tz) + + gitStream.write("committer %s\n" % committer) + + gitStream.write("data < 0: + gitStream.write("from %s\n" % parent) for file in files: path = file["path"] if not path.startswith(branchPrefix): - if not silent: - print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) +# if not silent: +# print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) continue rev = file["rev"] depotPath = path + "#" + rev @@ -292,7 +326,7 @@ def commit(details, files, branch, branchPrefix): if file["type"].startswith("x"): mode = 755 - data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() + data = p4File(depotPath) gitStream.write("M %s inline %s\n" % (mode, relPath)) gitStream.write("data %s\n" % len(data)) @@ -410,10 +444,22 @@ else: for branch in branchesForCommit(files): knownBranches.add(branch) branchPrefix = globalPrefix + branch + "/" + + parent = "" + ########### remove cnt!!! + if branch not in createdBranches and cnt > 2: + createdBranches.add(branch) + parent = findBranchParent(branchPrefix, files) + if parent == branch: + parent = "" +# elif len(parent) > 0: +# print "%s branched off of %s" % (branch, parent) + branch = "refs/heads/" + branch - commit(description, files, branch, branchPrefix) + commit(description, files, branch, branchPrefix, parent) else: - commit(description, files, branch, globalPrefix) + commit(description, files, branch, globalPrefix, initialParent) + initialParent = "" # except: # print gitError.read() # sys.exit(1) From 934371385c6e2e26132190542f4082c1655587e7 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 8 Mar 2007 21:34:40 +0100 Subject: [PATCH 050/260] Changed --known-branches to take a file as argument instead of a comma separated list. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index a2cca31173..5d4ed5cf9c 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -44,8 +44,8 @@ for o, a in opts: elif o == "--silent": silent= True elif o == "--known-branches": - for branch in o.split(","): - knownBranches.add(branch) + for branch in open(a).readlines(): + knownBranches.add(branch[:-1]) if len(args) == 0 and len(globalPrefix) != 0: if not silent: From a0f22e996c9d2f0e2a4a8feae852478f4aa667b5 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 09:49:19 +0100 Subject: [PATCH 051/260] Fixed p4-debug file extension. Signed-off-by: Simon Hausmann --- contrib/fast-import/{p4-debug.p4 => p4-debug.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename contrib/fast-import/{p4-debug.p4 => p4-debug.py} (100%) diff --git a/contrib/fast-import/p4-debug.p4 b/contrib/fast-import/p4-debug.py similarity index 100% rename from contrib/fast-import/p4-debug.p4 rename to contrib/fast-import/p4-debug.py From 59f1d2b52d71c5082359f4caa9af5752f90d26a9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 10:25:34 +0100 Subject: [PATCH 052/260] Make the p4 data/command cache configurable through the --cache-debug commandline option. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 5d4ed5cf9c..3d2b42b636 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -29,7 +29,8 @@ if len(globalPrefix) != 0: globalPrefix = globalPrefix[:-1] try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=" ]) + opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=", + "cache-debug" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -46,6 +47,8 @@ for o, a in opts: elif o == "--known-branches": for branch in open(a).readlines(): knownBranches.add(branch[:-1]) + elif o == "--cache-debug": + cacheDebug = True if len(args) == 0 and len(globalPrefix) != 0: if not silent: From 478764bc8261857e7eca2deac409e34c2778347a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 10:53:07 +0100 Subject: [PATCH 053/260] Minor code cleanups. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 3d2b42b636..bd2f03064b 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -340,6 +340,16 @@ def commit(details, files, branch, branchPrefix, parent): lastChange = int(details["change"]) +def extractFilesInCommitToBranch(files, branchPrefix): + newFiles = [] + + for file in files: + path = file["path"] + if path.startswith(branchPrefix): + newFiles.append(file) + + return newFiles + def getUserMap(): users = {} @@ -448,6 +458,8 @@ else: knownBranches.add(branch) branchPrefix = globalPrefix + branch + "/" + filesForCommit = extractFilesInCommitToBranch(files, branchPrefix) + parent = "" ########### remove cnt!!! if branch not in createdBranches and cnt > 2: @@ -458,10 +470,11 @@ else: # elif len(parent) > 0: # print "%s branched off of %s" % (branch, parent) + branch = "refs/heads/" + branch commit(description, files, branch, branchPrefix, parent) else: - commit(description, files, branch, globalPrefix, initialParent) + commit(description, filesForCommit, branch, globalPrefix, initialParent) initialParent = "" # except: # print gitError.read() From 85a8f1ac3ba10e20fab0e7f38e0d74213535c7ae Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 11:46:26 +0100 Subject: [PATCH 054/260] More code cleanups and preparations for more branch detection heuristics. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 61 ++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index bd2f03064b..65b7fca4b6 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -278,7 +278,7 @@ def findBranchParent(branchPrefix, files): for branch in knownBranches: if isSubPathOf(relPath, branch): # print "determined parent branch branch %s due to change in file %s" % (branch, source) - return "refs/heads/%s" % branch + return branch # else: # print "%s is not a subpath of branch %s" % (relPath, branch) @@ -350,6 +350,57 @@ def extractFilesInCommitToBranch(files, branchPrefix): return newFiles +def findBranchSourceHeuristic(files, branch, branchPrefix): + for file in files: + action = file["action"] + if action != "integrate" and action != "branch": + continue + path = file["path"] + rev = file["rev"] + depotPath = path + "#" + rev + + log = p4CmdList("filelog \"%s\"" % depotPath) + if len(log) != 1: + print "eek! I got confused by the filelog of %s" % depotPath + sys.exit(1); + + log = log[0] + if log["action0"] != action: + print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) + sys.exit(1); + + branchAction = log["how0,0"] + + if not branchAction.endswith(" from"): + continue # ignore for branching +# print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) +# sys.exit(1); + + source = log["file0,0"] + if source.startswith(branchPrefix): + continue + + lastSourceRev = log["erev0,0"] + + sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) + if len(sourceLog) != 1: + print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) + sys.exit(1); + sourceLog = sourceLog[0] + + relPath = source[len(globalPrefix):] + # strip off the filename + relPath = relPath[0:relPath.rfind("/")] + + for candidate in knownBranches: + if isSubPathOf(relPath, candidate) and candidate != branch: + return candidate + + return "" + +def changeIsBranchMerge(sourceBranch, destinationBranch, change): + return False + def getUserMap(): users = {} @@ -470,8 +521,16 @@ else: # elif len(parent) > 0: # print "%s branched off of %s" % (branch, parent) + if len(parent) == 0: + parent = findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) + if len(parent) > 0: + print "change %s could be a merge from %s into %s" % (description["change"], parent, branch) + if not changeIsBranchMerge(parent, branch, description["change"]): + parent = "" branch = "refs/heads/" + branch + if len(parent) > 0: + parent = "refs/heads/" + parent commit(description, files, branch, branchPrefix, parent) else: commit(description, filesForCommit, branch, globalPrefix, initialParent) From 43cc31e8a252d3ed326faa28d9a5d406c96d7b0d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 17:46:49 +0100 Subject: [PATCH 055/260] More work on branch detection by implementing changeIsBranchMerge(). Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 131 +++++++++++++++++++------- 1 file changed, 96 insertions(+), 35 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 65b7fca4b6..d0832e8c3d 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -284,7 +284,7 @@ def findBranchParent(branchPrefix, files): return "" -def commit(details, files, branch, branchPrefix, parent): +def commit(details, files, branch, branchPrefix, parent, merged = ""): global users global lastChange global committedChanges @@ -293,7 +293,7 @@ def commit(details, files, branch, branchPrefix, parent): author = details["user"] gitStream.write("commit %s\n" % branch) - gitStream.write("mark :%s\n" % details["change"]) +# gitStream.write("mark :%s\n" % details["change"]) committedChanges.add(int(details["change"])) committer = "" if author in users: @@ -311,6 +311,9 @@ def commit(details, files, branch, branchPrefix, parent): if len(parent) > 0: gitStream.write("from %s\n" % parent) + if len(merged) > 0: + gitStream.write("merge %s\n" % merged) + for file in files: path = file["path"] if not path.startswith(branchPrefix): @@ -399,7 +402,62 @@ def findBranchSourceHeuristic(files, branch, branchPrefix): return "" def changeIsBranchMerge(sourceBranch, destinationBranch, change): - return False + sourceFiles = {} + for file in p4CmdList("files %s...@%s" % (globalPrefix + sourceBranch + "/", change)): + if file["action"] == "delete": + continue + sourceFiles[file["depotFile"]] = file + + destinationFiles = {} + for file in p4CmdList("files %s...@%s" % (globalPrefix + destinationBranch + "/", change)): + destinationFiles[file["depotFile"]] = file + + for fileName in sourceFiles.keys(): + integrations = [] + deleted = False + for integration in p4CmdList("integrated \"%s\"" % fileName): + toFile = integration["fromFile"] # yes, it's true, it's fromFile + if not toFile in destinationFiles: + continue + destFile = destinationFiles[toFile] + if destFile["action"] == "delete": +# print "file %s has been deleted in %s" % (fileName, toFile) + deleted = True + break + + if int(integration["change"]) == change: + integrations.append(integration) + continue + + destRev = int(destFile["rev"]) + + startRev = integration["startFromRev"][1:] + if startRev == "none": + startRev = 0 + else: + startRev = int(startRev) + + endRev = integration["endFromRev"][1:] + if endRev == "none": + endRev = 0 + else: + endRev = int(endRev) + + initialBranch = (destRev == 1 and integration["how"] != "branch into") + inRange = (destRev >= startRev and destRev <= endRev) + newer = (destRev > startRev and destRev > endRev) + + if initialBranch or inRange or newer: + integrations.append(integration) + + if deleted: + continue + + if len(integrations) == 0: + print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch) + return False + + return True def getUserMap(): users = {} @@ -502,42 +560,45 @@ else: sys.stdout.flush() cnt = cnt + 1 -# try: - files = extractFilesFromCommit(description) - if detectBranches: - for branch in branchesForCommit(files): - knownBranches.add(branch) - branchPrefix = globalPrefix + branch + "/" + try: + files = extractFilesFromCommit(description) + if detectBranches: + for branch in branchesForCommit(files): + knownBranches.add(branch) + branchPrefix = globalPrefix + branch + "/" - filesForCommit = extractFilesInCommitToBranch(files, branchPrefix) + filesForCommit = extractFilesInCommitToBranch(files, branchPrefix) - parent = "" - ########### remove cnt!!! - if branch not in createdBranches and cnt > 2: - createdBranches.add(branch) - parent = findBranchParent(branchPrefix, files) - if parent == branch: - parent = "" -# elif len(parent) > 0: -# print "%s branched off of %s" % (branch, parent) - - if len(parent) == 0: - parent = findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) - if len(parent) > 0: - print "change %s could be a merge from %s into %s" % (description["change"], parent, branch) - if not changeIsBranchMerge(parent, branch, description["change"]): + merged = "" + parent = "" + ########### remove cnt!!! + if branch not in createdBranches and cnt > 2: + createdBranches.add(branch) + parent = findBranchParent(branchPrefix, files) + if parent == branch: parent = "" + # elif len(parent) > 0: + # print "%s branched off of %s" % (branch, parent) - branch = "refs/heads/" + branch - if len(parent) > 0: - parent = "refs/heads/" + parent - commit(description, files, branch, branchPrefix, parent) - else: - commit(description, filesForCommit, branch, globalPrefix, initialParent) - initialParent = "" -# except: -# print gitError.read() -# sys.exit(1) + if len(parent) == 0: + merged = findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) + if len(merged) > 0: + print "change %s could be a merge from %s into %s" % (description["change"], merged, branch) + if not changeIsBranchMerge(merged, branch, int(description["change"])): + merged = "" + + branch = "refs/heads/" + branch + if len(parent) > 0: + parent = "refs/heads/" + parent + if len(merged) > 0: + merged = "refs/heads/" + merged + commit(description, files, branch, branchPrefix, parent, merged) + else: + commit(description, filesForCommit, branch, globalPrefix, initialParent) + initialParent = "" + except IOError: + print gitError.read() + sys.exit(1) if not silent: print "" From dd87020bd3969dbac5ae26c527bcf50fe0ebb845 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 21:23:49 +0100 Subject: [PATCH 056/260] Reduce the number of false "merges" by skipping "branch from" entries in the integrated output as well as by ignoring integrations of future (newer) changes. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index d0832e8c3d..a45068d362 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -415,6 +415,7 @@ def changeIsBranchMerge(sourceBranch, destinationBranch, change): for fileName in sourceFiles.keys(): integrations = [] deleted = False + integrationCount = 0 for integration in p4CmdList("integrated \"%s\"" % fileName): toFile = integration["fromFile"] # yes, it's true, it's fromFile if not toFile in destinationFiles: @@ -424,10 +425,15 @@ def changeIsBranchMerge(sourceBranch, destinationBranch, change): # print "file %s has been deleted in %s" % (fileName, toFile) deleted = True break + integrationCount += 1 + if integration["how"] == "branch from": + continue if int(integration["change"]) == change: integrations.append(integration) continue + if int(integration["change"]) > change: + continue destRev = int(destFile["rev"]) @@ -453,7 +459,7 @@ def changeIsBranchMerge(sourceBranch, destinationBranch, change): if deleted: continue - if len(integrations) == 0: + if len(integrations) == 0 and integrationCount > 1: print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch) return False From 4fe2ca17f779e14554ebca1208310ae4eb0806b5 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 10 Mar 2007 21:30:24 +0100 Subject: [PATCH 057/260] Split up the cache commandline options into (command) cache and data cache. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index a45068d362..26cd648ef7 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -15,7 +15,8 @@ import os, string, sys, time, os.path import marshal, popen2, getopt, sha from sets import Set; -cacheDebug = False +dataCache = False +commandCache = False silent = False knownBranches = Set() @@ -30,7 +31,7 @@ if len(globalPrefix) != 0: try: opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=", - "cache-debug" ]) + "cache", "command-cache" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -47,8 +48,11 @@ for o, a in opts: elif o == "--known-branches": for branch in open(a).readlines(): knownBranches.add(branch[:-1]) - elif o == "--cache-debug": - cacheDebug = True + elif o == "--cache": + dataCache = True + commandCache = True + elif o == "--command-cache": + commandCache = True if len(args) == 0 and len(globalPrefix) != 0: if not silent: @@ -103,12 +107,12 @@ def p4File(depotPath): data = 0 try: - if not cacheDebug: + if not dataCache: raise data = open(cacheKey, "rb").read() except: data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() - if cacheDebug: + if dataCache: open(cacheKey, "wb").write(data) return data @@ -122,7 +126,7 @@ def p4CmdList(cmd): cached = True pipe = 0 try: - if not cacheDebug: + if not commandCache: raise pipe = open(cacheKey, "rb") except: @@ -138,7 +142,7 @@ def p4CmdList(cmd): pass pipe.close() - if not cached and cacheDebug: + if not cached and commandCache: pipe = open(cacheKey, "wb") for r in result: marshal.dump(r, pipe) From 0bcff6121d68d8733e9b572eba357bcd8e91e23e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 12 Mar 2007 23:00:34 +0100 Subject: [PATCH 058/260] First version of a new script to submit changes back to perforce from git repositories. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 208 +++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100755 contrib/fast-import/p4-git-sync.py diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py new file mode 100755 index 0000000000..8982e45345 --- /dev/null +++ b/contrib/fast-import/p4-git-sync.py @@ -0,0 +1,208 @@ +#!/usr/bin/python +# +# p4-git-sync.py +# +# Author: Simon Hausmann +# License: MIT +# + +import os, string, shelve, stat +import getopt, sys, marshal + +def p4CmdList(cmd): + cmd = "p4 -G %s" % cmd + pipe = os.popen(cmd, "rb") + + result = [] + try: + while True: + entry = marshal.load(pipe) + result.append(entry) + except EOFError: + pass + pipe.close() + + return result + +def p4Cmd(cmd): + list = p4CmdList(cmd) + result = {} + for entry in list: + result.update(entry) + return result; + +try: + opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "--git-dir=", "origin=", "reset", "master=", + "submit-log-subst=" ]) +except getopt.GetoptError: + print "fixme, syntax error" + sys.exit(1) + +logSubstitutions = {} +logSubstitutions[""] = "%log%" +logSubstitutions["\tDetails:"] = "\tDetails: %log%" +gitdir = os.environ.get("GIT_DIR", "") +origin = "origin" +master = "master" +firstTime = True +reset = False + +for o, a in opts: + if o == "--git-dir": + gitdir = a + elif o == "--origin": + origin = a + elif o == "--master": + master = a + elif o == "--continue": + firstTime = False + elif o == "--reset": + reset = True + firstTime = True + elif o == "--submit-log-subst": + key = a.split("%")[0] + value = a.split("%")[1] + logSubstitutions[key] = value + +if len(gitdir) == 0: + gitdir = ".git" +else: + os.environ["GIT_DIR"] = gitdir + +configFile = gitdir + "/p4-git-sync.cfg" + +origin = "origin" +if len(args) == 1: + origin = args[0] + +def die(msg): + sys.stderr.write(msg + "\n") + sys.exit(1) + +def system(cmd): + if os.system(cmd) != 0: + die("command failed: %s" % cmd) + +def check(): + return + if len(p4CmdList("opened ...")) > 0: + die("You have files opened with perforce! Close them before starting the sync.") + +def start(config): + if len(config) > 0 and not reset: + die("Cannot start sync. Previous sync config found at %s" % configFile) + + #if len(os.popen("git-update-index --refresh").read()) > 0: + # die("Your working tree is not clean. Check with git status!") + + commits = [] + for line in os.popen("git-rev-list --no-merges %s..%s" % (origin, master)).readlines(): + commits.append(line[:-1]) + commits.reverse() + + config["commits"] = commits + +# print "Cleaning index..." +# system("git checkout -f") + +def prepareLogMessage(template, message): + result = "" + + substs = logSubstitutions + for k in substs.keys(): + substs[k] = substs[k].replace("%log%", message) + + for line in template.split("\n"): + if line.startswith("#"): + result += line + "\n" + continue + + substituted = False + for key in substs.keys(): + if line.find(key) != -1: + value = substs[key] + if value != "@remove@": + result += line.replace(key, value) + "\n" + substituted = True + break + + if not substituted: + result += line + "\n" + + return result + +def apply(id): + print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read()) + diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() + filesToAdd = set() + filesToDelete = set() + for line in diff: + modifier = line[0] + path = line[1:].strip() + if modifier == "M": + system("p4 edit %s" % path) + elif modifier == "A": + filesToAdd.add(path) + if path in filesToDelete: + filesToDelete.remove(path) + elif modifier == "D": + filesToDelete.add(path) + if path in filesToAdd: + filesToAdd.remove(path) + else: + die("unknown modifier %s for %s" % (modifier, path)) + + system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") + system("git cherry-pick --no-commit \"%s\"" % id) + + for f in filesToAdd: + system("p4 add %s" % f) + for f in filesToDelete: + system("p4 revert %s" % f) + system("p4 delete %s" % f) + + logMessage = "" + foundTitle = False + for log in os.popen("git-cat-file commit %s" % id).readlines(): + log = log[:-1] + if not foundTitle: + if len(log) == 0: + foundTitle = 1 + continue + + if len(logMessage) > 0: + logMessage += "\t" + logMessage += log + "\n" + + template = os.popen("p4 change -o").read() + fileName = "submit.txt" + file = open(fileName, "w+") + file.write(prepareLogMessage(template, logMessage)) + file.close() + print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) + +check() + +config = shelve.open(configFile, writeback=True) + +if firstTime: + start(config) + +commits = config.get("commits", []) + +if len(commits) > 0: + firstTime = False + commit = commits[0] + commits = commits[1:] + config["commits"] = commits + apply(commit) + +config.close() + +if len(commits) == 0: + if firstTime: + print "No changes found to apply between %s and current HEAD" % origin + else: + print "All changes applied!" + os.remove(configFile) + From 5aba82fd5056fa57f467f3c5fe81fe71ee0e8b99 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 13 Mar 2007 09:14:45 +0100 Subject: [PATCH 059/260] Fix git-dir option and allow reading log substitutions from a file Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index 8982e45345..0c0f629a1d 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -32,8 +32,8 @@ def p4Cmd(cmd): return result; try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "--git-dir=", "origin=", "reset", "master=", - "submit-log-subst=" ]) + opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "git-dir=", "origin=", "reset", "master=", + "submit-log-subst=", "log-substitutions=" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -63,6 +63,10 @@ for o, a in opts: key = a.split("%")[0] value = a.split("%")[1] logSubstitutions[key] = value + elif o == "--log-substitutions": + for line in open(a, "r").readlines(): + tokens = line[:-1].split("=") + logSubstitutions[tokens[0]] = tokens[1] if len(gitdir) == 0: gitdir = ".git" From 09a14fb52410cfda029b51832316616b44348505 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 13 Mar 2007 16:36:10 +0100 Subject: [PATCH 060/260] Lots of bugfixes to p4-git-sync. Added interactive and dry-run mode. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 90 ++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 17 deletions(-) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index 0c0f629a1d..5e307af8c6 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -3,11 +3,13 @@ # p4-git-sync.py # # Author: Simon Hausmann +# Copyright: 2007 Simon Hausmann +# 2007 Trolltech ASA # License: MIT # import os, string, shelve, stat -import getopt, sys, marshal +import getopt, sys, marshal, tempfile def p4CmdList(cmd): cmd = "p4 -G %s" % cmd @@ -33,7 +35,8 @@ def p4Cmd(cmd): try: opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "git-dir=", "origin=", "reset", "master=", - "submit-log-subst=", "log-substitutions=" ]) + "submit-log-subst=", "log-substitutions=", "interactive", + "dry-run" ]) except getopt.GetoptError: print "fixme, syntax error" sys.exit(1) @@ -46,6 +49,8 @@ origin = "origin" master = "master" firstTime = True reset = False +interactive = False +dryRun = False for o, a in opts: if o == "--git-dir": @@ -67,6 +72,10 @@ for o, a in opts: for line in open(a, "r").readlines(): tokens = line[:-1].split("=") logSubstitutions[tokens[0]] = tokens[1] + elif o == "--interactive": + interactive = True + elif o == "--dry-run": + dryRun = True if len(gitdir) == 0: gitdir = ".git" @@ -112,19 +121,16 @@ def start(config): def prepareLogMessage(template, message): result = "" - substs = logSubstitutions - for k in substs.keys(): - substs[k] = substs[k].replace("%log%", message) - for line in template.split("\n"): if line.startswith("#"): result += line + "\n" continue substituted = False - for key in substs.keys(): + for key in logSubstitutions.keys(): if line.find(key) != -1: - value = substs[key] + value = logSubstitutions[key] + value = value.replace("%log%", message) if value != "@remove@": result += line.replace(key, value) + "\n" substituted = True @@ -136,6 +142,7 @@ def prepareLogMessage(template, message): return result def apply(id): + global interactive print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read()) diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() @@ -158,6 +165,9 @@ def apply(id): system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") system("git cherry-pick --no-commit \"%s\"" % id) + #system("git format-patch --stdout -k \"%s^\"..\"%s\" | git-am -k" % (id, id)) + #system("git branch -D tmp") + #system("git checkout -f -b tmp \"%s^\"" % id) for f in filesToAdd: system("p4 add %s" % f) @@ -168,22 +178,66 @@ def apply(id): logMessage = "" foundTitle = False for log in os.popen("git-cat-file commit %s" % id).readlines(): - log = log[:-1] if not foundTitle: - if len(log) == 0: + if len(log) == 1: foundTitle = 1 continue if len(logMessage) > 0: logMessage += "\t" - logMessage += log + "\n" + logMessage += log template = os.popen("p4 change -o").read() - fileName = "submit.txt" - file = open(fileName, "w+") - file.write(prepareLogMessage(template, logMessage)) - file.close() - print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) + + if interactive: + submitTemplate = prepareLogMessage(template, logMessage) + diff = os.popen("p4 diff -du ...").read() + + for newFile in filesToAdd: + diff += "==== new file ====\n" + diff += "--- /dev/null\n" + diff += "+++ %s\n" % newFile + f = open(newFile, "r") + for line in f.readlines(): + diff += "+" + line + f.close() + + pipe = os.popen("less", "w") + pipe.write(submitTemplate + diff) + pipe.close() + + response = "e" + while response == "e": + response = raw_input("Do you want to submit this change (y/e/n)? ") + if response == "e": + [handle, fileName] = tempfile.mkstemp() + tmpFile = os.fdopen(handle, "w+") + tmpFile.write(submitTemplate) + tmpFile.close() + editor = os.environ.get("EDITOR", "vi") + system(editor + " " + fileName) + tmpFile = open(fileName, "r") + submitTemplate = tmpFile.read() + tmpFile.close() + os.remove(fileName) + + if response == "y" or response == "yes": + if dryRun: + print submitTemplate + raw_input("Press return to continue...") + else: + pipe = os.popen("p4 submit -i", "w") + pipe.write(submitTemplate) + pipe.close() + else: + print "Not submitting!" + interactive = False + else: + fileName = "submit.txt" + file = open(fileName, "w+") + file.write(prepareLogMessage(template, logMessage)) + file.close() + print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) check() @@ -194,12 +248,14 @@ if firstTime: commits = config.get("commits", []) -if len(commits) > 0: +while len(commits) > 0: firstTime = False commit = commits[0] commits = commits[1:] config["commits"] = commits apply(commit) + if not interactive: + break config.close() From 794a913a0096a383560eb06e0fdf1d8331e12c0a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 14 Mar 2007 17:29:46 +0100 Subject: [PATCH 061/260] Automatically operate on a temporary branch, needed for cherry-pick to work when applying changes to files that are deleted in the future. Also do some Perforce cleaning Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index 5e307af8c6..02f4b36af3 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -97,7 +97,6 @@ def system(cmd): die("command failed: %s" % cmd) def check(): - return if len(p4CmdList("opened ...")) > 0: die("You have files opened with perforce! Close them before starting the sync.") @@ -115,6 +114,9 @@ def start(config): config["commits"] = commits + print "Creating temporary p4-sync branch from %s ..." % origin + system("git checkout -f -b p4-sync %s" % origin) + # print "Cleaning index..." # system("git checkout -f") @@ -264,5 +266,11 @@ if len(commits) == 0: print "No changes found to apply between %s and current HEAD" % origin else: print "All changes applied!" + print "Deleting temporary p4-sync branch and going back to %s" % master + system("git checkout %s" % master) + system("git branch -D p4-sync") + print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert -a ..." + system("p4 edit ...") + system("p4 revert -a ...") os.remove(configFile) From d7873afdf426b2f430f5635ff460012eb9c00d05 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 14 Mar 2007 17:33:46 +0100 Subject: [PATCH 062/260] Be nice and use /usr/bin/env python for the git-p4 scripts Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 2 +- contrib/fast-import/p4-git-sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 26cd648ef7..6f7bbb0f32 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # p4-fast-export.py # diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index 02f4b36af3..a4d126fd46 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # p4-git-sync.py # From 4d9e5fcea65f22c6bd9e1f3e5ae2f79c3af613db Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 14 Mar 2007 19:03:16 +0100 Subject: [PATCH 063/260] Ignore Apple resource files when importing from perforce to git. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index 6f7bbb0f32..a6dbd3b98d 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -329,6 +329,10 @@ def commit(details, files, branch, branchPrefix, parent, merged = ""): relPath = path[len(branchPrefix):] action = file["action"] + if file["type"] == "apple": + print "\nfile %s is a strange apple file that forks. Ignoring!" %s path + continue + if action == "delete": gitStream.write("D %s\n" % relPath) else: From d566209e7fecffb61a3209766cc937bf027e2c6c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 14 Mar 2007 23:30:23 +0100 Subject: [PATCH 064/260] Auto-detect the current git branch before submitting back to perforce. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index a4d126fd46..4f02079895 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -33,6 +33,10 @@ def p4Cmd(cmd): result.update(entry) return result; +def die(msg): + sys.stderr.write(msg + "\n") + sys.exit(1) + try: opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "git-dir=", "origin=", "reset", "master=", "submit-log-subst=", "log-substitutions=", "interactive", @@ -46,7 +50,7 @@ logSubstitutions[""] = "%log%" logSubstitutions["\tDetails:"] = "\tDetails: %log%" gitdir = os.environ.get("GIT_DIR", "") origin = "origin" -master = "master" +master = "" firstTime = True reset = False interactive = False @@ -88,9 +92,12 @@ origin = "origin" if len(args) == 1: origin = args[0] -def die(msg): - sys.stderr.write(msg + "\n") - sys.exit(1) +if len(master) == 0: + sys.stdout.write("Auto-detecting current branch: ") + master = os.popen("git-name-rev HEAD").read().split(" ")[1][:-1] + if len(master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, master)): + die("\nFailed to detect current branch! Aborting!"); + sys.stdout.write("%s\n" % master) def system(cmd): if os.system(cmd) != 0: From f72537f97e81398b597c05f12c112c52e8201b63 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 15 Mar 2007 19:07:06 +0100 Subject: [PATCH 065/260] Use p4 revert ... instead of revert -a ... after submitting, to make sure the p4 checkout is clean. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index 4f02079895..e07fcee42c 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -276,8 +276,8 @@ if len(commits) == 0: print "Deleting temporary p4-sync branch and going back to %s" % master system("git checkout %s" % master) system("git branch -D p4-sync") - print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert -a ..." - system("p4 edit ...") - system("p4 revert -a ...") + print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." + system("p4 edit ... >/dev/null") + system("p4 revert ... >/dev/null") os.remove(configFile) From 228d36c92b022ae7941b542731c89db5d9de3eac Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 16 Mar 2007 13:47:46 +0100 Subject: [PATCH 066/260] Default to interactive syncing Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index e07fcee42c..ed88debd45 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -39,7 +39,7 @@ def die(msg): try: opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "git-dir=", "origin=", "reset", "master=", - "submit-log-subst=", "log-substitutions=", "interactive", + "submit-log-subst=", "log-substitutions=", "noninteractive", "dry-run" ]) except getopt.GetoptError: print "fixme, syntax error" @@ -53,7 +53,7 @@ origin = "origin" master = "" firstTime = True reset = False -interactive = False +interactive = True dryRun = False for o, a in opts: @@ -76,8 +76,8 @@ for o, a in opts: for line in open(a, "r").readlines(): tokens = line[:-1].split("=") logSubstitutions[tokens[0]] = tokens[1] - elif o == "--interactive": - interactive = True + elif o == "--noninteractive": + interactive = False elif o == "--dry-run": dryRun = True From 09e16455e028ae264347067a8c86a7b9a1e5889c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 19 Mar 2007 11:57:07 +0100 Subject: [PATCH 067/260] Improved the git dir detection. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-git-sync.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py index ed88debd45..5a3bf90485 100755 --- a/contrib/fast-import/p4-git-sync.py +++ b/contrib/fast-import/p4-git-sync.py @@ -37,6 +37,11 @@ def die(msg): sys.stderr.write(msg + "\n") sys.exit(1) +def tryGitDir(path): + if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): + return True; + return False + try: opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "git-dir=", "origin=", "reset", "master=", "submit-log-subst=", "log-substitutions=", "noninteractive", @@ -86,6 +91,14 @@ if len(gitdir) == 0: else: os.environ["GIT_DIR"] = gitdir +if not tryGitDir(gitdir): + if tryGitDir(gitdir + "/.git"): + gitdir += "/.git" + os.environ["GIT_DIR"] = gitdir + else: + die("fatal: %s seems not to be a git repository." % gitdir) + + configFile = gitdir + "/p4-git-sync.cfg" origin = "origin" From 95d27cb75d70331cc775f37d1f3396cd01e5e238 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 19 Mar 2007 12:04:37 +0100 Subject: [PATCH 068/260] Pass the right number of arguments to commit, fixes single-branch imports. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py index a6dbd3b98d..9adb88fade 100755 --- a/contrib/fast-import/p4-fast-export.py +++ b/contrib/fast-import/p4-fast-export.py @@ -330,7 +330,7 @@ def commit(details, files, branch, branchPrefix, parent, merged = ""): action = file["action"] if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring!" %s path + print "\nfile %s is a strange apple file that forks. Ignoring!" % path continue if action == "delete": @@ -608,7 +608,7 @@ else: merged = "refs/heads/" + merged commit(description, files, branch, branchPrefix, parent, merged) else: - commit(description, filesForCommit, branch, globalPrefix, initialParent) + commit(description, files, branch, globalPrefix, initialParent) initialParent = "" except IOError: print gitError.read() From 86949eef4088d7b57fe7433568d573a926816f5c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 19 Mar 2007 20:59:12 +0100 Subject: [PATCH 069/260] Start moving the git-p4 tools into one single script. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 115 +++++++++++++++++++++++++++ contrib/fast-import/p4-clean-tags.py | 43 ---------- contrib/fast-import/p4-debug.py | 25 ------ 3 files changed, 115 insertions(+), 68 deletions(-) create mode 100755 contrib/fast-import/git-p4.py delete mode 100755 contrib/fast-import/p4-clean-tags.py delete mode 100755 contrib/fast-import/p4-debug.py diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py new file mode 100755 index 0000000000..8008156043 --- /dev/null +++ b/contrib/fast-import/git-p4.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# +# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. +# +# Author: Simon Hausmann +# License: MIT +# + +import optparse, sys, os, marshal, popen2 + +def p4CmdList(cmd): + cmd = "p4 -G %s" % cmd + pipe = os.popen(cmd, "rb") + + result = [] + try: + while True: + entry = marshal.load(pipe) + result.append(entry) + except EOFError: + pass + pipe.close() + + return result + +def p4Cmd(cmd): + list = p4CmdList(cmd) + result = {} + for entry in list: + result.update(entry) + return result; + +def die(msg): + sys.stderr.write(msg + "\n") + sys.exit(1) + +def currentGitBranch(): + return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1] + +class P4Debug: + def __init__(self): + self.options = [ + ] + + def run(self, args): + for output in p4CmdList(" ".join(args)): + print output + +class P4CleanTags: + def __init__(self): + self.options = [ +# optparse.make_option("--branch", dest="branch", default="refs/heads/master") + ] + def run(self, args): + branch = currentGitBranch() + print "Cleaning out stale p4 import tags..." + sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) + output = sout.read() + try: + tagIdx = output.index(" tags/p4/") + except: + print "Cannot find any p4/* tag. Nothing to do." + sys.exit(0) + + try: + caretIdx = output.index("^") + except: + caretIdx = len(output) - 1 + rev = int(output[tagIdx + 9 : caretIdx]) + + allTags = os.popen("git tag -l p4/").readlines() + for i in range(len(allTags)): + allTags[i] = int(allTags[i][3:-1]) + + allTags.sort() + + allTags.remove(rev) + + for rev in allTags: + print os.popen("git tag -d p4/%s" % rev).read() + + print "%s tags removed." % len(allTags) + +def printUsage(commands): + print "usage: %s [options]" % sys.argv[0] + print "" + print "valid commands: %s" % ", ".join(commands) + print "" + print "Try %s --help for command specific help." % sys.argv[0] + print "" + +commands = { + "debug" : P4Debug(), + "clean-tags" : P4CleanTags() +} + +if len(sys.argv[1:]) == 0: + printUsage(commands.keys()) + sys.exit(2) + +cmd = "" +cmdName = sys.argv[1] +try: + cmd = commands[cmdName] +except KeyError: + print "unknown command %s" % cmdName + print "" + printUsage(commands.keys()) + sys.exit(2) + +parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", cmd.options) + +(cmd, args) = parser.parse_args(sys.argv[2:], cmd); + +cmd.run(args) diff --git a/contrib/fast-import/p4-clean-tags.py b/contrib/fast-import/p4-clean-tags.py deleted file mode 100755 index 924ff89cc6..0000000000 --- a/contrib/fast-import/p4-clean-tags.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/python -# -# p4-debug.py -# -# Author: Simon Hausmann -# License: MIT -# -# removes unused p4 import tags -# -import os, string, sys -import popen2, getopt - -branch = "refs/heads/master" - -try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=" ]) -except getopt.GetoptError: - print "fixme, syntax error" - sys.exit(1) - -for o, a in opts: - if o == "--branch": - branch = "refs/heads/" + a - -sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) -output = sout.read() -tagIdx = output.index(" tags/p4/") -try: - caretIdx = output.index("^") -except: - caretIdx = len(output) - 1 -rev = int(output[tagIdx + 9 : caretIdx]) - -allTags = os.popen("git tag -l p4/").readlines() -for i in range(len(allTags)): - allTags[i] = int(allTags[i][3:-1]) - -allTags.sort() - -allTags.remove(rev) - -for rev in allTags: - print os.popen("git tag -d p4/%s" % rev).read() diff --git a/contrib/fast-import/p4-debug.py b/contrib/fast-import/p4-debug.py deleted file mode 100755 index 8fb159fd6a..0000000000 --- a/contrib/fast-import/p4-debug.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/python -# -# p4-debug.py -# -# Author: Simon Hausmann -# License: MIT -# -# executes a p4 command with -G and prints the resulting python dicts -# -import os, string, sys -import marshal, popen2 - -cmd = "" -for arg in sys.argv[1:]: - cmd += arg + " " - -pipe = os.popen("p4 -G %s" % cmd, "rb") -try: - while True: - entry = marshal.load(pipe) - print entry -except EOFError: - pass -pipe.close() - From c8c3911685de1185ca7c0afe62bb3964ebc82d38 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 19 Mar 2007 21:02:30 +0100 Subject: [PATCH 070/260] Provide a little bit of help description for the git-p4 "tools". Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 8008156043..42efc7d9aa 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -41,6 +41,7 @@ class P4Debug: def __init__(self): self.options = [ ] + self.description = "A tool to debug the output of p4 -G." def run(self, args): for output in p4CmdList(" ".join(args)): @@ -51,6 +52,7 @@ class P4CleanTags: self.options = [ # optparse.make_option("--branch", dest="branch", default="refs/heads/master") ] + self.description = "A tool to remove stale unused tags from incremental perforce imports." def run(self, args): branch = currentGitBranch() print "Cleaning out stale p4 import tags..." @@ -108,7 +110,8 @@ except KeyError: printUsage(commands.keys()) sys.exit(2) -parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", cmd.options) +parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", cmd.options, + description = cmd.description) (cmd, args) = parser.parse_args(sys.argv[2:], cmd); From 4f5cf76a55ecb3252b0924d0c4a16f3b037908cd Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 19 Mar 2007 22:25:17 +0100 Subject: [PATCH 071/260] First (untested) attempt at migrating p4-git-sync into the final git-p4 script Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 243 +++++++++++++++++++++++++++++++++- 1 file changed, 240 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 42efc7d9aa..593bda822f 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -6,7 +6,10 @@ # License: MIT # -import optparse, sys, os, marshal, popen2 +import optparse, sys, os, marshal, popen2, shelve +import tempfile + +gitdir = os.environ.get("GIT_DIR", "") def p4CmdList(cmd): cmd = "p4 -G %s" % cmd @@ -37,6 +40,15 @@ def die(msg): def currentGitBranch(): return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1] +def isValidGitDir(path): + if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): + return True; + return False + +def system(cmd): + if os.system(cmd) != 0: + die("command failed: %s" % cmd) + class P4Debug: def __init__(self): self.options = [ @@ -83,6 +95,214 @@ class P4CleanTags: print "%s tags removed." % len(allTags) +class P4Sync: + def __init__(self): + self.options = [ + optparse.make_option("--continue", action="store_false", dest="firstTime"), + optparse.make_option("--origin", dest="origin"), + optparse.make_option("--reset", action="store_true", dest="reset"), + optparse.make_option("--master", dest="master"), + optparse.make_option("--log-substitutions", dest="substFile"), + optparse.make_option("--noninteractive", action="store_false"), + optparse.make_option("--dry-run", action="store_true") + ] + self.description = "Submit changes from git to the perforce depot." + self.firstTime = True + self.reset = False + self.interactive = True + self.dryRun = False + self.substFile = "" + self.firstTime = True + self.origin = "origin" + self.master = "" + + self.logSubstitutions = {} + self.logSubstitutions[""] = "%log%" + self.logSubstitutions["\tDetails:"] = "\tDetails: %log%" + + def check(self): + if len(p4CmdList("opened ...")) > 0: + die("You have files opened with perforce! Close them before starting the sync.") + + def start(self): + if len(self.config) > 0 and not self.reset: + die("Cannot start sync. Previous sync config found at %s" % self.configFile) + + commits = [] + for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): + commits.append(line[:-1]) + commits.reverse() + + self.config["commits"] = commits + + print "Creating temporary p4-sync branch from %s ..." % self.origin + system("git checkout -f -b p4-sync %s" % self.origin) + + def prepareLogMessage(self, template, message): + result = "" + + for line in template.split("\n"): + if line.startswith("#"): + result += line + "\n" + continue + + substituted = False + for key in self.logSubstitutions.keys(): + if line.find(key) != -1: + value = self.logSubstitutions[key] + value = value.replace("%log%", message) + if value != "@remove@": + result += line.replace(key, value) + "\n" + substituted = True + break + + if not substituted: + result += line + "\n" + + return result + + def apply(self, id): + print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read()) + diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() + filesToAdd = set() + filesToDelete = set() + for line in diff: + modifier = line[0] + path = line[1:].strip() + if modifier == "M": + system("p4 edit %s" % path) + elif modifier == "A": + filesToAdd.add(path) + if path in filesToDelete: + filesToDelete.remove(path) + elif modifier == "D": + filesToDelete.add(path) + if path in filesToAdd: + filesToAdd.remove(path) + else: + die("unknown modifier %s for %s" % (modifier, path)) + + system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") + system("git cherry-pick --no-commit \"%s\"" % id) + + for f in filesToAdd: + system("p4 add %s" % f) + for f in filesToDelete: + system("p4 revert %s" % f) + system("p4 delete %s" % f) + + logMessage = "" + foundTitle = False + for log in os.popen("git-cat-file commit %s" % id).readlines(): + if not foundTitle: + if len(log) == 1: + foundTitle = 1 + continue + + if len(logMessage) > 0: + logMessage += "\t" + logMessage += log + + template = os.popen("p4 change -o").read() + + if self.interactive: + submitTemplate = self.prepareLogMessage(template, logMessage) + diff = os.popen("p4 diff -du ...").read() + + for newFile in filesToAdd: + diff += "==== new file ====\n" + diff += "--- /dev/null\n" + diff += "+++ %s\n" % newFile + f = open(newFile, "r") + for line in f.readlines(): + diff += "+" + line + f.close() + + pipe = os.popen("less", "w") + pipe.write(submitTemplate + diff) + pipe.close() + + response = "e" + while response == "e": + response = raw_input("Do you want to submit this change (y/e/n)? ") + if response == "e": + [handle, fileName] = tempfile.mkstemp() + tmpFile = os.fdopen(handle, "w+") + tmpFile.write(submitTemplate) + tmpFile.close() + editor = os.environ.get("EDITOR", "vi") + system(editor + " " + fileName) + tmpFile = open(fileName, "r") + submitTemplate = tmpFile.read() + tmpFile.close() + os.remove(fileName) + + if response == "y" or response == "yes": + if self.dryRun: + print submitTemplate + raw_input("Press return to continue...") + else: + pipe = os.popen("p4 submit -i", "w") + pipe.write(submitTemplate) + pipe.close() + else: + print "Not submitting!" + self.interactive = False + else: + fileName = "submit.txt" + file = open(fileName, "w+") + file.write(self.prepareLogMessage(template, logMessage)) + file.close() + print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) + + def run(self, args): + if self.reset: + self.firstTime = True + + if len(self.substFile) > 0: + for line in open(self.substFile, "r").readlines(): + tokens = line[:-1].split("=") + self.logSubstitutions[tokens[0]] = tokens[1] + + if len(self.master) == 0: + self.master = currentGitBranch() + if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)): + die("Detecting current git branch failed!") + + self.check() + self.configFile = gitdir + "/p4-git-sync.cfg" + self.config = shelve.open(self.configFile, writeback=True) + + if self.firstTime: + self.start() + + commits = self.config.get("commits", []) + + while len(commits) > 0: + self.firstTime = False + commit = commits[0] + commits = commits[1:] + self.config["commits"] = commits + self.apply(commit) + if not self.interactive: + break + + self.config.close() + + if len(commits) == 0: + if self.firstTime: + print "No changes found to apply between %s and current HEAD" % self.origin + else: + print "All changes applied!" + print "Deleting temporary p4-sync branch and going back to %s" % self.master + system("git checkout %s" % self.master) + system("git branch -D p4-sync") + print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." + system("p4 edit ... >/dev/null") + system("p4 revert ... >/dev/null") + os.remove(self.configFile) + + def printUsage(commands): print "usage: %s [options]" % sys.argv[0] print "" @@ -93,7 +313,8 @@ def printUsage(commands): commands = { "debug" : P4Debug(), - "clean-tags" : P4CleanTags() + "clean-tags" : P4CleanTags(), + "sync-to-perforce" : P4Sync() } if len(sys.argv[1:]) == 0: @@ -110,9 +331,25 @@ except KeyError: printUsage(commands.keys()) sys.exit(2) -parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", cmd.options, +options = cmd.options +cmd.gitdir = gitdir +options.append(optparse.make_option("--git-dir", dest="gitdir")) + +parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", options, description = cmd.description) (cmd, args) = parser.parse_args(sys.argv[2:], cmd); +gitdir = cmd.gitdir +if len(gitdir) == 0: + gitdir = ".git" + +if not isValidGitDir(gitdir): + if isValidGitDir(gitdir + "/.git"): + gitdir += "/.git" + else: + dir("fatal: cannot locate git repository at %s" % gitdir) + +os.environ["GIT_DIR"] = gitdir + cmd.run(args) From 83dce55af3c792134787bdc807f932dc8a67ea74 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 19 Mar 2007 22:26:36 +0100 Subject: [PATCH 072/260] Part of the code is copyright by Trolltech ASA. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 593bda822f..fa1b19fbbc 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -3,6 +3,8 @@ # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. # # Author: Simon Hausmann +# Copyright: 2007 Simon Hausmann +# 2007 Trolltech ASA # License: MIT # From 05140f342e1df7319dd3da2ef8157bfd5760fee6 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 20 Mar 2007 18:32:47 +0100 Subject: [PATCH 073/260] sync-to-perforce is now called submit and fixed the gitdir check a little bit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index fa1b19fbbc..aca1bcaab2 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -316,7 +316,7 @@ def printUsage(commands): commands = { "debug" : P4Debug(), "clean-tags" : P4CleanTags(), - "sync-to-perforce" : P4Sync() + "submit" : P4Sync() } if len(sys.argv[1:]) == 0: @@ -350,7 +350,7 @@ if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"): gitdir += "/.git" else: - dir("fatal: cannot locate git repository at %s" % gitdir) + die("fatal: cannot locate git repository at %s" % gitdir) os.environ["GIT_DIR"] = gitdir From b984733c8048423537540fdca681ddc448b60fcc Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 20 Mar 2007 20:54:23 +0100 Subject: [PATCH 074/260] Completely untested "merge" of p4-fast-export.py into git-p4.py Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 590 +++++++++++++++++++++++++++++++++- 1 file changed, 582 insertions(+), 8 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index aca1bcaab2..477238fa1c 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -9,7 +9,8 @@ # import optparse, sys, os, marshal, popen2, shelve -import tempfile +import tempfile, getopt, sha, os.path, time +from sets import Set; gitdir = os.environ.get("GIT_DIR", "") @@ -51,7 +52,11 @@ def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) -class P4Debug: +class Command: + def __init__(self): + self.usage = "usage: %prog [options]" + +class P4Debug(Command): def __init__(self): self.options = [ ] @@ -60,9 +65,11 @@ class P4Debug: def run(self, args): for output in p4CmdList(" ".join(args)): print output + return True -class P4CleanTags: +class P4CleanTags(Command): def __init__(self): + Command.__init__(self) self.options = [ # optparse.make_option("--branch", dest="branch", default="refs/heads/master") ] @@ -96,9 +103,11 @@ class P4CleanTags: print os.popen("git tag -d p4/%s" % rev).read() print "%s tags removed." % len(allTags) + return True -class P4Sync: +class P4Sync(Command): def __init__(self): + Command.__init__(self) self.options = [ optparse.make_option("--continue", action="store_false", dest="firstTime"), optparse.make_option("--origin", dest="origin"), @@ -304,6 +313,566 @@ class P4Sync: system("p4 revert ... >/dev/null") os.remove(self.configFile) + return True + +class GitSync(Command): + def __init__(self): + Command.__init__(self) + self.options = [ + optparse.make_option("--branch", dest="branch"), + optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"), + optparse.make_option("--changesfile", dest="changesFile"), + optparse.make_option("--silent", dest="silent", action="store_true"), + optparse.make_option("--known-branches", dest="knownBranches"), + optparse.make_option("--cache", dest="doCache", action="store_true"), + optparse.make_option("--command-cache", dest="commandCache", action="store_true") + ] + self.description = """Imports from Perforce into a git repository.\n + example: + //depot/my/project/ -- to import the current head + //depot/my/project/@all -- to import everything + //depot/my/project/@1,6 -- to import only from revision 1 to 6 + + (a ... is not needed in the path p4 specification, it's added implicitly)""" + + self.usage += " //depot/path[@revRange]" + + self.dataCache = False + self.commandCache = False + self.silent = False + self.knownBranches = Set() + self.createdBranches = Set() + self.committedChanges = Set() + self.branch = "master" + self.detectBranches = False + self.changesFile = "" + + def p4File(self, depotPath): + return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() + + def extractFilesFromCommit(self, commit): + files = [] + fnum = 0 + while commit.has_key("depotFile%s" % fnum): + path = commit["depotFile%s" % fnum] + if not path.startswith(self.globalPrefix): + # if not self.silent: + # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.globalPrefix, change) + fnum = fnum + 1 + continue + + file = {} + file["path"] = path + file["rev"] = commit["rev%s" % fnum] + file["action"] = commit["action%s" % fnum] + file["type"] = commit["type%s" % fnum] + files.append(file) + fnum = fnum + 1 + return files + + def isSubPathOf(self, first, second): + if not first.startswith(second): + return False + if first == second: + return True + return first[len(second)] == "/" + + def branchesForCommit(self, files): + branches = Set() + + for file in files: + relativePath = file["path"][len(self.globalPrefix):] + # strip off the filename + relativePath = relativePath[0:relativePath.rfind("/")] + + # if len(branches) == 0: + # branches.add(relativePath) + # knownBranches.add(relativePath) + # continue + + ###### this needs more testing :) + knownBranch = False + for branch in branches: + if relativePath == branch: + knownBranch = True + break + # if relativePath.startswith(branch): + if self.isSubPathOf(relativePath, branch): + knownBranch = True + break + # if branch.startswith(relativePath): + if self.isSubPathOf(branch, relativePath): + branches.remove(branch) + break + + if knownBranch: + continue + + for branch in knownBranches: + #if relativePath.startswith(branch): + if self.isSubPathOf(relativePath, branch): + if len(branches) == 0: + relativePath = branch + else: + knownBranch = True + break + + if knownBranch: + continue + + branches.add(relativePath) + self.knownBranches.add(relativePath) + + return branches + + def findBranchParent(self, branchPrefix, files): + for file in files: + path = file["path"] + if not path.startswith(branchPrefix): + continue + action = file["action"] + if action != "integrate" and action != "branch": + continue + rev = file["rev"] + depotPath = path + "#" + rev + + log = p4CmdList("filelog \"%s\"" % depotPath) + if len(log) != 1: + print "eek! I got confused by the filelog of %s" % depotPath + sys.exit(1); + + log = log[0] + if log["action0"] != action: + print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) + sys.exit(1); + + branchAction = log["how0,0"] + # if branchAction == "branch into" or branchAction == "ignored": + # continue # ignore for branching + + if not branchAction.endswith(" from"): + continue # ignore for branching + # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) + # sys.exit(1); + + source = log["file0,0"] + if source.startswith(branchPrefix): + continue + + lastSourceRev = log["erev0,0"] + + sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) + if len(sourceLog) != 1: + print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) + sys.exit(1); + sourceLog = sourceLog[0] + + relPath = source[len(self.globalPrefix):] + # strip off the filename + relPath = relPath[0:relPath.rfind("/")] + + for branch in self.knownBranches: + if self.isSubPathOf(relPath, branch): + # print "determined parent branch branch %s due to change in file %s" % (branch, source) + return branch + # else: + # print "%s is not a subpath of branch %s" % (relPath, branch) + + return "" + + def commit(self, details, files, branch, branchPrefix, parent, merged = ""): + epoch = details["time"] + author = details["user"] + + self.gitStream.write("commit %s\n" % branch) + # gitStream.write("mark :%s\n" % details["change"]) + self.committedChanges.add(int(details["change"])) + committer = "" + if author in self.users: + committer = "%s %s %s" % (self.users[author], epoch, tz) + else: + committer = "%s %s %s" % (author, epoch, tz) + + self.gitStream.write("committer %s\n" % committer) + + self.gitStream.write("data < 0: + self.gitStream.write("from %s\n" % parent) + + if len(merged) > 0: + self.gitStream.write("merge %s\n" % merged) + + for file in files: + path = file["path"] + if not path.startswith(branchPrefix): + # if not silent: + # print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) + continue + rev = file["rev"] + depotPath = path + "#" + rev + relPath = path[len(branchPrefix):] + action = file["action"] + + if file["type"] == "apple": + print "\nfile %s is a strange apple file that forks. Ignoring!" % path + continue + + if action == "delete": + self.gitStream.write("D %s\n" % relPath) + else: + mode = 644 + if file["type"].startswith("x"): + mode = 755 + + data = self.p4File(depotPath) + + self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + self.gitStream.write("data %s\n" % len(data)) + self.gitStream.write(data) + self.gitStream.write("\n") + + self.gitStream.write("\n") + + self.lastChange = int(details["change"]) + + def extractFilesInCommitToBranch(self, files, branchPrefix): + newFiles = [] + + for file in files: + path = file["path"] + if path.startswith(branchPrefix): + newFiles.append(file) + + return newFiles + + def findBranchSourceHeuristic(self, files, branch, branchPrefix): + for file in files: + action = file["action"] + if action != "integrate" and action != "branch": + continue + path = file["path"] + rev = file["rev"] + depotPath = path + "#" + rev + + log = p4CmdList("filelog \"%s\"" % depotPath) + if len(log) != 1: + print "eek! I got confused by the filelog of %s" % depotPath + sys.exit(1); + + log = log[0] + if log["action0"] != action: + print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) + sys.exit(1); + + branchAction = log["how0,0"] + + if not branchAction.endswith(" from"): + continue # ignore for branching + # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) + # sys.exit(1); + + source = log["file0,0"] + if source.startswith(branchPrefix): + continue + + lastSourceRev = log["erev0,0"] + + sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) + if len(sourceLog) != 1: + print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) + sys.exit(1); + sourceLog = sourceLog[0] + + relPath = source[len(self.globalPrefix):] + # strip off the filename + relPath = relPath[0:relPath.rfind("/")] + + for candidate in self.knownBranches: + if self.isSubPathOf(relPath, candidate) and candidate != branch: + return candidate + + return "" + + def changeIsBranchMerge(self, sourceBranch, destinationBranch, change): + sourceFiles = {} + for file in p4CmdList("files %s...@%s" % (self.globalPrefix + sourceBranch + "/", change)): + if file["action"] == "delete": + continue + sourceFiles[file["depotFile"]] = file + + destinationFiles = {} + for file in p4CmdList("files %s...@%s" % (self.globalPrefix + destinationBranch + "/", change)): + destinationFiles[file["depotFile"]] = file + + for fileName in sourceFiles.keys(): + integrations = [] + deleted = False + integrationCount = 0 + for integration in p4CmdList("integrated \"%s\"" % fileName): + toFile = integration["fromFile"] # yes, it's true, it's fromFile + if not toFile in destinationFiles: + continue + destFile = destinationFiles[toFile] + if destFile["action"] == "delete": + # print "file %s has been deleted in %s" % (fileName, toFile) + deleted = True + break + integrationCount += 1 + if integration["how"] == "branch from": + continue + + if int(integration["change"]) == change: + integrations.append(integration) + continue + if int(integration["change"]) > change: + continue + + destRev = int(destFile["rev"]) + + startRev = integration["startFromRev"][1:] + if startRev == "none": + startRev = 0 + else: + startRev = int(startRev) + + endRev = integration["endFromRev"][1:] + if endRev == "none": + endRev = 0 + else: + endRev = int(endRev) + + initialBranch = (destRev == 1 and integration["how"] != "branch into") + inRange = (destRev >= startRev and destRev <= endRev) + newer = (destRev > startRev and destRev > endRev) + + if initialBranch or inRange or newer: + integrations.append(integration) + + if deleted: + continue + + if len(integrations) == 0 and integrationCount > 1: + print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch) + return False + + return True + + def getUserMap(self): + self.users = {} + + for output in p4CmdList("users"): + if not output.has_key("User"): + continue + self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" + + def run(self, args): + self.branch = "refs/heads/" + self.branch + self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() + if len(self.globalPrefix) != 0: + self.globalPrefix = self.globalPrefix[:-1] + + if len(args) == 0 and len(self.globalPrefix) != 0: + if not self.silent: + print "[using previously specified depot path %s]" % self.globalPrefix + elif len(args) != 1: + return False + else: + if len(self.globalPrefix) != 0 and self.globalPrefix != args[0]: + print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.globalPrefix, args[0]) + sys.exit(1) + self.globalPrefix = args[0] + + self.changeRange = "" + self.revision = "" + self.users = {} + self.initialParent = "" + self.lastChange = 0 + self.initialTag = "" + + if self.globalPrefix.find("@") != -1: + atIdx = self.globalPrefix.index("@") + self.changeRange = self.globalPrefix[atIdx:] + if self.changeRange == "@all": + self.changeRange = "" + elif self.changeRange.find(",") == -1: + self.revision = self.changeRange + self.changeRange = "" + self.globalPrefix = self.globalPrefix[0:atIdx] + elif self.globalPrefix.find("#") != -1: + hashIdx = self.globalPrefix.index("#") + self.revision = self.globalPrefix[hashIdx:] + self.globalPrefix = self.globalPrefix[0:hashIdx] + elif len(self.previousDepotPath) == 0: + self.revision = "#head" + + if self.globalPrefix.endswith("..."): + self.globalPrefix = self.globalPrefix[:-3] + + if not self.globalPrefix.endswith("/"): + self.globalPrefix += "/" + + self.getUserMap() + + if len(self.changeRange) == 0: + try: + sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % self.branch) + output = sout.read() + if output.endswith("\n"): + output = output[:-1] + tagIdx = output.index(" tags/p4/") + caretIdx = output.find("^") + endPos = len(output) + if caretIdx != -1: + endPos = caretIdx + self.rev = int(output[tagIdx + 9 : endPos]) + 1 + self.changeRange = "@%s,#head" % self.rev + self.initialParent = os.popen("git-rev-parse %s" % self.branch).read()[:-1] + self.initialTag = "p4/%s" % (int(self.rev) - 1) + except: + pass + + tz = - time.timezone / 36 + tzsign = ("%s" % tz)[0] + if tzsign != '+' and tzsign != '-': + tz = "+" + ("%s" % tz) + + self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git-fast-import") + + if len(self.revision) > 0: + print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision) + + details = { "user" : "git perforce import user", "time" : int(time.time()) } + details["desc"] = "Initial import of %s from the state at revision %s" % (self.globalPrefix, self.revision) + details["change"] = self.revision + newestRevision = 0 + + fileCnt = 0 + for info in p4CmdList("files %s...%s" % (self.globalPrefix, self.revision)): + change = int(info["change"]) + if change > newestRevision: + newestRevision = change + + if info["action"] == "delete": + continue + + for prop in [ "depotFile", "rev", "action", "type" ]: + details["%s%s" % (prop, fileCnt)] = info[prop] + + fileCnt = fileCnt + 1 + + details["change"] = newestRevision + + try: + self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix) + except: + print self.gitError.read() + + else: + changes = [] + + if len(changesFile) > 0: + output = open(self.changesFile).readlines() + changeSet = Set() + for line in output: + changeSet.add(int(line)) + + for change in changeSet: + changes.append(change) + + changes.sort() + else: + output = os.popen("p4 changes %s...%s" % (self.globalPrefix, self.changeRange)).readlines() + + for line in output: + changeNum = line.split(" ")[1] + changes.append(changeNum) + + changes.reverse() + + if len(changes) == 0: + if not silent: + print "no changes to import!" + sys.exit(1) + + cnt = 1 + for change in changes: + description = p4Cmd("describe %s" % change) + + if not silent: + sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.flush() + cnt = cnt + 1 + + try: + files = self.extractFilesFromCommit(description) + if self.detectBranches: + for branch in self.branchesForCommit(files): + self.knownBranches.add(branch) + branchPrefix = self.globalPrefix + branch + "/" + + filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix) + + merged = "" + parent = "" + ########### remove cnt!!! + if branch not in self.createdBranches and cnt > 2: + self.createdBranches.add(branch) + parent = self.findBranchParent(branchPrefix, files) + if parent == branch: + parent = "" + # elif len(parent) > 0: + # print "%s branched off of %s" % (branch, parent) + + if len(parent) == 0: + merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) + if len(merged) > 0: + print "change %s could be a merge from %s into %s" % (description["change"], merged, branch) + if not self.changeIsBranchMerge(merged, branch, int(description["change"])): + merged = "" + + branch = "refs/heads/" + branch + if len(parent) > 0: + parent = "refs/heads/" + parent + if len(merged) > 0: + merged = "refs/heads/" + merged + self.commit(description, files, branch, branchPrefix, parent, merged) + else: + self.commit(description, files, branch, globalPrefix, initialParent) + self.initialParent = "" + except IOError: + print self.gitError.read() + sys.exit(1) + + if not self.silent: + print "" + + self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange) + self.gitStream.write("from %s\n\n" % self.branch); + + + self.gitStream.close() + self.gitOutput.close() + self.gitError.close() + + os.popen("git-repo-config p4.depotpath %s" % self.globalPrefix).read() + if len(self.initialTag) > 0: + os.popen("git tag -d %s" % self.initialTag).read() + + return True + +class HelpFormatter(optparse.IndentedHelpFormatter): + def __init__(self): + optparse.IndentedHelpFormatter.__init__(self) + + def format_description(self, description): + if description: + return description + "\n" + else: + return "" def printUsage(commands): print "usage: %s [options]" % sys.argv[0] @@ -316,7 +885,8 @@ def printUsage(commands): commands = { "debug" : P4Debug(), "clean-tags" : P4CleanTags(), - "submit" : P4Sync() + "submit" : P4Sync(), + "sync" : GitSync() } if len(sys.argv[1:]) == 0: @@ -337,8 +907,10 @@ options = cmd.options cmd.gitdir = gitdir options.append(optparse.make_option("--git-dir", dest="gitdir")) -parser = optparse.OptionParser("usage: %prog " + cmdName + " [options]", options, - description = cmd.description) +parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), + options, + description = cmd.description, + formatter = HelpFormatter()) (cmd, args) = parser.parse_args(sys.argv[2:], cmd); @@ -354,4 +926,6 @@ if not isValidGitDir(gitdir): os.environ["GIT_DIR"] = gitdir -cmd.run(args) +if not cmd.run(args): + parser.print_help() + From 0828ab140360c4c831e112f8d244923a7f032e74 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 20 Mar 2007 20:59:30 +0100 Subject: [PATCH 075/260] Added missing "self"s to make the script evaluate correctly. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 477238fa1c..8cb63f9540 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -489,9 +489,9 @@ class GitSync(Command): self.committedChanges.add(int(details["change"])) committer = "" if author in self.users: - committer = "%s %s %s" % (self.users[author], epoch, tz) + committer = "%s %s %s" % (self.users[author], epoch, self.tz) else: - committer = "%s %s %s" % (author, epoch, tz) + committer = "%s %s %s" % (author, epoch, self.tz) self.gitStream.write("committer %s\n" % committer) @@ -735,10 +735,10 @@ class GitSync(Command): except: pass - tz = - time.timezone / 36 - tzsign = ("%s" % tz)[0] + self.tz = - time.timezone / 36 + tzsign = ("%s" % self.tz)[0] if tzsign != '+' and tzsign != '-': - tz = "+" + ("%s" % tz) + self.tz = "+" + ("%s" % self.tz) self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git-fast-import") @@ -774,7 +774,7 @@ class GitSync(Command): else: changes = [] - if len(changesFile) > 0: + if len(self.changesFile) > 0: output = open(self.changesFile).readlines() changeSet = Set() for line in output: @@ -794,7 +794,7 @@ class GitSync(Command): changes.reverse() if len(changes) == 0: - if not silent: + if not self.silent: print "no changes to import!" sys.exit(1) @@ -802,7 +802,7 @@ class GitSync(Command): for change in changes: description = p4Cmd("describe %s" % change) - if not silent: + if not self.silent: sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) sys.stdout.flush() cnt = cnt + 1 @@ -841,7 +841,7 @@ class GitSync(Command): merged = "refs/heads/" + merged self.commit(description, files, branch, branchPrefix, parent, merged) else: - self.commit(description, files, branch, globalPrefix, initialParent) + self.commit(description, files, self.branch, self.globalPrefix, self.initialParent) self.initialParent = "" except IOError: print self.gitError.read() From c715706b15426c7b106677868ce48e4bd78b94d6 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 20 Mar 2007 21:13:49 +0100 Subject: [PATCH 076/260] Fixed the initial version import by getting the file index correct by correctly skipping deleted files. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 8cb63f9540..54ddf73dac 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -480,7 +480,7 @@ class GitSync(Command): return "" - def commit(self, details, files, branch, branchPrefix, parent, merged = ""): + def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""): epoch = details["time"] author = details["user"] @@ -757,6 +757,7 @@ class GitSync(Command): newestRevision = change if info["action"] == "delete": + fileCnt = fileCnt + 1 continue for prop in [ "depotFile", "rev", "action", "type" ]: @@ -768,7 +769,7 @@ class GitSync(Command): try: self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix) - except: + except IOError: print self.gitError.read() else: From c5fdcbcc20ebf0716fd17e3ac4f73baacf38a699 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 20 Mar 2007 22:09:27 +0100 Subject: [PATCH 077/260] Removed p4-fast-export and p4-git-sync as they've been integrated into git-p4 now. Signed-off-by: Simon Hausmann --- contrib/fast-import/p4-fast-export.py | 632 -------------------------- contrib/fast-import/p4-git-sync.py | 296 ------------ 2 files changed, 928 deletions(-) delete mode 100755 contrib/fast-import/p4-fast-export.py delete mode 100755 contrib/fast-import/p4-git-sync.py diff --git a/contrib/fast-import/p4-fast-export.py b/contrib/fast-import/p4-fast-export.py deleted file mode 100755 index 9adb88fade..0000000000 --- a/contrib/fast-import/p4-fast-export.py +++ /dev/null @@ -1,632 +0,0 @@ -#!/usr/bin/env python -# -# p4-fast-export.py -# -# Author: Simon Hausmann -# License: MIT -# -# TODO: -# - support integrations (at least p4i) -# - support p4 submit (hah!) -# - emulate p4's delete behavior: if a directory becomes empty delete it. continue -# with parent dir until non-empty dir is found. -# -import os, string, sys, time, os.path -import marshal, popen2, getopt, sha -from sets import Set; - -dataCache = False -commandCache = False - -silent = False -knownBranches = Set() -createdBranches = Set() -committedChanges = Set() -branch = "refs/heads/master" -globalPrefix = previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() -detectBranches = False -changesFile = "" -if len(globalPrefix) != 0: - globalPrefix = globalPrefix[:-1] - -try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "branch=", "detect-branches", "changesfile=", "silent", "known-branches=", - "cache", "command-cache" ]) -except getopt.GetoptError: - print "fixme, syntax error" - sys.exit(1) - -for o, a in opts: - if o == "--branch": - branch = "refs/heads/" + a - elif o == "--detect-branches": - detectBranches = True - elif o == "--changesfile": - changesFile = a - elif o == "--silent": - silent= True - elif o == "--known-branches": - for branch in open(a).readlines(): - knownBranches.add(branch[:-1]) - elif o == "--cache": - dataCache = True - commandCache = True - elif o == "--command-cache": - commandCache = True - -if len(args) == 0 and len(globalPrefix) != 0: - if not silent: - print "[using previously specified depot path %s]" % globalPrefix -elif len(args) != 1: - print "usage: %s //depot/path[@revRange]" % sys.argv[0] - print "\n example:" - print " %s //depot/my/project/ -- to import the current head" - print " %s //depot/my/project/@all -- to import everything" - print " %s //depot/my/project/@1,6 -- to import only from revision 1 to 6" - print "" - print " (a ... is not needed in the path p4 specification, it's added implicitly)" - print "" - sys.exit(1) -else: - if len(globalPrefix) != 0 and globalPrefix != args[0]: - print "previous import used depot path %s and now %s was specified. this doesn't work!" % (globalPrefix, args[0]) - sys.exit(1) - globalPrefix = args[0] - -changeRange = "" -revision = "" -users = {} -initialParent = "" -lastChange = 0 -initialTag = "" - -if globalPrefix.find("@") != -1: - atIdx = globalPrefix.index("@") - changeRange = globalPrefix[atIdx:] - if changeRange == "@all": - changeRange = "" - elif changeRange.find(",") == -1: - revision = changeRange - changeRange = "" - globalPrefix = globalPrefix[0:atIdx] -elif globalPrefix.find("#") != -1: - hashIdx = globalPrefix.index("#") - revision = globalPrefix[hashIdx:] - globalPrefix = globalPrefix[0:hashIdx] -elif len(previousDepotPath) == 0: - revision = "#head" - -if globalPrefix.endswith("..."): - globalPrefix = globalPrefix[:-3] - -if not globalPrefix.endswith("/"): - globalPrefix += "/" - -def p4File(depotPath): - cacheKey = "/tmp/p4cache/data-" + sha.new(depotPath).hexdigest() - - data = 0 - try: - if not dataCache: - raise - data = open(cacheKey, "rb").read() - except: - data = os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() - if dataCache: - open(cacheKey, "wb").write(data) - - return data - -def p4CmdList(cmd): - fullCmd = "p4 -G %s" % cmd; - - cacheKey = sha.new(fullCmd).hexdigest() - cacheKey = "/tmp/p4cache/cmd-" + cacheKey - - cached = True - pipe = 0 - try: - if not commandCache: - raise - pipe = open(cacheKey, "rb") - except: - cached = False - pipe = os.popen(fullCmd, "rb") - - result = [] - try: - while True: - entry = marshal.load(pipe) - result.append(entry) - except EOFError: - pass - pipe.close() - - if not cached and commandCache: - pipe = open(cacheKey, "wb") - for r in result: - marshal.dump(r, pipe) - pipe.close() - - return result - -def p4Cmd(cmd): - list = p4CmdList(cmd) - result = {} - for entry in list: - result.update(entry) - return result; - -def extractFilesFromCommit(commit): - files = [] - fnum = 0 - while commit.has_key("depotFile%s" % fnum): - path = commit["depotFile%s" % fnum] - if not path.startswith(globalPrefix): -# if not silent: -# print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, globalPrefix, change) - fnum = fnum + 1 - continue - - file = {} - file["path"] = path - file["rev"] = commit["rev%s" % fnum] - file["action"] = commit["action%s" % fnum] - file["type"] = commit["type%s" % fnum] - files.append(file) - fnum = fnum + 1 - return files - -def isSubPathOf(first, second): - if not first.startswith(second): - return False - if first == second: - return True - return first[len(second)] == "/" - -def branchesForCommit(files): - global knownBranches - branches = Set() - - for file in files: - relativePath = file["path"][len(globalPrefix):] - # strip off the filename - relativePath = relativePath[0:relativePath.rfind("/")] - -# if len(branches) == 0: -# branches.add(relativePath) -# knownBranches.add(relativePath) -# continue - - ###### this needs more testing :) - knownBranch = False - for branch in branches: - if relativePath == branch: - knownBranch = True - break -# if relativePath.startswith(branch): - if isSubPathOf(relativePath, branch): - knownBranch = True - break -# if branch.startswith(relativePath): - if isSubPathOf(branch, relativePath): - branches.remove(branch) - break - - if knownBranch: - continue - - for branch in knownBranches: - #if relativePath.startswith(branch): - if isSubPathOf(relativePath, branch): - if len(branches) == 0: - relativePath = branch - else: - knownBranch = True - break - - if knownBranch: - continue - - branches.add(relativePath) - knownBranches.add(relativePath) - - return branches - -def findBranchParent(branchPrefix, files): - for file in files: - path = file["path"] - if not path.startswith(branchPrefix): - continue - action = file["action"] - if action != "integrate" and action != "branch": - continue - rev = file["rev"] - depotPath = path + "#" + rev - - log = p4CmdList("filelog \"%s\"" % depotPath) - if len(log) != 1: - print "eek! I got confused by the filelog of %s" % depotPath - sys.exit(1); - - log = log[0] - if log["action0"] != action: - print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) - sys.exit(1); - - branchAction = log["how0,0"] -# if branchAction == "branch into" or branchAction == "ignored": -# continue # ignore for branching - - if not branchAction.endswith(" from"): - continue # ignore for branching -# print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) -# sys.exit(1); - - source = log["file0,0"] - if source.startswith(branchPrefix): - continue - - lastSourceRev = log["erev0,0"] - - sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) - if len(sourceLog) != 1: - print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) - sys.exit(1); - sourceLog = sourceLog[0] - - relPath = source[len(globalPrefix):] - # strip off the filename - relPath = relPath[0:relPath.rfind("/")] - - for branch in knownBranches: - if isSubPathOf(relPath, branch): -# print "determined parent branch branch %s due to change in file %s" % (branch, source) - return branch -# else: -# print "%s is not a subpath of branch %s" % (relPath, branch) - - return "" - -def commit(details, files, branch, branchPrefix, parent, merged = ""): - global users - global lastChange - global committedChanges - - epoch = details["time"] - author = details["user"] - - gitStream.write("commit %s\n" % branch) -# gitStream.write("mark :%s\n" % details["change"]) - committedChanges.add(int(details["change"])) - committer = "" - if author in users: - committer = "%s %s %s" % (users[author], epoch, tz) - else: - committer = "%s %s %s" % (author, epoch, tz) - - gitStream.write("committer %s\n" % committer) - - gitStream.write("data < 0: - gitStream.write("from %s\n" % parent) - - if len(merged) > 0: - gitStream.write("merge %s\n" % merged) - - for file in files: - path = file["path"] - if not path.startswith(branchPrefix): -# if not silent: -# print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) - continue - rev = file["rev"] - depotPath = path + "#" + rev - relPath = path[len(branchPrefix):] - action = file["action"] - - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring!" % path - continue - - if action == "delete": - gitStream.write("D %s\n" % relPath) - else: - mode = 644 - if file["type"].startswith("x"): - mode = 755 - - data = p4File(depotPath) - - gitStream.write("M %s inline %s\n" % (mode, relPath)) - gitStream.write("data %s\n" % len(data)) - gitStream.write(data) - gitStream.write("\n") - - gitStream.write("\n") - - lastChange = int(details["change"]) - -def extractFilesInCommitToBranch(files, branchPrefix): - newFiles = [] - - for file in files: - path = file["path"] - if path.startswith(branchPrefix): - newFiles.append(file) - - return newFiles - -def findBranchSourceHeuristic(files, branch, branchPrefix): - for file in files: - action = file["action"] - if action != "integrate" and action != "branch": - continue - path = file["path"] - rev = file["rev"] - depotPath = path + "#" + rev - - log = p4CmdList("filelog \"%s\"" % depotPath) - if len(log) != 1: - print "eek! I got confused by the filelog of %s" % depotPath - sys.exit(1); - - log = log[0] - if log["action0"] != action: - print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) - sys.exit(1); - - branchAction = log["how0,0"] - - if not branchAction.endswith(" from"): - continue # ignore for branching -# print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) -# sys.exit(1); - - source = log["file0,0"] - if source.startswith(branchPrefix): - continue - - lastSourceRev = log["erev0,0"] - - sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) - if len(sourceLog) != 1: - print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) - sys.exit(1); - sourceLog = sourceLog[0] - - relPath = source[len(globalPrefix):] - # strip off the filename - relPath = relPath[0:relPath.rfind("/")] - - for candidate in knownBranches: - if isSubPathOf(relPath, candidate) and candidate != branch: - return candidate - - return "" - -def changeIsBranchMerge(sourceBranch, destinationBranch, change): - sourceFiles = {} - for file in p4CmdList("files %s...@%s" % (globalPrefix + sourceBranch + "/", change)): - if file["action"] == "delete": - continue - sourceFiles[file["depotFile"]] = file - - destinationFiles = {} - for file in p4CmdList("files %s...@%s" % (globalPrefix + destinationBranch + "/", change)): - destinationFiles[file["depotFile"]] = file - - for fileName in sourceFiles.keys(): - integrations = [] - deleted = False - integrationCount = 0 - for integration in p4CmdList("integrated \"%s\"" % fileName): - toFile = integration["fromFile"] # yes, it's true, it's fromFile - if not toFile in destinationFiles: - continue - destFile = destinationFiles[toFile] - if destFile["action"] == "delete": -# print "file %s has been deleted in %s" % (fileName, toFile) - deleted = True - break - integrationCount += 1 - if integration["how"] == "branch from": - continue - - if int(integration["change"]) == change: - integrations.append(integration) - continue - if int(integration["change"]) > change: - continue - - destRev = int(destFile["rev"]) - - startRev = integration["startFromRev"][1:] - if startRev == "none": - startRev = 0 - else: - startRev = int(startRev) - - endRev = integration["endFromRev"][1:] - if endRev == "none": - endRev = 0 - else: - endRev = int(endRev) - - initialBranch = (destRev == 1 and integration["how"] != "branch into") - inRange = (destRev >= startRev and destRev <= endRev) - newer = (destRev > startRev and destRev > endRev) - - if initialBranch or inRange or newer: - integrations.append(integration) - - if deleted: - continue - - if len(integrations) == 0 and integrationCount > 1: - print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch) - return False - - return True - -def getUserMap(): - users = {} - - for output in p4CmdList("users"): - if not output.has_key("User"): - continue - users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" - return users - -users = getUserMap() - -if len(changeRange) == 0: - try: - sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) - output = sout.read() - if output.endswith("\n"): - output = output[:-1] - tagIdx = output.index(" tags/p4/") - caretIdx = output.find("^") - endPos = len(output) - if caretIdx != -1: - endPos = caretIdx - rev = int(output[tagIdx + 9 : endPos]) + 1 - changeRange = "@%s,#head" % rev - initialParent = os.popen("git-rev-parse %s" % branch).read()[:-1] - initialTag = "p4/%s" % (int(rev) - 1) - except: - pass - -tz = - time.timezone / 36 -tzsign = ("%s" % tz)[0] -if tzsign != '+' and tzsign != '-': - tz = "+" + ("%s" % tz) - -gitOutput, gitStream, gitError = popen2.popen3("git-fast-import") - -if len(revision) > 0: - print "Doing initial import of %s from revision %s" % (globalPrefix, revision) - - details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = "Initial import of %s from the state at revision %s" % (globalPrefix, revision) - details["change"] = revision - newestRevision = 0 - - fileCnt = 0 - for info in p4CmdList("files %s...%s" % (globalPrefix, revision)): - change = int(info["change"]) - if change > newestRevision: - newestRevision = change - - if info["action"] == "delete": - continue - - for prop in [ "depotFile", "rev", "action", "type" ]: - details["%s%s" % (prop, fileCnt)] = info[prop] - - fileCnt = fileCnt + 1 - - details["change"] = newestRevision - - try: - commit(details, extractFilesFromCommit(details), branch, globalPrefix) - except: - print gitError.read() - -else: - changes = [] - - if len(changesFile) > 0: - output = open(changesFile).readlines() - changeSet = Set() - for line in output: - changeSet.add(int(line)) - - for change in changeSet: - changes.append(change) - - changes.sort() - else: - output = os.popen("p4 changes %s...%s" % (globalPrefix, changeRange)).readlines() - - for line in output: - changeNum = line.split(" ")[1] - changes.append(changeNum) - - changes.reverse() - - if len(changes) == 0: - if not silent: - print "no changes to import!" - sys.exit(1) - - cnt = 1 - for change in changes: - description = p4Cmd("describe %s" % change) - - if not silent: - sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) - sys.stdout.flush() - cnt = cnt + 1 - - try: - files = extractFilesFromCommit(description) - if detectBranches: - for branch in branchesForCommit(files): - knownBranches.add(branch) - branchPrefix = globalPrefix + branch + "/" - - filesForCommit = extractFilesInCommitToBranch(files, branchPrefix) - - merged = "" - parent = "" - ########### remove cnt!!! - if branch not in createdBranches and cnt > 2: - createdBranches.add(branch) - parent = findBranchParent(branchPrefix, files) - if parent == branch: - parent = "" - # elif len(parent) > 0: - # print "%s branched off of %s" % (branch, parent) - - if len(parent) == 0: - merged = findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) - if len(merged) > 0: - print "change %s could be a merge from %s into %s" % (description["change"], merged, branch) - if not changeIsBranchMerge(merged, branch, int(description["change"])): - merged = "" - - branch = "refs/heads/" + branch - if len(parent) > 0: - parent = "refs/heads/" + parent - if len(merged) > 0: - merged = "refs/heads/" + merged - commit(description, files, branch, branchPrefix, parent, merged) - else: - commit(description, files, branch, globalPrefix, initialParent) - initialParent = "" - except IOError: - print gitError.read() - sys.exit(1) - -if not silent: - print "" - -gitStream.write("reset refs/tags/p4/%s\n" % lastChange) -gitStream.write("from %s\n\n" % branch); - - -gitStream.close() -gitOutput.close() -gitError.close() - -os.popen("git-repo-config p4.depotpath %s" % globalPrefix).read() -if len(initialTag) > 0: - os.popen("git tag -d %s" % initialTag).read() - -sys.exit(0) diff --git a/contrib/fast-import/p4-git-sync.py b/contrib/fast-import/p4-git-sync.py deleted file mode 100755 index 5a3bf90485..0000000000 --- a/contrib/fast-import/p4-git-sync.py +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/env python -# -# p4-git-sync.py -# -# Author: Simon Hausmann -# Copyright: 2007 Simon Hausmann -# 2007 Trolltech ASA -# License: MIT -# - -import os, string, shelve, stat -import getopt, sys, marshal, tempfile - -def p4CmdList(cmd): - cmd = "p4 -G %s" % cmd - pipe = os.popen(cmd, "rb") - - result = [] - try: - while True: - entry = marshal.load(pipe) - result.append(entry) - except EOFError: - pass - pipe.close() - - return result - -def p4Cmd(cmd): - list = p4CmdList(cmd) - result = {} - for entry in list: - result.update(entry) - return result; - -def die(msg): - sys.stderr.write(msg + "\n") - sys.exit(1) - -def tryGitDir(path): - if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): - return True; - return False - -try: - opts, args = getopt.getopt(sys.argv[1:], "", [ "continue", "git-dir=", "origin=", "reset", "master=", - "submit-log-subst=", "log-substitutions=", "noninteractive", - "dry-run" ]) -except getopt.GetoptError: - print "fixme, syntax error" - sys.exit(1) - -logSubstitutions = {} -logSubstitutions[""] = "%log%" -logSubstitutions["\tDetails:"] = "\tDetails: %log%" -gitdir = os.environ.get("GIT_DIR", "") -origin = "origin" -master = "" -firstTime = True -reset = False -interactive = True -dryRun = False - -for o, a in opts: - if o == "--git-dir": - gitdir = a - elif o == "--origin": - origin = a - elif o == "--master": - master = a - elif o == "--continue": - firstTime = False - elif o == "--reset": - reset = True - firstTime = True - elif o == "--submit-log-subst": - key = a.split("%")[0] - value = a.split("%")[1] - logSubstitutions[key] = value - elif o == "--log-substitutions": - for line in open(a, "r").readlines(): - tokens = line[:-1].split("=") - logSubstitutions[tokens[0]] = tokens[1] - elif o == "--noninteractive": - interactive = False - elif o == "--dry-run": - dryRun = True - -if len(gitdir) == 0: - gitdir = ".git" -else: - os.environ["GIT_DIR"] = gitdir - -if not tryGitDir(gitdir): - if tryGitDir(gitdir + "/.git"): - gitdir += "/.git" - os.environ["GIT_DIR"] = gitdir - else: - die("fatal: %s seems not to be a git repository." % gitdir) - - -configFile = gitdir + "/p4-git-sync.cfg" - -origin = "origin" -if len(args) == 1: - origin = args[0] - -if len(master) == 0: - sys.stdout.write("Auto-detecting current branch: ") - master = os.popen("git-name-rev HEAD").read().split(" ")[1][:-1] - if len(master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, master)): - die("\nFailed to detect current branch! Aborting!"); - sys.stdout.write("%s\n" % master) - -def system(cmd): - if os.system(cmd) != 0: - die("command failed: %s" % cmd) - -def check(): - if len(p4CmdList("opened ...")) > 0: - die("You have files opened with perforce! Close them before starting the sync.") - -def start(config): - if len(config) > 0 and not reset: - die("Cannot start sync. Previous sync config found at %s" % configFile) - - #if len(os.popen("git-update-index --refresh").read()) > 0: - # die("Your working tree is not clean. Check with git status!") - - commits = [] - for line in os.popen("git-rev-list --no-merges %s..%s" % (origin, master)).readlines(): - commits.append(line[:-1]) - commits.reverse() - - config["commits"] = commits - - print "Creating temporary p4-sync branch from %s ..." % origin - system("git checkout -f -b p4-sync %s" % origin) - -# print "Cleaning index..." -# system("git checkout -f") - -def prepareLogMessage(template, message): - result = "" - - for line in template.split("\n"): - if line.startswith("#"): - result += line + "\n" - continue - - substituted = False - for key in logSubstitutions.keys(): - if line.find(key) != -1: - value = logSubstitutions[key] - value = value.replace("%log%", message) - if value != "@remove@": - result += line.replace(key, value) + "\n" - substituted = True - break - - if not substituted: - result += line + "\n" - - return result - -def apply(id): - global interactive - print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read()) - diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() - filesToAdd = set() - filesToDelete = set() - for line in diff: - modifier = line[0] - path = line[1:].strip() - if modifier == "M": - system("p4 edit %s" % path) - elif modifier == "A": - filesToAdd.add(path) - if path in filesToDelete: - filesToDelete.remove(path) - elif modifier == "D": - filesToDelete.add(path) - if path in filesToAdd: - filesToAdd.remove(path) - else: - die("unknown modifier %s for %s" % (modifier, path)) - - system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") - system("git cherry-pick --no-commit \"%s\"" % id) - #system("git format-patch --stdout -k \"%s^\"..\"%s\" | git-am -k" % (id, id)) - #system("git branch -D tmp") - #system("git checkout -f -b tmp \"%s^\"" % id) - - for f in filesToAdd: - system("p4 add %s" % f) - for f in filesToDelete: - system("p4 revert %s" % f) - system("p4 delete %s" % f) - - logMessage = "" - foundTitle = False - for log in os.popen("git-cat-file commit %s" % id).readlines(): - if not foundTitle: - if len(log) == 1: - foundTitle = 1 - continue - - if len(logMessage) > 0: - logMessage += "\t" - logMessage += log - - template = os.popen("p4 change -o").read() - - if interactive: - submitTemplate = prepareLogMessage(template, logMessage) - diff = os.popen("p4 diff -du ...").read() - - for newFile in filesToAdd: - diff += "==== new file ====\n" - diff += "--- /dev/null\n" - diff += "+++ %s\n" % newFile - f = open(newFile, "r") - for line in f.readlines(): - diff += "+" + line - f.close() - - pipe = os.popen("less", "w") - pipe.write(submitTemplate + diff) - pipe.close() - - response = "e" - while response == "e": - response = raw_input("Do you want to submit this change (y/e/n)? ") - if response == "e": - [handle, fileName] = tempfile.mkstemp() - tmpFile = os.fdopen(handle, "w+") - tmpFile.write(submitTemplate) - tmpFile.close() - editor = os.environ.get("EDITOR", "vi") - system(editor + " " + fileName) - tmpFile = open(fileName, "r") - submitTemplate = tmpFile.read() - tmpFile.close() - os.remove(fileName) - - if response == "y" or response == "yes": - if dryRun: - print submitTemplate - raw_input("Press return to continue...") - else: - pipe = os.popen("p4 submit -i", "w") - pipe.write(submitTemplate) - pipe.close() - else: - print "Not submitting!" - interactive = False - else: - fileName = "submit.txt" - file = open(fileName, "w+") - file.write(prepareLogMessage(template, logMessage)) - file.close() - print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) - -check() - -config = shelve.open(configFile, writeback=True) - -if firstTime: - start(config) - -commits = config.get("commits", []) - -while len(commits) > 0: - firstTime = False - commit = commits[0] - commits = commits[1:] - config["commits"] = commits - apply(commit) - if not interactive: - break - -config.close() - -if len(commits) == 0: - if firstTime: - print "No changes found to apply between %s and current HEAD" % origin - else: - print "All changes applied!" - print "Deleting temporary p4-sync branch and going back to %s" % master - system("git checkout %s" % master) - system("git branch -D p4-sync") - print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." - system("p4 edit ... >/dev/null") - system("p4 revert ... >/dev/null") - os.remove(configFile) - From 0b69b469257424bcf015b7d3d03303c015c133f8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 20 Mar 2007 22:41:00 +0100 Subject: [PATCH 078/260] Start of the git-p4 documentation. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 74 ++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 contrib/fast-import/git-p4.txt diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt new file mode 100644 index 0000000000..b87efc6af9 --- /dev/null +++ b/contrib/fast-import/git-p4.txt @@ -0,0 +1,74 @@ +git-p4.py - Perforce <-> Git converter using git-fast-import + +Usage +===== + +git-p4 supports two main modes: Importing from Perforce to a Git repository is +done using "git-p4.py sync". Submitting changes from Git back to Perforce is +done using "git-p4.py submit". + +Importing +========= + +The procedure is simple: + + mkdir repo-git + cd repo-git + git init + git-p4.py sync //path/in/your/perforce/depot + +This will import the current head revision of the specified depot path into the +master branch of your git repository. You can use the --branch=mybranch option +to let git-p4 import from Perforce into a git branch of your choice. + +If you want to import the entire history of a given depot path just use + + git-p4.py sync //path/in/depot@all + + +Support for Perforce integrations is still work in progress. Don't bother +trying it unless you want to hack on it :) + + +Incremental Imports +=================== + +After an initial import you can easily synchronize your git repository with +newer changes from the Perforce depot by just calling + + git-p4.p4 sync + +in your git repository. git-p4 stores the depot path of the original import in +the .git/config file and remembers the last imported p4 revision as a git tag +called p4/ . + +Submitting +========== + +git-p4 has EXPERIMENTAL support for submitting changes from a git repository +back to a Perforce depot. This requires a Perforce checkout separate to your +git repository. This is the basic procedure: + + cd path/to/your/perforce/checkout + git-p4.py submit --git-dir=/path/to/your/git/repository + +This will create a temporary git branch, use git-rev-list to find out which git +commits are in your current branch but not in the "origin" branch. You can +override the name of the "origin" branch by using the --origin=mybranch option. +The "origin" branch has to be the branch populated with git-p4's sync +operation. + +After some preparations (which might take a while) git-p4 enters a loop where +it will first show a Perforce submit template and a diff of the change to +apply. After quitting the pager with 'q' git-p4 asks for confirmation for +issuing the "p4 submit" command and also gives you the option of editing the +submit template using "e". + +If a submit fails you may have to "p4 resolve" and submit manually. You can +continue importing the remaining changes with + + git-p4.py submit --git-dir=/path/to/your/git/repository --continue + +After submitting you should sync your origin branch from Perforce using +git-p4's sync command. + From b4aa8d12b48af84242a4a484673bc41326566ac9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Mar 2007 08:27:33 +0100 Subject: [PATCH 079/260] Documentation enhancements. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index b87efc6af9..4b4fcde72b 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -25,6 +25,8 @@ If you want to import the entire history of a given depot path just use git-p4.py sync //path/in/depot@all +To achieve optimal compression you may want to run 'git repack -a -d -f' after +a big import. This may take a while. Support for Perforce integrations is still work in progress. Don't bother trying it unless you want to hack on it :) @@ -42,6 +44,10 @@ in your git repository. git-p4 stores the depot path of the original import in the .git/config file and remembers the last imported p4 revision as a git tag called p4/ . +It is recommended to run 'git repack -a -d -f' from time to time when using +incremental imports to optimally combine the individual git packs that each +incremental import creates through the use of git-fast-import. + Submitting ========== From 04219c04b7554d13b049d4aba534ce02a12abeb4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Mar 2007 10:11:20 +0100 Subject: [PATCH 080/260] Added experimental but super-fast --apply-as-patch option to git-p4 submit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 54ddf73dac..2009dce23b 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -115,7 +115,8 @@ class P4Sync(Command): optparse.make_option("--master", dest="master"), optparse.make_option("--log-substitutions", dest="substFile"), optparse.make_option("--noninteractive", action="store_false"), - optparse.make_option("--dry-run", action="store_true") + optparse.make_option("--dry-run", action="store_true"), + optparse.make_option("--apply-as-patch", action="store_true", dest="applyAsPatch") ] self.description = "Submit changes from git to the perforce depot." self.firstTime = True @@ -126,6 +127,7 @@ class P4Sync(Command): self.firstTime = True self.origin = "origin" self.master = "" + self.applyAsPatch = False self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -146,8 +148,9 @@ class P4Sync(Command): self.config["commits"] = commits - print "Creating temporary p4-sync branch from %s ..." % self.origin - system("git checkout -f -b p4-sync %s" % self.origin) + if not self.applyAsPatch: + print "Creating temporary p4-sync branch from %s ..." % self.origin + system("git checkout -f -b p4-sync %s" % self.origin) def prepareLogMessage(self, template, message): result = "" @@ -193,8 +196,11 @@ class P4Sync(Command): else: die("unknown modifier %s for %s" % (modifier, path)) - system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") - system("git cherry-pick --no-commit \"%s\"" % id) + if self.applyAsPatch: + system("git-diff-tree -p \"%s^\" \"%s\" | patch -p1" % (id, id)) + else: + system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") + system("git cherry-pick --no-commit \"%s\"" % id) for f in filesToAdd: system("p4 add %s" % f) @@ -305,12 +311,13 @@ class P4Sync(Command): print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" - print "Deleting temporary p4-sync branch and going back to %s" % self.master - system("git checkout %s" % self.master) - system("git branch -D p4-sync") - print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." - system("p4 edit ... >/dev/null") - system("p4 revert ... >/dev/null") + if not self.applyAsPatch: + print "Deleting temporary p4-sync branch and going back to %s" % self.master + system("git checkout %s" % self.master) + system("git branch -D p4-sync") + print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." + system("p4 edit ... >/dev/null") + system("p4 revert ... >/dev/null") os.remove(self.configFile) return True From 5d0b6042d4a33dbd78840a8e0d8957441e2cfa72 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Mar 2007 10:57:54 +0100 Subject: [PATCH 081/260] Fix support for deletions in git-p4 submit when using --apply-as-patch by filtering out deletions in the diff-tree output. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 2009dce23b..336c3eab52 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -197,7 +197,7 @@ class P4Sync(Command): die("unknown modifier %s for %s" % (modifier, path)) if self.applyAsPatch: - system("git-diff-tree -p \"%s^\" \"%s\" | patch -p1" % (id, id)) + system("git-diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) else: system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") system("git cherry-pick --no-commit \"%s\"" % id) From 1932a6ac7c3725aec20574942e5eed9b1e1c5dee Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Mar 2007 11:01:18 +0100 Subject: [PATCH 082/260] Made --apply-as-patch the default for git-p4 submit as it's significantly faster. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index 336c3eab52..a07534058c 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -127,7 +127,7 @@ class P4Sync(Command): self.firstTime = True self.origin = "origin" self.master = "" - self.applyAsPatch = False + self.applyAsPatch = True self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" From 20618650058741117d909e086c2e78c003e5cfcd Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Mar 2007 13:05:30 +0100 Subject: [PATCH 083/260] Make it possible to invoke git-p4 from within subdirectories of a git working tree. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index a07534058c..f940e93bad 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -925,6 +925,10 @@ parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), gitdir = cmd.gitdir if len(gitdir) == 0: gitdir = ".git" + if not isValidGitDir(gitdir): + cdup = os.popen("git-rev-parse --show-cdup").read()[:-1] + if isValidGitDir(cdup + "/" + gitdir): + os.chdir(cdup) if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"): From 53150250b13e7621c8ade1d59d19c946b745dd39 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 21 Mar 2007 21:04:12 +0100 Subject: [PATCH 084/260] Don't show the submit template and the diff first in less but show it in $editor right away Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4.py index f940e93bad..06858844e5 100755 --- a/contrib/fast-import/git-p4.py +++ b/contrib/fast-import/git-p4.py @@ -235,24 +235,26 @@ class P4Sync(Command): diff += "+" + line f.close() - pipe = os.popen("less", "w") - pipe.write(submitTemplate + diff) - pipe.close() + separatorLine = "######## everything below this line is just the diff #######\n" response = "e" + firstIteration = True while response == "e": - response = raw_input("Do you want to submit this change (y/e/n)? ") + if not firstIteration: + response = raw_input("Do you want to submit this change (y/e/n)? ") + firstIteration = False if response == "e": [handle, fileName] = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+") - tmpFile.write(submitTemplate) + tmpFile.write(submitTemplate + separatorLine + diff) tmpFile.close() editor = os.environ.get("EDITOR", "vi") system(editor + " " + fileName) tmpFile = open(fileName, "r") - submitTemplate = tmpFile.read() + message = tmpFile.read() tmpFile.close() os.remove(fileName) + submitTemplate = message[:message.index(separatorLine)] if response == "y" or response == "yes": if self.dryRun: From e7f0d0d9b9dc99a3f748ed686625cc9f87d9a267 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 09:13:01 +0100 Subject: [PATCH 085/260] Removed the .py extension from git-p4 as it's annoying to type every time. Signed-off-by: Simon Hausmann --- contrib/fast-import/{git-p4.py => git-p4} | 0 contrib/fast-import/git-p4.txt | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) rename contrib/fast-import/{git-p4.py => git-p4} (100%) diff --git a/contrib/fast-import/git-p4.py b/contrib/fast-import/git-p4 similarity index 100% rename from contrib/fast-import/git-p4.py rename to contrib/fast-import/git-p4 diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 4b4fcde72b..5786bffad4 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -1,11 +1,11 @@ -git-p4.py - Perforce <-> Git converter using git-fast-import +git-p4 - Perforce <-> Git converter using git-fast-import Usage ===== git-p4 supports two main modes: Importing from Perforce to a Git repository is -done using "git-p4.py sync". Submitting changes from Git back to Perforce is -done using "git-p4.py submit". +done using "git-p4 sync". Submitting changes from Git back to Perforce is +done using "git-p4 submit". Importing ========= @@ -15,7 +15,7 @@ The procedure is simple: mkdir repo-git cd repo-git git init - git-p4.py sync //path/in/your/perforce/depot + git-p4 sync //path/in/your/perforce/depot This will import the current head revision of the specified depot path into the master branch of your git repository. You can use the --branch=mybranch option @@ -23,7 +23,7 @@ to let git-p4 import from Perforce into a git branch of your choice. If you want to import the entire history of a given depot path just use - git-p4.py sync //path/in/depot@all + git-p4 sync //path/in/depot@all To achieve optimal compression you may want to run 'git repack -a -d -f' after a big import. This may take a while. @@ -56,7 +56,7 @@ back to a Perforce depot. This requires a Perforce checkout separate to your git repository. This is the basic procedure: cd path/to/your/perforce/checkout - git-p4.py submit --git-dir=/path/to/your/git/repository + git-p4 submit --git-dir=/path/to/your/git/repository This will create a temporary git branch, use git-rev-list to find out which git commits are in your current branch but not in the "origin" branch. You can @@ -73,7 +73,7 @@ submit template using "e". If a submit fails you may have to "p4 resolve" and submit manually. You can continue importing the remaining changes with - git-p4.py submit --git-dir=/path/to/your/git/repository --continue + git-p4 submit --git-dir=/path/to/your/git/repository --continue After submitting you should sync your origin branch from Perforce using git-p4's sync command. From a559b289bd78d5a3fac0f908be7d5ff92ad09dcb Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 20:27:41 +0100 Subject: [PATCH 086/260] Changed the format of the imported log message slightly, so that it's easier to parse again. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 06858844e5..a5b6d94d1a 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -506,7 +506,7 @@ class GitSync(Command): self.gitStream.write("data < 0: From f5816a5522763f46e075cc618eab12110107a917 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 20:36:28 +0100 Subject: [PATCH 087/260] Changed the default branch for imports from "master" to "p4" Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- contrib/fast-import/git-p4.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index a5b6d94d1a..669cf95d74 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -352,7 +352,7 @@ class GitSync(Command): self.knownBranches = Set() self.createdBranches = Set() self.committedChanges = Set() - self.branch = "master" + self.branch = "p4" self.detectBranches = False self.changesFile = "" diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 5786bffad4..0d30aff64c 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -17,9 +17,9 @@ The procedure is simple: git init git-p4 sync //path/in/your/perforce/depot -This will import the current head revision of the specified depot path into the -master branch of your git repository. You can use the --branch=mybranch option -to let git-p4 import from Perforce into a git branch of your choice. +This will import the current head revision of the specified depot path into a +"p4" branch of your git repository. You can use the --branch=mybranch option +to use a different branch. If you want to import the entire history of a given depot path just use From 6ae8de88f53f92dd593c5d03b67d0194e7d4eefe Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 21:10:25 +0100 Subject: [PATCH 088/260] Added some helper function(s) to parse the depot path and change number from the log message Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 48 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 669cf95d74..6ead1c4173 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -52,12 +52,44 @@ def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) +def extractLogMessageFromGitCommit(commit): + logMessage = "" + foundTitle = False + for log in os.popen("git-cat-file commit %s" % commit).readlines(): + if not foundTitle: + if len(log) == 1: + foundTitle = 1 + continue + + logMessage += log + return logMessage + +def extractDepotPathAndChangeFromGitLog(log): + values = {} + for line in log.split("\n"): + line = line.strip() + if line.startswith("[git-p4:") and line.endswith("]"): + line = line[8:-1].strip() + for assignment in line.split(":"): + variable = assignment.strip() + value = "" + equalPos = assignment.find("=") + if equalPos != -1: + variable = assignment[:equalPos].strip() + value = assignment[equalPos + 1:].strip() + if value.startswith("\"") and value.endswith("\""): + value = value[1:-1] + values[variable] = value + + return values.get("depot-path"), values.get("change") + class Command: def __init__(self): self.usage = "usage: %prog [options]" class P4Debug(Command): def __init__(self): + Command.__init__(self) self.options = [ ] self.description = "A tool to debug the output of p4 -G." @@ -208,17 +240,9 @@ class P4Sync(Command): system("p4 revert %s" % f) system("p4 delete %s" % f) - logMessage = "" - foundTitle = False - for log in os.popen("git-cat-file commit %s" % id).readlines(): - if not foundTitle: - if len(log) == 1: - foundTitle = 1 - continue - - if len(logMessage) > 0: - logMessage += "\t" - logMessage += log + logMessage = extractLogMessageFromGitCommit(id) + logMessage = logMessage.replace("\n", "\n\t") + logMessage = logMessage[:-1] template = os.popen("p4 change -o").read() @@ -506,7 +530,7 @@ class GitSync(Command): self.gitStream.write("data < 0: From 8136a6399c886dec8d1b806a7d8728324e906729 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 21:27:14 +0100 Subject: [PATCH 089/260] Helper function to check the existance of a revision Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6ead1c4173..b21dee3199 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -83,6 +83,9 @@ def extractDepotPathAndChangeFromGitLog(log): return values.get("depot-path"), values.get("change") +def gitBranchExists(branch): + return os.system("git-rev-parse %s 2>/dev/null >/dev/null") == 0 + class Command: def __init__(self): self.usage = "usage: %prog [options]" From 569d1bd4092aacd44a826852e087d0f0e9928ce8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 21:34:16 +0100 Subject: [PATCH 090/260] Set the default branch in run, not in the constructor Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b21dee3199..e6a34f4f47 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -379,7 +379,7 @@ class GitSync(Command): self.knownBranches = Set() self.createdBranches = Set() self.committedChanges = Set() - self.branch = "p4" + self.branch = "" self.detectBranches = False self.changesFile = "" @@ -706,6 +706,9 @@ class GitSync(Command): self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" def run(self, args): + if len(self.branch) == 0: + self.branch = "p4" + self.branch = "refs/heads/" + self.branch self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() if len(self.globalPrefix) != 0: From 179caebff4a917dc35c8166ec183bc5c76df53e1 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 22 Mar 2007 22:17:42 +0100 Subject: [PATCH 091/260] Brand new smart incremental import that doesn't need tags or git repo-config :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 38 +++++++++++++++++++++++++++------- contrib/fast-import/git-p4.txt | 6 ++---- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e6a34f4f47..8684e4b20f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -84,7 +84,9 @@ def extractDepotPathAndChangeFromGitLog(log): return values.get("depot-path"), values.get("change") def gitBranchExists(branch): - return os.system("git-rev-parse %s 2>/dev/null >/dev/null") == 0 + if os.system("git-rev-parse %s 2>/dev/null >/dev/null" % branch) == 0: + return True + return False class Command: def __init__(self): @@ -706,17 +708,40 @@ class GitSync(Command): self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" def run(self, args): + self.globalPrefix = "" + self.changeRange = "" + self.initialParent = "" + self.tagLastChange = True + if len(self.branch) == 0: self.branch = "p4" + if len(args) == 0: + if not gitBranchExists(self.branch) and gitBranchExists("origin"): + if not self.silent: + print "Creating %s branch in git repository based on origin" % self.branch + system("git branch %s origin" % self.branch) + + [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) + if len(self.previousDepotPath) > 0 and len(p4Change) > 0: + p4Change = int(p4Change) + 1 + self.globalPrefix = self.previousDepotPath + self.changeRange = "@%s,#head" % p4Change + self.initialParent = self.branch + self.tagLastChange = False + if not self.silent: + print "Performing incremental import into %s git branch" % self.branch self.branch = "refs/heads/" + self.branch - self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() + + if len(self.globalPrefix) == 0: + self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() + if len(self.globalPrefix) != 0: self.globalPrefix = self.globalPrefix[:-1] if len(args) == 0 and len(self.globalPrefix) != 0: if not self.silent: - print "[using previously specified depot path %s]" % self.globalPrefix + print "Depot path: %s" % self.globalPrefix elif len(args) != 1: return False else: @@ -725,10 +750,8 @@ class GitSync(Command): sys.exit(1) self.globalPrefix = args[0] - self.changeRange = "" self.revision = "" self.users = {} - self.initialParent = "" self.lastChange = 0 self.initialTag = "" @@ -890,8 +913,9 @@ class GitSync(Command): if not self.silent: print "" - self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange) - self.gitStream.write("from %s\n\n" % self.branch); + if self.tagLastChange: + self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange) + self.gitStream.write("from %s\n\n" % self.branch); self.gitStream.close() diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 0d30aff64c..4319c82dcd 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -38,11 +38,9 @@ Incremental Imports After an initial import you can easily synchronize your git repository with newer changes from the Perforce depot by just calling - git-p4.p4 sync + git-p4 sync -in your git repository. git-p4 stores the depot path of the original import in -the .git/config file and remembers the last imported p4 revision as a git tag -called p4/ . +in your git repository. It is recommended to run 'git repack -a -d -f' from time to time when using incremental imports to optimally combine the individual git packs that each From 9512497bcf574a2f70e43be0bcb58e35fb6aaba8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 23 Mar 2007 09:16:07 +0100 Subject: [PATCH 092/260] Make it possible to run git-p4 submit from within the git repository Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 38 +++++++++++++++++++++++++++++++++- contrib/fast-import/git-p4.txt | 27 ++++++++++++------------ 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 8684e4b20f..a8f7cce25d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -162,7 +162,7 @@ class P4Sync(Command): self.dryRun = False self.substFile = "" self.firstTime = True - self.origin = "origin" + self.origin = "" self.master = "" self.applyAsPatch = True @@ -304,6 +304,42 @@ class P4Sync(Command): print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) def run(self, args): + global gitdir + # make gitdir absolute so we can cd out into the perforce checkout + gitdir = os.path.abspath(gitdir) + os.environ["GIT_DIR"] = gitdir + depotPath = "" + if gitBranchExists("p4"): + [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) + if len(depotPath) == 0 and gitBranchExists("origin"): + [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) + + if len(depotPath) == 0: + print "Internal error: cannot locate perforce depot path from existing branches" + sys.exit(128) + + if not depotPath.endswith("/"): + depotPath += "/" + clientPath = p4Cmd("where %s..." % depotPath).get("path") + if clientPath.endswith("..."): + clientPath = clientPath[:-3] + + if len(clientPath) == 0: + print "Error: Cannot locate perforce checkout of %s in client view" % depotPath + sys.exit(128) + + print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath) + os.chdir(clientPath) + response = raw_input("Do you want to sync %s with p4 sync? (y/n)" % clientPath) + if response == "y" or response == "yes": + system("p4 sync ...") + + if len(self.origin) == 0: + if gitBranchExists("p4"): + self.origin = "p4" + else: + self.origin = "origin" + if self.reset: self.firstTime = True diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 4319c82dcd..8bf0805c74 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -51,28 +51,27 @@ Submitting git-p4 has EXPERIMENTAL support for submitting changes from a git repository back to a Perforce depot. This requires a Perforce checkout separate to your -git repository. This is the basic procedure: +git repository. All it should take is calling - cd path/to/your/perforce/checkout - git-p4 submit --git-dir=/path/to/your/git/repository + git-p4 submit -This will create a temporary git branch, use git-rev-list to find out which git -commits are in your current branch but not in the "origin" branch. You can -override the name of the "origin" branch by using the --origin=mybranch option. -The "origin" branch has to be the branch populated with git-p4's sync -operation. +in your git repository. This will attempt to locate the perforce checkout +corresponding to your imported depot path. By default the changes between your +current branch and the "p4" branch will be submitted. If there is no "p4" +branch the "origin" branch will be used as reference instead. You can override +this with the --origin=mysourcebranch option. The "origin" branch has to be the +branch populated with git-p4's sync operation. After some preparations (which might take a while) git-p4 enters a loop where it will first show a Perforce submit template and a diff of the change to -apply. After quitting the pager with 'q' git-p4 asks for confirmation for -issuing the "p4 submit" command and also gives you the option of editing the -submit template using "e". +apply in the editor. After saving and exiting the editor you will be asked whether +you really want to submit the change or not. If a submit fails you may have to "p4 resolve" and submit manually. You can continue importing the remaining changes with - git-p4 submit --git-dir=/path/to/your/git/repository --continue + git-p4 submit --continue -After submitting you should sync your origin branch from Perforce using -git-p4's sync command. +After submitting you should sync your perforce import branch ("p4" or "origin") +from Perforce using git-p4's sync command. From 967f72e21b9f38725d64a7520bca99bbef9e8ab3 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 23 Mar 2007 09:30:41 +0100 Subject: [PATCH 093/260] Use the new incremental import style by default Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index a8f7cce25d..61978c3da6 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -751,21 +751,22 @@ class GitSync(Command): if len(self.branch) == 0: self.branch = "p4" - if len(args) == 0: - if not gitBranchExists(self.branch) and gitBranchExists("origin"): - if not self.silent: - print "Creating %s branch in git repository based on origin" % self.branch - system("git branch %s origin" % self.branch) - [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) - if len(self.previousDepotPath) > 0 and len(p4Change) > 0: - p4Change = int(p4Change) + 1 - self.globalPrefix = self.previousDepotPath - self.changeRange = "@%s,#head" % p4Change - self.initialParent = self.branch - self.tagLastChange = False - if not self.silent: - print "Performing incremental import into %s git branch" % self.branch + if len(args) == 0: + if not gitBranchExists(self.branch) and gitBranchExists("origin"): + if not self.silent: + print "Creating %s branch in git repository based on origin" % self.branch + system("git branch %s origin" % self.branch) + + [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) + if len(self.previousDepotPath) > 0 and len(p4Change) > 0: + p4Change = int(p4Change) + 1 + self.globalPrefix = self.previousDepotPath + self.changeRange = "@%s,#head" % p4Change + self.initialParent = self.branch + self.tagLastChange = False + if not self.silent: + print "Performing incremental import into %s git branch" % self.branch self.branch = "refs/heads/" + self.branch From cb2c9db507cda6804f85ebbacb58a7458ab127a6 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 24 Mar 2007 09:15:11 +0100 Subject: [PATCH 094/260] Different versions of p4 have different output for the where command ;( Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 61978c3da6..9503786207 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -36,6 +36,22 @@ def p4Cmd(cmd): result.update(entry) return result; +def p4Where(depotPath): + if not depotPath.endswith("/"): + depotPath += "/" + output = p4Cmd("where %s..." % depotPath) + clientPath = "" + if "path" in output: + clientPath = output.get("path") + elif "data" in output: + data = output.get("data") + lastSpace = data.rfind(" ") + clientPath = data[lastSpace + 1:] + + if clientPath.endswith("..."): + clientPath = clientPath[:-3] + return clientPath + def die(msg): sys.stderr.write(msg + "\n") sys.exit(1) @@ -318,11 +334,7 @@ class P4Sync(Command): print "Internal error: cannot locate perforce depot path from existing branches" sys.exit(128) - if not depotPath.endswith("/"): - depotPath += "/" - clientPath = p4Cmd("where %s..." % depotPath).get("path") - if clientPath.endswith("..."): - clientPath = clientPath[:-3] + clientPath = p4Where(depotPath) if len(clientPath) == 0: print "Error: Cannot locate perforce checkout of %s in client view" % depotPath From 274917a3d65109cfdf225615177fe27624d8461a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 24 Mar 2007 09:18:20 +0100 Subject: [PATCH 095/260] Minor cosmetic fixlet for the git-p4 submit sync question. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 9503786207..aaa5d5ee5f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -342,7 +342,7 @@ class P4Sync(Command): print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath) os.chdir(clientPath) - response = raw_input("Do you want to sync %s with p4 sync? (y/n)" % clientPath) + response = raw_input("Do you want to sync %s with p4 sync? (y/n) " % clientPath) if response == "y" or response == "yes": system("p4 sync ...") From 9863f4055e1219904f6d26c2803a72fc6c8ce561 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 24 Mar 2007 16:35:05 +0100 Subject: [PATCH 096/260] Prefer git command over git-command. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index aaa5d5ee5f..09990be373 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -57,7 +57,7 @@ def die(msg): sys.exit(1) def currentGitBranch(): - return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1] + return os.popen("git name-rev HEAD").read().split(" ")[1][:-1] def isValidGitDir(path): if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): @@ -71,7 +71,7 @@ def system(cmd): def extractLogMessageFromGitCommit(commit): logMessage = "" foundTitle = False - for log in os.popen("git-cat-file commit %s" % commit).readlines(): + for log in os.popen("git cat-file commit %s" % commit).readlines(): if not foundTitle: if len(log) == 1: foundTitle = 1 @@ -100,7 +100,7 @@ def extractDepotPathAndChangeFromGitLog(log): return values.get("depot-path"), values.get("change") def gitBranchExists(branch): - if os.system("git-rev-parse %s 2>/dev/null >/dev/null" % branch) == 0: + if os.system("git rev-parse %s 2>/dev/null >/dev/null" % branch) == 0: return True return False @@ -130,7 +130,7 @@ class P4CleanTags(Command): def run(self, args): branch = currentGitBranch() print "Cleaning out stale p4 import tags..." - sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch) + sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % branch) output = sout.read() try: tagIdx = output.index(" tags/p4/") @@ -195,7 +195,7 @@ class P4Sync(Command): die("Cannot start sync. Previous sync config found at %s" % self.configFile) commits = [] - for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): + for line in os.popen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): commits.append(line[:-1]) commits.reverse() @@ -229,7 +229,7 @@ class P4Sync(Command): return result def apply(self, id): - print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read()) + print "Applying %s" % (os.popen("git log --max-count=1 --pretty=oneline %s" % id).read()) diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() filesToDelete = set() @@ -250,9 +250,9 @@ class P4Sync(Command): die("unknown modifier %s for %s" % (modifier, path)) if self.applyAsPatch: - system("git-diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) + system("git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) else: - system("git-diff-files --name-only -z | git-update-index --remove -z --stdin") + system("git diff-files --name-only -z | git update-index --remove -z --stdin") system("git cherry-pick --no-commit \"%s\"" % id) for f in filesToAdd: @@ -783,7 +783,7 @@ class GitSync(Command): self.branch = "refs/heads/" + self.branch if len(self.globalPrefix) == 0: - self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read() + self.globalPrefix = self.previousDepotPath = os.popen("git repo-config --get p4.depotpath").read() if len(self.globalPrefix) != 0: self.globalPrefix = self.globalPrefix[:-1] @@ -830,7 +830,7 @@ class GitSync(Command): if len(self.changeRange) == 0: try: - sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % self.branch) + sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % self.branch) output = sout.read() if output.endswith("\n"): output = output[:-1] @@ -841,7 +841,7 @@ class GitSync(Command): endPos = caretIdx self.rev = int(output[tagIdx + 9 : endPos]) + 1 self.changeRange = "@%s,#head" % self.rev - self.initialParent = os.popen("git-rev-parse %s" % self.branch).read()[:-1] + self.initialParent = os.popen("git rev-parse %s" % self.branch).read()[:-1] self.initialTag = "p4/%s" % (int(self.rev) - 1) except: pass @@ -851,7 +851,7 @@ class GitSync(Command): if tzsign != '+' and tzsign != '-': self.tz = "+" + ("%s" % self.tz) - self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git-fast-import") + self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git fast-import") if len(self.revision) > 0: print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision) @@ -971,7 +971,7 @@ class GitSync(Command): self.gitOutput.close() self.gitError.close() - os.popen("git-repo-config p4.depotpath %s" % self.globalPrefix).read() + os.popen("git repo-config p4.depotpath %s" % self.globalPrefix).read() if len(self.initialTag) > 0: os.popen("git tag -d %s" % self.initialTag).read() @@ -1031,7 +1031,7 @@ gitdir = cmd.gitdir if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): - cdup = os.popen("git-rev-parse --show-cdup").read()[:-1] + cdup = os.popen("git rev-parse --show-cdup").read()[:-1] if isValidGitDir(cdup + "/" + gitdir): os.chdir(cdup) From e20a9e530a02fefaec31b484bd224784b8814554 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 26 Mar 2007 00:13:51 +0200 Subject: [PATCH 097/260] Don't try to parse any options with git-p4 debug but pass it straight on to p4 Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 09990be373..5b023b1b7a 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1018,14 +1018,18 @@ except KeyError: options = cmd.options cmd.gitdir = gitdir -options.append(optparse.make_option("--git-dir", dest="gitdir")) -parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), - options, - description = cmd.description, - formatter = HelpFormatter()) +args = sys.argv[2:] -(cmd, args) = parser.parse_args(sys.argv[2:], cmd); +if len(options) > 0: + options.append(optparse.make_option("--git-dir", dest="gitdir")) + + parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), + options, + description = cmd.description, + formatter = HelpFormatter()) + + (cmd, args) = parser.parse_args(sys.argv[2:], cmd); gitdir = cmd.gitdir if len(gitdir) == 0: From 8910ac0e888daeefdbe6f7391bece150b12b1ad0 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 26 Mar 2007 08:18:55 +0200 Subject: [PATCH 098/260] git-p4 debug doesn't need a git repository Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 5b023b1b7a..eb5b40aa98 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -107,6 +107,7 @@ def gitBranchExists(branch): class Command: def __init__(self): self.usage = "usage: %prog [options]" + self.needsGit = True class P4Debug(Command): def __init__(self): @@ -114,6 +115,7 @@ class P4Debug(Command): self.options = [ ] self.description = "A tool to debug the output of p4 -G." + self.needsGit = False def run(self, args): for output in p4CmdList(" ".join(args)): @@ -1031,21 +1033,22 @@ if len(options) > 0: (cmd, args) = parser.parse_args(sys.argv[2:], cmd); -gitdir = cmd.gitdir -if len(gitdir) == 0: - gitdir = ".git" +if cmd.needsGit: + gitdir = cmd.gitdir + if len(gitdir) == 0: + gitdir = ".git" + if not isValidGitDir(gitdir): + cdup = os.popen("git rev-parse --show-cdup").read()[:-1] + if isValidGitDir(cdup + "/" + gitdir): + os.chdir(cdup) + if not isValidGitDir(gitdir): - cdup = os.popen("git rev-parse --show-cdup").read()[:-1] - if isValidGitDir(cdup + "/" + gitdir): - os.chdir(cdup) + if isValidGitDir(gitdir + "/.git"): + gitdir += "/.git" + else: + die("fatal: cannot locate git repository at %s" % gitdir) -if not isValidGitDir(gitdir): - if isValidGitDir(gitdir + "/.git"): - gitdir += "/.git" - else: - die("fatal: cannot locate git repository at %s" % gitdir) - -os.environ["GIT_DIR"] = gitdir + os.environ["GIT_DIR"] = gitdir if not cmd.run(args): parser.print_help() From 1f4ba1cbfc1d08a9c120012c9199caaec3dc5f58 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 26 Mar 2007 22:34:34 +0200 Subject: [PATCH 099/260] Added support for mapping p4 labels to git tags Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 58 +++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index eb5b40aa98..eab5990548 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -625,7 +625,47 @@ class GitSync(Command): self.gitStream.write("\n") - self.lastChange = int(details["change"]) + change = int(details["change"]) + + self.lastChange = change + + if change in self.labels: + label = self.labels[change] + labelDetails = label[0] + labelRevisions = label[1] + + files = p4CmdList("files %s...@%s" % (branchPrefix, change)) + + if len(files) == len(labelRevisions): + + cleanedFiles = {} + for info in files: + if info["action"] == "delete": + continue + cleanedFiles[info["depotFile"]] = info["rev"] + + if cleanedFiles == labelRevisions: + self.gitStream.write("tag tag_%s\n" % labelDetails["label"]) + self.gitStream.write("from %s\n" % branch) + + owner = labelDetails["Owner"] + tagger = "" + if author in self.users: + tagger = "%s %s %s" % (self.users[owner], epoch, self.tz) + else: + tagger = "%s %s %s" % (owner, epoch, self.tz) + self.gitStream.write("tagger %s\n" % tagger) + self.gitStream.write("data <" + def getLabels(self): + self.labels = {} + + for output in p4CmdList("labels %s..." % self.globalPrefix): + label = output["label"] + revisions = {} + newestChange = 0 + for file in p4CmdList("files //...@%s" % label): + revisions[file["depotFile"]] = file["rev"] + change = int(file["change"]) + if change > newestChange: + newestChange = change + + self.labels[newestChange] = [output, revisions] + def run(self, args): self.globalPrefix = "" self.changeRange = "" @@ -829,6 +884,7 @@ class GitSync(Command): self.globalPrefix += "/" self.getUserMap() + self.getLabels(); if len(self.changeRange) == 0: try: From a46668faf7f02c3fb55bf877e16c7ffdf8c9a129 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 28 Mar 2007 17:05:38 +0200 Subject: [PATCH 100/260] Fix variable usage in tag import Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index eab5990548..60c4b3dc6c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -660,11 +660,11 @@ class GitSync(Command): self.gitStream.write("EOT\n\n") else: - if not silent: + if not self.silent: print "Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change) else: - if not silent: + if not self.silent: print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change) def extractFilesInCommitToBranch(self, files, branchPrefix): From c9b50e6307aa9931e18b6cd4d00de817b8e4a4c2 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 29 Mar 2007 19:15:24 +0200 Subject: [PATCH 101/260] Fix the docs for git-p4 submit and turn git-p4 submit --master=foo into simply git-p4 submit mytopicbranch. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 18 +++++++++++------- contrib/fast-import/git-p4.txt | 23 +++++++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 60c4b3dc6c..59c3edae19 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -167,13 +167,13 @@ class P4Sync(Command): optparse.make_option("--continue", action="store_false", dest="firstTime"), optparse.make_option("--origin", dest="origin"), optparse.make_option("--reset", action="store_true", dest="reset"), - optparse.make_option("--master", dest="master"), optparse.make_option("--log-substitutions", dest="substFile"), optparse.make_option("--noninteractive", action="store_false"), optparse.make_option("--dry-run", action="store_true"), optparse.make_option("--apply-as-patch", action="store_true", dest="applyAsPatch") ] self.description = "Submit changes from git to the perforce depot." + self.usage += " [name of git branch to submit into perforce depot]" self.firstTime = True self.reset = False self.interactive = True @@ -181,7 +181,6 @@ class P4Sync(Command): self.substFile = "" self.firstTime = True self.origin = "" - self.master = "" self.applyAsPatch = True self.logSubstitutions = {} @@ -326,6 +325,16 @@ class P4Sync(Command): # make gitdir absolute so we can cd out into the perforce checkout gitdir = os.path.abspath(gitdir) os.environ["GIT_DIR"] = gitdir + + if len(args) == 0: + self.master = currentGitBranch() + if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)): + die("Detecting current git branch failed!") + elif len(args) == 1: + self.master = args[0] + else: + return False + depotPath = "" if gitBranchExists("p4"): [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) @@ -362,11 +371,6 @@ class P4Sync(Command): tokens = line[:-1].split("=") self.logSubstitutions[tokens[0]] = tokens[1] - if len(self.master) == 0: - self.master = currentGitBranch() - if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)): - die("Detecting current git branch failed!") - self.check() self.configFile = gitdir + "/p4-git-sync.cfg" self.config = shelve.open(self.configFile, writeback=True) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 8bf0805c74..30e2cb9a55 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -49,23 +49,19 @@ incremental import creates through the use of git-fast-import. Submitting ========== -git-p4 has EXPERIMENTAL support for submitting changes from a git repository -back to a Perforce depot. This requires a Perforce checkout separate to your -git repository. All it should take is calling +git-p4 has support for submitting changes from a git repository back to the +Perforce depot. This requires a Perforce checkout separate to your git +repository. To submit all changes that are in the current git branch but not in +the "p4" branch (or "origin" if "p4" doesn't exist) simply call git-p4 submit -in your git repository. This will attempt to locate the perforce checkout -corresponding to your imported depot path. By default the changes between your -current branch and the "p4" branch will be submitted. If there is no "p4" -branch the "origin" branch will be used as reference instead. You can override -this with the --origin=mysourcebranch option. The "origin" branch has to be the -branch populated with git-p4's sync operation. +in your git repository. If you want to submit changes in a specific branch that +is not your current git branch you can also pass that as an argument: -After some preparations (which might take a while) git-p4 enters a loop where -it will first show a Perforce submit template and a diff of the change to -apply in the editor. After saving and exiting the editor you will be asked whether -you really want to submit the change or not. + git-p4 submit mytopicbranch + +You can override the reference branch with the --origin=mysourcebranch option. If a submit fails you may have to "p4 resolve" and submit manually. You can continue importing the remaining changes with @@ -74,4 +70,3 @@ continue importing the remaining changes with After submitting you should sync your perforce import branch ("p4" or "origin") from Perforce using git-p4's sync command. - From 2a9489c0249c8aef6c6ead501bae25771e710e12 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 1 Apr 2007 13:39:39 +0200 Subject: [PATCH 102/260] Fix "compilation" :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 59c3edae19..0443337359 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -416,7 +416,7 @@ class GitSync(Command): optparse.make_option("--changesfile", dest="changesFile"), optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--known-branches", dest="knownBranches"), - optparse.make_option("--cache", dest="doCache", action="store_true"), + optparse.make_option("--data-cache", dest="dataCache", action="store_true"), optparse.make_option("--command-cache", dest="commandCache", action="store_true") ] self.description = """Imports from Perforce into a git repository.\n @@ -500,7 +500,7 @@ class GitSync(Command): if knownBranch: continue - for branch in knownBranches: + for branch in self.knownBranches: #if relativePath.startswith(branch): if self.isSubPathOf(relativePath, branch): if len(branches) == 0: From 711544b00c22b1c2559333e8925e449812f9e5cf Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 1 Apr 2007 15:40:46 +0200 Subject: [PATCH 103/260] Clean up python class names. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0443337359..24c8e66e87 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -160,7 +160,7 @@ class P4CleanTags(Command): print "%s tags removed." % len(allTags) return True -class P4Sync(Command): +class P4Submit(Command): def __init__(self): Command.__init__(self) self.options = [ @@ -407,7 +407,7 @@ class P4Sync(Command): return True -class GitSync(Command): +class P4Sync(Command): def __init__(self): Command.__init__(self) self.options = [ @@ -1060,8 +1060,8 @@ def printUsage(commands): commands = { "debug" : P4Debug(), "clean-tags" : P4CleanTags(), - "submit" : P4Sync(), - "sync" : GitSync() + "submit" : P4Submit(), + "sync" : P4Sync() } if len(sys.argv[1:]) == 0: From 01ce1fe9676e3f714f3a2d068bf8cfe021f4073c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 7 Apr 2007 23:46:50 +0200 Subject: [PATCH 104/260] Added git-p4 rebase convenience Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 22 ++++++++++++++++++++-- contrib/fast-import/git-p4.txt | 14 +++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 24c8e66e87..aa85800d69 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -804,7 +804,11 @@ class P4Sync(Command): def getLabels(self): self.labels = {} - for output in p4CmdList("labels %s..." % self.globalPrefix): + l = p4CmdList("labels %s..." % self.globalPrefix) + if len(l) > 0: + print "Finding files belonging to labels in %s" % self.globalPrefix + + for output in l: label = output["label"] revisions = {} newestChange = 0 @@ -1039,6 +1043,19 @@ class P4Sync(Command): return True +class P4Rebase(Command): + def __init__(self): + Command.__init__(self) + self.options = [ ] + self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it" + + def run(self, args): + sync = P4Sync() + sync.run([]) + print "Rebasing the current branch" + system("git rebase p4") + return True + class HelpFormatter(optparse.IndentedHelpFormatter): def __init__(self): optparse.IndentedHelpFormatter.__init__(self) @@ -1061,7 +1078,8 @@ commands = { "debug" : P4Debug(), "clean-tags" : P4CleanTags(), "submit" : P4Submit(), - "sync" : P4Sync() + "sync" : P4Sync(), + "rebase" : P4Rebase() } if len(sys.argv[1:]) == 0: diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 30e2cb9a55..5f7251c2d6 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -40,12 +40,24 @@ newer changes from the Perforce depot by just calling git-p4 sync -in your git repository. +in your git repository. By default the "p4" branch is updated. It is recommended to run 'git repack -a -d -f' from time to time when using incremental imports to optimally combine the individual git packs that each incremental import creates through the use of git-fast-import. +Updating +======== + +A common working pattern is to fetch the latest changes from the Perforce depot +and merge them with local uncommitted changes. The recommended way is to use +git's rebase mechanism to preserve linear history. git-p4 provides a convenient + + git-p4 rebase + +command that calls git-p4 sync followed by git rebase to rebase the current +working branch. + Submitting ========== From 1f52af6c732bc17415474c426dc009f9e6a36a83 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 00:07:02 +0200 Subject: [PATCH 105/260] Provide a tree summary after git-p4 rebase Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index aa85800d69..170af90abc 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -974,7 +974,7 @@ class P4Sync(Command): if len(changes) == 0: if not self.silent: print "no changes to import!" - sys.exit(1) + return True cnt = 1 for change in changes: @@ -1053,7 +1053,9 @@ class P4Rebase(Command): sync = P4Sync() sync.run([]) print "Rebasing the current branch" + oldHead = os.popen("git rev-parse HEAD").read()[:-1] system("git rebase p4") + system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True class HelpFormatter(optparse.IndentedHelpFormatter): From cb53e1f8e9a78ab3f980250d2388a62ae2a42d77 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 00:12:02 +0200 Subject: [PATCH 106/260] Turn off potentially slow label detection by default Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 170af90abc..fcdfe22a45 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -417,7 +417,8 @@ class P4Sync(Command): optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--known-branches", dest="knownBranches"), optparse.make_option("--data-cache", dest="dataCache", action="store_true"), - optparse.make_option("--command-cache", dest="commandCache", action="store_true") + optparse.make_option("--command-cache", dest="commandCache", action="store_true"), + optparse.make_option("--detect-labels", dest="detectLabels", action="store_true") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -437,6 +438,7 @@ class P4Sync(Command): self.committedChanges = Set() self.branch = "" self.detectBranches = False + self.detectLabels = False self.changesFile = "" def p4File(self, depotPath): @@ -892,7 +894,9 @@ class P4Sync(Command): self.globalPrefix += "/" self.getUserMap() - self.getLabels(); + self.labels = {} + if self.detectLabels: + self.getLabels(); if len(self.changeRange) == 0: try: From 68ed351ab51c8b5731665fe6a8a7cc9651edf6c9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 09:00:55 +0200 Subject: [PATCH 107/260] Honor --silent for labels Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index fcdfe22a45..65660e1351 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -807,7 +807,7 @@ class P4Sync(Command): self.labels = {} l = p4CmdList("labels %s..." % self.globalPrefix) - if len(l) > 0: + if len(l) > 0 and not silent: print "Finding files belonging to labels in %s" % self.globalPrefix for output in l: From f9a3a4f796461276bbbcfef965984086e8e00b46 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 10:08:26 +0200 Subject: [PATCH 108/260] Added git-p4 clone convenience command Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 57 ++++++++++++++++++++++++++++++++-- contrib/fast-import/git-p4.txt | 23 +++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 65660e1351..0a22d9a2e4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -440,6 +440,7 @@ class P4Sync(Command): self.detectBranches = False self.detectLabels = False self.changesFile = "" + self.tagLastChange = True def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -826,7 +827,6 @@ class P4Sync(Command): self.globalPrefix = "" self.changeRange = "" self.initialParent = "" - self.tagLastChange = True if len(self.branch) == 0: self.branch = "p4" @@ -1062,6 +1062,58 @@ class P4Rebase(Command): system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True +class P4Clone(P4Sync): + def __init__(self): + P4Sync.__init__(self) + self.description = "Creates a new git repository and imports from Perforce into it" + self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]" + self.needsGit = False + self.tagLastChange = False + + def run(self, args): + if len(args) < 1: + return False + depotPath = args[0] + dir = "" + if len(args) == 2: + dir = args[1] + elif len(args) > 2: + return False + + if not depotPath.startswith("//"): + return False + + if len(dir) == 0: + dir = depotPath + atPos = dir.rfind("@") + if atPos != -1: + dir = dir[0:atPos] + hashPos = dir.rfind("#") + if hashPos != -1: + dir = dir[0:hashPos] + + if dir.endswith("..."): + dir = dir[:-3] + + if dir.endswith("/"): + dir = dir[:-1] + + slashPos = dir.rfind("/") + if slashPos != -1: + dir = dir[slashPos + 1:] + + print "Importing from %s into %s" % (depotPath, dir) + os.makedirs(dir) + os.chdir(dir) + system("git init") + if not P4Sync.run(self, [depotPath]): + return False + os.wait() + if self.branch != "master": + system("git branch master p4") + system("git checkout -f") + return True + class HelpFormatter(optparse.IndentedHelpFormatter): def __init__(self): optparse.IndentedHelpFormatter.__init__(self) @@ -1085,7 +1137,8 @@ commands = { "clean-tags" : P4CleanTags(), "submit" : P4Submit(), "sync" : P4Sync(), - "rebase" : P4Rebase() + "rebase" : P4Rebase(), + "clone" : P4Clone() } if len(sys.argv[1:]) == 0: diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 5f7251c2d6..99ae85bd7b 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -10,7 +10,25 @@ done using "git-p4 submit". Importing ========= -The procedure is simple: +You can simply start with + + git-p4 clone //depot/path/project + +or + + git-p4 clone //depot/path/project myproject + +This will create an empty git repository in a subdirectory called "project" (or +"myproject" with the second command), import the head revision from the +specified perforce path into a git "p4" branch, create a master branch off it +and check it out. If you want the entire history (not just the head revision) then +you can simply append a "@all" to the depot path: + + git-p4 clone //depot/project/main@all myproject + + + +If you want more control you can also use the git-p4 sync command directly: mkdir repo-git cd repo-git @@ -31,6 +49,9 @@ a big import. This may take a while. Support for Perforce integrations is still work in progress. Don't bother trying it unless you want to hack on it :) +For convenience there's also the git-p4 clone command that works similar to +git-clone and combines the creation of the git repository with the the initial +import and the branch setup Incremental Imports =================== From c45b1cfe1eda2cc67c4f07a0cd6986911d7c2fd8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 10:13:32 +0200 Subject: [PATCH 109/260] Fix file determination for #head imports Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0a22d9a2e4..28b088544b 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -938,7 +938,8 @@ class P4Sync(Command): newestRevision = change if info["action"] == "delete": - fileCnt = fileCnt + 1 + # don't increase the file cnt, otherwise details["depotFile123"] will have gaps! + #fileCnt = fileCnt + 1 continue for prop in [ "depotFile", "rev", "action", "type" ]: From 10c3211b81a2f67c3853900e462333933f910696 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 10:15:47 +0200 Subject: [PATCH 110/260] fix variable usage (oops) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 28b088544b..72fa48af04 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -808,7 +808,7 @@ class P4Sync(Command): self.labels = {} l = p4CmdList("labels %s..." % self.globalPrefix) - if len(l) > 0 and not silent: + if len(l) > 0 and not self.silent: print "Finding files belonging to labels in %s" % self.globalPrefix for output in l: From 7243b350b3469eb78be950331290ea16e57c3de8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 8 Apr 2007 10:21:56 +0200 Subject: [PATCH 111/260] Added a simple example of usage to the "documentation" :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 99ae85bd7b..4f6a680f95 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -103,3 +103,25 @@ continue importing the remaining changes with After submitting you should sync your perforce import branch ("p4" or "origin") from Perforce using git-p4's sync command. + + +Example +======= + +# Clone a repository + git-p4 clone //depot/path/project +# Enter the newly cloned directory + cd project +# Do some work... + vi foo.h +# ... and commit locally to gi + git commit foo.h +# In the meantime somebody submitted changes to the Perforce depot. Rebase your latest +# changes against the latest changes in Perforce: + git-p4 rebase +# Submit your locally committed changes back to Perforce + git-p4 submit +# ... and synchronize with Perforce + git-p4 rebase + + From 80b5910fac79b22ccc8eabb2262562c913d5603c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 9 Apr 2007 12:43:40 +0200 Subject: [PATCH 112/260] Allow for convenient rebasing after git-p4 submit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 72fa48af04..4fbfaf11b6 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -352,6 +352,7 @@ class P4Submit(Command): sys.exit(128) print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath) + oldWorkingDirectory = os.getcwd() os.chdir(clientPath) response = raw_input("Do you want to sync %s with p4 sync? (y/n) " % clientPath) if response == "y" or response == "yes": @@ -403,6 +404,11 @@ class P4Submit(Command): print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." system("p4 edit ... >/dev/null") system("p4 revert ... >/dev/null") + response = raw_input("Do you want to sync from Perforce now using git-p4 rebase (y/n)? ") + if response == "y" or response == "yes": + os.chdir(oldWorkingDirectory) + rebase = P4Rebase() + rebase.run([]) os.remove(self.configFile) return True From fd4ca86a0b920ddde464e787ef2350126f083d75 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 13 Apr 2007 22:21:10 +0200 Subject: [PATCH 113/260] Print an error message of some sort if git fast-import fails. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 4fbfaf11b6..6db757ae43 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -958,6 +958,7 @@ class P4Sync(Command): try: self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix) except IOError: + print "IO error with git fast-import. Is your git version recent enough?" print self.gitError.read() else: From f291b4e3d47fb03a65c0dfcfbd28f8fabb6187c1 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 14 Apr 2007 11:21:50 +0200 Subject: [PATCH 114/260] Fix the timezone formatting. Now qgit also displays (parses) it correctly. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6db757ae43..44a07c27ce 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -922,10 +922,7 @@ class P4Sync(Command): except: pass - self.tz = - time.timezone / 36 - tzsign = ("%s" % self.tz)[0] - if tzsign != '+' and tzsign != '-': - self.tz = "+" + ("%s" % self.tz) + self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git fast-import") From 8b72ca0f76213eb375840e6b9069b260d97f8286 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 14 Apr 2007 16:05:54 +0200 Subject: [PATCH 115/260] Removed the old patch apply code from git-p4 submit. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 44a07c27ce..3202a819ca 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -170,7 +170,6 @@ class P4Submit(Command): optparse.make_option("--log-substitutions", dest="substFile"), optparse.make_option("--noninteractive", action="store_false"), optparse.make_option("--dry-run", action="store_true"), - optparse.make_option("--apply-as-patch", action="store_true", dest="applyAsPatch") ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" @@ -181,7 +180,6 @@ class P4Submit(Command): self.substFile = "" self.firstTime = True self.origin = "" - self.applyAsPatch = True self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -202,10 +200,6 @@ class P4Submit(Command): self.config["commits"] = commits - if not self.applyAsPatch: - print "Creating temporary p4-sync branch from %s ..." % self.origin - system("git checkout -f -b p4-sync %s" % self.origin) - def prepareLogMessage(self, template, message): result = "" @@ -250,11 +244,7 @@ class P4Submit(Command): else: die("unknown modifier %s for %s" % (modifier, path)) - if self.applyAsPatch: - system("git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) - else: - system("git diff-files --name-only -z | git update-index --remove -z --stdin") - system("git cherry-pick --no-commit \"%s\"" % id) + system("git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) for f in filesToAdd: system("p4 add %s" % f) @@ -397,13 +387,6 @@ class P4Submit(Command): print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" - if not self.applyAsPatch: - print "Deleting temporary p4-sync branch and going back to %s" % self.master - system("git checkout %s" % self.master) - system("git branch -D p4-sync") - print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." - system("p4 edit ... >/dev/null") - system("p4 revert ... >/dev/null") response = raw_input("Do you want to sync from Perforce now using git-p4 rebase (y/n)? ") if response == "y" or response == "yes": os.chdir(oldWorkingDirectory) From 5e80dd4d7e34fdd507997615d7fdefc2411571eb Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 14 Apr 2007 16:09:43 +0200 Subject: [PATCH 116/260] Slightly improved formatting of the raw_input questions. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 3202a819ca..9c9852c75f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -277,7 +277,7 @@ class P4Submit(Command): firstIteration = True while response == "e": if not firstIteration: - response = raw_input("Do you want to submit this change (y/e/n)? ") + response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o ") firstIteration = False if response == "e": [handle, fileName] = tempfile.mkstemp() @@ -344,7 +344,7 @@ class P4Submit(Command): print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath) oldWorkingDirectory = os.getcwd() os.chdir(clientPath) - response = raw_input("Do you want to sync %s with p4 sync? (y/n) " % clientPath) + response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % clientPath) if response == "y" or response == "yes": system("p4 sync ...") @@ -387,7 +387,7 @@ class P4Submit(Command): print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" - response = raw_input("Do you want to sync from Perforce now using git-p4 rebase (y/n)? ") + response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") if response == "y" or response == "yes": os.chdir(oldWorkingDirectory) rebase = P4Rebase() From 90865adc01213520980459ddc261a019f673cce8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 15 Apr 2007 09:34:15 +0200 Subject: [PATCH 117/260] A new attempt at fixing the child-fast-import-process-not-finished race condition in the clone command Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 9c9852c75f..b77cb20e3f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -907,7 +907,10 @@ class P4Sync(Command): self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) - self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git fast-import") + importProcess = popen2.Popen3("git fast-import", capturestderr = True) + self.gitOutput = importProcess.fromchild + self.gitStream = importProcess.tochild + self.gitError = importProcess.childerr if len(self.revision) > 0: print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision) @@ -1028,6 +1031,7 @@ class P4Sync(Command): self.gitStream.close() self.gitOutput.close() self.gitError.close() + importProcess.wait() os.popen("git repo-config p4.depotpath %s" % self.globalPrefix).read() if len(self.initialTag) > 0: @@ -1096,7 +1100,6 @@ class P4Clone(P4Sync): system("git init") if not P4Sync.run(self, [depotPath]): return False - os.wait() if self.branch != "master": system("git branch master p4") system("git checkout -f") From 51a2640afdd12475642728dc3576966abe0dba6d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 15 Apr 2007 09:59:56 +0200 Subject: [PATCH 118/260] Handle patch errors in git-p4 submit better. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b77cb20e3f..fb13469ce2 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -244,7 +244,33 @@ class P4Submit(Command): else: die("unknown modifier %s for %s" % (modifier, path)) - system("git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) + diffcmd = "git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\"" % (id, id) + patchcmd = diffcmd + " | patch -p1" + + if os.system(patchcmd + " --dry-run --silent") != 0: + print "Unfortunately applying the change failed!" + print "What do you want to do?" + response = "x" + while response != "s" and response != "a" and response != "w": + response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ") + if response == "s": + print "Skipping! Good luck with the next patches..." + return + elif response == "a": + os.system(patchcmd) + if len(filesToAdd) > 0: + print "You may also want to call p4 add on the following files:" + print " ".join(filesToAdd) + if len(filesToDelete): + print "The following files should be scheduled for deletion with p4 delete:" + print " ".join(filesToDelete) + die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") + elif response == "w": + system(diffcmd + " > patch.txt") + print "Patch saved to patch.txt in %s !" % self.clientPath + die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") + + system(patchcmd) for f in filesToAdd: system("p4 add %s" % f) @@ -335,16 +361,16 @@ class P4Submit(Command): print "Internal error: cannot locate perforce depot path from existing branches" sys.exit(128) - clientPath = p4Where(depotPath) + self.clientPath = p4Where(depotPath) - if len(clientPath) == 0: + if len(self.clientPath) == 0: print "Error: Cannot locate perforce checkout of %s in client view" % depotPath sys.exit(128) - print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath) + print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath) oldWorkingDirectory = os.getcwd() - os.chdir(clientPath) - response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % clientPath) + os.chdir(self.clientPath) + response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath) if response == "y" or response == "yes": system("p4 sync ...") From ff5dba20e324fb394b7077d0b1d1c5eb4357959f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 1 May 2007 18:28:38 +0200 Subject: [PATCH 119/260] Doc cleanups. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 4f6a680f95..d36a1cf18c 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -4,8 +4,8 @@ Usage ===== git-p4 supports two main modes: Importing from Perforce to a Git repository is -done using "git-p4 sync". Submitting changes from Git back to Perforce is -done using "git-p4 submit". +done using "git-p4 sync" or "git-p4 rebase". Submitting changes from Git back +to Perforce is done using "git-p4 submit". Importing ========= @@ -49,10 +49,6 @@ a big import. This may take a while. Support for Perforce integrations is still work in progress. Don't bother trying it unless you want to hack on it :) -For convenience there's also the git-p4 clone command that works similar to -git-clone and combines the creation of the git repository with the the initial -import and the branch setup - Incremental Imports =================== From 1c094184da5c7bf572c907633279890bd15d1952 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 1 May 2007 23:15:48 +0200 Subject: [PATCH 120/260] Micro cleanup Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index fb13469ce2..9adc66a05d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -74,7 +74,7 @@ def extractLogMessageFromGitCommit(commit): for log in os.popen("git cat-file commit %s" % commit).readlines(): if not foundTitle: if len(log) == 1: - foundTitle = 1 + foundTitle = True continue logMessage += log From 8f8725314dbed75dc1e603646ea6a2c630025f6e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 1 May 2007 23:23:00 +0200 Subject: [PATCH 121/260] cleanup, renamed self.globalPrefix to self.depotPath Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 9adc66a05d..6b74feeb64 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -465,9 +465,9 @@ class P4Sync(Command): fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] - if not path.startswith(self.globalPrefix): + if not path.startswith(self.depotPath): # if not self.silent: - # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.globalPrefix, change) + # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) fnum = fnum + 1 continue @@ -491,7 +491,7 @@ class P4Sync(Command): branches = Set() for file in files: - relativePath = file["path"][len(self.globalPrefix):] + relativePath = file["path"][len(self.depotPath):] # strip off the filename relativePath = relativePath[0:relativePath.rfind("/")] @@ -577,7 +577,7 @@ class P4Sync(Command): sys.exit(1); sourceLog = sourceLog[0] - relPath = source[len(self.globalPrefix):] + relPath = source[len(self.depotPath):] # strip off the filename relPath = relPath[0:relPath.rfind("/")] @@ -737,7 +737,7 @@ class P4Sync(Command): sys.exit(1); sourceLog = sourceLog[0] - relPath = source[len(self.globalPrefix):] + relPath = source[len(self.depotPath):] # strip off the filename relPath = relPath[0:relPath.rfind("/")] @@ -749,13 +749,13 @@ class P4Sync(Command): def changeIsBranchMerge(self, sourceBranch, destinationBranch, change): sourceFiles = {} - for file in p4CmdList("files %s...@%s" % (self.globalPrefix + sourceBranch + "/", change)): + for file in p4CmdList("files %s...@%s" % (self.depotPath + sourceBranch + "/", change)): if file["action"] == "delete": continue sourceFiles[file["depotFile"]] = file destinationFiles = {} - for file in p4CmdList("files %s...@%s" % (self.globalPrefix + destinationBranch + "/", change)): + for file in p4CmdList("files %s...@%s" % (self.depotPath + destinationBranch + "/", change)): destinationFiles[file["depotFile"]] = file for fileName in sourceFiles.keys(): @@ -822,9 +822,9 @@ class P4Sync(Command): def getLabels(self): self.labels = {} - l = p4CmdList("labels %s..." % self.globalPrefix) + l = p4CmdList("labels %s..." % self.depotPath) if len(l) > 0 and not self.silent: - print "Finding files belonging to labels in %s" % self.globalPrefix + print "Finding files belonging to labels in %s" % self.depotPath for output in l: label = output["label"] @@ -839,7 +839,7 @@ class P4Sync(Command): self.labels[newestChange] = [output, revisions] def run(self, args): - self.globalPrefix = "" + self.depotPath = "" self.changeRange = "" self.initialParent = "" @@ -855,7 +855,7 @@ class P4Sync(Command): [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) if len(self.previousDepotPath) > 0 and len(p4Change) > 0: p4Change = int(p4Change) + 1 - self.globalPrefix = self.previousDepotPath + self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change self.initialParent = self.branch self.tagLastChange = False @@ -864,49 +864,49 @@ class P4Sync(Command): self.branch = "refs/heads/" + self.branch - if len(self.globalPrefix) == 0: - self.globalPrefix = self.previousDepotPath = os.popen("git repo-config --get p4.depotpath").read() + if len(self.depotPath) == 0: + self.depotPath = self.previousDepotPath = os.popen("git repo-config --get p4.depotpath").read() - if len(self.globalPrefix) != 0: - self.globalPrefix = self.globalPrefix[:-1] + if len(self.depotPath) != 0: + self.depotPath = self.depotPath[:-1] - if len(args) == 0 and len(self.globalPrefix) != 0: + if len(args) == 0 and len(self.depotPath) != 0: if not self.silent: - print "Depot path: %s" % self.globalPrefix + print "Depot path: %s" % self.depotPath elif len(args) != 1: return False else: - if len(self.globalPrefix) != 0 and self.globalPrefix != args[0]: - print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.globalPrefix, args[0]) + if len(self.depotPath) != 0 and self.depotPath != args[0]: + print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0]) sys.exit(1) - self.globalPrefix = args[0] + self.depotPath = args[0] self.revision = "" self.users = {} self.lastChange = 0 self.initialTag = "" - if self.globalPrefix.find("@") != -1: - atIdx = self.globalPrefix.index("@") - self.changeRange = self.globalPrefix[atIdx:] + if self.depotPath.find("@") != -1: + atIdx = self.depotPath.index("@") + self.changeRange = self.depotPath[atIdx:] if self.changeRange == "@all": self.changeRange = "" elif self.changeRange.find(",") == -1: self.revision = self.changeRange self.changeRange = "" - self.globalPrefix = self.globalPrefix[0:atIdx] - elif self.globalPrefix.find("#") != -1: - hashIdx = self.globalPrefix.index("#") - self.revision = self.globalPrefix[hashIdx:] - self.globalPrefix = self.globalPrefix[0:hashIdx] + self.depotPath = self.depotPath[0:atIdx] + elif self.depotPath.find("#") != -1: + hashIdx = self.depotPath.index("#") + self.revision = self.depotPath[hashIdx:] + self.depotPath = self.depotPath[0:hashIdx] elif len(self.previousDepotPath) == 0: self.revision = "#head" - if self.globalPrefix.endswith("..."): - self.globalPrefix = self.globalPrefix[:-3] + if self.depotPath.endswith("..."): + self.depotPath = self.depotPath[:-3] - if not self.globalPrefix.endswith("/"): - self.globalPrefix += "/" + if not self.depotPath.endswith("/"): + self.depotPath += "/" self.getUserMap() self.labels = {} @@ -939,15 +939,15 @@ class P4Sync(Command): self.gitError = importProcess.childerr if len(self.revision) > 0: - print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision) + print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = "Initial import of %s from the state at revision %s" % (self.globalPrefix, self.revision) + details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision) details["change"] = self.revision newestRevision = 0 fileCnt = 0 - for info in p4CmdList("files %s...%s" % (self.globalPrefix, self.revision)): + for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)): change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -965,7 +965,7 @@ class P4Sync(Command): details["change"] = newestRevision try: - self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix) + self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath) except IOError: print "IO error with git fast-import. Is your git version recent enough?" print self.gitError.read() @@ -984,7 +984,7 @@ class P4Sync(Command): changes.sort() else: - output = os.popen("p4 changes %s...%s" % (self.globalPrefix, self.changeRange)).readlines() + output = os.popen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() for line in output: changeNum = line.split(" ")[1] @@ -1011,7 +1011,7 @@ class P4Sync(Command): if self.detectBranches: for branch in self.branchesForCommit(files): self.knownBranches.add(branch) - branchPrefix = self.globalPrefix + branch + "/" + branchPrefix = self.depotPath + branch + "/" filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix) @@ -1040,7 +1040,7 @@ class P4Sync(Command): merged = "refs/heads/" + merged self.commit(description, files, branch, branchPrefix, parent, merged) else: - self.commit(description, files, self.branch, self.globalPrefix, self.initialParent) + self.commit(description, files, self.branch, self.depotPath, self.initialParent) self.initialParent = "" except IOError: print self.gitError.read() @@ -1059,7 +1059,7 @@ class P4Sync(Command): self.gitError.close() importProcess.wait() - os.popen("git repo-config p4.depotpath %s" % self.globalPrefix).read() + os.popen("git repo-config p4.depotpath %s" % self.depotPath).read() if len(self.initialTag) > 0: os.popen("git tag -d %s" % self.initialTag).read() From 28359251395c1647bf40adcdabbe4d85d7b085af Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 1 May 2007 23:26:19 +0200 Subject: [PATCH 122/260] Cleanup, removed the old tagging code Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6b74feeb64..9927fa142e 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -455,7 +455,6 @@ class P4Sync(Command): self.detectBranches = False self.detectLabels = False self.changesFile = "" - self.tagLastChange = True def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -858,15 +857,11 @@ class P4Sync(Command): self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change self.initialParent = self.branch - self.tagLastChange = False if not self.silent: print "Performing incremental import into %s git branch" % self.branch self.branch = "refs/heads/" + self.branch - if len(self.depotPath) == 0: - self.depotPath = self.previousDepotPath = os.popen("git repo-config --get p4.depotpath").read() - if len(self.depotPath) != 0: self.depotPath = self.depotPath[:-1] @@ -884,7 +879,6 @@ class P4Sync(Command): self.revision = "" self.users = {} self.lastChange = 0 - self.initialTag = "" if self.depotPath.find("@") != -1: atIdx = self.depotPath.index("@") @@ -927,7 +921,6 @@ class P4Sync(Command): self.rev = int(output[tagIdx + 9 : endPos]) + 1 self.changeRange = "@%s,#head" % self.rev self.initialParent = os.popen("git rev-parse %s" % self.branch).read()[:-1] - self.initialTag = "p4/%s" % (int(self.rev) - 1) except: pass @@ -1049,20 +1042,12 @@ class P4Sync(Command): if not self.silent: print "" - if self.tagLastChange: - self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange) - self.gitStream.write("from %s\n\n" % self.branch); - self.gitStream.close() self.gitOutput.close() self.gitError.close() importProcess.wait() - os.popen("git repo-config p4.depotpath %s" % self.depotPath).read() - if len(self.initialTag) > 0: - os.popen("git tag -d %s" % self.initialTag).read() - return True class P4Rebase(Command): @@ -1086,7 +1071,6 @@ class P4Clone(P4Sync): self.description = "Creates a new git repository and imports from Perforce into it" self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]" self.needsGit = False - self.tagLastChange = False def run(self, args): if len(args) < 1: From a844b7406f17ac6e069e617509a2997d7e128500 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 7 May 2007 20:14:17 +0200 Subject: [PATCH 123/260] Document some implementation details, for the curious... :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index d36a1cf18c..ff0da9d0c8 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -121,3 +121,20 @@ Example git-p4 rebase +Implementation Details... +========================= + +* Changesets from Perforce are imported using git fast-import. +* The import does not require anything from the Perforce client view as it just uses + "p4 print //depot/path/file#revision" to get the actual file contents. +* Every imported changeset has a special [git-p4...] line at the + end of the log message that gives information about the corresponding + Perforce change number and is also used by git-p4 itself to find out + where to continue importing when doing incremental imports. + Basically when syncing it extracts the perforce change number of the + latest commit in the "p4" branch and uses "p4 changes //depot/path/...@changenum,#head" + to find out which changes need to be imported. +* git-p4 submit uses "git rev-list" to pick the commits between the "p4" branch + and the current branch. + The commits themselves are applied using git diff-tree ... | patch -p1 + From 084835805565726c825f0853626a33a0263066bf Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 14:31:06 +0200 Subject: [PATCH 124/260] Use the subprocess module instead of popen2 to make it work on Windows. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 9927fa142e..a2f582f8c9 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -8,7 +8,7 @@ # License: MIT # -import optparse, sys, os, marshal, popen2, shelve +import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time from sets import Set; @@ -926,10 +926,10 @@ class P4Sync(Command): self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) - importProcess = popen2.Popen3("git fast-import", capturestderr = True) - self.gitOutput = importProcess.fromchild - self.gitStream = importProcess.tochild - self.gitError = importProcess.childerr + importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); + self.gitOutput = importProcess.stdout + self.gitStream = importProcess.stdin + self.gitError = importProcess.stderr if len(self.revision) > 0: print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision) From ac1fde55a71e166a86468582961908ab0d01413b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 14:42:56 +0200 Subject: [PATCH 125/260] Added a little .bat wrapper from Marius Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.bat | 1 + 1 file changed, 1 insertion(+) create mode 100644 contrib/fast-import/git-p4.bat diff --git a/contrib/fast-import/git-p4.bat b/contrib/fast-import/git-p4.bat new file mode 100644 index 0000000000..666eb73016 --- /dev/null +++ b/contrib/fast-import/git-p4.bat @@ -0,0 +1 @@ +python "%~d0%~p0git-p4" %* From caace111129545559bf798d83ac8cfd596d36f66 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 14:57:57 +0200 Subject: [PATCH 126/260] Make sure all popen calls use binary mode (for Windows) and also make gitBranchExists work on Windows. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index a2f582f8c9..d83ce1ae95 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -14,6 +14,9 @@ from sets import Set; gitdir = os.environ.get("GIT_DIR", "") +def mypopen(command): + return os.popen(command, "rb"); + def p4CmdList(cmd): cmd = "p4 -G %s" % cmd pipe = os.popen(cmd, "rb") @@ -57,7 +60,7 @@ def die(msg): sys.exit(1) def currentGitBranch(): - return os.popen("git name-rev HEAD").read().split(" ")[1][:-1] + return mypopen("git name-rev HEAD").read().split(" ")[1][:-1] def isValidGitDir(path): if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): @@ -71,7 +74,7 @@ def system(cmd): def extractLogMessageFromGitCommit(commit): logMessage = "" foundTitle = False - for log in os.popen("git cat-file commit %s" % commit).readlines(): + for log in mypopen("git cat-file commit %s" % commit).readlines(): if not foundTitle: if len(log) == 1: foundTitle = True @@ -100,9 +103,8 @@ def extractDepotPathAndChangeFromGitLog(log): return values.get("depot-path"), values.get("change") def gitBranchExists(branch): - if os.system("git rev-parse %s 2>/dev/null >/dev/null" % branch) == 0: - return True - return False + proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE); + return proc.wait() == 0; class Command: def __init__(self): @@ -146,7 +148,7 @@ class P4CleanTags(Command): caretIdx = len(output) - 1 rev = int(output[tagIdx + 9 : caretIdx]) - allTags = os.popen("git tag -l p4/").readlines() + allTags = mypopen("git tag -l p4/").readlines() for i in range(len(allTags)): allTags[i] = int(allTags[i][3:-1]) @@ -155,7 +157,7 @@ class P4CleanTags(Command): allTags.remove(rev) for rev in allTags: - print os.popen("git tag -d p4/%s" % rev).read() + print mypopen("git tag -d p4/%s" % rev).read() print "%s tags removed." % len(allTags) return True @@ -194,7 +196,7 @@ class P4Submit(Command): die("Cannot start sync. Previous sync config found at %s" % self.configFile) commits = [] - for line in os.popen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): + for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): commits.append(line[:-1]) commits.reverse() @@ -224,8 +226,8 @@ class P4Submit(Command): return result def apply(self, id): - print "Applying %s" % (os.popen("git log --max-count=1 --pretty=oneline %s" % id).read()) - diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() + print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read()) + diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() filesToDelete = set() for line in diff: @@ -282,11 +284,11 @@ class P4Submit(Command): logMessage = logMessage.replace("\n", "\n\t") logMessage = logMessage[:-1] - template = os.popen("p4 change -o").read() + template = mypopen("p4 change -o").read() if self.interactive: submitTemplate = self.prepareLogMessage(template, logMessage) - diff = os.popen("p4 diff -du ...").read() + diff = mypopen("p4 diff -du ...").read() for newFile in filesToAdd: diff += "==== new file ====\n" @@ -323,7 +325,7 @@ class P4Submit(Command): print submitTemplate raw_input("Press return to continue...") else: - pipe = os.popen("p4 submit -i", "w") + pipe = mypopen("p4 submit -i", "w") pipe.write(submitTemplate) pipe.close() else: @@ -920,7 +922,7 @@ class P4Sync(Command): endPos = caretIdx self.rev = int(output[tagIdx + 9 : endPos]) + 1 self.changeRange = "@%s,#head" % self.rev - self.initialParent = os.popen("git rev-parse %s" % self.branch).read()[:-1] + self.initialParent = mypopen("git rev-parse %s" % self.branch).read()[:-1] except: pass @@ -977,7 +979,7 @@ class P4Sync(Command): changes.sort() else: - output = os.popen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() + output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() for line in output: changeNum = line.split(" ")[1] @@ -1060,7 +1062,7 @@ class P4Rebase(Command): sync = P4Sync() sync.run([]) print "Rebasing the current branch" - oldHead = os.popen("git rev-parse HEAD").read()[:-1] + oldHead = mypopen("git rev-parse HEAD").read()[:-1] system("git rebase p4") system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True @@ -1176,7 +1178,7 @@ if cmd.needsGit: if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): - cdup = os.popen("git rev-parse --show-cdup").read()[:-1] + cdup = mypopen("git rev-parse --show-cdup").read()[:-1] if isValidGitDir(cdup + "/" + gitdir): os.chdir(cdup) From 25df95cce470d74cdc5e8905a298bbc36c7ec6d0 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 15:15:39 +0200 Subject: [PATCH 127/260] Make submitting work on Windows. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d83ce1ae95..d134a28189 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -9,7 +9,7 @@ # import optparse, sys, os, marshal, popen2, subprocess, shelve -import tempfile, getopt, sha, os.path, time +import tempfile, getopt, sha, os.path, time, platform from sets import Set; gitdir = os.environ.get("GIT_DIR", "") @@ -299,7 +299,10 @@ class P4Submit(Command): diff += "+" + line f.close() - separatorLine = "######## everything below this line is just the diff #######\n" + separatorLine = "######## everything below this line is just the diff #######" + if platform.system() == "Windows": + separatorLine += "\r" + separatorLine += "\n" response = "e" firstIteration = True @@ -312,9 +315,12 @@ class P4Submit(Command): tmpFile = os.fdopen(handle, "w+") tmpFile.write(submitTemplate + separatorLine + diff) tmpFile.close() - editor = os.environ.get("EDITOR", "vi") + defaultEditor = "vi" + if platform.system() == "Windows": + defaultEditor = "notepad" + editor = os.environ.get("EDITOR", defaultEditor); system(editor + " " + fileName) - tmpFile = open(fileName, "r") + tmpFile = open(fileName, "rb") message = tmpFile.read() tmpFile.close() os.remove(fileName) @@ -325,7 +331,7 @@ class P4Submit(Command): print submitTemplate raw_input("Press return to continue...") else: - pipe = mypopen("p4 submit -i", "w") + pipe = os.popen("p4 submit -i", "wb") pipe.write(submitTemplate) pipe.close() else: From 42890f6291ab8b718dec5a3346875c45724d4ebb Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 16:07:02 +0200 Subject: [PATCH 128/260] Converted to unix newlines Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4.bat b/contrib/fast-import/git-p4.bat index 666eb73016..6524a54c67 100644 --- a/contrib/fast-import/git-p4.bat +++ b/contrib/fast-import/git-p4.bat @@ -1 +1 @@ -python "%~d0%~p0git-p4" %* +python "%~d0%~p0git-p4" %* From 95962f318e0f00cd4e2ebb54531c3d738aa39ece Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Tue, 15 May 2007 15:51:25 +0200 Subject: [PATCH 129/260] Make the command call silent Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4.bat b/contrib/fast-import/git-p4.bat index 6524a54c67..9f97e884f5 100644 --- a/contrib/fast-import/git-p4.bat +++ b/contrib/fast-import/git-p4.bat @@ -1 +1 @@ -python "%~d0%~p0git-p4" %* +@python "%~d0%~p0git-p4" %* From cd6cc0d31845576082fabc7f246a99988aca6d26 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 16:15:26 +0200 Subject: [PATCH 130/260] Fix git-p4 clone //depot/project (head import) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d134a28189..2633f29942 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -849,6 +849,7 @@ class P4Sync(Command): self.depotPath = "" self.changeRange = "" self.initialParent = "" + self.previousDepotPath = "" if len(self.branch) == 0: self.branch = "p4" From 81f2373f89895a47ed0251ac0856798514cfb618 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 15 May 2007 23:06:43 +0200 Subject: [PATCH 131/260] Make git-p4 work with bare repositories. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 2633f29942..84cdca1aa2 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1185,9 +1185,7 @@ if cmd.needsGit: if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): - cdup = mypopen("git rev-parse --show-cdup").read()[:-1] - if isValidGitDir(cdup + "/" + gitdir): - os.chdir(cdup) + gitdir = mypopen("git rev-parse --git-dir").read()[:-1] if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"): From d336c15835c924ba8d153edbfceca88d5c748582 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 16 May 2007 09:41:26 +0200 Subject: [PATCH 132/260] Added the possibility of skipping patches during git-p4 submit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 84cdca1aa2..eba7a67c68 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -230,11 +230,13 @@ class P4Submit(Command): diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() filesToDelete = set() + editedFiles = set() for line in diff: modifier = line[0] path = line[1:].strip() if modifier == "M": - system("p4 edit %s" % path) + system("p4 edit \"%s\"" % path) + editedFiles.add(path) elif modifier == "A": filesToAdd.add(path) if path in filesToDelete: @@ -308,7 +310,7 @@ class P4Submit(Command): firstIteration = True while response == "e": if not firstIteration: - response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o ") + response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ") firstIteration = False if response == "e": [handle, fileName] = tempfile.mkstemp() @@ -334,6 +336,15 @@ class P4Submit(Command): pipe = os.popen("p4 submit -i", "wb") pipe.write(submitTemplate) pipe.close() + elif response == "s": + for f in editedFiles: + system("p4 revert \"%s\"" % f); + for f in filesToAdd: + system("p4 revert \"%s\"" % f); + system("rm %s" %f) + for f in filesToDelete: + system("p4 delete \"%s\"" % f); + return else: print "Not submitting!" self.interactive = False From c3c46244518178bac49caf21d6ba3a782292bb10 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 16 May 2007 09:43:13 +0200 Subject: [PATCH 133/260] Give a better hint if git-p4 submit fails Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index eba7a67c68..c48b257577 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -193,7 +193,7 @@ class P4Submit(Command): def start(self): if len(self.config) > 0 and not self.reset: - die("Cannot start sync. Previous sync config found at %s" % self.configFile) + die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile) commits = [] for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): From dc1a93b6dce1d38e6e0ddd8980d8ccd64937fcb1 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 16 May 2007 12:12:39 +0200 Subject: [PATCH 134/260] Fix calling git-p4 rebase from within a subdirectory (git rebase wants to be in toplevel) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index c48b257577..ca6c623809 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1197,6 +1197,8 @@ if cmd.needsGit: gitdir = ".git" if not isValidGitDir(gitdir): gitdir = mypopen("git rev-parse --git-dir").read()[:-1] + if os.path.exists(gitdir): + os.chdir(mypopen("git rev-parse --show-cdup").read()[:-1]); if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"): From ca0affe7bbc0ef507e2ec01c75e4f54d309dfad7 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 16 May 2007 13:15:34 +0200 Subject: [PATCH 135/260] A little todo note before I forget it :), based on a suggestion from Lars. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ca6c623809..70366ff8c0 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -7,6 +7,8 @@ # 2007 Trolltech ASA # License: MIT # +# TODO: Add an option to sync/rebase to fetch and rebase from origin first. +# import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform From 5c4153e4880a90c05612521e9ee575a308d207db Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 07:42:38 +0200 Subject: [PATCH 136/260] Fixing syncing (gitdir discovery / cd) for bare repositories Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 70366ff8c0..239304857d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1200,7 +1200,9 @@ if cmd.needsGit: if not isValidGitDir(gitdir): gitdir = mypopen("git rev-parse --git-dir").read()[:-1] if os.path.exists(gitdir): - os.chdir(mypopen("git rev-parse --show-cdup").read()[:-1]); + cdup = mypopen("git rev-parse --show-cdup").read()[:-1]; + if len(cdup) > 0: + os.chdir(cdup); if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"): From f9162f6a4cc9f4af3e527412b364a8eda4502920 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 09:02:45 +0200 Subject: [PATCH 137/260] Always pass a sha1 for the initial parent so that git-fast-import doesn't think it's creating a new branch from itself. It's a sensible error in general but in the case of incremental imports we have to apply force :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 239304857d..1f549b5c62 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -865,7 +865,7 @@ class P4Sync(Command): self.previousDepotPath = "" if len(self.branch) == 0: - self.branch = "p4" + self.branch = "refs/remotes/p4" if len(args) == 0: if not gitBranchExists(self.branch) and gitBranchExists("origin"): @@ -878,11 +878,12 @@ class P4Sync(Command): p4Change = int(p4Change) + 1 self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change - self.initialParent = self.branch + self.initialParent = mypopen("git rev-parse %s" % self.branch).read()[:-1] if not self.silent: print "Performing incremental import into %s git branch" % self.branch - self.branch = "refs/heads/" + self.branch + if not self.branch.startswith("refs/"): + self.branch = "refs/heads/" + self.branch if len(self.depotPath) != 0: self.depotPath = self.depotPath[:-1] From 463e8af65577b1df4495cb08640840234ac80c86 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 09:13:54 +0200 Subject: [PATCH 138/260] Clean up code duplication for revision parsing and fix previous commit to not import into remotes/p4 (yet!). Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 1f549b5c62..e18f3cb8ab 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -69,6 +69,9 @@ def isValidGitDir(path): return True; return False +def parseRevision(ref): + return mypopen("git rev-parse %s" % ref).read()[:-1] + def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) @@ -865,7 +868,7 @@ class P4Sync(Command): self.previousDepotPath = "" if len(self.branch) == 0: - self.branch = "refs/remotes/p4" + self.branch = "p4" if len(args) == 0: if not gitBranchExists(self.branch) and gitBranchExists("origin"): @@ -878,7 +881,7 @@ class P4Sync(Command): p4Change = int(p4Change) + 1 self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change - self.initialParent = mypopen("git rev-parse %s" % self.branch).read()[:-1] + self.initialParent = parseRevision(self.branch) if not self.silent: print "Performing incremental import into %s git branch" % self.branch @@ -943,7 +946,7 @@ class P4Sync(Command): endPos = caretIdx self.rev = int(output[tagIdx + 9 : endPos]) + 1 self.changeRange = "@%s,#head" % self.rev - self.initialParent = mypopen("git rev-parse %s" % self.branch).read()[:-1] + self.initialParent = parseRevision(self.branch) except: pass From 8a2820def44f163ec8909863aefa5e8f98a236ea Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 19:44:50 +0200 Subject: [PATCH 139/260] Removed cleantags command. It doesn't have any meaning anymore. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 39 -------------------------------------- 1 file changed, 39 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e18f3cb8ab..910c69b534 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -129,44 +129,6 @@ class P4Debug(Command): print output return True -class P4CleanTags(Command): - def __init__(self): - Command.__init__(self) - self.options = [ -# optparse.make_option("--branch", dest="branch", default="refs/heads/master") - ] - self.description = "A tool to remove stale unused tags from incremental perforce imports." - def run(self, args): - branch = currentGitBranch() - print "Cleaning out stale p4 import tags..." - sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % branch) - output = sout.read() - try: - tagIdx = output.index(" tags/p4/") - except: - print "Cannot find any p4/* tag. Nothing to do." - sys.exit(0) - - try: - caretIdx = output.index("^") - except: - caretIdx = len(output) - 1 - rev = int(output[tagIdx + 9 : caretIdx]) - - allTags = mypopen("git tag -l p4/").readlines() - for i in range(len(allTags)): - allTags[i] = int(allTags[i][3:-1]) - - allTags.sort() - - allTags.remove(rev) - - for rev in allTags: - print mypopen("git tag -d p4/%s" % rev).read() - - print "%s tags removed." % len(allTags) - return True - class P4Submit(Command): def __init__(self): Command.__init__(self) @@ -1161,7 +1123,6 @@ def printUsage(commands): commands = { "debug" : P4Debug(), - "clean-tags" : P4CleanTags(), "submit" : P4Submit(), "sync" : P4Sync(), "rebase" : P4Rebase(), From 1c9d393d30797decad703b642fa87b67a5236af4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 20:15:47 +0200 Subject: [PATCH 140/260] Removed ancient and unused code to find the last imported revision from previous imports to use for the current import by looking at the p4 tags. The current approach of using the log message works better. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 910c69b534..a19ba47481 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -895,23 +895,6 @@ class P4Sync(Command): if self.detectLabels: self.getLabels(); - if len(self.changeRange) == 0: - try: - sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % self.branch) - output = sout.read() - if output.endswith("\n"): - output = output[:-1] - tagIdx = output.index(" tags/p4/") - caretIdx = output.find("^") - endPos = len(output) - if caretIdx != -1: - endPos = caretIdx - self.rev = int(output[tagIdx + 9 : endPos]) + 1 - self.changeRange = "@%s,#head" % self.rev - self.initialParent = parseRevision(self.branch) - except: - pass - self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); From 8ead4fda3fbaba93aae46931285e9613a058c08b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 20:26:58 +0200 Subject: [PATCH 141/260] Create the origin based import branch using git update-ref instead of git branch so that it's possible to have the import branch in refs/remotes. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index a19ba47481..4cd486eb3a 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -836,7 +836,10 @@ class P4Sync(Command): if not gitBranchExists(self.branch) and gitBranchExists("origin"): if not self.silent: print "Creating %s branch in git repository based on origin" % self.branch - system("git branch %s origin" % self.branch) + branch = self.branch + if not branch.startswith("refs"): + branch = "refs/heads/" + branch + system("git update-ref %s origin" % branch) [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) if len(self.previousDepotPath) > 0 and len(p4Change) > 0: From c6d44cb1a1a9bca58d91ef414b9dbaa393d2de3a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 20:57:05 +0200 Subject: [PATCH 142/260] Changed the default p4 import branch to be refs/remotes/p4/{HEAD,master} instead of refs/heads/p4. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 10 +++++++++- contrib/fast-import/git-p4.txt | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 4cd486eb3a..3cc6481378 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -828,9 +828,15 @@ class P4Sync(Command): self.changeRange = "" self.initialParent = "" self.previousDepotPath = "" + # importing into default remotes/p4/* layout? + defaultImport = False if len(self.branch) == 0: - self.branch = "p4" + if gitBranchExists("refs/heads/p4"): + self.branch = "p4" + else: + self.branch = "refs/remotes/p4/master" + defaultImport = True if len(args) == 0: if not gitBranchExists(self.branch) and gitBranchExists("origin"): @@ -840,6 +846,8 @@ class P4Sync(Command): if not branch.startswith("refs"): branch = "refs/heads/" + branch system("git update-ref %s origin" % branch) + if defaultImport: + system("git symbolic-ref refs/remotes/p4/HEAD %s" % branch) [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) if len(self.previousDepotPath) > 0 and len(p4Change) > 0: diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index ff0da9d0c8..32aeb0ac0b 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -20,9 +20,9 @@ or This will create an empty git repository in a subdirectory called "project" (or "myproject" with the second command), import the head revision from the -specified perforce path into a git "p4" branch, create a master branch off it -and check it out. If you want the entire history (not just the head revision) then -you can simply append a "@all" to the depot path: +specified perforce path into a git "p4" branch (remotes/p4 actually), create a +master branch off it and check it out. If you want the entire history (not just +the head revision) then you can simply append a "@all" to the depot path: git-p4 clone //depot/project/main@all myproject From 48df6fd850b2d9dc52a8a89a9e6deecd2cf1d351 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 21:18:53 +0200 Subject: [PATCH 143/260] Bite the bullet and automatically convert old style refs/heads/p4 repositories to the new style refs/remotes/p4 branching. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 3cc6481378..cb9961a571 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -832,10 +832,12 @@ class P4Sync(Command): defaultImport = False if len(self.branch) == 0: + self.branch = "refs/remotes/p4/master" if gitBranchExists("refs/heads/p4"): - self.branch = "p4" + system("git update-ref %s refs/heads/p4" % self.branch) + system("git symbolic-ref refs/remotes/p4/HEAD refs/remotes/p4/master") + system("git branch -D p4"); else: - self.branch = "refs/remotes/p4/master" defaultImport = True if len(args) == 0: From ef48f9093ced75ca20295f42b993bd390873573d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 22:17:49 +0200 Subject: [PATCH 144/260] Added support for git-p4 sync/rebase --with-origin. See git-p4.txt for details :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 23 +++++++++++++++++++++-- contrib/fast-import/git-p4.txt | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index cb9961a571..e653e8cd37 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -419,7 +419,8 @@ class P4Sync(Command): optparse.make_option("--known-branches", dest="knownBranches"), optparse.make_option("--data-cache", dest="dataCache", action="store_true"), optparse.make_option("--command-cache", dest="commandCache", action="store_true"), - optparse.make_option("--detect-labels", dest="detectLabels", action="store_true") + optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), + optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -441,6 +442,7 @@ class P4Sync(Command): self.detectBranches = False self.detectLabels = False self.changesFile = "" + self.syncWithOrigin = False def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -831,6 +833,21 @@ class P4Sync(Command): # importing into default remotes/p4/* layout? defaultImport = False + if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master"): + print "Syncing with origin first as requested by calling git fetch origin" + system("git fetch origin") + [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) + [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) + if len(originPreviousDepotPath) > 0 and len(originP4Change) > 0 and len(p4Change) > 0: + if originPreviousDepotPath == p4PreviousDepotPath: + originP4Change = int(originP4Change) + p4Change = int(p4Change) + if originP4Change > p4Change: + print "origin (%s) is newer than p4 (%s). Updating p4 branch from origin." % (originP4Change, p4Change) + system("git update-ref refs/remotes/p4/master origin"); + else: + print "Cannot sync with origin. It was imported from %s while remotes/p4 was imported from %s" % (originPreviousDepotPath, p4PreviousDepotPath) + if len(self.branch) == 0: self.branch = "refs/remotes/p4/master" if gitBranchExists("refs/heads/p4"): @@ -1037,11 +1054,13 @@ class P4Sync(Command): class P4Rebase(Command): def __init__(self): Command.__init__(self) - self.options = [ ] + self.options = [ optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ] self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it" + self.syncWithOrigin = False def run(self, args): sync = P4Sync() + sync.syncWithOrigin = self.syncWithOrigin sync.run([]) print "Rebasing the current branch" oldHead = mypopen("git rev-parse HEAD").read()[:-1] diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index 32aeb0ac0b..ac8e6cff0b 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -63,6 +63,26 @@ It is recommended to run 'git repack -a -d -f' from time to time when using incremental imports to optimally combine the individual git packs that each incremental import creates through the use of git-fast-import. + +A useful setup may be that you have a periodically updated git repository +somewhere that contains a complete import of a Perforce project. That git +repository can be used to clone the working repository from and one would +import from Perforce directly after cloning using git-p4. If the connection to +the Perforce server is slow and the working repository hasn't been synced for a +while it may be desirable to fetch changes from the origin git repository using +the efficient git protocol. git-p4 supports this through + + git-p4 sync --with-origin + +or + + git-p4 rebase --with-origin + +In that case "git fetch origin" is called and if it turns out that the origin +branch is newer than the git "p4" import branch then the latter is updated from +the former and the direct import from Perforce is resumed, which will result in +fewer changes to be imported using the slower perforce connection. + Updating ======== From 71bd9bacec6a18c2db582e1f32b1902c82359376 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 17 May 2007 22:22:26 +0200 Subject: [PATCH 145/260] Removed todo item that is implemented :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e653e8cd37..a21d902ea9 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -7,8 +7,6 @@ # 2007 Trolltech ASA # License: MIT # -# TODO: Add an option to sync/rebase to fetch and rebase from origin first. -# import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform From 05094f987c2f141ec9b873ad6d6303b0aca173a4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 18 May 2007 20:32:35 +0200 Subject: [PATCH 146/260] Fix branch setup after initial clone. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index a21d902ea9..2f3615bd72 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -828,8 +828,6 @@ class P4Sync(Command): self.changeRange = "" self.initialParent = "" self.previousDepotPath = "" - # importing into default remotes/p4/* layout? - defaultImport = False if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master"): print "Syncing with origin first as requested by calling git fetch origin" @@ -850,10 +848,9 @@ class P4Sync(Command): self.branch = "refs/remotes/p4/master" if gitBranchExists("refs/heads/p4"): system("git update-ref %s refs/heads/p4" % self.branch) - system("git symbolic-ref refs/remotes/p4/HEAD refs/remotes/p4/master") system("git branch -D p4"); - else: - defaultImport = True + if not gitBranchExists("refs/remotes/p4/HEAD"): + system("git symbolic-ref refs/remotes/p4/HEAD %s" % self.branch) if len(args) == 0: if not gitBranchExists(self.branch) and gitBranchExists("origin"): @@ -863,8 +860,6 @@ class P4Sync(Command): if not branch.startswith("refs"): branch = "refs/heads/" + branch system("git update-ref %s origin" % branch) - if defaultImport: - system("git symbolic-ref refs/remotes/p4/HEAD %s" % branch) [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) if len(self.previousDepotPath) > 0 and len(p4Change) > 0: From 66c6a9b559a883460b1aeed7dadec713be17588c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 18 May 2007 20:39:38 +0200 Subject: [PATCH 147/260] Removed unused cache variables. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 2f3615bd72..e164edef55 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -415,8 +415,6 @@ class P4Sync(Command): optparse.make_option("--changesfile", dest="changesFile"), optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--known-branches", dest="knownBranches"), - optparse.make_option("--data-cache", dest="dataCache", action="store_true"), - optparse.make_option("--command-cache", dest="commandCache", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ] @@ -430,8 +428,6 @@ class P4Sync(Command): self.usage += " //depot/path[@revRange]" - self.dataCache = False - self.commandCache = False self.silent = False self.knownBranches = Set() self.createdBranches = Set() From 4b97ffb1e462c2334b38047fbf2b7e8b24c36aed Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 18 May 2007 21:45:23 +0200 Subject: [PATCH 148/260] Started rewriting the branch detection, based on "p4 branches" and "p4 branch -o foo". Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 287 ++++++------------------------------- 1 file changed, 45 insertions(+), 242 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e164edef55..e993d3f693 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -414,9 +414,9 @@ class P4Sync(Command): optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"), optparse.make_option("--changesfile", dest="changesFile"), optparse.make_option("--silent", dest="silent", action="store_true"), - optparse.make_option("--known-branches", dest="knownBranches"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), - optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") + optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"), + optparse.make_option("--verbose", dest="verbose", action="store_true") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -429,7 +429,6 @@ class P4Sync(Command): self.usage += " //depot/path[@revRange]" self.silent = False - self.knownBranches = Set() self.createdBranches = Set() self.committedChanges = Set() self.branch = "" @@ -437,6 +436,7 @@ class P4Sync(Command): self.detectLabels = False self.changesFile = "" self.syncWithOrigin = False + self.verbose = False def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -461,120 +461,25 @@ class P4Sync(Command): fnum = fnum + 1 return files - def isSubPathOf(self, first, second): - if not first.startswith(second): - return False - if first == second: - return True - return first[len(second)] == "/" - def branchesForCommit(self, files): branches = Set() for file in files: - relativePath = file["path"][len(self.depotPath):] - # strip off the filename - relativePath = relativePath[0:relativePath.rfind("/")] + path = file["path"][len(self.depotPath):] - # if len(branches) == 0: - # branches.add(relativePath) - # knownBranches.add(relativePath) - # continue - - ###### this needs more testing :) - knownBranch = False - for branch in branches: - if relativePath == branch: - knownBranch = True - break - # if relativePath.startswith(branch): - if self.isSubPathOf(relativePath, branch): - knownBranch = True - break - # if branch.startswith(relativePath): - if self.isSubPathOf(branch, relativePath): - branches.remove(branch) - break - - if knownBranch: - continue - - for branch in self.knownBranches: - #if relativePath.startswith(branch): - if self.isSubPathOf(relativePath, branch): - if len(branches) == 0: - relativePath = branch - else: - knownBranch = True - break - - if knownBranch: - continue - - branches.add(relativePath) - self.knownBranches.add(relativePath) + for branch in self.knownBranches.keys(): + if path.startswith(branch): + branches.add(branch) return branches - def findBranchParent(self, branchPrefix, files): - for file in files: - path = file["path"] - if not path.startswith(branchPrefix): - continue - action = file["action"] - if action != "integrate" and action != "branch": - continue - rev = file["rev"] - depotPath = path + "#" + rev - - log = p4CmdList("filelog \"%s\"" % depotPath) - if len(log) != 1: - print "eek! I got confused by the filelog of %s" % depotPath - sys.exit(1); - - log = log[0] - if log["action0"] != action: - print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) - sys.exit(1); - - branchAction = log["how0,0"] - # if branchAction == "branch into" or branchAction == "ignored": - # continue # ignore for branching - - if not branchAction.endswith(" from"): - continue # ignore for branching - # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) - # sys.exit(1); - - source = log["file0,0"] - if source.startswith(branchPrefix): - continue - - lastSourceRev = log["erev0,0"] - - sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) - if len(sourceLog) != 1: - print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) - sys.exit(1); - sourceLog = sourceLog[0] - - relPath = source[len(self.depotPath):] - # strip off the filename - relPath = relPath[0:relPath.rfind("/")] - - for branch in self.knownBranches: - if self.isSubPathOf(relPath, branch): - # print "determined parent branch branch %s due to change in file %s" % (branch, source) - return branch - # else: - # print "%s is not a subpath of branch %s" % (relPath, branch) - - return "" - - def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""): + def commit(self, details, files, branch, branchPrefix, parent = ""): epoch = details["time"] author = details["user"] + if self.verbose: + print "commit into %s" % branch + self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) @@ -592,11 +497,10 @@ class P4Sync(Command): self.gitStream.write("EOT\n\n") if len(parent) > 0: + if self.verbose: + print "parent %s" % parent self.gitStream.write("from %s\n" % parent) - if len(merged) > 0: - self.gitStream.write("merge %s\n" % merged) - for file in files: path = file["path"] if not path.startswith(branchPrefix): @@ -680,118 +584,6 @@ class P4Sync(Command): return newFiles - def findBranchSourceHeuristic(self, files, branch, branchPrefix): - for file in files: - action = file["action"] - if action != "integrate" and action != "branch": - continue - path = file["path"] - rev = file["rev"] - depotPath = path + "#" + rev - - log = p4CmdList("filelog \"%s\"" % depotPath) - if len(log) != 1: - print "eek! I got confused by the filelog of %s" % depotPath - sys.exit(1); - - log = log[0] - if log["action0"] != action: - print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) - sys.exit(1); - - branchAction = log["how0,0"] - - if not branchAction.endswith(" from"): - continue # ignore for branching - # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) - # sys.exit(1); - - source = log["file0,0"] - if source.startswith(branchPrefix): - continue - - lastSourceRev = log["erev0,0"] - - sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) - if len(sourceLog) != 1: - print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) - sys.exit(1); - sourceLog = sourceLog[0] - - relPath = source[len(self.depotPath):] - # strip off the filename - relPath = relPath[0:relPath.rfind("/")] - - for candidate in self.knownBranches: - if self.isSubPathOf(relPath, candidate) and candidate != branch: - return candidate - - return "" - - def changeIsBranchMerge(self, sourceBranch, destinationBranch, change): - sourceFiles = {} - for file in p4CmdList("files %s...@%s" % (self.depotPath + sourceBranch + "/", change)): - if file["action"] == "delete": - continue - sourceFiles[file["depotFile"]] = file - - destinationFiles = {} - for file in p4CmdList("files %s...@%s" % (self.depotPath + destinationBranch + "/", change)): - destinationFiles[file["depotFile"]] = file - - for fileName in sourceFiles.keys(): - integrations = [] - deleted = False - integrationCount = 0 - for integration in p4CmdList("integrated \"%s\"" % fileName): - toFile = integration["fromFile"] # yes, it's true, it's fromFile - if not toFile in destinationFiles: - continue - destFile = destinationFiles[toFile] - if destFile["action"] == "delete": - # print "file %s has been deleted in %s" % (fileName, toFile) - deleted = True - break - integrationCount += 1 - if integration["how"] == "branch from": - continue - - if int(integration["change"]) == change: - integrations.append(integration) - continue - if int(integration["change"]) > change: - continue - - destRev = int(destFile["rev"]) - - startRev = integration["startFromRev"][1:] - if startRev == "none": - startRev = 0 - else: - startRev = int(startRev) - - endRev = integration["endFromRev"][1:] - if endRev == "none": - endRev = 0 - else: - endRev = int(endRev) - - initialBranch = (destRev == 1 and integration["how"] != "branch into") - inRange = (destRev >= startRev and destRev <= endRev) - newer = (destRev > startRev and destRev > endRev) - - if initialBranch or inRange or newer: - integrations.append(integration) - - if deleted: - continue - - if len(integrations) == 0 and integrationCount > 1: - print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch) - return False - - return True - def getUserMap(self): self.users = {} @@ -819,6 +611,27 @@ class P4Sync(Command): self.labels[newestChange] = [output, revisions] + def getBranchMapping(self): + # map from branch depot path to parent branch + self.knownBranches = {} + + for info in p4CmdList("branches"): + details = p4Cmd("branch -o %s" % info["branch"]) + viewIdx = 0 + while details.has_key("View%s" % viewIdx): + paths = details["View%s" % viewIdx].split(" ") + viewIdx = viewIdx + 1 + # require standard //depot/foo/... //depot/bar/... mapping + if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."): + continue + source = paths[0] + destination = paths[1] + if source.startswith(self.depotPath) and destination.startswith(self.depotPath): + source = source[len(self.depotPath):-4] + destination = destination[len(self.depotPath):-4] + self.knownBranches[destination] = source + self.knownBranches[source] = source + def run(self, args): self.depotPath = "" self.changeRange = "" @@ -914,6 +727,9 @@ class P4Sync(Command): if self.detectLabels: self.getLabels(); + if self.detectBranches: + self.getBranchMapping(); + self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); @@ -993,35 +809,22 @@ class P4Sync(Command): files = self.extractFilesFromCommit(description) if self.detectBranches: for branch in self.branchesForCommit(files): - self.knownBranches.add(branch) branchPrefix = self.depotPath + branch + "/" - filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix) - - merged = "" parent = "" - ########### remove cnt!!! - if branch not in self.createdBranches and cnt > 2: + + filesForCommit = self.extractFilesInCommitToBranch(files, branch) + + if branch not in self.createdBranches : self.createdBranches.add(branch) - parent = self.findBranchParent(branchPrefix, files) + parent = self.knownBranches[branch] if parent == branch: parent = "" - # elif len(parent) > 0: - # print "%s branched off of %s" % (branch, parent) - if len(parent) == 0: - merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) - if len(merged) > 0: - print "change %s could be a merge from %s into %s" % (description["change"], merged, branch) - if not self.changeIsBranchMerge(merged, branch, int(description["change"])): - merged = "" - - branch = "refs/heads/" + branch + branch = "refs/remotes/p4/" + branch if len(parent) > 0: - parent = "refs/heads/" + parent - if len(merged) > 0: - merged = "refs/heads/" + merged - self.commit(description, files, branch, branchPrefix, parent, merged) + parent = "refs/remotes/p4/" + parent + self.commit(description, files, branch, branchPrefix, parent) else: self.commit(description, files, self.branch, self.depotPath, self.initialParent) self.initialParent = "" From 8f9b2e082b54787676bdad793ca013ba4fb4e407 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 18 May 2007 22:13:26 +0200 Subject: [PATCH 149/260] Give branches a nice project prefix and don't bail out on clone if we failed to detect the master branch. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e993d3f693..7d42739637 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -729,6 +729,7 @@ class P4Sync(Command): if self.detectBranches: self.getBranchMapping(); + self.branchPrefix = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) @@ -815,12 +816,23 @@ class P4Sync(Command): filesForCommit = self.extractFilesInCommitToBranch(files, branch) - if branch not in self.createdBranches : + if branch not in self.createdBranches: self.createdBranches.add(branch) parent = self.knownBranches[branch] if parent == branch: parent = "" + # main branch? use master + if branch == "main": + branch = "master" + else: + branch = self.branchPrefix + branch + + if parent == "main": + parent = "master" + elif len(parent) > 0: + parent = self.branchPrefix + parent + branch = "refs/remotes/p4/" + branch if len(parent) > 0: parent = "refs/remotes/p4/" + parent @@ -906,8 +918,11 @@ class P4Clone(P4Sync): if not P4Sync.run(self, [depotPath]): return False if self.branch != "master": - system("git branch master p4") - system("git checkout -f") + if gitBranchExists("refs/remotes/p4/master"): + system("git branch master refs/remotes/p4/master") + system("git checkout -f") + else: + print "Could not detect main branch. No checkout/master branch created." return True class HelpFormatter(optparse.IndentedHelpFormatter): From 29bdbac1cd5fc4126b62c9a030403d56ae43c204 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 19 May 2007 10:23:12 +0200 Subject: [PATCH 150/260] More work on the incremental importing of multiple branches. Improved error detection by checking the exit code of git-fast-import. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 90 ++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 7d42739637..2c1dc9e2b3 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -612,8 +612,7 @@ class P4Sync(Command): self.labels[newestChange] = [output, revisions] def getBranchMapping(self): - # map from branch depot path to parent branch - self.knownBranches = {} + self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) @@ -629,16 +628,34 @@ class P4Sync(Command): if source.startswith(self.depotPath) and destination.startswith(self.depotPath): source = source[len(self.depotPath):-4] destination = destination[len(self.depotPath):-4] - self.knownBranches[destination] = source - self.knownBranches[source] = source + if destination not in self.knownBranches: + self.knownBranches[destination] = source + if source not in self.knownBranches: + self.knownBranches[source] = source + + def listExistingP4GitBranches(self): + self.p4BranchesInGit = [] + + for line in mypopen("git rev-parse --symbolic --remotes").readlines(): + if line.startswith("p4/") and line != "p4/HEAD\n": + branch = line[3:-1] + self.p4BranchesInGit.append(branch) + self.initialParents["refs/remotes/p4/" + branch] = parseRevision(line[:-1]) def run(self, args): self.depotPath = "" self.changeRange = "" self.initialParent = "" self.previousDepotPath = "" + # map from branch depot path to parent branch + self.knownBranches = {} + self.initialParents = {} + + self.listExistingP4GitBranches() + + if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master") and not self.detectBranches: + ### needs to be ported to multi branch import - if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master"): print "Syncing with origin first as requested by calling git fetch origin" system("git fetch origin") [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) @@ -662,7 +679,8 @@ class P4Sync(Command): system("git symbolic-ref refs/remotes/p4/HEAD %s" % self.branch) if len(args) == 0: - if not gitBranchExists(self.branch) and gitBranchExists("origin"): + if not gitBranchExists(self.branch) and gitBranchExists("origin") and not self.detectBranches: + ### needs to be ported to multi branch import if not self.silent: print "Creating %s branch in git repository based on origin" % self.branch branch = self.branch @@ -670,11 +688,33 @@ class P4Sync(Command): branch = "refs/heads/" + branch system("git update-ref %s origin" % branch) - [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) - if len(self.previousDepotPath) > 0 and len(p4Change) > 0: - p4Change = int(p4Change) + 1 + if self.verbose: + print "branches: %s" % self.p4BranchesInGit + + p4Change = 0 + for branch in self.p4BranchesInGit: + depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("refs/remotes/p4/" + branch)) + + if self.verbose: + print "path %s change %s" % (depotPath, change) + + if len(depotPath) > 0 and len(change) > 0: + change = int(change) + 1 + p4Change = max(p4Change, change) + + if len(self.previousDepotPath) == 0: + self.previousDepotPath = depotPath + else: + i = 0 + l = min(len(self.previousDepotPath), len(depotPath)) + while i < l and self.previousDepotPath[i] == depotPath[i]: + i = i + 1 + self.previousDepotPath = self.previousDepotPath[:i] + + if p4Change > 0: self.depotPath = self.previousDepotPath - self.changeRange = "@%s,#head" % p4Change + #self.changeRange = "@%s,#head" % p4Change + self.changeRange = "@%s,%s" % (p4Change, p4Change + 10) self.initialParent = parseRevision(self.branch) if not self.silent: print "Performing incremental import into %s git branch" % self.branch @@ -729,7 +769,13 @@ class P4Sync(Command): if self.detectBranches: self.getBranchMapping(); - self.branchPrefix = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] + if self.verbose: + print "p4-git branches: %s" % self.p4BranchesInGit + print "initial parents: %s" % self.initialParents + for b in self.p4BranchesInGit: + if b != "master": + b = b[len(self.projectName):] + self.createdBranches.add(b) self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) @@ -784,6 +830,8 @@ class P4Sync(Command): changes.sort() else: + if self.verbose: + print "Getting p4 changes for %s...%s" % (self.depotPath, self.changeRange) output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() for line in output: @@ -816,26 +864,39 @@ class P4Sync(Command): filesForCommit = self.extractFilesInCommitToBranch(files, branch) + if self.verbose: + print "branch is %s" % branch + if branch not in self.createdBranches: self.createdBranches.add(branch) parent = self.knownBranches[branch] if parent == branch: parent = "" + elif self.verbose: + print "parent determined through known branches: %s" % parent # main branch? use master if branch == "main": branch = "master" else: - branch = self.branchPrefix + branch + branch = self.projectName + branch if parent == "main": parent = "master" elif len(parent) > 0: - parent = self.branchPrefix + parent + parent = self.projectName + parent branch = "refs/remotes/p4/" + branch if len(parent) > 0: parent = "refs/remotes/p4/" + parent + + if self.verbose: + print "looking for initial parent for %s; current parent is %s" % (branch, parent) + + if len(parent) == 0 and branch in self.initialParents: + parent = self.initialParents[branch] + del self.initialParents[branch] + self.commit(description, files, branch, branchPrefix, parent) else: self.commit(description, files, self.branch, self.depotPath, self.initialParent) @@ -849,9 +910,10 @@ class P4Sync(Command): self.gitStream.close() + if importProcess.wait() != 0: + die("fast-import failed: %s" % self.gitError.read()) self.gitOutput.close() self.gitError.close() - importProcess.wait() return True From d5904674d1a52620a702b4f5efa1afe21f8a5947 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 19 May 2007 11:07:32 +0200 Subject: [PATCH 151/260] Cleanup/speed up the branch<> file split and removed change range limitation that I added for debugging (oops). Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 2c1dc9e2b3..35c5f9c696 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -461,15 +461,17 @@ class P4Sync(Command): fnum = fnum + 1 return files - def branchesForCommit(self, files): - branches = Set() + def splitFilesIntoBranches(self, files): + branches = {} for file in files: path = file["path"][len(self.depotPath):] for branch in self.knownBranches.keys(): if path.startswith(branch): - branches.add(branch) + if branch not in branches: + branches[branch] = [] + branches[branch].append(file["path"]) return branches @@ -574,16 +576,6 @@ class P4Sync(Command): if not self.silent: print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change) - def extractFilesInCommitToBranch(self, files, branchPrefix): - newFiles = [] - - for file in files: - path = file["path"] - if path.startswith(branchPrefix): - newFiles.append(file) - - return newFiles - def getUserMap(self): self.users = {} @@ -713,8 +705,7 @@ class P4Sync(Command): if p4Change > 0: self.depotPath = self.previousDepotPath - #self.changeRange = "@%s,#head" % p4Change - self.changeRange = "@%s,%s" % (p4Change, p4Change + 10) + self.changeRange = "@%s,#head" % p4Change self.initialParent = parseRevision(self.branch) if not self.silent: print "Performing incremental import into %s git branch" % self.branch @@ -857,12 +848,13 @@ class P4Sync(Command): try: files = self.extractFilesFromCommit(description) if self.detectBranches: - for branch in self.branchesForCommit(files): + branches = self.splitFilesIntoBranches(files) + for branch in branches.keys(): branchPrefix = self.depotPath + branch + "/" parent = "" - filesForCommit = self.extractFilesInCommitToBranch(files, branch) + filesForCommit = branches[branch] if self.verbose: print "branch is %s" % branch From 71b112d4a4e070170af7b5a50647f7dac9b48e56 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 19 May 2007 11:54:11 +0200 Subject: [PATCH 152/260] More cleanups and speedups for labels and branches Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 39 ++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 35c5f9c696..49114d2c6f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -461,17 +461,32 @@ class P4Sync(Command): fnum = fnum + 1 return files - def splitFilesIntoBranches(self, files): + def splitFilesIntoBranches(self, commit): branches = {} - for file in files: - path = file["path"][len(self.depotPath):] + fnum = 0 + while commit.has_key("depotFile%s" % fnum): + path = commit["depotFile%s" % fnum] + if not path.startswith(self.depotPath): + # if not self.silent: + # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) + fnum = fnum + 1 + continue + + file = {} + file["path"] = path + file["rev"] = commit["rev%s" % fnum] + file["action"] = commit["action%s" % fnum] + file["type"] = commit["type%s" % fnum] + fnum = fnum + 1 + + relPath = path[len(self.depotPath):] for branch in self.knownBranches.keys(): - if path.startswith(branch): + if relPath.startswith(branch): if branch not in branches: branches[branch] = [] - branches[branch].append(file["path"]) + branches[branch].append(file) return branches @@ -542,6 +557,8 @@ class P4Sync(Command): label = self.labels[change] labelDetails = label[0] labelRevisions = label[1] + if self.verbose: + print "Change %s is labelled %s" % (change, labelDetails) files = p4CmdList("files %s...@%s" % (branchPrefix, change)) @@ -595,13 +612,15 @@ class P4Sync(Command): label = output["label"] revisions = {} newestChange = 0 - for file in p4CmdList("files //...@%s" % label): + if self.verbose: + print "Querying files for label %s" % label + for file in p4CmdList("files %s...@%s" % (self.depotPath, label)): revisions[file["depotFile"]] = file["rev"] change = int(file["change"]) if change > newestChange: newestChange = change - self.labels[newestChange] = [output, revisions] + self.labels[int(newestChange)] = [output, revisions] def getBranchMapping(self): self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] @@ -846,9 +865,8 @@ class P4Sync(Command): cnt = cnt + 1 try: - files = self.extractFilesFromCommit(description) if self.detectBranches: - branches = self.splitFilesIntoBranches(files) + branches = self.splitFilesIntoBranches(description) for branch in branches.keys(): branchPrefix = self.depotPath + branch + "/" @@ -889,8 +907,9 @@ class P4Sync(Command): parent = self.initialParents[branch] del self.initialParents[branch] - self.commit(description, files, branch, branchPrefix, parent) + self.commit(description, filesForCommit, branch, branchPrefix, parent) else: + files = self.extractFilesFromCommit(description) self.commit(description, files, self.branch, self.depotPath, self.initialParent) self.initialParent = "" except IOError: From 9bda3a8556681c90309ec34e2ee5ae50306b61a6 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 19 May 2007 12:05:40 +0200 Subject: [PATCH 153/260] Removed unused variable, more cleanups Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 49114d2c6f..f76d198d64 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -551,9 +551,7 @@ class P4Sync(Command): change = int(details["change"]) - self.lastChange = change - - if change in self.labels: + if self.labels.has_key(change): label = self.labels[change] labelDetails = label[0] labelRevisions = label[1] @@ -620,7 +618,10 @@ class P4Sync(Command): if change > newestChange: newestChange = change - self.labels[int(newestChange)] = [output, revisions] + self.labels[newestChange] = [output, revisions] + + if self.verbose: + print "Label changes: %s" % self.labels.keys() def getBranchMapping(self): self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] @@ -748,7 +749,6 @@ class P4Sync(Command): self.revision = "" self.users = {} - self.lastChange = 0 if self.depotPath.find("@") != -1: atIdx = self.depotPath.index("@") From b607e71efdd23cd676645b5d7bf49d985834fab8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 10:55:54 +0200 Subject: [PATCH 154/260] Cache the output of "p4 users" for faster syncs on high latency links. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index f76d198d64..e5e7c6be12 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -501,6 +501,8 @@ class P4Sync(Command): # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) committer = "" + if author not in self.users: + self.getUserMapFromPerforceServer() if author in self.users: committer = "%s %s %s" % (self.users[author], epoch, self.tz) else: @@ -591,7 +593,7 @@ class P4Sync(Command): if not self.silent: print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change) - def getUserMap(self): + def getUserMapFromPerforceServer(self): self.users = {} for output in p4CmdList("users"): @@ -599,6 +601,23 @@ class P4Sync(Command): continue self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" + cache = open(gitdir + "/p4-usercache.txt", "wb") + for user in self.users.keys(): + cache.write("%s\t%s\n" % (user, self.users[user])) + cache.close(); + + def loadUserMapFromCache(self): + self.users = {} + try: + cache = open(gitdir + "/p4-usercache.txt", "rb") + lines = cache.readlines() + cache.close() + for line in lines: + entry = line[:-1].split("\t") + self.users[entry[0]] = entry[1] + except IOError: + self.getUserMapFromPerforceServer() + def getLabels(self): self.labels = {} @@ -772,7 +791,7 @@ class P4Sync(Command): if not self.depotPath.endswith("/"): self.depotPath += "/" - self.getUserMap() + self.loadUserMapFromCache() self.labels = {} if self.detectLabels: self.getLabels(); From 59fa4171090ae8dcc7087ac617dca40e66f9b33a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 15:15:34 +0200 Subject: [PATCH 155/260] Fix gitdir not being set when cloning. Needed for writing the p4 users cache. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e5e7c6be12..14be55bcfc 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -972,6 +972,8 @@ class P4Clone(P4Sync): self.needsGit = False def run(self, args): + global gitdir + if len(args) < 1: return False depotPath = args[0] @@ -1007,6 +1009,7 @@ class P4Clone(P4Sync): os.makedirs(dir) os.chdir(dir) system("git init") + gitdir = os.getcwd() if not P4Sync.run(self, [depotPath]): return False if self.branch != "master": From 64ffb06a9c940e57a215d313cd58f702d695e919 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 15:24:01 +0200 Subject: [PATCH 156/260] Oops, not only /set/ gitdir on clone, also set it /correctly/ :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 14be55bcfc..80d966fb30 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1009,7 +1009,7 @@ class P4Clone(P4Sync): os.makedirs(dir) os.chdir(dir) system("git init") - gitdir = os.getcwd() + gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, [depotPath]): return False if self.branch != "master": From 47a130b7bf74b8f61d9fee01f5dbb745d8f44b29 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 16:33:21 +0200 Subject: [PATCH 157/260] Use git format-patch and git apply --apply when extracting patches from git and applying them to a Perforce checkout. This should make it possible to apply git commits with binary files that cannot be handled by path. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 80d966fb30..0b1df09cb8 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -213,10 +213,13 @@ class P4Submit(Command): else: die("unknown modifier %s for %s" % (modifier, path)) - diffcmd = "git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\"" % (id, id) - patchcmd = diffcmd + " | patch -p1" + diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) + patchcmd = diffcmd + " | git apply " + tryPatchCmd = diffcmd + "--check -" + applyPatchCmd = diffcmd + "--check --apply -" + print mypopen(diffcmd).read() - if os.system(patchcmd + " --dry-run --silent") != 0: + if os.system(tryPatchCmd) != 0: print "Unfortunately applying the change failed!" print "What do you want to do?" response = "x" @@ -226,7 +229,7 @@ class P4Submit(Command): print "Skipping! Good luck with the next patches..." return elif response == "a": - os.system(patchcmd) + os.system(applyPatchCmd) if len(filesToAdd) > 0: print "You may also want to call p4 add on the following files:" print " ".join(filesToAdd) @@ -239,7 +242,7 @@ class P4Submit(Command): print "Patch saved to patch.txt in %s !" % self.clientPath die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") - system(patchcmd) + system(applyPatchCmd) for f in filesToAdd: system("p4 add %s" % f) From c1b296b9f19a62aa9e444cc454b3d62ffaa98da4 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 16:55:05 +0200 Subject: [PATCH 158/260] Added support for git-p4 submit --direct (experimental) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 61 +++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0b1df09cb8..bcea4cf3de 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -137,6 +137,7 @@ class P4Submit(Command): optparse.make_option("--log-substitutions", dest="substFile"), optparse.make_option("--noninteractive", action="store_false"), optparse.make_option("--dry-run", action="store_true"), + optparse.make_option("--direct", dest="directSubmit", action="store_true"), ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" @@ -147,6 +148,7 @@ class P4Submit(Command): self.substFile = "" self.firstTime = True self.origin = "" + self.directSubmit = False self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -161,9 +163,12 @@ class P4Submit(Command): die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile) commits = [] - for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): - commits.append(line[:-1]) - commits.reverse() + if self.directSubmit: + commits.append("0") + else: + for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): + commits.append(line[:-1]) + commits.reverse() self.config["commits"] = commits @@ -191,8 +196,12 @@ class P4Submit(Command): return result def apply(self, id): - print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read()) - diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() + if self.directSubmit: + print "Applying local change in working directory/index" + diff = self.diffStatus + else: + print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read()) + diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() filesToDelete = set() editedFiles = set() @@ -213,11 +222,13 @@ class P4Submit(Command): else: die("unknown modifier %s for %s" % (modifier, path)) - diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) + if self.directSubmit: + diffcmd = "cat \"%s\"" % self.diffFile + else: + diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) patchcmd = diffcmd + " | git apply " - tryPatchCmd = diffcmd + "--check -" - applyPatchCmd = diffcmd + "--check --apply -" - print mypopen(diffcmd).read() + tryPatchCmd = patchcmd + "--check -" + applyPatchCmd = patchcmd + "--check --apply -" if os.system(tryPatchCmd) != 0: print "Unfortunately applying the change failed!" @@ -250,9 +261,11 @@ class P4Submit(Command): system("p4 revert %s" % f) system("p4 delete %s" % f) - logMessage = extractLogMessageFromGitCommit(id) - logMessage = logMessage.replace("\n", "\n\t") - logMessage = logMessage[:-1] + logMessage = "" + if not self.directSubmit: + logMessage = extractLogMessageFromGitCommit(id) + logMessage = logMessage.replace("\n", "\n\t") + logMessage = logMessage[:-1] template = mypopen("p4 change -o").read() @@ -356,6 +369,15 @@ class P4Submit(Command): print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath) oldWorkingDirectory = os.getcwd() + + if self.directSubmit: + self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines() + patch = mypopen("git diff -p --binary --diff-filter=ACMRTUXB HEAD").read() + self.diffFile = gitdir + "/p4-git-diff" + f = open(self.diffFile, "wb") + f.write(patch) + f.close(); + os.chdir(self.clientPath) response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath) if response == "y" or response == "yes": @@ -395,14 +417,25 @@ class P4Submit(Command): self.config.close() + if self.directSubmit: + os.remove(self.diffFile) + if len(commits) == 0: if self.firstTime: print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" - response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") + response = "" + os.chdir(oldWorkingDirectory) + + if self.directSubmit: + response = raw_input("Do you want to DISCARD your git WORKING DIRECTORY CHANGES and sync from Perforce now using git-p4 rebase? [y]es/[n]o ") + if response == "y" or response == "yes": + system("git reset --hard") + + if len(response) == 0: + response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") if response == "y" or response == "yes": - os.chdir(oldWorkingDirectory) rebase = P4Rebase() rebase.run([]) os.remove(self.configFile) From 8a5fc95b43ab25e7ce0ca14cf733669e5708b448 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 23:39:40 +0200 Subject: [PATCH 159/260] Specifying --detect-branches is now only needed for the initial clone/sync. Afterwards it's turned on implicitly if more p4 branches than remotes/p4/master are found. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index bcea4cf3de..d4bf67333f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -719,6 +719,9 @@ class P4Sync(Command): self.initialParents = {} self.listExistingP4GitBranches() + if len(self.p4BranchesInGit) > 1: + print "Importing from/into multiple branches" + self.detectBranches = True if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master") and not self.detectBranches: ### needs to be ported to multi branch import From 24f7b53fdd0bcc76453fc087932ca2cdecf47f9d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 23:42:22 +0200 Subject: [PATCH 160/260] Had an idea for debugging, record it :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d4bf67333f..92522417cd 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -7,6 +7,10 @@ # 2007 Trolltech ASA # License: MIT # +# TODO: * implement git-p4 rollback for debugging +# to roll back all p4 remote branches to a commit older or equal to +# the specified change. +# import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform From b1561ee25628c074b3afac738ead387181d3b311 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 20 May 2007 23:52:51 +0200 Subject: [PATCH 161/260] Another (potentially life-saving) idea for submit --direct Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 92522417cd..5055f32140 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -10,6 +10,12 @@ # TODO: * implement git-p4 rollback for debugging # to roll back all p4 remote branches to a commit older or equal to # the specified change. +# * for git-p4 submit --direct it would be nice to still create a +# git commit without updating HEAD before submitting to perforce. +# With the commit sha1 printed (or recoded in a .git/foo file?) +# it's possible to recover if anything goes wrong instead of potentially +# loosing a change entirely because it was never comitted to git and +# the p4 submit failed (or resulted in lots of conflicts, etc.) # import optparse, sys, os, marshal, popen2, subprocess, shelve From 341dc1c1793455875086373a865db4d0a132ef6e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 00:39:16 +0200 Subject: [PATCH 162/260] Improved output for multi branch imports and noted another little todo item Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 5055f32140..beb6529b44 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -16,6 +16,8 @@ # it's possible to recover if anything goes wrong instead of potentially # loosing a change entirely because it was never comitted to git and # the p4 submit failed (or resulted in lots of conflicts, etc.) +# * Consider making --with-origin the default, assuming that the git +# protocol is always more efficient. (needs manual testing first :) # import optparse, sys, os, marshal, popen2, subprocess, shelve @@ -729,7 +731,7 @@ class P4Sync(Command): self.initialParents = {} self.listExistingP4GitBranches() - if len(self.p4BranchesInGit) > 1: + if len(self.p4BranchesInGit) > 1 and not self.silent: print "Importing from/into multiple branches" self.detectBranches = True @@ -795,7 +797,7 @@ class P4Sync(Command): self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change self.initialParent = parseRevision(self.branch) - if not self.silent: + if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch if not self.branch.startswith("refs/"): @@ -920,15 +922,17 @@ class P4Sync(Command): if len(changes) == 0: if not self.silent: - print "no changes to import!" + print "No changes to import!" return True + self.updatedBranches = set() + cnt = 1 for change in changes: description = p4Cmd("describe %s" % change) if not self.silent: - sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) sys.stdout.flush() cnt = cnt + 1 @@ -945,6 +949,8 @@ class P4Sync(Command): if self.verbose: print "branch is %s" % branch + self.updatedBranches.add(branch) + if branch not in self.createdBranches: self.createdBranches.add(branch) parent = self.knownBranches[branch] @@ -984,8 +990,13 @@ class P4Sync(Command): print self.gitError.read() sys.exit(1) - if not self.silent: - print "" + if not self.silent: + print "" + if len(self.updatedBranches) > 0: + sys.stdout.write("Updated branches: ") + for b in self.updatedBranches: + sys.stdout.write("%s " % b) + sys.stdout.write("\n") self.gitStream.close() From 33be3e6550c7051c3ac4bc624c7dacfad6974b4f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 08:44:16 +0200 Subject: [PATCH 163/260] Fix conversion from old style heads/p4 to remotes/p4/master Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index beb6529b44..7489c91081 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -730,11 +730,6 @@ class P4Sync(Command): self.knownBranches = {} self.initialParents = {} - self.listExistingP4GitBranches() - if len(self.p4BranchesInGit) > 1 and not self.silent: - print "Importing from/into multiple branches" - self.detectBranches = True - if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master") and not self.detectBranches: ### needs to be ported to multi branch import @@ -760,6 +755,12 @@ class P4Sync(Command): if not gitBranchExists("refs/remotes/p4/HEAD"): system("git symbolic-ref refs/remotes/p4/HEAD %s" % self.branch) + # this needs to be called after the conversion from heads/p4 to remotes/p4/master + self.listExistingP4GitBranches() + if len(self.p4BranchesInGit) > 1 and not self.silent: + print "Importing from/into multiple branches" + self.detectBranches = True + if len(args) == 0: if not gitBranchExists(self.branch) and gitBranchExists("origin") and not self.detectBranches: ### needs to be ported to multi branch import From dc5240369610a6f72eab9b59447889dfd69b31c5 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 09:34:56 +0200 Subject: [PATCH 164/260] Fix error detection with git-p4 submit when the requested depot path is not in the client view. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 7489c91081..73da5d2b27 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -55,6 +55,8 @@ def p4Where(depotPath): if not depotPath.endswith("/"): depotPath += "/" output = p4Cmd("where %s..." % depotPath) + if output["code"] == "error": + return "" clientPath = "" if "path" in output: clientPath = output.get("path") From faf1bd202640263b06c99c335269971aa9d9ead1 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 10:05:30 +0200 Subject: [PATCH 165/260] Fix git symbolic-ref warning on initial clone Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 73da5d2b27..35a513fcb8 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -732,6 +732,8 @@ class P4Sync(Command): self.knownBranches = {} self.initialParents = {} + createP4HeadRef = False; + if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master") and not self.detectBranches: ### needs to be ported to multi branch import @@ -754,8 +756,9 @@ class P4Sync(Command): if gitBranchExists("refs/heads/p4"): system("git update-ref %s refs/heads/p4" % self.branch) system("git branch -D p4"); + # create it /after/ importing, when master exists if not gitBranchExists("refs/remotes/p4/HEAD"): - system("git symbolic-ref refs/remotes/p4/HEAD %s" % self.branch) + createP4HeadRef = True # this needs to be called after the conversion from heads/p4 to remotes/p4/master self.listExistingP4GitBranches() @@ -1008,6 +1011,9 @@ class P4Sync(Command): self.gitOutput.close() self.gitError.close() + if createP4HeadRef: + system("git symbolic-ref refs/remotes/p4/HEAD %s" % self.branch) + return True class P4Rebase(Command): From cbf5efa61a30d6454f5739cee3498f98a01210da Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 10:08:11 +0200 Subject: [PATCH 166/260] Detect with git-p4 submit --direct when there are no changes in the working directory Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 35a513fcb8..f08ee6da44 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -386,6 +386,9 @@ class P4Submit(Command): if self.directSubmit: self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines() + if len(self.diffStatus) == 0: + print "No changes in working directory to submit." + return True patch = mypopen("git diff -p --binary --diff-filter=ACMRTUXB HEAD").read() self.diffFile = gitdir + "/p4-git-diff" f = open(self.diffFile, "wb") From 7944f1425c0665eef6a5b9f5cc92e15cddb42984 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 11:04:26 +0200 Subject: [PATCH 167/260] Make git-p4 submit --direct safer by also creating a git commit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index f08ee6da44..b32d8dbfd3 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -10,12 +10,6 @@ # TODO: * implement git-p4 rollback for debugging # to roll back all p4 remote branches to a commit older or equal to # the specified change. -# * for git-p4 submit --direct it would be nice to still create a -# git commit without updating HEAD before submitting to perforce. -# With the commit sha1 printed (or recoded in a .git/foo file?) -# it's possible to recover if anything goes wrong instead of potentially -# loosing a change entirely because it was never comitted to git and -# the p4 submit failed (or resulted in lots of conflicts, etc.) # * Consider making --with-origin the default, assuming that the git # protocol is always more efficient. (needs manual testing first :) # @@ -328,9 +322,17 @@ class P4Submit(Command): print submitTemplate raw_input("Press return to continue...") else: - pipe = os.popen("p4 submit -i", "wb") - pipe.write(submitTemplate) - pipe.close() + if self.directSubmit: + print "Submitting to git first" + os.chdir(self.oldWorkingDirectory) + pipe = os.popen("git commit -a -F -", "wb") + pipe.write(submitTemplate) + pipe.close() + os.chdir(self.clientPath) + + pipe = os.popen("p4 submit -i", "wb") + pipe.write(submitTemplate) + pipe.close() elif response == "s": for f in editedFiles: system("p4 revert \"%s\"" % f); @@ -382,7 +384,7 @@ class P4Submit(Command): sys.exit(128) print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath) - oldWorkingDirectory = os.getcwd() + self.oldWorkingDirectory = os.getcwd() if self.directSubmit: self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines() @@ -442,16 +444,8 @@ class P4Submit(Command): print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" - response = "" - os.chdir(oldWorkingDirectory) - - if self.directSubmit: - response = raw_input("Do you want to DISCARD your git WORKING DIRECTORY CHANGES and sync from Perforce now using git-p4 rebase? [y]es/[n]o ") - if response == "y" or response == "yes": - system("git reset --hard") - - if len(response) == 0: - response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") + os.chdir(self.oldWorkingDirectory) + response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") if response == "y" or response == "yes": rebase = P4Rebase() rebase.run([]) From 5834684d51ee6e0c21cce8d1b5df06a34a36a4f9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 22:57:06 +0200 Subject: [PATCH 168/260] Added a rollback command for debugging. It sets back the heads of the p4 branches to the specified p4 change number or earlier. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b32d8dbfd3..40264cdc18 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -7,10 +7,7 @@ # 2007 Trolltech ASA # License: MIT # -# TODO: * implement git-p4 rollback for debugging -# to roll back all p4 remote branches to a commit older or equal to -# the specified change. -# * Consider making --with-origin the default, assuming that the git +# TODO: * Consider making --with-origin the default, assuming that the git # protocol is always more efficient. (needs manual testing first :) # @@ -135,6 +132,35 @@ class P4Debug(Command): print output return True +class P4RollBack(Command): + def __init__(self): + Command.__init__(self) + self.options = [ + ] + self.description = "A tool to debug the multi-branch import. Don't use :)" + + def run(self, args): + if len(args) != 1: + return False + maxChange = int(args[0]) + for line in mypopen("git rev-parse --symbolic --remotes").readlines(): + if line.startswith("p4/") and line != "p4/HEAD\n": + ref = "refs/remotes/" + line[:-1] + log = extractLogMessageFromGitCommit(ref) + depotPath, change = extractDepotPathAndChangeFromGitLog(log) + changed = False + while len(change) > 0 and int(change) > maxChange: + changed = True + print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) + system("git update-ref %s \"%s^\"" % (ref, ref)) + log = extractLogMessageFromGitCommit(ref) + depotPath, change = extractDepotPathAndChangeFromGitLog(log) + + if changed: + print "%s is at %s" % (ref, change) + + return True + class P4Submit(Command): def __init__(self): Command.__init__(self) @@ -1109,7 +1135,8 @@ commands = { "submit" : P4Submit(), "sync" : P4Sync(), "rebase" : P4Rebase(), - "clone" : P4Clone() + "clone" : P4Clone(), + "rollback" : P4RollBack() } if len(sys.argv[1:]) == 0: From af8da89cb7225c7938a836de0812467c059ca52d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 23:25:51 +0200 Subject: [PATCH 169/260] Fix branch detection in multi-branch imports Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 40264cdc18..515f7a906f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -556,7 +556,7 @@ class P4Sync(Command): relPath = path[len(self.depotPath):] for branch in self.knownBranches.keys(): - if relPath.startswith(branch): + if relPath.startswith(branch + "/"): # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2 if branch not in branches: branches[branch] = [] branches[branch].append(file) From 52102d4784a5f16fd84eaf98b61f714460b68693 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 21 May 2007 23:44:24 +0200 Subject: [PATCH 170/260] Fixes for rollback, delete branches that did not exist at the specified p4 change Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 515f7a906f..1457396abd 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -136,8 +136,10 @@ class P4RollBack(Command): def __init__(self): Command.__init__(self) self.options = [ + optparse.make_option("--verbose", dest="verbose", action="store_true") ] self.description = "A tool to debug the multi-branch import. Don't use :)" + self.verbose = False def run(self, args): if len(args) != 1: @@ -149,15 +151,22 @@ class P4RollBack(Command): log = extractLogMessageFromGitCommit(ref) depotPath, change = extractDepotPathAndChangeFromGitLog(log) changed = False + + if len(p4Cmd("changes -m 1 %s...@%s" % (depotPath, maxChange))) == 0: + print "Branch %s did not exist at change %s, deleting." % (ref, maxChange) + system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) + continue + while len(change) > 0 and int(change) > maxChange: changed = True - print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) + if self.verbose: + print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) system("git update-ref %s \"%s^\"" % (ref, ref)) log = extractLogMessageFromGitCommit(ref) depotPath, change = extractDepotPathAndChangeFromGitLog(log) if changed: - print "%s is at %s" % (ref, change) + print "%s rewound to %s" % (ref, change) return True From a028a98e9ae83231f0657fdb112f7d9c0cf0b98c Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 00:03:08 +0200 Subject: [PATCH 171/260] Added support for importing multiple branches into refs/heads instead of just refs/remotes using --import-local. Needs some further microfix but seems to work otherwise. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 1457396abd..969c1fe45b 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -498,7 +498,8 @@ class P4Sync(Command): optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"), - optparse.make_option("--verbose", dest="verbose", action="store_true") + optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -519,6 +520,7 @@ class P4Sync(Command): self.changesFile = "" self.syncWithOrigin = False self.verbose = False + self.importIntoRemotes = True def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -749,11 +751,17 @@ class P4Sync(Command): def listExistingP4GitBranches(self): self.p4BranchesInGit = [] - for line in mypopen("git rev-parse --symbolic --remotes").readlines(): + cmdline = "git rev-parse --symbolic " + if self.importIntoRemotes: + cmdline += " --remotes" + else: + cmdline += " --branches" + + for line in mypopen(cmdline).readlines(): if line.startswith("p4/") and line != "p4/HEAD\n": branch = line[3:-1] self.p4BranchesInGit.append(branch) - self.initialParents["refs/remotes/p4/" + branch] = parseRevision(line[:-1]) + self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) def run(self, args): self.depotPath = "" @@ -764,9 +772,14 @@ class P4Sync(Command): self.knownBranches = {} self.initialParents = {} + if self.importIntoRemotes: + self.refPrefix = "refs/remotes/p4/" + else: + self.refPrefix = "refs/heads/p4/" + createP4HeadRef = False; - if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists("refs/remotes/p4/master") and not self.detectBranches: + if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists(self.refPrefix + "master") and not self.detectBranches: ### needs to be ported to multi branch import print "Syncing with origin first as requested by calling git fetch origin" @@ -779,17 +792,17 @@ class P4Sync(Command): p4Change = int(p4Change) if originP4Change > p4Change: print "origin (%s) is newer than p4 (%s). Updating p4 branch from origin." % (originP4Change, p4Change) - system("git update-ref refs/remotes/p4/master origin"); + system("git update-ref " + self.refPrefix + "master origin"); else: print "Cannot sync with origin. It was imported from %s while remotes/p4 was imported from %s" % (originPreviousDepotPath, p4PreviousDepotPath) if len(self.branch) == 0: - self.branch = "refs/remotes/p4/master" - if gitBranchExists("refs/heads/p4"): + self.branch = self.refPrefix + "master" + if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: system("git update-ref %s refs/heads/p4" % self.branch) system("git branch -D p4"); # create it /after/ importing, when master exists - if not gitBranchExists("refs/remotes/p4/HEAD"): + if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: createP4HeadRef = True # this needs to be called after the conversion from heads/p4 to remotes/p4/master @@ -813,7 +826,7 @@ class P4Sync(Command): p4Change = 0 for branch in self.p4BranchesInGit: - depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("refs/remotes/p4/" + branch)) + depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.refPrefix + branch)) if self.verbose: print "path %s change %s" % (depotPath, change) @@ -1008,9 +1021,9 @@ class P4Sync(Command): elif len(parent) > 0: parent = self.projectName + parent - branch = "refs/remotes/p4/" + branch + branch = self.refPrefix + branch if len(parent) > 0: - parent = "refs/remotes/p4/" + parent + parent = self.refPrefix + parent if self.verbose: print "looking for initial parent for %s; current parent is %s" % (branch, parent) @@ -1044,7 +1057,7 @@ class P4Sync(Command): self.gitError.close() if createP4HeadRef: - system("git symbolic-ref refs/remotes/p4/HEAD %s" % self.branch) + system("git symbolic-ref %s/HEAD %s" % (self.refPrefix, self.branch)) return True From 01a9c9c5a80c6a1e82298870ca8e5bc73b2a828d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 00:07:35 +0200 Subject: [PATCH 172/260] Added support for --max-changes= to ease import debugging Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 969c1fe45b..3d97ce1a24 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -499,7 +499,8 @@ class P4Sync(Command): optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false") + optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), + optparse.make_option("--max-changes", dest="maxChanges") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -521,6 +522,7 @@ class P4Sync(Command): self.syncWithOrigin = False self.verbose = False self.importIntoRemotes = True + self.maxChanges = "" def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -971,6 +973,9 @@ class P4Sync(Command): changes.reverse() + if len(self.maxChanges) > 0: + changes = changes[0:min(int(self.maxChanges), len(changes))] + if len(changes) == 0: if not self.silent: print "No changes to import!" From 57284050a8123a4c2d22d435a8c6daf1b53011e8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 00:15:50 +0200 Subject: [PATCH 173/260] Use refs/heads/* instead of refs/heads/p4/* for local imports Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 3d97ce1a24..152c3c1ca5 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -760,10 +760,15 @@ class P4Sync(Command): cmdline += " --branches" for line in mypopen(cmdline).readlines(): - if line.startswith("p4/") and line != "p4/HEAD\n": + if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): + continue + if self.importIntoRemotes: + # strip off p4 branch = line[3:-1] - self.p4BranchesInGit.append(branch) - self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) + else: + branch = line[:-1] + self.p4BranchesInGit.append(branch) + self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) def run(self, args): self.depotPath = "" @@ -777,11 +782,11 @@ class P4Sync(Command): if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" else: - self.refPrefix = "refs/heads/p4/" + self.refPrefix = "refs/heads/" createP4HeadRef = False; - if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists(self.refPrefix + "master") and not self.detectBranches: + if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists(self.refPrefix + "master") and not self.detectBranches and self.importIntoRemotes: ### needs to be ported to multi branch import print "Syncing with origin first as requested by calling git fetch origin" From a396b292678838b4cae1b358c62da6b4e0929fc2 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 00:33:34 +0200 Subject: [PATCH 174/260] Doc updates Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index ac8e6cff0b..aa9f31e5fc 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -120,6 +120,13 @@ continue importing the remaining changes with After submitting you should sync your perforce import branch ("p4" or "origin") from Perforce using git-p4's sync command. +If you have changes in your working directory that you haven't committed into +git yet but that you want to commit to Perforce directly ("quick fixes") then +you do not have to go through the intermediate step of creating a git commit +first but you can just call + + git-p4 submit --direct + Example ======= @@ -156,5 +163,5 @@ Implementation Details... to find out which changes need to be imported. * git-p4 submit uses "git rev-list" to pick the commits between the "p4" branch and the current branch. - The commits themselves are applied using git diff-tree ... | patch -p1 + The commits themselves are applied using git diff/format-patch ... | git apply From 65d2ade95e6ef5a15a30d5115073477dfe03fb85 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 16:41:46 +0200 Subject: [PATCH 175/260] Avoid calling git symbolic-ref refs/heads/p4//HEAD (double slash) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 152c3c1ca5..5cea6cf9ac 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1067,7 +1067,7 @@ class P4Sync(Command): self.gitError.close() if createP4HeadRef: - system("git symbolic-ref %s/HEAD %s" % (self.refPrefix, self.branch)) + system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) return True From 0c66a783936d5cd7b1f28400c6b192c37690304a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 20:07:57 +0200 Subject: [PATCH 176/260] Make rollback work with locally imported branches Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 5cea6cf9ac..f12ad8baff 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -136,18 +136,28 @@ class P4RollBack(Command): def __init__(self): Command.__init__(self) self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true") + optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") ] self.description = "A tool to debug the multi-branch import. Don't use :)" self.verbose = False + self.rollbackLocalBranches = False def run(self, args): if len(args) != 1: return False maxChange = int(args[0]) - for line in mypopen("git rev-parse --symbolic --remotes").readlines(): - if line.startswith("p4/") and line != "p4/HEAD\n": - ref = "refs/remotes/" + line[:-1] + + if self.rollbackLocalBranches: + refPrefix = "refs/heads/" + lines = mypopen("git rev-parse --symbolic --branches").readlines() + else: + refPrefix = "refs/remotes/" + lines = mypopen("git rev-parse --symbolic --remotes").readlines() + + for line in lines: + if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): + ref = refPrefix + line[:-1] log = extractLogMessageFromGitCommit(ref) depotPath, change = extractDepotPathAndChangeFromGitLog(log) changed = False From a6d5da36af9a7087cf14e717b72c66ef2c34eb3b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 23:27:31 +0200 Subject: [PATCH 177/260] Don't make len(p4Cmd("p4 changes -m 1 //foo/...")) == 0 succeed when the p4 command itself failed. When the p4 command failed write out the exit code in the returned dict. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index f12ad8baff..89a85ebb19 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -31,7 +31,9 @@ def p4CmdList(cmd): result.append(entry) except EOFError: pass - pipe.close() + exitCode = pipe.close() + if exitCode != None: + result["p4ExitCode"] = exitCode return result From ac3e0d79eef4535bb61d79315688fb1d225dea3b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 23:32:32 +0200 Subject: [PATCH 178/260] Oops, fill the /list/ correct with the p4 exit code. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 89a85ebb19..6ae3bc6e5d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -33,7 +33,9 @@ def p4CmdList(cmd): pass exitCode = pipe.close() if exitCode != None: - result["p4ExitCode"] = exitCode + entry = {} + entry["p4ExitCode"] = exitCode + result.append(entry) return result From 66a2f523958129e9b697d30ed44a5174010cb42a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 23:40:48 +0200 Subject: [PATCH 179/260] Catch p4 errors in rollback early enough (before deleting refs!) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6ae3bc6e5d..6d016b83d4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -152,6 +152,9 @@ class P4RollBack(Command): return False maxChange = int(args[0]) + if "p4ExitCode" in p4Cmd("p4 changes -m 1"): + die("Problems executing p4"); + if self.rollbackLocalBranches: refPrefix = "refs/heads/" lines = mypopen("git rev-parse --symbolic --branches").readlines() From ad192f2888816454df14b8a35945fb418bcb9e13 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 23:44:19 +0200 Subject: [PATCH 180/260] Fix p4 execution in git-p4 rollback. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6d016b83d4..b2341b7ec4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -152,7 +152,7 @@ class P4RollBack(Command): return False maxChange = int(args[0]) - if "p4ExitCode" in p4Cmd("p4 changes -m 1"): + if "p4ExitCode" in p4Cmd("changes -m 1"): die("Problems executing p4"); if self.rollbackLocalBranches: From b3fd1b28083cded86fd141c812cfd6d215ef4d5b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 23 May 2007 23:53:14 +0200 Subject: [PATCH 181/260] Fix multi-branch import with --silent. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b2341b7ec4..d8b7080b4b 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -831,8 +831,9 @@ class P4Sync(Command): # this needs to be called after the conversion from heads/p4 to remotes/p4/master self.listExistingP4GitBranches() - if len(self.p4BranchesInGit) > 1 and not self.silent: - print "Importing from/into multiple branches" + if len(self.p4BranchesInGit) > 1: + if not self.silent: + print "Importing from/into multiple branches" self.detectBranches = True if len(args) == 0: From ebd8116870247fbc9fb27656d28f1b8a43aae766 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 24 May 2007 00:24:52 +0200 Subject: [PATCH 182/260] Load the user map from p4 only once at run-time. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d8b7080b4b..9e9d623a3c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -697,6 +697,8 @@ class P4Sync(Command): print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change) def getUserMapFromPerforceServer(self): + if self.userMapFromPerforceServer: + return self.users = {} for output in p4CmdList("users"): @@ -708,9 +710,11 @@ class P4Sync(Command): for user in self.users.keys(): cache.write("%s\t%s\n" % (user, self.users[user])) cache.close(); + self.userMapFromPerforceServer = True def loadUserMapFromCache(self): self.users = {} + self.userMapFromPerforceServer = False try: cache = open(gitdir + "/p4-usercache.txt", "rb") lines = cache.readlines() From c1f9197f372a5378c3dbb36adeedf37ae175ecf1 Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Thu, 24 May 2007 14:07:55 +0200 Subject: [PATCH 183/260] Replace \r\n with \n when importing from p4 on Windows Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 9e9d623a3c..09b3cb5573 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -540,6 +540,7 @@ class P4Sync(Command): self.verbose = False self.importIntoRemotes = True self.maxChanges = "" + self.isWindows = (platform.system() == "Windows") def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -647,6 +648,9 @@ class P4Sync(Command): data = self.p4File(depotPath) + if self.isWindows and file["type"].endswith("text"): + data = data.replace("\r\n", "\n") + self.gitStream.write("M %s inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) From d1874ed33bc346dca8b86891757703c908634aad Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 24 May 2007 21:23:04 +0200 Subject: [PATCH 184/260] Fix creating the remotes/p4 branches based on origin/* for the multi-branch import Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 09b3cb5573..b587e79b48 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -795,6 +795,20 @@ class P4Sync(Command): self.p4BranchesInGit.append(branch) self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) + def createBranchesFromOrigin(self): + if not self.silent: + print "Creating branch(es) in %s based on origin branch(es)" % self.refPrefix + + for line in mypopen("git rev-parse --symbolic --remotes"): + if (not line.startswith("origin/")) or line.endswith("HEAD\n"): + continue + headName = line[len("origin/"):-1] + remoteHead = self.refPrefix + headName + if not os.path.exists(gitdir + "/" + remoteHead): + if self.verbose: + print "creating %s" % remoteHead + system("git update-ref %s origin/%s" % (remoteHead, headName)) + def run(self, args): self.depotPath = "" self.changeRange = "" @@ -841,18 +855,14 @@ class P4Sync(Command): self.listExistingP4GitBranches() if len(self.p4BranchesInGit) > 1: if not self.silent: - print "Importing from/into multiple branches" + print "Importing from/into multiple branches" self.detectBranches = True if len(args) == 0: - if not gitBranchExists(self.branch) and gitBranchExists("origin") and not self.detectBranches: - ### needs to be ported to multi branch import - if not self.silent: - print "Creating %s branch in git repository based on origin" % self.branch - branch = self.branch - if not branch.startswith("refs"): - branch = "refs/heads/" + branch - system("git update-ref %s origin" % branch) + if len(self.p4BranchesInGit) == 0: + self.createBranchesFromOrigin() + self.listExistingP4GitBranches() + return True if self.verbose: print "branches: %s" % self.p4BranchesInGit From 2cc58fd99ae5bc5845792393f232853da1ff6082 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 24 May 2007 22:10:40 +0200 Subject: [PATCH 185/260] Forgot to remove this return statement from debugging Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b587e79b48..bfd950d53d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -862,7 +862,6 @@ class P4Sync(Command): if len(self.p4BranchesInGit) == 0: self.createBranchesFromOrigin() self.listExistingP4GitBranches() - return True if self.verbose: print "branches: %s" % self.p4BranchesInGit From abcd790fe9bc168041dcc2a0e678d22a135d33c8 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 24 May 2007 22:25:36 +0200 Subject: [PATCH 186/260] Added support for --with-origin with multi-branch imports Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 58 ++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index bfd950d53d..0597daa849 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -795,19 +795,37 @@ class P4Sync(Command): self.p4BranchesInGit.append(branch) self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) - def createBranchesFromOrigin(self): + def createOrUpdateBranchesFromOrigin(self): if not self.silent: - print "Creating branch(es) in %s based on origin branch(es)" % self.refPrefix + print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix for line in mypopen("git rev-parse --symbolic --remotes"): if (not line.startswith("origin/")) or line.endswith("HEAD\n"): continue headName = line[len("origin/"):-1] remoteHead = self.refPrefix + headName + originHead = "origin/" + headName + + update = False if not os.path.exists(gitdir + "/" + remoteHead): if self.verbose: print "creating %s" % remoteHead - system("git update-ref %s origin/%s" % (remoteHead, headName)) + update = True + else: + [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) + [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) + if len(originPreviousDepotPath) > 0 and len(originP4Change) > 0 and len(p4Change) > 0: + if originPreviousDepotPath == p4PreviousDepotPath: + originP4Change = int(originP4Change) + p4Change = int(p4Change) + if originP4Change > p4Change: + print "%s (%s) is newer than %s (%s). Updating p4 branch from origin." % (originHead, originP4Change, remoteHead, p4Change) + update = True + else: + print "Ignoring: %s was imported from %s while %s was imported from %s" % (originHead, originPreviousDepotPath, remoteHead, p4PreviousDepotPath) + + if update: + system("git update-ref %s %s" % (remoteHead, originHead)) def run(self, args): self.depotPath = "" @@ -825,23 +843,6 @@ class P4Sync(Command): createP4HeadRef = False; - if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists(self.refPrefix + "master") and not self.detectBranches and self.importIntoRemotes: - ### needs to be ported to multi branch import - - print "Syncing with origin first as requested by calling git fetch origin" - system("git fetch origin") - [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) - [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) - if len(originPreviousDepotPath) > 0 and len(originP4Change) > 0 and len(p4Change) > 0: - if originPreviousDepotPath == p4PreviousDepotPath: - originP4Change = int(originP4Change) - p4Change = int(p4Change) - if originP4Change > p4Change: - print "origin (%s) is newer than p4 (%s). Updating p4 branch from origin." % (originP4Change, p4Change) - system("git update-ref " + self.refPrefix + "master origin"); - else: - print "Cannot sync with origin. It was imported from %s while remotes/p4 was imported from %s" % (originPreviousDepotPath, p4PreviousDepotPath) - if len(self.branch) == 0: self.branch = self.refPrefix + "master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: @@ -851,17 +852,14 @@ class P4Sync(Command): if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: createP4HeadRef = True - # this needs to be called after the conversion from heads/p4 to remotes/p4/master - self.listExistingP4GitBranches() - if len(self.p4BranchesInGit) > 1: - if not self.silent: - print "Importing from/into multiple branches" - self.detectBranches = True - if len(args) == 0: - if len(self.p4BranchesInGit) == 0: - self.createBranchesFromOrigin() - self.listExistingP4GitBranches() + self.createOrUpdateBranchesFromOrigin() + self.listExistingP4GitBranches() + + if len(self.p4BranchesInGit) > 1: + if not self.silent: + print "Importing from/into multiple branches" + self.detectBranches = True if self.verbose: print "branches: %s" % self.p4BranchesInGit From 10f880f8d4e2c6390928a0bcc8b43161b5f845b2 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 24 May 2007 22:28:28 +0200 Subject: [PATCH 187/260] Oops, fix --with-origin to /really/ also call git fetch :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0597daa849..08af23f9fd 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -841,6 +841,10 @@ class P4Sync(Command): else: self.refPrefix = "refs/heads/" + if self.syncWithOrigin: + print "Syncing with origin first as requested by calling git fetch origin" + system("git fetch origin") + createP4HeadRef = False; if len(self.branch) == 0: From 65c5f3e3f2dfb470c16628032235222a492e3239 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 08:44:41 +0200 Subject: [PATCH 188/260] Avoid creating non-p4 branches in remotes/p4 off of remotes/origin Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 08af23f9fd..1cce38a6f7 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -802,19 +802,23 @@ class P4Sync(Command): for line in mypopen("git rev-parse --symbolic --remotes"): if (not line.startswith("origin/")) or line.endswith("HEAD\n"): continue + headName = line[len("origin/"):-1] remoteHead = self.refPrefix + headName originHead = "origin/" + headName + [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) + if len(originPreviousDepotPath) == 0 or len(originP4Change) == 0: + continue + update = False if not os.path.exists(gitdir + "/" + remoteHead): if self.verbose: print "creating %s" % remoteHead update = True else: - [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) - if len(originPreviousDepotPath) > 0 and len(originP4Change) > 0 and len(p4Change) > 0: + if len(p4Change) > 0: if originPreviousDepotPath == p4PreviousDepotPath: originP4Change = int(originP4Change) p4Change = int(p4Change) From 4280e5333354c6dddcd994bdacd3c6a11ac2da5e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 08:49:18 +0200 Subject: [PATCH 189/260] Make git-p4 work with packed refs (don't use os.path.exists to check for the existance of a ref) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 1cce38a6f7..e8a5c1fa31 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -412,7 +412,7 @@ class P4Submit(Command): if len(args) == 0: self.master = currentGitBranch() - if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)): + if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): die("Detecting current git branch failed!") elif len(args) == 1: self.master = args[0] @@ -812,7 +812,7 @@ class P4Sync(Command): continue update = False - if not os.path.exists(gitdir + "/" + remoteHead): + if not gitBranchExists(remoteHead): if self.verbose: print "creating %s" % remoteHead update = True From 417a7a6fc8c3d77e3e9a3bb0e11cb2eea978e20b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 10:28:46 +0200 Subject: [PATCH 190/260] Make --with-origin also work without origin :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e8a5c1fa31..30ee68c609 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -846,8 +846,9 @@ class P4Sync(Command): self.refPrefix = "refs/heads/" if self.syncWithOrigin: - print "Syncing with origin first as requested by calling git fetch origin" - system("git fetch origin") + if gitBranchExists("origin"): + print "Syncing with origin first as requested by calling git fetch origin" + system("git fetch origin") createP4HeadRef = False; From 01265103fe3966abd720ebb0bba1882a5701f327 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 10:36:10 +0200 Subject: [PATCH 191/260] Make --with-origin the default for syncing. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 16 ++++++++++------ contrib/fast-import/git-p4.txt | 14 +++----------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 30ee68c609..ed5a5c593c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -118,6 +118,9 @@ def gitBranchExists(branch): proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE); return proc.wait() == 0; +def gitConfig(key): + return mypopen("git config %s" % key).read()[:-1] + class Command: def __init__(self): self.usage = "usage: %prog [options]" @@ -514,7 +517,6 @@ class P4Sync(Command): optparse.make_option("--changesfile", dest="changesFile"), optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), - optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), optparse.make_option("--max-changes", dest="maxChanges") @@ -536,12 +538,15 @@ class P4Sync(Command): self.detectBranches = False self.detectLabels = False self.changesFile = "" - self.syncWithOrigin = False + self.syncWithOrigin = True self.verbose = False self.importIntoRemotes = True self.maxChanges = "" self.isWindows = (platform.system() == "Windows") + if gitConfig("git-p4.syncFromOrigin") == "false": + self.syncWithOrigin = False + def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -847,7 +852,8 @@ class P4Sync(Command): if self.syncWithOrigin: if gitBranchExists("origin"): - print "Syncing with origin first as requested by calling git fetch origin" + if not self.silent: + print "Syncing with origin first by calling git fetch origin" system("git fetch origin") createP4HeadRef = False; @@ -1116,13 +1122,11 @@ class P4Sync(Command): class P4Rebase(Command): def __init__(self): Command.__init__(self) - self.options = [ optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ] + self.options = [ ] self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it" - self.syncWithOrigin = False def run(self, args): sync = P4Sync() - sync.syncWithOrigin = self.syncWithOrigin sync.run([]) print "Rebasing the current branch" oldHead = mypopen("git rev-parse HEAD").read()[:-1] diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index aa9f31e5fc..c315158d8d 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -70,18 +70,10 @@ repository can be used to clone the working repository from and one would import from Perforce directly after cloning using git-p4. If the connection to the Perforce server is slow and the working repository hasn't been synced for a while it may be desirable to fetch changes from the origin git repository using -the efficient git protocol. git-p4 supports this through +the efficient git protocol. git-p4 supports this setup by calling "git fetch origin" +by default if there is an origin branch. You can disable this using - git-p4 sync --with-origin - -or - - git-p4 rebase --with-origin - -In that case "git fetch origin" is called and if it turns out that the origin -branch is newer than the git "p4" import branch then the latter is updated from -the former and the direct import from Perforce is resumed, which will result in -fewer changes to be imported using the slower perforce connection. + git config git-p4.syncFromOrigin false Updating ======== From d414c74afd8610c8c22dadf62b5ba8c6efa59e75 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 11:36:42 +0200 Subject: [PATCH 192/260] Shortcut the case where we have no origin branch Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ed5a5c593c..d99237c632 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -844,6 +844,7 @@ class P4Sync(Command): # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} + self.hasOrigin = gitBranchExists("origin") if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" @@ -851,7 +852,7 @@ class P4Sync(Command): self.refPrefix = "refs/heads/" if self.syncWithOrigin: - if gitBranchExists("origin"): + if self.hasOrigin: if not self.silent: print "Syncing with origin first by calling git fetch origin" system("git fetch origin") @@ -868,7 +869,8 @@ class P4Sync(Command): createP4HeadRef = True if len(args) == 0: - self.createOrUpdateBranchesFromOrigin() + if self.hasOrigin: + self.createOrUpdateBranchesFromOrigin() self.listExistingP4GitBranches() if len(self.p4BranchesInGit) > 1: From 877db584aae9816671147da45c30c31748ef287f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 19:43:38 +0200 Subject: [PATCH 193/260] Forgot to remove this TODO item when I made --with-origin the default :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 --- 1 file changed, 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d99237c632..0946965043 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -7,9 +7,6 @@ # 2007 Trolltech ASA # License: MIT # -# TODO: * Consider making --with-origin the default, assuming that the git -# protocol is always more efficient. (needs manual testing first :) -# import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform From cb4f1280dd5691890abea4521bff0a3d6e3facfd Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 25 May 2007 22:34:30 +0200 Subject: [PATCH 194/260] Added git-p4 submit --trust-me-like-a-fool for the adventurous users :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0946965043..dbd5b3bcf1 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -195,9 +195,9 @@ class P4Submit(Command): optparse.make_option("--origin", dest="origin"), optparse.make_option("--reset", action="store_true", dest="reset"), optparse.make_option("--log-substitutions", dest="substFile"), - optparse.make_option("--noninteractive", action="store_false"), optparse.make_option("--dry-run", action="store_true"), optparse.make_option("--direct", dest="directSubmit", action="store_true"), + optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"), ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" @@ -209,6 +209,7 @@ class P4Submit(Command): self.firstTime = True self.origin = "" self.directSubmit = False + self.trustMeLikeAFool = False self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -348,6 +349,9 @@ class P4Submit(Command): separatorLine += "\n" response = "e" + if self.trustMeLikeAFool: + response = "y" + firstIteration = True while response == "e": if not firstIteration: From a3c55c09ecae2a9c852c8c10e668c901639e845b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 27 May 2007 15:48:01 +0200 Subject: [PATCH 195/260] Fix creation of refs/remotes/p4/HEAD symbolic ref Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index dbd5b3bcf1..aeefadcd66 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -858,8 +858,6 @@ class P4Sync(Command): print "Syncing with origin first by calling git fetch origin" system("git fetch origin") - createP4HeadRef = False; - if len(self.branch) == 0: self.branch = self.refPrefix + "master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: @@ -867,7 +865,7 @@ class P4Sync(Command): system("git branch -D p4"); # create it /after/ importing, when master exists if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: - createP4HeadRef = True + system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) if len(args) == 0: if self.hasOrigin: @@ -1117,9 +1115,6 @@ class P4Sync(Command): self.gitOutput.close() self.gitError.close() - if createP4HeadRef: - system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - return True class P4Rebase(Command): From ce6f33c83537a04a3fb7557d7a74fe81ebccd7e4 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 16:46:29 -0300 Subject: [PATCH 196/260] Cleanups - don't use dir (python builtin) - use re for munging depotPath into destination Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index aeefadcd66..910e4ff3cd 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -10,6 +10,7 @@ import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform +import re from sets import Set; gitdir = os.environ.get("GIT_DIR", "") @@ -842,6 +843,7 @@ class P4Sync(Command): self.changeRange = "" self.initialParent = "" self.previousDepotPath = "" + # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} @@ -1145,37 +1147,26 @@ class P4Clone(P4Sync): if len(args) < 1: return False depotPath = args[0] - dir = "" + destination = "" if len(args) == 2: - dir = args[1] + destination = args[1] elif len(args) > 2: return False if not depotPath.startswith("//"): return False - if len(dir) == 0: - dir = depotPath - atPos = dir.rfind("@") - if atPos != -1: - dir = dir[0:atPos] - hashPos = dir.rfind("#") - if hashPos != -1: - dir = dir[0:hashPos] + depotDir = re.sub("(@[^@]*)$", "", depotPath) + depotDir = re.sub("(#[^#]*)$", "", depotDir) + depotDir = re.sub(r"\.\.\.$,", "", depotDir) + depotDir = re.sub(r"/$", "", depotDir) - if dir.endswith("..."): - dir = dir[:-3] + if not destination: + destination = os.path.split(depotDir)[-1] - if dir.endswith("/"): - dir = dir[:-1] - - slashPos = dir.rfind("/") - if slashPos != -1: - dir = dir[slashPos + 1:] - - print "Importing from %s into %s" % (depotPath, dir) - os.makedirs(dir) - os.chdir(dir) + print "Importing from %s into %s" % (depotPath, destination) + os.makedirs(destination) + os.chdir(destination) system("git init") gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, [depotPath]): From cebdf5af319ce638862fe2e2cd2797840962ddbb Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 16:53:11 -0300 Subject: [PATCH 197/260] reformatting: break long lines. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 48 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 910e4ff3cd..aba4752d4e 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -222,7 +222,9 @@ class P4Submit(Command): def start(self): if len(self.config) > 0 and not self.reset: - die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile) + die("Cannot start sync. Previous sync config found at %s\n" + "If you want to start submitting again from scratch " + "maybe you want to call git-p4 submit --reset" % self.configFile) commits = [] if self.directSubmit: @@ -297,7 +299,8 @@ class P4Submit(Command): print "What do you want to do?" response = "x" while response != "s" and response != "a" and response != "w": - response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ") + response = raw_input("[s]kip this patch / [a]pply the patch forcibly " + "and with .rej files / [w]rite the patch to a file (patch.txt) ") if response == "s": print "Skipping! Good luck with the next patches..." return @@ -309,11 +312,13 @@ class P4Submit(Command): if len(filesToDelete): print "The following files should be scheduled for deletion with p4 delete:" print " ".join(filesToDelete) - die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") + die("Please resolve and submit the conflict manually and " + + "continue afterwards with git-p4 submit --continue") elif response == "w": system(diffcmd + " > patch.txt") print "Patch saved to patch.txt in %s !" % self.clientPath - die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") + die("Please resolve and submit the conflict manually and " + "continue afterwards with git-p4 submit --continue") system(applyPatchCmd) @@ -407,7 +412,9 @@ class P4Submit(Command): file = open(fileName, "w+") file.write(self.prepareLogMessage(template, logMessage)) file.close() - print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName) + print ("Perforce submit template written as %s. " + + "Please review/edit and then use p4 submit -i < %s to submit directly!" + % (fileName, fileName)) def run(self, args): global gitdir @@ -634,7 +641,6 @@ class P4Sync(Command): for file in files: path = file["path"] if not path.startswith(branchPrefix): - # if not silent: # print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) continue rev = file["rev"] @@ -701,11 +707,13 @@ class P4Sync(Command): else: if not self.silent: - print "Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change) + print ("Tag %s does not match with change %s: files do not match." + % (labelDetails["label"], change)) else: if not self.silent: - print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change) + print ("Tag %s does not match with change %s: file count is different." + % (labelDetails["label"], change)) def getUserMapFromPerforceServer(self): if self.userMapFromPerforceServer: @@ -854,11 +862,10 @@ class P4Sync(Command): else: self.refPrefix = "refs/heads/" - if self.syncWithOrigin: - if self.hasOrigin: - if not self.silent: - print "Syncing with origin first by calling git fetch origin" - system("git fetch origin") + if self.syncWithOrigin and self.hasOrigin: + if not self.silent: + print "Syncing with origin first by calling git fetch origin" + system("git fetch origin") if len(self.branch) == 0: self.branch = self.refPrefix + "master" @@ -884,7 +891,8 @@ class P4Sync(Command): p4Change = 0 for branch in self.p4BranchesInGit: - depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.refPrefix + branch)) + logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) + (depotPath, change) = extractDepotPathAndChangeFromGitLog(logMsg) if self.verbose: print "path %s change %s" % (depotPath, change) @@ -922,7 +930,8 @@ class P4Sync(Command): return False else: if len(self.depotPath) != 0 and self.depotPath != args[0]: - print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0]) + print ("previous import used depot path %s and now %s was specified. " + "This doesn't work!" % (self.depotPath, args[0])) sys.exit(1) self.depotPath = args[0] @@ -968,7 +977,8 @@ class P4Sync(Command): self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) - importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); + importProcess = subprocess.Popen(["git", "fast-import"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); self.gitOutput = importProcess.stdout self.gitStream = importProcess.stdin self.gitError = importProcess.stderr @@ -977,7 +987,8 @@ class P4Sync(Command): print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision) + details["desc"] = ("Initial import of %s from the state at revision %s" + % (self.depotPath, self.revision)) details["change"] = self.revision newestRevision = 0 @@ -1123,7 +1134,8 @@ class P4Rebase(Command): def __init__(self): Command.__init__(self) self.options = [ ] - self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it" + self.description = ("Fetches the latest revision from perforce and " + + "rebases the current work (branch) against it") def run(self, args): sync = P4Sync() From 7cb5cbefd2c8b4c24bc87c6e30906907f94726ad Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 16:55:48 -0300 Subject: [PATCH 198/260] rename apply() to applyCommit(); apply is a python builtin Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index aba4752d4e..bd0ea54929 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -259,7 +259,7 @@ class P4Submit(Command): return result - def apply(self, id): + def applyCommit(self, id): if self.directSubmit: print "Applying local change in working directory/index" diff = self.diffStatus @@ -494,7 +494,7 @@ class P4Submit(Command): commit = commits[0] commits = commits[1:] self.config["commits"] = commits - self.apply(commit) + self.applyCommit(commit) if not self.interactive: break From c8cbbee980ac3961d5cdae164f2cb0a6b88075e0 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 28 May 2007 14:43:25 +0200 Subject: [PATCH 199/260] Fix my email address, this isn't really KDE related :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index bd0ea54929..400edce459 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -2,8 +2,8 @@ # # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git. # -# Author: Simon Hausmann -# Copyright: 2007 Simon Hausmann +# Author: Simon Hausmann +# Copyright: 2007 Simon Hausmann # 2007 Trolltech ASA # License: MIT # From b016d39756f7d82e51c35e281673ed30b95cc586 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 17:10:46 -0300 Subject: [PATCH 201/260] Robustness fixes for pipes - add read_pipe(), read_pipe_lines(), write_pipe(), which check pipe.close() - use throughout Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 85 ++++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 400edce459..05b308f5be 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -14,9 +14,43 @@ import re from sets import Set; gitdir = os.environ.get("GIT_DIR", "") +silent = False -def mypopen(command): - return os.popen(command, "rb"); +def write_pipe (c, str): + if not silent: + sys.stderr.write ('writing pipe: %s\n' % c) + + ## todo: check return status + pipe = os.popen (c, 'w') + val = pipe.write(str) + if pipe.close (): + sys.stderr.write ('Command failed') + sys.exit (1) + + return val + +def read_pipe (c): + sys.stderr.write ('reading pipe: %s\n' % c) + ## todo: check return status + pipe = os.popen (c, 'rb') + val = pipe.read() + if pipe.close (): + sys.stderr.write ('Command failed') + sys.exit (1) + + return val + + +def read_pipe_lines (c): + sys.stderr.write ('reading pipe: %s\n' % c) + ## todo: check return status + pipe = os.popen (c, 'rb') + val = pipe.readlines() + if pipe.close (): + sys.stderr.write ('Command failed') + sys.exit (1) + + return val def p4CmdList(cmd): cmd = "p4 -G %s" % cmd @@ -67,7 +101,7 @@ def die(msg): sys.exit(1) def currentGitBranch(): - return mypopen("git name-rev HEAD").read().split(" ")[1][:-1] + return read_pipe("git name-rev HEAD").split(" ")[1][:-1] def isValidGitDir(path): if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): @@ -75,7 +109,7 @@ def isValidGitDir(path): return False def parseRevision(ref): - return mypopen("git rev-parse %s" % ref).read()[:-1] + return read_pipe("git rev-parse %s" % ref)[:-1] def system(cmd): if os.system(cmd) != 0: @@ -83,8 +117,10 @@ def system(cmd): def extractLogMessageFromGitCommit(commit): logMessage = "" + + ## fixme: title is first line of commit, not 1st paragraph. foundTitle = False - for log in mypopen("git cat-file commit %s" % commit).readlines(): + for log in read_pipe_lines("git cat-file commit %s" % commit): if not foundTitle: if len(log) == 1: foundTitle = True @@ -158,10 +194,10 @@ class P4RollBack(Command): if self.rollbackLocalBranches: refPrefix = "refs/heads/" - lines = mypopen("git rev-parse --symbolic --branches").readlines() + lines = read_pipe_lines("git rev-parse --symbolic --branches") else: refPrefix = "refs/remotes/" - lines = mypopen("git rev-parse --symbolic --remotes").readlines() + lines = read_pipe_lines("git rev-parse --symbolic --remotes") for line in lines: if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): @@ -230,7 +266,7 @@ class P4Submit(Command): if self.directSubmit: commits.append("0") else: - for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): + for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)): commits.append(line[:-1]) commits.reverse() @@ -264,8 +300,8 @@ class P4Submit(Command): print "Applying local change in working directory/index" diff = self.diffStatus else: - print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read()) - diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() + print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id)) + diff = read_pipe_lines("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)) filesToAdd = set() filesToDelete = set() editedFiles = set() @@ -334,11 +370,11 @@ class P4Submit(Command): logMessage = logMessage.replace("\n", "\n\t") logMessage = logMessage[:-1] - template = mypopen("p4 change -o").read() + template = read_pipe("p4 change -o") if self.interactive: submitTemplate = self.prepareLogMessage(template, logMessage) - diff = mypopen("p4 diff -du ...").read() + diff = read_pipe("p4 diff -du ...") for newFile in filesToAdd: diff += "==== new file ====\n" @@ -387,14 +423,10 @@ class P4Submit(Command): if self.directSubmit: print "Submitting to git first" os.chdir(self.oldWorkingDirectory) - pipe = os.popen("git commit -a -F -", "wb") - pipe.write(submitTemplate) - pipe.close() + write_pipe("git commit -a -F -", submitTemplate) os.chdir(self.clientPath) - pipe = os.popen("p4 submit -i", "wb") - pipe.write(submitTemplate) - pipe.close() + write_pipe("p4 submit -i", submitTemplate) elif response == "s": for f in editedFiles: system("p4 revert \"%s\"" % f); @@ -451,11 +483,11 @@ class P4Submit(Command): self.oldWorkingDirectory = os.getcwd() if self.directSubmit: - self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines() + self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD") if len(self.diffStatus) == 0: print "No changes in working directory to submit." return True - patch = mypopen("git diff -p --binary --diff-filter=ACMRTUXB HEAD").read() + patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD") self.diffFile = gitdir + "/p4-git-diff" f = open(self.diffFile, "wb") f.write(patch) @@ -557,7 +589,7 @@ class P4Sync(Command): self.syncWithOrigin = False def p4File(self, depotPath): - return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() + return read_pipe("p4 print -q \"%s\"" % depotPath) def extractFilesFromCommit(self, commit): files = [] @@ -799,7 +831,7 @@ class P4Sync(Command): else: cmdline += " --branches" - for line in mypopen(cmdline).readlines(): + for line in read_pipe_lines(cmdline): if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): continue if self.importIntoRemotes: @@ -1032,7 +1064,7 @@ class P4Sync(Command): else: if self.verbose: print "Getting p4 changes for %s...%s" % (self.depotPath, self.changeRange) - output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() + output = read_pipe_lines("p4 changes %s...%s" % (self.depotPath, self.changeRange)) for line in output: changeNum = line.split(" ")[1] @@ -1141,7 +1173,7 @@ class P4Rebase(Command): sync = P4Sync() sync.run([]) print "Rebasing the current branch" - oldHead = mypopen("git rev-parse HEAD").read()[:-1] + oldHead = read_pipe("git rev-parse HEAD")[:-1] system("git rebase p4") system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True @@ -1252,9 +1284,9 @@ if cmd.needsGit: if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): - gitdir = mypopen("git rev-parse --git-dir").read()[:-1] + gitdir = read_pipe("git rev-parse --git-dir")[:-1] if os.path.exists(gitdir): - cdup = mypopen("git rev-parse --show-cdup").read()[:-1]; + cdup = read_pipe("git rev-parse --show-cdup")[:-1]; if len(cdup) > 0: os.chdir(cdup); @@ -1268,4 +1300,3 @@ if cmd.needsGit: if not cmd.run(args): parser.print_help() - From bce4c5fc0b6aa9e599fc181c5350b461dafecee2 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 17:14:33 -0300 Subject: [PATCH 202/260] cleanup - use re.sub() iso. if for stripping ... - spacing nits Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 40 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 05b308f5be..8d649dd762 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -16,39 +16,39 @@ from sets import Set; gitdir = os.environ.get("GIT_DIR", "") silent = False -def write_pipe (c, str): +def write_pipe(c, str): if not silent: - sys.stderr.write ('writing pipe: %s\n' % c) + sys.stderr.write('writing pipe: %s\n' % c) ## todo: check return status - pipe = os.popen (c, 'w') + pipe = os.popen(c, 'w') val = pipe.write(str) - if pipe.close (): - sys.stderr.write ('Command failed') - sys.exit (1) + if pipe.close(): + sys.stderr.write('Command failed') + sys.exit(1) return val -def read_pipe (c): - sys.stderr.write ('reading pipe: %s\n' % c) +def read_pipe(c): + sys.stderr.write('reading pipe: %s\n' % c) ## todo: check return status - pipe = os.popen (c, 'rb') + pipe = os.popen(c, 'rb') val = pipe.read() - if pipe.close (): - sys.stderr.write ('Command failed') - sys.exit (1) + if pipe.close(): + sys.stderr.write('Command failed') + sys.exit(1) return val -def read_pipe_lines (c): - sys.stderr.write ('reading pipe: %s\n' % c) +def read_pipe_lines(c): + sys.stderr.write('reading pipe: %s\n' % c) ## todo: check return status - pipe = os.popen (c, 'rb') + pipe = os.popen(c, 'rb') val = pipe.readlines() - if pipe.close (): - sys.stderr.write ('Command failed') - sys.exit (1) + if pipe.close(): + sys.stderr.write('Command failed') + sys.exit(1) return val @@ -986,9 +986,7 @@ class P4Sync(Command): elif len(self.previousDepotPath) == 0: self.revision = "#head" - if self.depotPath.endswith("..."): - self.depotPath = self.depotPath[:-3] - + self.depotPath = re.sub ("\.\.\.$", "", self.depotPath) if not self.depotPath.endswith("/"): self.depotPath += "/" From 6754a299d8d977326e396d641106cc5db4d29db1 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 17:41:50 -0300 Subject: [PATCH 203/260] minor cleanups Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 8d649dd762..93c9d6417c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -30,7 +30,8 @@ def write_pipe(c, str): return val def read_pipe(c): - sys.stderr.write('reading pipe: %s\n' % c) + if not silent: + sys.stderr.write('reading pipe: %s\n' % c) ## todo: check return status pipe = os.popen(c, 'rb') val = pipe.read() @@ -42,7 +43,8 @@ def read_pipe(c): def read_pipe_lines(c): - sys.stderr.write('reading pipe: %s\n' % c) + if not silent: + sys.stderr.write('reading pipe: %s\n' % c) ## todo: check return status pipe = os.popen(c, 'rb') val = pipe.readlines() @@ -52,6 +54,14 @@ def read_pipe_lines(c): return val +def system(cmd): + if not silent: + sys.stderr.write("executing %s" % cmd) + if os.system(cmd) != 0: + die("command failed: %s" % cmd) + + + def p4CmdList(cmd): cmd = "p4 -G %s" % cmd pipe = os.popen(cmd, "rb") @@ -111,10 +121,6 @@ def isValidGitDir(path): def parseRevision(ref): return read_pipe("git rev-parse %s" % ref)[:-1] -def system(cmd): - if os.system(cmd) != 0: - die("command failed: %s" % cmd) - def extractLogMessageFromGitCommit(commit): logMessage = "" @@ -571,7 +577,6 @@ class P4Sync(Command): (a ... is not needed in the path p4 specification, it's added implicitly)""" self.usage += " //depot/path[@revRange]" - self.silent = False self.createdBranches = Set() self.committedChanges = Set() @@ -584,6 +589,7 @@ class P4Sync(Command): self.importIntoRemotes = True self.maxChanges = "" self.isWindows = (platform.system() == "Windows") + self.depotPath = None if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False @@ -611,9 +617,11 @@ class P4Sync(Command): fnum = fnum + 1 return files + def stripRepoPath(self, path): + return path[len(self.depotPath):] + def splitFilesIntoBranches(self, commit): branches = {} - fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] @@ -630,10 +638,12 @@ class P4Sync(Command): file["type"] = commit["type%s" % fnum] fnum = fnum + 1 - relPath = path[len(self.depotPath):] + relPath = self.stripRepoPath(path) for branch in self.knownBranches.keys(): - if relPath.startswith(branch + "/"): # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2 + + # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2 + if relPath.startswith(branch + "/"): if branch not in branches: branches[branch] = [] branches[branch].append(file) From 8b41a97f8a3e2778ce733cd3d7e181bc28cc43a0 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:20:53 -0300 Subject: [PATCH 204/260] clone and sync --keep-path to keep perforce path to module. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 93c9d6417c..51d117b789 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -11,6 +11,7 @@ import optparse, sys, os, marshal, popen2, subprocess, shelve import tempfile, getopt, sha, os.path, time, platform import re + from sets import Set; gitdir = os.environ.get("GIT_DIR", "") @@ -20,7 +21,6 @@ def write_pipe(c, str): if not silent: sys.stderr.write('writing pipe: %s\n' % c) - ## todo: check return status pipe = os.popen(c, 'w') val = pipe.write(str) if pipe.close(): @@ -32,7 +32,7 @@ def write_pipe(c, str): def read_pipe(c): if not silent: sys.stderr.write('reading pipe: %s\n' % c) - ## todo: check return status + pipe = os.popen(c, 'rb') val = pipe.read() if pipe.close(): @@ -60,8 +60,6 @@ def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) - - def p4CmdList(cmd): cmd = "p4 -G %s" % cmd pipe = os.popen(cmd, "rb") @@ -566,7 +564,8 @@ class P4Sync(Command): optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), - optparse.make_option("--max-changes", dest="maxChanges") + optparse.make_option("--max-changes", dest="maxChanges"), + optparse.make_option("--keep-path", dest="keepRepoPath") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -590,6 +589,7 @@ class P4Sync(Command): self.maxChanges = "" self.isWindows = (platform.system() == "Windows") self.depotPath = None + self.keepRepoPath = False if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False @@ -617,8 +617,11 @@ class P4Sync(Command): fnum = fnum + 1 return files - def stripRepoPath(self, path): - return path[len(self.depotPath):] + def stripRepoPath(self, path, prefix): + if self.keepRepoPath: + prefix = re.sub("^(//[^/]+/).*", r'\1', prefix) + + return path[len(prefix):] def splitFilesIntoBranches(self, commit): branches = {} @@ -638,7 +641,7 @@ class P4Sync(Command): file["type"] = commit["type%s" % fnum] fnum = fnum + 1 - relPath = self.stripRepoPath(path) + relPath = self.stripRepoPath(path, self.depotPath) for branch in self.knownBranches.keys(): @@ -687,7 +690,7 @@ class P4Sync(Command): continue rev = file["rev"] depotPath = path + "#" + rev - relPath = path[len(branchPrefix):] + relPath = self.stripRepoPath(path, branchPrefix) action = file["action"] if file["type"] == "apple": From b76f0565bfcfedba9e8920aa5120fc97796b642f Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:29:34 -0300 Subject: [PATCH 205/260] use string.strip() iso. slicing. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 51d117b789..ac446a8dc4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -845,15 +845,15 @@ class P4Sync(Command): cmdline += " --branches" for line in read_pipe_lines(cmdline): + lie = line.strip() if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): continue if self.importIntoRemotes: # strip off p4 - branch = line[3:-1] - else: - branch = line[:-1] + branch = re.sub ("^p4/", "", line) + self.p4BranchesInGit.append(branch) - self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) + self.initialParents[self.refPrefix + branch] = parseRevision(line) def createOrUpdateBranchesFromOrigin(self): if not self.silent: From b25b20656dceff6145a1286f10618ab25d717537 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 206/260] use strip() iso. slicing for removing \n Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ac446a8dc4..efa2fce29e 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -15,7 +15,7 @@ import re from sets import Set; gitdir = os.environ.get("GIT_DIR", "") -silent = False +silent = True def write_pipe(c, str): if not silent: @@ -109,7 +109,7 @@ def die(msg): sys.exit(1) def currentGitBranch(): - return read_pipe("git name-rev HEAD").split(" ")[1][:-1] + return read_pipe("git name-rev HEAD").split(" ")[1].strip() def isValidGitDir(path): if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): @@ -117,7 +117,7 @@ def isValidGitDir(path): return False def parseRevision(ref): - return read_pipe("git rev-parse %s" % ref)[:-1] + return read_pipe("git rev-parse %s" % ref).strip() def extractLogMessageFromGitCommit(commit): logMessage = "" @@ -205,7 +205,8 @@ class P4RollBack(Command): for line in lines: if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): - ref = refPrefix + line[:-1] + line = line.strip() + ref = refPrefix + line log = extractLogMessageFromGitCommit(ref) depotPath, change = extractDepotPathAndChangeFromGitLog(log) changed = False @@ -271,7 +272,7 @@ class P4Submit(Command): commits.append("0") else: for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)): - commits.append(line[:-1]) + commits.append(line.strip()) commits.reverse() self.config["commits"] = commits @@ -372,7 +373,7 @@ class P4Submit(Command): if not self.directSubmit: logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.replace("\n", "\n\t") - logMessage = logMessage[:-1] + logMessage = logMessage.strip() template = read_pipe("p4 change -o") @@ -513,7 +514,7 @@ class P4Submit(Command): if len(self.substFile) > 0: for line in open(self.substFile, "r").readlines(): - tokens = line[:-1].split("=") + tokens = line.strip().split("=") self.logSubstitutions[tokens[0]] = tokens[1] self.check() @@ -784,7 +785,7 @@ class P4Sync(Command): lines = cache.readlines() cache.close() for line in lines: - entry = line[:-1].split("\t") + entry = line.strip().split("\t") self.users[entry[0]] = entry[1] except IOError: self.getUserMapFromPerforceServer() @@ -814,7 +815,7 @@ class P4Sync(Command): print "Label changes: %s" % self.labels.keys() def getBranchMapping(self): - self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] + self.projectName = self.depotPath[self.depotPath.strip().rfind("/") + 1:] for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) @@ -848,6 +849,7 @@ class P4Sync(Command): lie = line.strip() if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): continue + if self.importIntoRemotes: # strip off p4 branch = re.sub ("^p4/", "", line) @@ -966,7 +968,7 @@ class P4Sync(Command): self.branch = "refs/heads/" + self.branch if len(self.depotPath) != 0: - self.depotPath = self.depotPath[:-1] + self.depotPath = self.depotPath.strip() if len(args) == 0 and len(self.depotPath) != 0: if not self.silent: @@ -1184,7 +1186,7 @@ class P4Rebase(Command): sync = P4Sync() sync.run([]) print "Rebasing the current branch" - oldHead = read_pipe("git rev-parse HEAD")[:-1] + oldHead = read_pipe("git rev-parse HEAD").strip() system("git rebase p4") system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True @@ -1217,7 +1219,7 @@ class P4Clone(P4Sync): depotDir = re.sub(r"/$", "", depotDir) if not destination: - destination = os.path.split(depotDir)[-1] + destination = os.path.split(depotDir)[1] print "Importing from %s into %s" % (depotPath, destination) os.makedirs(destination) @@ -1295,9 +1297,9 @@ if cmd.needsGit: if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): - gitdir = read_pipe("git rev-parse --git-dir")[:-1] + gitdir = read_pipe("git rev-parse --git-dir").strip() if os.path.exists(gitdir): - cdup = read_pipe("git rev-parse --show-cdup")[:-1]; + cdup = read_pipe("git rev-parse --show-cdup").strip() if len(cdup) > 0: os.chdir(cdup); From 4addad229169ece62f36c6c17cc15bdf532a60bc Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 207/260] add --verbose to all commands. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index efa2fce29e..cf9db73da6 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -15,47 +15,47 @@ import re from sets import Set; gitdir = os.environ.get("GIT_DIR", "") -silent = True +verbose = False def write_pipe(c, str): - if not silent: + if verbose: sys.stderr.write('writing pipe: %s\n' % c) pipe = os.popen(c, 'w') val = pipe.write(str) if pipe.close(): - sys.stderr.write('Command failed') + sys.stderr.write('Command %s failed\n' % c) sys.exit(1) return val -def read_pipe(c): - if not silent: +def read_pipe(c, ignore_error=False): + if verbose: sys.stderr.write('reading pipe: %s\n' % c) pipe = os.popen(c, 'rb') val = pipe.read() - if pipe.close(): - sys.stderr.write('Command failed') + if pipe.close() and not ignore_error: + sys.stderr.write('Command %s failed\n' % c) sys.exit(1) return val def read_pipe_lines(c): - if not silent: + if verbose: sys.stderr.write('reading pipe: %s\n' % c) ## todo: check return status pipe = os.popen(c, 'rb') val = pipe.readlines() if pipe.close(): - sys.stderr.write('Command failed') + sys.stderr.write('Command %s failed\n' % c) sys.exit(1) return val def system(cmd): - if not silent: + if verbose: sys.stderr.write("executing %s" % cmd) if os.system(cmd) != 0: die("command failed: %s" % cmd) @@ -157,7 +157,7 @@ def gitBranchExists(branch): return proc.wait() == 0; def gitConfig(key): - return mypopen("git config %s" % key).read()[:-1] + return read_pipe("git config %s" % key, ignore_error=True).strip() class Command: def __init__(self): @@ -168,7 +168,8 @@ class P4Debug(Command): def __init__(self): Command.__init__(self) self.options = [ - ] + optparse.make_option("--verbose", dest="verbose", action="store_true"), + ] self.description = "A tool to debug the output of p4 -G." self.needsGit = False @@ -234,6 +235,7 @@ class P4Submit(Command): Command.__init__(self) self.options = [ optparse.make_option("--continue", action="store_false", dest="firstTime"), + optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--origin", dest="origin"), optparse.make_option("--reset", action="store_true", dest="reset"), optparse.make_option("--log-substitutions", dest="substFile"), @@ -861,11 +863,12 @@ class P4Sync(Command): if not self.silent: print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix - for line in mypopen("git rev-parse --symbolic --remotes"): + for line in read_pipe_lines("git rev-parse --symbolic --remotes"): + line = line.strip() if (not line.startswith("origin/")) or line.endswith("HEAD\n"): continue - headName = line[len("origin/"):-1] + headName = line[len("origin/")] remoteHead = self.refPrefix + headName originHead = "origin/" + headName @@ -1292,6 +1295,7 @@ if len(options) > 0: (cmd, args) = parser.parse_args(sys.argv[2:], cmd); +verbose = cmd.verbose if cmd.needsGit: gitdir = cmd.gitdir if len(gitdir) == 0: From 6326aa58662932212678a25c6f482a8da4195ac9 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 208/260] Extract multiple paths concurrently. This enables importing just the interesting bits of large repositories. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 258 ++++++++++++++++++++++--------------- 1 file changed, 151 insertions(+), 107 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index cf9db73da6..ad145e8595 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -133,24 +133,26 @@ def extractLogMessageFromGitCommit(commit): logMessage += log return logMessage -def extractDepotPathAndChangeFromGitLog(log): +def extractDepotPathsAndChangeFromGitLog(log): values = {} for line in log.split("\n"): line = line.strip() - if line.startswith("[git-p4:") and line.endswith("]"): - line = line[8:-1].strip() - for assignment in line.split(":"): - variable = assignment.strip() - value = "" - equalPos = assignment.find("=") - if equalPos != -1: - variable = assignment[:equalPos].strip() - value = assignment[equalPos + 1:].strip() - if value.startswith("\"") and value.endswith("\""): - value = value[1:-1] - values[variable] = value + m = re.search (r"^ *\[git-p4: (.*)\]$", line) + if not m: + continue - return values.get("depot-path"), values.get("change") + assignments = m.group(1).split (':') + for a in assignments: + vals = a.split ('=') + key = vals[0].strip() + val = ('='.join (vals[1:])).strip() + if val.endswith ('\"') and val.startswith('"'): + val = val[1:-1] + + values[key] = val + + paths = values.get("depot-path").split(',') + return paths, values.get("change") def gitBranchExists(branch): proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE); @@ -209,10 +211,11 @@ class P4RollBack(Command): line = line.strip() ref = refPrefix + line log = extractLogMessageFromGitCommit(ref) - depotPath, change = extractDepotPathAndChangeFromGitLog(log) + depotPaths, change = extractDepotPathsAndChangeFromGitLog(log) changed = False - if len(p4Cmd("changes -m 1 %s...@%s" % (depotPath, maxChange))) == 0: + if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange) + for p in depotPaths]))) == 0: print "Branch %s did not exist at change %s, deleting." % (ref, maxChange) system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) continue @@ -223,7 +226,7 @@ class P4RollBack(Command): print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) system("git update-ref %s \"%s^\"" % (ref, ref)) log = extractLogMessageFromGitCommit(ref) - depotPath, change = extractDepotPathAndChangeFromGitLog(log) + depotPaths, change = extractDepotPathsAndChangeFromGitLog(log) if changed: print "%s rewound to %s" % (ref, change) @@ -472,9 +475,9 @@ class P4Submit(Command): depotPath = "" if gitBranchExists("p4"): - [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) + [depotPaths, dummy] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) if len(depotPath) == 0 and gitBranchExists("origin"): - [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) + [depotPaths, dummy] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" @@ -568,7 +571,7 @@ class P4Sync(Command): optparse.make_option("--verbose", dest="verbose", action="store_true"), optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), optparse.make_option("--max-changes", dest="maxChanges"), - optparse.make_option("--keep-path", dest="keepRepoPath") + optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true') ] self.description = """Imports from Perforce into a git repository.\n example: @@ -591,8 +594,8 @@ class P4Sync(Command): self.importIntoRemotes = True self.maxChanges = "" self.isWindows = (platform.system() == "Windows") - self.depotPath = None self.keepRepoPath = False + self.depotPaths = None if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False @@ -605,9 +608,10 @@ class P4Sync(Command): fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] - if not path.startswith(self.depotPath): - # if not self.silent: - # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) + + found = [p for p in self.depotPaths + if path.startswith (p)] + if not found: fnum = fnum + 1 continue @@ -620,20 +624,24 @@ class P4Sync(Command): fnum = fnum + 1 return files - def stripRepoPath(self, path, prefix): + def stripRepoPath(self, path, prefixes): if self.keepRepoPath: - prefix = re.sub("^(//[^/]+/).*", r'\1', prefix) + prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])] - return path[len(prefix):] + for p in prefixes: + if path.startswith(p): + path = path[len(p):] + + return path def splitFilesIntoBranches(self, commit): branches = {} fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] - if not path.startswith(self.depotPath): - # if not self.silent: - # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) + found = [p for p in self.depotPaths + if path.startswith (p)] + if not found: fnum = fnum + 1 continue @@ -644,7 +652,7 @@ class P4Sync(Command): file["type"] = commit["type%s" % fnum] fnum = fnum + 1 - relPath = self.stripRepoPath(path, self.depotPath) + relPath = self.stripRepoPath(path, self.depotPaths) for branch in self.knownBranches.keys(): @@ -656,7 +664,7 @@ class P4Sync(Command): return branches - def commit(self, details, files, branch, branchPrefix, parent = ""): + def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] author = details["user"] @@ -678,7 +686,8 @@ class P4Sync(Command): self.gitStream.write("data < 0: @@ -688,12 +697,13 @@ class P4Sync(Command): for file in files: path = file["path"] - if not path.startswith(branchPrefix): - # print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"]) + + + if not [p for p in branchPrefixes if path.startswith(p)]: continue rev = file["rev"] depotPath = path + "#" + rev - relPath = self.stripRepoPath(path, branchPrefix) + relPath = self.stripRepoPath(path, branchPrefixes) action = file["action"] if file["type"] == "apple": @@ -728,7 +738,8 @@ class P4Sync(Command): if self.verbose: print "Change %s is labelled %s" % (change, labelDetails) - files = p4CmdList("files %s...@%s" % (branchPrefix, change)) + files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change) + for p in branchPrefixes])) if len(files) == len(labelRevisions): @@ -795,9 +806,9 @@ class P4Sync(Command): def getLabels(self): self.labels = {} - l = p4CmdList("labels %s..." % self.depotPath) + l = p4CmdList("labels %s..." % ' '.join (self.depotPaths)) if len(l) > 0 and not self.silent: - print "Finding files belonging to labels in %s" % self.depotPath + print "Finding files belonging to labels in %s" % `self.depotPath` for output in l: label = output["label"] @@ -805,7 +816,9 @@ class P4Sync(Command): newestChange = 0 if self.verbose: print "Querying files for label %s" % label - for file in p4CmdList("files %s...@%s" % (self.depotPath, label)): + for file in p4CmdList("files " + + ' '.join (["%s...@%s" % (p, label) + for p in self.depotPaths])): revisions[file["depotFile"]] = file["rev"] change = int(file["change"]) if change > newestChange: @@ -817,6 +830,8 @@ class P4Sync(Command): print "Label changes: %s" % self.labels.keys() def getBranchMapping(self): + + ## FIXME - what's a P4 projectName ? self.projectName = self.depotPath[self.depotPath.strip().rfind("/") + 1:] for info in p4CmdList("branches"): @@ -872,8 +887,8 @@ class P4Sync(Command): remoteHead = self.refPrefix + headName originHead = "origin/" + headName - [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) - if len(originPreviousDepotPath) == 0 or len(originP4Change) == 0: + [originPreviousDepotPaths, originP4Change] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) + if len(originPreviousDepotPaths) == 0 or len(originP4Change) == 0: continue update = False @@ -882,25 +897,26 @@ class P4Sync(Command): print "creating %s" % remoteHead update = True else: - [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) + [p4PreviousDepotPaths, p4Change] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) if len(p4Change) > 0: - if originPreviousDepotPath == p4PreviousDepotPath: + if originPreviousDepotPaths == p4PreviousDepotPaths: originP4Change = int(originP4Change) p4Change = int(p4Change) if originP4Change > p4Change: print "%s (%s) is newer than %s (%s). Updating p4 branch from origin." % (originHead, originP4Change, remoteHead, p4Change) update = True else: - print "Ignoring: %s was imported from %s while %s was imported from %s" % (originHead, originPreviousDepotPath, remoteHead, p4PreviousDepotPath) + print "Ignoring: %s was imported from %s while %s was imported from %s" % (originHead, originPreviousDepotPaths, remoteHead, p4PreviousDepotPaths) if update: system("git update-ref %s %s" % (remoteHead, originHead)) + def run(self, args): - self.depotPath = "" + self.depotPaths = [] self.changeRange = "" self.initialParent = "" - self.previousDepotPath = "" + self.previousDepotPaths = [] # map from branch depot path to parent branch self.knownBranches = {} @@ -926,7 +942,7 @@ class P4Sync(Command): if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - if len(args) == 0: + if args == []: if self.hasOrigin: self.createOrUpdateBranchesFromOrigin() self.listExistingP4GitBranches() @@ -942,26 +958,31 @@ class P4Sync(Command): p4Change = 0 for branch in self.p4BranchesInGit: logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) - (depotPath, change) = extractDepotPathAndChangeFromGitLog(logMsg) + (depotPaths, change) = extractDepotPathsAndChangeFromGitLog(logMsg) if self.verbose: - print "path %s change %s" % (depotPath, change) + print "path %s change %s" % (','.join(depotPaths), change) - if len(depotPath) > 0 and len(change) > 0: + if len(depotPaths) > 0 and len(change) > 0: change = int(change) + 1 p4Change = max(p4Change, change) - if len(self.previousDepotPath) == 0: - self.previousDepotPath = depotPath + if len(self.previousDepotPaths) == 0: + self.previousDepotPaths = depotPaths else: - i = 0 - l = min(len(self.previousDepotPath), len(depotPath)) - while i < l and self.previousDepotPath[i] == depotPath[i]: - i = i + 1 - self.previousDepotPath = self.previousDepotPath[:i] + ## FIXME + paths = [] + for (prev, cur) in zip(self.previousDepotPaths, depotPaths): + for i in range(0, max(len(cur), len(prev))): + if cur[i] <> prev[i]: + break + + paths.append (cur[:i]) + + self.previousDepotPaths = paths if p4Change > 0: - self.depotPath = self.previousDepotPath + self.depotPaths = self.previousDepotPaths self.changeRange = "@%s,#head" % p4Change self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: @@ -970,43 +991,47 @@ class P4Sync(Command): if not self.branch.startswith("refs/"): self.branch = "refs/heads/" + self.branch - if len(self.depotPath) != 0: - self.depotPath = self.depotPath.strip() - - if len(args) == 0 and len(self.depotPath) != 0: + if len(args) == 0 and self.depotPaths: if not self.silent: - print "Depot path: %s" % self.depotPath - elif len(args) != 1: - return False + print "Depot paths: %s" % ' '.join(self.depotPaths) else: - if len(self.depotPath) != 0 and self.depotPath != args[0]: + if self.depotPaths and self.depotPaths != args: print ("previous import used depot path %s and now %s was specified. " - "This doesn't work!" % (self.depotPath, args[0])) + "This doesn't work!" % (' '.join (self.depotPaths), + ' '.join (args))) sys.exit(1) - self.depotPath = args[0] + + self.depotPaths = args self.revision = "" self.users = {} - if self.depotPath.find("@") != -1: - atIdx = self.depotPath.index("@") - self.changeRange = self.depotPath[atIdx:] - if self.changeRange == "@all": - self.changeRange = "" - elif self.changeRange.find(",") == -1: - self.revision = self.changeRange - self.changeRange = "" - self.depotPath = self.depotPath[0:atIdx] - elif self.depotPath.find("#") != -1: - hashIdx = self.depotPath.index("#") - self.revision = self.depotPath[hashIdx:] - self.depotPath = self.depotPath[0:hashIdx] - elif len(self.previousDepotPath) == 0: - self.revision = "#head" + newPaths = [] + for p in self.depotPaths: + if p.find("@") != -1: + atIdx = p.index("@") + self.changeRange = p[atIdx:] + if self.changeRange == "@all": + self.changeRange = "" + elif self.changeRange.find(",") == -1: + self.revision = self.changeRange + self.changeRange = "" + p = p[0:atIdx] + elif p.find("#") != -1: + hashIdx = p.index("#") + self.revision = p[hashIdx:] + p = p[0:hashIdx] + elif self.previousDepotPaths == []: + self.revision = "#head" + + p = re.sub ("\.\.\.$", "", p) + if not p.endswith("/"): + p += "/" + + newPaths.append(p) + + self.depotPaths = newPaths - self.depotPath = re.sub ("\.\.\.$", "", self.depotPath) - if not self.depotPath.endswith("/"): - self.depotPath += "/" self.loadUserMapFromCache() self.labels = {} @@ -1020,28 +1045,34 @@ class P4Sync(Command): print "initial parents: %s" % self.initialParents for b in self.p4BranchesInGit: if b != "master": + + ## FIXME b = b[len(self.projectName):] self.createdBranches.add(b) self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) importProcess = subprocess.Popen(["git", "fast-import"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE); self.gitOutput = importProcess.stdout self.gitStream = importProcess.stdin self.gitError = importProcess.stderr if len(self.revision) > 0: - print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision) + print "Doing initial import of %s from revision %s" % (' '.join(self.depotPaths), self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } details["desc"] = ("Initial import of %s from the state at revision %s" - % (self.depotPath, self.revision)) + % (' '.join(self.depotPaths), self.revision)) details["change"] = self.revision newestRevision = 0 fileCnt = 0 - for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)): + for info in p4CmdList("files " + + ' '.join(["%s...%s" + % (p, self.revision) + for p in self.depotPaths])): change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -1059,7 +1090,7 @@ class P4Sync(Command): details["change"] = newestRevision try: - self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath) + self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths) except IOError: print "IO error with git fast-import. Is your git version recent enough?" print self.gitError.read() @@ -1079,8 +1110,11 @@ class P4Sync(Command): changes.sort() else: if self.verbose: - print "Getting p4 changes for %s...%s" % (self.depotPath, self.changeRange) - output = read_pipe_lines("p4 changes %s...%s" % (self.depotPath, self.changeRange)) + print "Getting p4 changes for %s...%s" % (`self.depotPaths`, + self.changeRange) + assert self.depotPaths + output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, self.changeRange) + for p in self.depotPaths])) for line in output: changeNum = line.split(" ")[1] @@ -1111,7 +1145,8 @@ class P4Sync(Command): if self.detectBranches: branches = self.splitFilesIntoBranches(description) for branch in branches.keys(): - branchPrefix = self.depotPath + branch + "/" + ## HACK --hwn + branchPrefix = self.depotPaths[0] + branch + "/" parent = "" @@ -1134,11 +1169,14 @@ class P4Sync(Command): if branch == "main": branch = "master" else: + + ## FIXME branch = self.projectName + branch if parent == "main": parent = "master" elif len(parent) > 0: + ## FIXME parent = self.projectName + parent branch = self.refPrefix + branch @@ -1155,7 +1193,8 @@ class P4Sync(Command): self.commit(description, filesForCommit, branch, branchPrefix, parent) else: files = self.extractFilesFromCommit(description) - self.commit(description, files, self.branch, self.depotPath, self.initialParent) + self.commit(description, files, self.branch, self.depotPaths, + self.initialParent) self.initialParent = "" except IOError: print self.gitError.read() @@ -1206,30 +1245,35 @@ class P4Clone(P4Sync): if len(args) < 1: return False - depotPath = args[0] destination = "" - if len(args) == 2: + if self.keepRepoPath: + destination = args[-1] + args = args[:-1] + elif len(args) == 2: destination = args[1] elif len(args) > 2: return False - if not depotPath.startswith("//"): - return False - - depotDir = re.sub("(@[^@]*)$", "", depotPath) - depotDir = re.sub("(#[^#]*)$", "", depotDir) - depotDir = re.sub(r"\.\.\.$,", "", depotDir) - depotDir = re.sub(r"/$", "", depotDir) + depotPaths = args + for p in depotPaths: + if not p.startswith("//"): + return False if not destination: + depotPath = args[0] + depotDir = re.sub("(@[^@]*)$", "", depotPath) + depotDir = re.sub("(#[^#]*)$", "", depotDir) + depotDir = re.sub(r"\.\.\.$,", "", depotDir) + depotDir = re.sub(r"/$", "", depotDir) + destination = os.path.split(depotDir)[1] - print "Importing from %s into %s" % (depotPath, destination) + print "Importing from %s into %s" % (`depotPaths`, destination) os.makedirs(destination) os.chdir(destination) system("git init") gitdir = os.getcwd() + "/.git" - if not P4Sync.run(self, [depotPath]): + if not P4Sync.run(self, depotPaths): return False if self.branch != "master": if gitBranchExists("refs/remotes/p4/master"): From 9226c03c3279d66e2dea21a7a3e9f187e5ee9b9a Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 28 May 2007 19:23:19 +0200 Subject: [PATCH 209/260] In *_pipe print the command that failed if it fails. Fixed old calls to mypopen. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index efa2fce29e..0f1285b39b 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -24,7 +24,7 @@ def write_pipe(c, str): pipe = os.popen(c, 'w') val = pipe.write(str) if pipe.close(): - sys.stderr.write('Command failed') + sys.stderr.write('Command failed: %s' % c) sys.exit(1) return val @@ -36,7 +36,7 @@ def read_pipe(c): pipe = os.popen(c, 'rb') val = pipe.read() if pipe.close(): - sys.stderr.write('Command failed') + sys.stderr.write('Command failed: %s' % c) sys.exit(1) return val @@ -49,7 +49,7 @@ def read_pipe_lines(c): pipe = os.popen(c, 'rb') val = pipe.readlines() if pipe.close(): - sys.stderr.write('Command failed') + sys.stderr.write('Command failed: %s' % c) sys.exit(1) return val @@ -157,7 +157,7 @@ def gitBranchExists(branch): return proc.wait() == 0; def gitConfig(key): - return mypopen("git config %s" % key).read()[:-1] + return os.popen("git config %s" % key, "rb").read()[:-1] class Command: def __init__(self): @@ -861,7 +861,7 @@ class P4Sync(Command): if not self.silent: print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix - for line in mypopen("git rev-parse --symbolic --remotes"): + for line in read_pipe_lines("git rev-parse --symbolic --remotes"): if (not line.startswith("origin/")) or line.endswith("HEAD\n"): continue From cfeb59be256bf9cd2853ed04d4af056b2f0eff31 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 28 May 2007 19:24:57 +0200 Subject: [PATCH 210/260] Fix typo in listExistingP4Branches that broke sync. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 0f1285b39b..794286ee8e 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -846,8 +846,8 @@ class P4Sync(Command): cmdline += " --branches" for line in read_pipe_lines(cmdline): - lie = line.strip() - if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): + line = line.strip() + if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD"): continue if self.importIntoRemotes: From bb6e09b27afeaae780dabfda7a07d59fc7efc4cf Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 211/260] Diverse cleanups - print commands with \n - extractDepotPathsAndChangeFromGitLog -> extractSettings, returning dict. - store keepRepoPath in [git-p4: ] line - create a main() function, so git-p4 can be pychecked - use --destination for clone destination. This simplifies logic for --keep-path Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 216 ++++++++++++++++++++++--------------- 1 file changed, 130 insertions(+), 86 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ad145e8595..b280e97742 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -56,7 +56,7 @@ def read_pipe_lines(c): def system(cmd): if verbose: - sys.stderr.write("executing %s" % cmd) + sys.stderr.write("executing %s\n" % cmd) if os.system(cmd) != 0: die("command failed: %s" % cmd) @@ -112,7 +112,8 @@ def currentGitBranch(): return read_pipe("git name-rev HEAD").split(" ")[1].strip() def isValidGitDir(path): - if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): + if (os.path.exists(path + "/HEAD") + and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")): return True; return False @@ -133,7 +134,7 @@ def extractLogMessageFromGitCommit(commit): logMessage += log return logMessage -def extractDepotPathsAndChangeFromGitLog(log): +def extractSettingsGitLog(log): values = {} for line in log.split("\n"): line = line.strip() @@ -151,11 +152,12 @@ def extractDepotPathsAndChangeFromGitLog(log): values[key] = val - paths = values.get("depot-path").split(',') - return paths, values.get("change") + values['depot-paths'] = values.get("depot-paths").split(',') + return values def gitBranchExists(branch): - proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE); + proc = subprocess.Popen(["git", "rev-parse", branch], + stderr=subprocess.PIPE, stdout=subprocess.PIPE); return proc.wait() == 0; def gitConfig(key): @@ -211,7 +213,11 @@ class P4RollBack(Command): line = line.strip() ref = refPrefix + line log = extractLogMessageFromGitCommit(ref) - depotPaths, change = extractDepotPathsAndChangeFromGitLog(log) + settings = extractSettingsGitLog(log) + + depotPaths = settings['depot-paths'] + change = settings['change'] + changed = False if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange) @@ -220,13 +226,17 @@ class P4RollBack(Command): system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) continue - while len(change) > 0 and int(change) > maxChange: + while change and int(change) > maxChange: changed = True if self.verbose: print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) system("git update-ref %s \"%s^\"" % (ref, ref)) log = extractLogMessageFromGitCommit(ref) - depotPaths, change = extractDepotPathsAndChangeFromGitLog(log) + settings = extractSettingsGitLog(log) + + + depotPaths = settings['depot-paths'] + change = settings['change'] if changed: print "%s rewound to %s" % (ref, change) @@ -474,10 +484,12 @@ class P4Submit(Command): return False depotPath = "" + settings = None if gitBranchExists("p4"): - [depotPaths, dummy] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit("p4")) + settings = extractSettingsGitLog(extractLogMessageFromGitCommit("p4")) if len(depotPath) == 0 and gitBranchExists("origin"): - [depotPaths, dummy] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit("origin")) + settings = extractSettingsGitLog(extractLogMessageFromGitCommit("origin")) + depotPaths = settings['depot-paths'] if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" @@ -686,8 +698,11 @@ class P4Sync(Command): self.gitStream.write("data < 0: @@ -883,12 +898,13 @@ class P4Sync(Command): if (not line.startswith("origin/")) or line.endswith("HEAD\n"): continue - headName = line[len("origin/")] + headName = line[len("origin/"):] remoteHead = self.refPrefix + headName originHead = "origin/" + headName - [originPreviousDepotPaths, originP4Change] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) - if len(originPreviousDepotPaths) == 0 or len(originP4Change) == 0: + original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) + if (not original.has_key('depot-paths') + or not original.has_key('change')): continue update = False @@ -897,20 +913,36 @@ class P4Sync(Command): print "creating %s" % remoteHead update = True else: - [p4PreviousDepotPaths, p4Change] = extractDepotPathsAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) - if len(p4Change) > 0: - if originPreviousDepotPaths == p4PreviousDepotPaths: - originP4Change = int(originP4Change) - p4Change = int(p4Change) + settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) + if settings.has_key('change') > 0: + if settings['depot-paths'] == original['depot-paths']: + originP4Change = int(original['change']) + p4Change = int(settings['change']) if originP4Change > p4Change: - print "%s (%s) is newer than %s (%s). Updating p4 branch from origin." % (originHead, originP4Change, remoteHead, p4Change) + print ("%s (%s) is newer than %s (%s). " + "Updating p4 branch from origin." + % (originHead, originP4Change, + remoteHead, p4Change)) update = True else: - print "Ignoring: %s was imported from %s while %s was imported from %s" % (originHead, originPreviousDepotPaths, remoteHead, p4PreviousDepotPaths) + print ("Ignoring: %s was imported from %s while " + "%s was imported from %s" + % (originHead, ','.join(original['depot-paths']), + remoteHead, ','.join(settings['depot-paths']))) if update: system("git update-ref %s %s" % (remoteHead, originHead)) + def updateOptionDict(self, d): + option_keys = {} + if self.keepRepoPath: + option_keys['keepRepoPath'] = 1 + + d["options"] = ' '.join(sorted(option_keys.keys())) + + def readOptions(self, d): + self.keepRepoPath = (d.has_key('options') + and ('keepRepoPath' in d['options'])) def run(self, args): self.depotPaths = [] @@ -942,7 +974,8 @@ class P4Sync(Command): if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - if args == []: + ### FIXME + if 1: if self.hasOrigin: self.createOrUpdateBranchesFromOrigin() self.listExistingP4GitBranches() @@ -958,19 +991,22 @@ class P4Sync(Command): p4Change = 0 for branch in self.p4BranchesInGit: logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) - (depotPaths, change) = extractDepotPathsAndChangeFromGitLog(logMsg) + + settings = extractSettingsGitLog(logMsg) if self.verbose: print "path %s change %s" % (','.join(depotPaths), change) - if len(depotPaths) > 0 and len(change) > 0: - change = int(change) + 1 + self.readOptions(settings) + if (settings.has_key('depot-paths') + and settings.has_key ('change')): + change = int(settings['change']) + 1 p4Change = max(p4Change, change) - if len(self.previousDepotPaths) == 0: + depotPaths = sorted(settings['depot-paths']) + if self.previousDepotPaths == []: self.previousDepotPaths = depotPaths else: - ## FIXME paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): for i in range(0, max(len(cur), len(prev))): @@ -982,7 +1018,7 @@ class P4Sync(Command): self.previousDepotPaths = paths if p4Change > 0: - self.depotPaths = self.previousDepotPaths + self.depotPaths = sorted(self.previousDepotPaths) self.changeRange = "@%s,#head" % p4Change self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: @@ -1001,7 +1037,7 @@ class P4Sync(Command): ' '.join (args))) sys.exit(1) - self.depotPaths = args + self.depotPaths = sorted(args) self.revision = "" self.users = {} @@ -1088,7 +1124,7 @@ class P4Sync(Command): fileCnt = fileCnt + 1 details["change"] = newestRevision - + self.updateOptionDict(details) try: self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths) except IOError: @@ -1135,6 +1171,7 @@ class P4Sync(Command): cnt = 1 for change in changes: description = p4Cmd("describe %s" % change) + self.updateOptionDict(description) if not self.silent: sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) @@ -1237,7 +1274,12 @@ class P4Clone(P4Sync): def __init__(self): P4Sync.__init__(self) self.description = "Creates a new git repository and imports from Perforce into it" - self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]" + self.usage = "usage: %prog [options] //depot/path[@revRange]" + self.options.append( + optparse.make_option("--destination", dest="cloneDestination", + action='store', default=None, + help="where to leave result of the clone")) + self.cloneDestination = None self.needsGit = False def run(self, args): @@ -1245,32 +1287,28 @@ class P4Clone(P4Sync): if len(args) < 1: return False - destination = "" - if self.keepRepoPath: - destination = args[-1] - args = args[:-1] - elif len(args) == 2: - destination = args[1] - elif len(args) > 2: - return False + + if self.keepRepoPath and not self.cloneDestination: + sys.stderr.write("Must specify destination for --keep-path\n") + sys.exit(1) depotPaths = args for p in depotPaths: if not p.startswith("//"): return False - if not destination: + if not self.cloneDestination: depotPath = args[0] depotDir = re.sub("(@[^@]*)$", "", depotPath) depotDir = re.sub("(#[^#]*)$", "", depotDir) depotDir = re.sub(r"\.\.\.$,", "", depotDir) depotDir = re.sub(r"/$", "", depotDir) - destination = os.path.split(depotDir)[1] + self.cloneDestination = os.path.split(depotDir)[1] - print "Importing from %s into %s" % (`depotPaths`, destination) - os.makedirs(destination) - os.chdir(destination) + print "Importing from %s into %s" % (`depotPaths`, self.cloneDestination) + os.makedirs(self.cloneDestination) + os.chdir(self.cloneDestination) system("git init") gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, depotPaths): @@ -1310,54 +1348,60 @@ commands = { "rollback" : P4RollBack() } -if len(sys.argv[1:]) == 0: - printUsage(commands.keys()) - sys.exit(2) -cmd = "" -cmdName = sys.argv[1] -try: - cmd = commands[cmdName] -except KeyError: - print "unknown command %s" % cmdName - print "" - printUsage(commands.keys()) - sys.exit(2) +def main(): + if len(sys.argv[1:]) == 0: + printUsage(commands.keys()) + sys.exit(2) -options = cmd.options -cmd.gitdir = gitdir + cmd = "" + cmdName = sys.argv[1] + try: + cmd = commands[cmdName] + except KeyError: + print "unknown command %s" % cmdName + print "" + printUsage(commands.keys()) + sys.exit(2) -args = sys.argv[2:] + options = cmd.options + cmd.gitdir = gitdir -if len(options) > 0: - options.append(optparse.make_option("--git-dir", dest="gitdir")) + args = sys.argv[2:] - parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), - options, - description = cmd.description, - formatter = HelpFormatter()) + if len(options) > 0: + options.append(optparse.make_option("--git-dir", dest="gitdir")) - (cmd, args) = parser.parse_args(sys.argv[2:], cmd); + parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName), + options, + description = cmd.description, + formatter = HelpFormatter()) + + (cmd, args) = parser.parse_args(sys.argv[2:], cmd); + global verbose + verbose = cmd.verbose + if cmd.needsGit: + gitdir = cmd.gitdir + if len(gitdir) == 0: + gitdir = ".git" + if not isValidGitDir(gitdir): + gitdir = read_pipe("git rev-parse --git-dir").strip() + if os.path.exists(gitdir): + cdup = read_pipe("git rev-parse --show-cdup").strip() + if len(cdup) > 0: + os.chdir(cdup); -verbose = cmd.verbose -if cmd.needsGit: - gitdir = cmd.gitdir - if len(gitdir) == 0: - gitdir = ".git" if not isValidGitDir(gitdir): - gitdir = read_pipe("git rev-parse --git-dir").strip() - if os.path.exists(gitdir): - cdup = read_pipe("git rev-parse --show-cdup").strip() - if len(cdup) > 0: - os.chdir(cdup); + if isValidGitDir(gitdir + "/.git"): + gitdir += "/.git" + else: + die("fatal: cannot locate git repository at %s" % gitdir) - if not isValidGitDir(gitdir): - if isValidGitDir(gitdir + "/.git"): - gitdir += "/.git" - else: - die("fatal: cannot locate git repository at %s" % gitdir) + os.environ["GIT_DIR"] = gitdir - os.environ["GIT_DIR"] = gitdir + if not cmd.run(args): + parser.print_help() -if not cmd.run(args): - parser.print_help() + +if __name__ == '__main__': + main() From b86f73782eafd1c61bb706ec5ca3d1ec548d82f5 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 212/260] remove global .gitdir Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 53 +++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 28ea53dea4..6501387657 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -14,7 +14,6 @@ import re from sets import Set; -gitdir = os.environ.get("GIT_DIR", "") verbose = False def write_pipe(c, str): @@ -469,9 +468,7 @@ class P4Submit(Command): % (fileName, fileName)) def run(self, args): - global gitdir # make gitdir absolute so we can cd out into the perforce checkout - gitdir = os.path.abspath(gitdir) os.environ["GIT_DIR"] = gitdir if len(args) == 0: @@ -510,7 +507,7 @@ class P4Submit(Command): print "No changes in working directory to submit." return True patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD") - self.diffFile = gitdir + "/p4-git-diff" + self.diffFile = self.gitdir + "/p4-git-diff" f = open(self.diffFile, "wb") f.write(patch) f.close(); @@ -535,7 +532,7 @@ class P4Submit(Command): self.logSubstitutions[tokens[0]] = tokens[1] self.check() - self.configFile = gitdir + "/p4-git-sync.cfg" + self.configFile = self.gitdir + "/p4-git-sync.cfg" self.config = shelve.open(self.configFile, writeback=True) if self.firstTime: @@ -799,7 +796,7 @@ class P4Sync(Command): continue self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" - cache = open(gitdir + "/p4-usercache.txt", "wb") + cache = open(self.gitdir + "/p4-usercache.txt", "wb") for user in self.users.keys(): cache.write("%s\t%s\n" % (user, self.users[user])) cache.close(); @@ -809,7 +806,7 @@ class P4Sync(Command): self.users = {} self.userMapFromPerforceServer = False try: - cache = open(gitdir + "/p4-usercache.txt", "rb") + cache = open(self.gitdir + "/p4-usercache.txt", "rb") lines = cache.readlines() cache.close() for line in lines: @@ -1283,8 +1280,6 @@ class P4Clone(P4Sync): self.needsGit = False def run(self, args): - global gitdir - if len(args) < 1: return False @@ -1310,7 +1305,7 @@ class P4Clone(P4Sync): os.makedirs(self.cloneDestination) os.chdir(self.cloneDestination) system("git init") - gitdir = os.getcwd() + "/.git" + self.gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, depotPaths): return False if self.branch != "master": @@ -1340,12 +1335,12 @@ def printUsage(commands): print "" commands = { - "debug" : P4Debug(), - "submit" : P4Submit(), - "sync" : P4Sync(), - "rebase" : P4Rebase(), - "clone" : P4Clone(), - "rollback" : P4RollBack() + "debug" : P4Debug, + "submit" : P4Submit, + "sync" : P4Sync, + "rebase" : P4Rebase, + "clone" : P4Clone, + "rollback" : P4RollBack } @@ -1357,7 +1352,8 @@ def main(): cmd = "" cmdName = sys.argv[1] try: - cmd = commands[cmdName] + klass = commands[cmdName] + cmd = klass() except KeyError: print "unknown command %s" % cmdName print "" @@ -1365,7 +1361,7 @@ def main(): sys.exit(2) options = cmd.options - cmd.gitdir = gitdir + cmd.gitdir = os.environ.get("GIT_DIR", None) args = sys.argv[2:] @@ -1381,23 +1377,22 @@ def main(): global verbose verbose = cmd.verbose if cmd.needsGit: - gitdir = cmd.gitdir - if len(gitdir) == 0: - gitdir = ".git" - if not isValidGitDir(gitdir): - gitdir = read_pipe("git rev-parse --git-dir").strip() - if os.path.exists(gitdir): + if cmd.gitdir == None: + cmd.gitdir = os.path.abspath(".git") + if not isValidGitDir(cmd.gitdir): + cmd.gitdir = read_pipe("git rev-parse --git-dir").strip() + if os.path.exists(cmd.gitdir): cdup = read_pipe("git rev-parse --show-cdup").strip() if len(cdup) > 0: os.chdir(cdup); - if not isValidGitDir(gitdir): - if isValidGitDir(gitdir + "/.git"): - gitdir += "/.git" + if not isValidGitDir(cmd.gitdir): + if isValidGitDir(cmd.gitdir + "/.git"): + cmd.gitdir += "/.git" else: - die("fatal: cannot locate git repository at %s" % gitdir) + die("fatal: cannot locate git repository at %s" % cmd.gitdir) - os.environ["GIT_DIR"] = gitdir + os.environ["GIT_DIR"] = cmd.gitdir if not cmd.run(args): parser.print_help() From 6a49f8e2e04317175060576d85a5d2062ebb43a4 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 213/260] Read p4 files in one batch. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 91 +++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6501387657..8a3c53eb8c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -61,6 +61,8 @@ def system(cmd): def p4CmdList(cmd): cmd = "p4 -G %s" % cmd + if verbose: + sys.stderr.write("Opening pipe: %s\n" % cmd) pipe = os.popen(cmd, "rb") result = [] @@ -609,9 +611,6 @@ class P4Sync(Command): if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False - def p4File(self, depotPath): - return read_pipe("p4 print -q \"%s\"" % depotPath) - def extractFilesFromCommit(self, commit): files = [] fnum = 0 @@ -673,6 +672,39 @@ class P4Sync(Command): return branches + ## Should move this out, doesn't use SELF. + def readP4Files(self, files): + specs = [(f['path'] + "#" + f['rev'], f) for f in files] + + data = read_pipe('p4 print %s' % ' '.join(['"%s"' % spec + for (spec, info) in specs])) + + idx = 0 + for j in range(0, len(specs)): + filespec, info = specs[j] + + assert idx < len(data) + if data[idx:idx + len(filespec)] != filespec: + assert False + idx = data.find ('\n', idx) + assert idx > 0 + idx += 1 + + start = idx + + end = -1 + if j < len(specs)-1: + next_spec, next_info = specs[j+1] + end = data.find(next_spec, start) + + assert end >= 0 + else: + end = len(specs) + + + info['data'] = data[start:end] + idx = end + def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] author = details["user"] @@ -681,7 +713,7 @@ class P4Sync(Command): print "commit into %s" % branch self.gitStream.write("commit %s\n" % branch) - # gitStream.write("mark :%s\n" % details["change"]) +# gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) committer = "" if author not in self.users: @@ -707,29 +739,30 @@ class P4Sync(Command): print "parent %s" % parent self.gitStream.write("from %s\n" % parent) + + new_files = [] + for f in files: + if [p for p in branchPrefixes if f['path'].startswith(p)]: + new_files.append (f) + else: + sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) + files = new_files + + self.readP4Files(files) for file in files: - path = file["path"] - - - if not [p for p in branchPrefixes if path.startswith(p)]: - continue - rev = file["rev"] - depotPath = path + "#" + rev - relPath = self.stripRepoPath(path, branchPrefixes) - action = file["action"] - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring!" % path + print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path'] continue - if action == "delete": + relPath = self.stripRepoPath(file['path'], branchPrefixes) + if file["action"] == "delete": self.gitStream.write("D %s\n" % relPath) else: mode = 644 if file["type"].startswith("x"): mode = 755 - data = self.p4File(depotPath) + data = file['data'] if self.isWindows and file["type"].endswith("text"): data = data.replace("\r\n", "\n") @@ -971,8 +1004,9 @@ class P4Sync(Command): if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - ### FIXME - if 1: + # TODO: should always look at previous commits, + # merge with previous imports, if possible. + if args == []: if self.hasOrigin: self.createOrUpdateBranchesFromOrigin() self.listExistingP4GitBranches() @@ -1046,7 +1080,7 @@ class P4Sync(Command): self.changeRange = p[atIdx:] if self.changeRange == "@all": self.changeRange = "" - elif self.changeRange.find(",") == -1: + elif ',' not in self.changeRange: self.revision = self.changeRange self.changeRange = "" p = p[0:atIdx] @@ -1279,6 +1313,15 @@ class P4Clone(P4Sync): self.cloneDestination = None self.needsGit = False + def defaultDestination(self, args): + ## TODO: use common prefix of args? + depotPath = args[0] + depotDir = re.sub("(@[^@]*)$", "", depotPath) + depotDir = re.sub("(#[^#]*)$", "", depotDir) + depotDir = re.sub(r"\.\.\.$,", "", depotDir) + depotDir = re.sub(r"/$", "", depotDir) + return os.path.split(depotDir)[1] + def run(self, args): if len(args) < 1: return False @@ -1293,13 +1336,7 @@ class P4Clone(P4Sync): return False if not self.cloneDestination: - depotPath = args[0] - depotDir = re.sub("(@[^@]*)$", "", depotPath) - depotDir = re.sub("(#[^#]*)$", "", depotDir) - depotDir = re.sub(r"\.\.\.$,", "", depotDir) - depotDir = re.sub(r"/$", "", depotDir) - - self.cloneDestination = os.path.split(depotDir)[1] + self.cloneDestination = self.defaultDestination() print "Importing from %s into %s" % (`depotPaths`, self.cloneDestination) os.makedirs(self.cloneDestination) From 9320da8dd492c13f1d32b7fde8c9e60bdbd10217 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 214/260] Thinko, fix buglet. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 8a3c53eb8c..f1f562fae4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -699,11 +699,11 @@ class P4Sync(Command): assert end >= 0 else: - end = len(specs) - + end = len(data) info['data'] = data[start:end] idx = end + assert idx == len(data) def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] From 183b8ef89be041cd50803427cfc46fe1afed17bb Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 215/260] store p4 user cache in home directory. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index f1f562fae4..bd1afb2964 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -819,6 +819,9 @@ class P4Sync(Command): print ("Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)) + def getUserCacheFilename(self): + return os.environ["HOME"] + "/.gitp4-usercache.txt") + def getUserMapFromPerforceServer(self): if self.userMapFromPerforceServer: return @@ -829,17 +832,19 @@ class P4Sync(Command): continue self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" - cache = open(self.gitdir + "/p4-usercache.txt", "wb") - for user in self.users.keys(): - cache.write("%s\t%s\n" % (user, self.users[user])) - cache.close(); + + s = '' + for (key, val) in self.users.items(): + s += "%s\t%s\n" % (key, val) + + open(self.getUserCacheFilename(), "wb").write(s) self.userMapFromPerforceServer = True def loadUserMapFromCache(self): self.users = {} self.userMapFromPerforceServer = False try: - cache = open(self.gitdir + "/p4-usercache.txt", "rb") + cache = open(self.getUserCacheFilename(), "rb") lines = cache.readlines() cache.close() for line in lines: From a3287be5bc40c1aa036eb5422db5a6a087d1736e Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 216/260] thinko. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index bd1afb2964..01efd92809 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -820,7 +820,7 @@ class P4Sync(Command): % (labelDetails["label"], change)) def getUserCacheFilename(self): - return os.environ["HOME"] + "/.gitp4-usercache.txt") + return os.environ["HOME"] + "/.gitp4-usercache.txt" def getUserMapFromPerforceServer(self): if self.userMapFromPerforceServer: From 96e07dd23c570fe5e65e619c0f1e3c87be2a8352 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 217/260] read files before creating the commit. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 01efd92809..d1989e6134 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -697,6 +697,9 @@ class P4Sync(Command): next_spec, next_info = specs[j+1] end = data.find(next_spec, start) + if end < 0: + print spec, next_spec + assert end >= 0 else: end = len(data) @@ -712,6 +715,20 @@ class P4Sync(Command): if self.verbose: print "commit into %s" % branch + # start with reading files; if that fails, we should not + # create a commit. + new_files = [] + for f in files: + if [p for p in branchPrefixes if f['path'].startswith(p)]: + new_files.append (f) + else: + sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) + files = new_files + self.readP4Files(files) + + + + self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) @@ -739,16 +756,6 @@ class P4Sync(Command): print "parent %s" % parent self.gitStream.write("from %s\n" % parent) - - new_files = [] - for f in files: - if [p for p in branchPrefixes if f['path'].startswith(p)]: - new_files.append (f) - else: - sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) - files = new_files - - self.readP4Files(files) for file in files: if file["type"] == "apple": print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path'] @@ -1030,9 +1037,6 @@ class P4Sync(Command): settings = extractSettingsGitLog(logMsg) - if self.verbose: - print "path %s change %s" % (','.join(depotPaths), change) - self.readOptions(settings) if (settings.has_key('depot-paths') and settings.has_key ('change')): @@ -1145,6 +1149,9 @@ class P4Sync(Command): + ' '.join(["%s...%s" % (p, self.revision) for p in self.depotPaths])): + + if not info.has_key("change"): + print info change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -1154,7 +1161,7 @@ class P4Sync(Command): #fileCnt = fileCnt + 1 continue - for prop in [ "depotFile", "rev", "action", "type" ]: + for prop in ["depotFile", "rev", "action", "type" ]: details["%s%s" % (prop, fileCnt)] = info[prop] fileCnt = fileCnt + 1 From 982bb8a30376d0024a1794426e6e2291a7a21294 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 218/260] don't p4 print deleted files. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d1989e6134..63d7a4c995 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -674,17 +674,18 @@ class P4Sync(Command): ## Should move this out, doesn't use SELF. def readP4Files(self, files): - specs = [(f['path'] + "#" + f['rev'], f) for f in files] + specs = [(f['path'] + "#" + f['rev'], f) for f in files + if f['action'] != 'delete'] - data = read_pipe('p4 print %s' % ' '.join(['"%s"' % spec - for (spec, info) in specs])) + data = read_pipe('p4 print %s' % ' '.join(['"%s"' % path + for (path, info) in specs])) idx = 0 for j in range(0, len(specs)): - filespec, info = specs[j] + (pathrev, info) = specs[j] assert idx < len(data) - if data[idx:idx + len(filespec)] != filespec: + if data[idx:idx + len(pathrev)] != pathrev: assert False idx = data.find ('\n', idx) assert idx > 0 @@ -694,11 +695,15 @@ class P4Sync(Command): end = -1 if j < len(specs)-1: - next_spec, next_info = specs[j+1] - end = data.find(next_spec, start) + (next_pathrev, next_info) = specs[j+1] + end = data.find(next_pathrev, start) if end < 0: - print spec, next_spec + print 'j' + print 'PATHREV', pathrev, specs[j] + print 'nextpathrev', next_pathrev, specs[j+1] + print 'start', start, len(data) + print 'end', end assert end >= 0 else: From f2eda79f6967363f9377ef3b137a35d0c86aca2c Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 219/260] only run p4 print if necessary Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 63d7a4c995..76bbe3fdbf 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -677,6 +677,9 @@ class P4Sync(Command): specs = [(f['path'] + "#" + f['rev'], f) for f in files if f['action'] != 'delete'] + if not specs: + return + data = read_pipe('p4 print %s' % ' '.join(['"%s"' % path for (path, info) in specs])) From d2c6dd30eff648aef30d003ef327ab9415f86db0 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 220/260] use p4CmdList() to get file contents in Python dicts. This is more robust. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 56 ++++++++++++++------------------------ 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 76bbe3fdbf..1d799708f6 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -580,7 +580,8 @@ class P4Sync(Command): optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), + optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false", + help="Import into refs/heads/ , not refs/remotes"), optparse.make_option("--max-changes", dest="maxChanges"), optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true') ] @@ -680,41 +681,22 @@ class P4Sync(Command): if not specs: return - data = read_pipe('p4 print %s' % ' '.join(['"%s"' % path - for (path, info) in specs])) + filedata = p4CmdList('print %s' % ' '.join(['"%s"' % path + for (path, info) in specs])) - idx = 0 - for j in range(0, len(specs)): - (pathrev, info) = specs[j] + j = 0; + contents = {} + while filedata[j:]: + stat = filedata[j] + text = filedata[j+1] + j += 2 - assert idx < len(data) - if data[idx:idx + len(pathrev)] != pathrev: - assert False - idx = data.find ('\n', idx) - assert idx > 0 - idx += 1 + assert stat['code'] == 'stat' and text['code'] == 'text' + contents[stat['depotFile']] = text['data'] - start = idx - - end = -1 - if j < len(specs)-1: - (next_pathrev, next_info) = specs[j+1] - end = data.find(next_pathrev, start) - - if end < 0: - print 'j' - print 'PATHREV', pathrev, specs[j] - print 'nextpathrev', next_pathrev, specs[j+1] - print 'start', start, len(data) - print 'end', end - - assert end >= 0 - else: - end = len(data) - - info['data'] = data[start:end] - idx = end - assert idx == len(data) + for f in files: + assert not f.has_key('data') + f['data'] = contents[f['path']] def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] @@ -1158,8 +1140,12 @@ class P4Sync(Command): % (p, self.revision) for p in self.depotPaths])): - if not info.has_key("change"): - print info + if info['code'] == 'error': + sys.stderr.write("p4 returned an error: %s\n" + % info['data']) + sys.exit(1) + + change = int(info["change"]) if change > newestRevision: newestRevision = change From 86dff6b6762149d5f3d4b44bb57f58a8399a33ee Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 221/260] Cleanups & import into p4/master for local import - import into master/local if --import-local is set - use Die() for exiting - if --verbose is set, raise Exception() - use joined strings iso. `list` for progress printing Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 54 ++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 1d799708f6..cc0b7013da 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -16,40 +16,44 @@ from sets import Set; verbose = False +def die(msg): + if verbose: + raise Exception(msg) + else: + sys.stderr.write(msg + "\n") + sys.exit(1) + def write_pipe(c, str): if verbose: - sys.stderr.write('writing pipe: %s\n' % c) + sys.stderr.write('Writing pipe: %s\n' % c) pipe = os.popen(c, 'w') val = pipe.write(str) if pipe.close(): - sys.stderr.write('Command failed: %s' % c) - sys.exit(1) + die('Command failed: %s' % c) return val def read_pipe(c, ignore_error=False): if verbose: - sys.stderr.write('reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % c) pipe = os.popen(c, 'rb') val = pipe.read() if pipe.close() and not ignore_error: - sys.stderr.write('Command failed: %s\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val def read_pipe_lines(c): if verbose: - sys.stderr.write('reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % c) ## todo: check return status pipe = os.popen(c, 'rb') val = pipe.readlines() if pipe.close(): - sys.stderr.write('Command failed: %s\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val @@ -105,10 +109,6 @@ def p4Where(depotPath): clientPath = clientPath[:-3] return clientPath -def die(msg): - sys.stderr.write(msg + "\n") - sys.exit(1) - def currentGitBranch(): return read_pipe("git name-rev HEAD").split(" ")[1].strip() @@ -583,7 +583,8 @@ class P4Sync(Command): optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false", help="Import into refs/heads/ , not refs/remotes"), optparse.make_option("--max-changes", dest="maxChanges"), - optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true') + optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true', + help="Keep entire BRANCH/DIR/SUBDIR prefix during import") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -876,10 +877,14 @@ class P4Sync(Command): if self.verbose: print "Label changes: %s" % self.labels.keys() + def guessProjectName(self): + for p in self.depotPaths: + return p [p.strip().rfind("/") + 1:] + def getBranchMapping(self): ## FIXME - what's a P4 projectName ? - self.projectName = self.depotPath[self.depotPath.strip().rfind("/") + 1:] + self.projectName = self.guessProjectName() for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) @@ -911,9 +916,11 @@ class P4Sync(Command): for line in read_pipe_lines(cmdline): line = line.strip() - if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD"): - continue + ## only import to p4/ + if not line.startswith('p4/'): + continue + branch = line if self.importIntoRemotes: # strip off p4 branch = re.sub ("^p4/", "", line) @@ -923,7 +930,8 @@ class P4Sync(Command): def createOrUpdateBranchesFromOrigin(self): if not self.silent: - print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix + print ("Creating/updating branch(es) in %s based on origin branch(es)" + % self.refPrefix) for line in read_pipe_lines("git rev-parse --symbolic --remotes"): line = line.strip() @@ -998,7 +1006,7 @@ class P4Sync(Command): system("git fetch origin") if len(self.branch) == 0: - self.branch = self.refPrefix + "master" + self.branch = self.refPrefix + "p4/master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: system("git update-ref %s refs/heads/p4" % self.branch) system("git branch -D p4"); @@ -1023,6 +1031,7 @@ class P4Sync(Command): p4Change = 0 for branch in self.p4BranchesInGit: + print self.p4BranchesInGit logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) settings = extractSettingsGitLog(logMsg) @@ -1125,7 +1134,7 @@ class P4Sync(Command): self.gitStream = importProcess.stdin self.gitError = importProcess.stderr - if len(self.revision) > 0: + if self.revision: print "Doing initial import of %s from revision %s" % (' '.join(self.depotPaths), self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } @@ -1183,7 +1192,7 @@ class P4Sync(Command): changes.sort() else: if self.verbose: - print "Getting p4 changes for %s...%s" % (`self.depotPaths`, + print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), self.changeRange) assert self.depotPaths output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, self.changeRange) @@ -1344,7 +1353,7 @@ class P4Clone(P4Sync): if not self.cloneDestination: self.cloneDestination = self.defaultDestination() - print "Importing from %s into %s" % (`depotPaths`, self.cloneDestination) + print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) os.makedirs(self.cloneDestination) os.chdir(self.cloneDestination) system("git init") @@ -1357,6 +1366,7 @@ class P4Clone(P4Sync): system("git checkout -f") else: print "Could not detect main branch. No checkout/master branch created." + return True class HelpFormatter(optparse.IndentedHelpFormatter): From b17f88b5445210c2223b0642917c43581c767761 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 222/260] remove debug print Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index cc0b7013da..5dae1f19ae 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1031,7 +1031,6 @@ class P4Sync(Command): p4Change = 0 for branch in self.p4BranchesInGit: - print self.p4BranchesInGit logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch) settings = extractSettingsGitLog(logMsg) From b1ce94472684957cd5aba759f495a889f154a9a2 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 223/260] thinko: really ignore deleted files. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 5dae1f19ae..294c2ecfca 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -173,13 +173,18 @@ class P4Debug(Command): def __init__(self): Command.__init__(self) self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--verbose", dest="verbose", action="store_true", + default=False), ] self.description = "A tool to debug the output of p4 -G." self.needsGit = False + self.verbose = False def run(self, args): + j = 0 for output in p4CmdList(" ".join(args)): + print 'Element: %d' % j + j += 1 print output return True @@ -676,24 +681,27 @@ class P4Sync(Command): ## Should move this out, doesn't use SELF. def readP4Files(self, files): - specs = [(f['path'] + "#" + f['rev'], f) for f in files + files = [f for f in files if f['action'] != 'delete'] - if not specs: + if not files: return - filedata = p4CmdList('print %s' % ' '.join(['"%s"' % path - for (path, info) in specs])) + filedata = p4CmdList('print %s' % ' '.join(['"%s#%s"' % (f['path'], + f['rev']) + for f in files])) j = 0; contents = {} - while filedata[j:]: + while j < len(filedata): stat = filedata[j] - text = filedata[j+1] - j += 2 + j += 1 + text = '' + while j < len(filedata) and filedata[j]['code'] == 'text': + text += filedata[j]['data'] + j += 1 - assert stat['code'] == 'stat' and text['code'] == 'text' - contents[stat['depotFile']] = text['data'] + contents[stat['depotFile']] = text for f in files: assert not f.has_key('data') From 7530a40ce2006082580865f4db6d32b956ca8dc0 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 224/260] look for 'text' and 'binary' files. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 294c2ecfca..e955ad4f44 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -697,7 +697,8 @@ class P4Sync(Command): stat = filedata[j] j += 1 text = '' - while j < len(filedata) and filedata[j]['code'] == 'text': + while j < len(filedata) and filedata[j]['code'] in ('text', + 'binary'): text += filedata[j]['data'] j += 1 @@ -773,7 +774,7 @@ class P4Sync(Command): if self.isWindows and file["type"].endswith("text"): data = data.replace("\r\n", "\n") - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + self.gitStream.write("M %d inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) self.gitStream.write("\n") From 845b42cb6c1c1e80c4283234ea2162cae809bd50 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 09:19:34 +0200 Subject: [PATCH 225/260] Fix support for "depot-path" in older git-p4 imports Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e955ad4f44..e576f2dd9e 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -153,7 +153,10 @@ def extractSettingsGitLog(log): values[key] = val - values['depot-paths'] = values.get("depot-paths").split(',') + paths = values.get("depot-paths") + if not paths: + paths = values.get("depot-path") + values['depot-paths'] = paths.split(',') return values def gitBranchExists(branch): From 583e170706f5d69770d0de220286f8f0f73667f3 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 09:37:13 +0200 Subject: [PATCH 226/260] Fix common path "calculation" from logs of multiple branches. Need to use min instead of max for prev/cur to avoid out-of-bounds string access. Also treat "i" as index of the last match instead of a length because in case of a complete match of the two strings i was off by one. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e576f2dd9e..ba34f0a61f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1059,11 +1059,12 @@ class P4Sync(Command): else: paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, max(len(cur), len(prev))): + for i in range(0, min(len(cur), len(prev))): if cur[i] <> prev[i]: + i = i - 1 break - paths.append (cur[:i]) + paths.append (cur[:i + 1]) self.previousDepotPaths = paths From 330f53b8d679ece8363a73ed6f6e211d99cd5fdf Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 09:39:51 +0200 Subject: [PATCH 227/260] Don't attempt to set the initialParent on multi-branch imports (useless). At some point the code paths should be unified, but for now I need a working git-p4 :) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ba34f0a61f..ff737d762c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1071,7 +1071,8 @@ class P4Sync(Command): if p4Change > 0: self.depotPaths = sorted(self.previousDepotPaths) self.changeRange = "@%s,#head" % p4Change - self.initialParent = parseRevision(self.branch) + if not self.detectBranches: + self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch From 6509e19cd1f6b5620d339a2be35b8a160ab30794 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 09:41:53 +0200 Subject: [PATCH 228/260] Hack to make the multi-branch import work again with self.depotPaths now that self.depotPath is gone Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ff737d762c..ececc44518 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -909,9 +909,10 @@ class P4Sync(Command): continue source = paths[0] destination = paths[1] - if source.startswith(self.depotPath) and destination.startswith(self.depotPath): - source = source[len(self.depotPath):-4] - destination = destination[len(self.depotPath):-4] + ## HACK + if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]): + source = source[len(self.depotPaths[0]):-4] + destination = destination[len(self.depotPaths[0]):-4] if destination not in self.knownBranches: self.knownBranches[destination] = source if source not in self.knownBranches: From 68c42153060cab9ea6c16256febcf736a2a710d9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 12:51:03 +0200 Subject: [PATCH 229/260] Fix git-p4 rebase Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ececc44518..50d92c0f42 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1320,6 +1320,7 @@ class P4Rebase(Command): self.options = [ ] self.description = ("Fetches the latest revision from perforce and " + "rebases the current work (branch) against it") + self.verbose = False def run(self, args): sync = P4Sync() From b0d10df77a1130bdf3b4f59fdc18312432b90823 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 13:09:14 +0200 Subject: [PATCH 230/260] Fix git-p4 submit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 50d92c0f42..8be0afe828 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -276,6 +276,7 @@ class P4Submit(Command): self.origin = "" self.directSubmit = False self.trustMeLikeAFool = False + self.verbose = False self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -478,9 +479,6 @@ class P4Submit(Command): % (fileName, fileName)) def run(self, args): - # make gitdir absolute so we can cd out into the perforce checkout - os.environ["GIT_DIR"] = gitdir - if len(args) == 0: self.master = currentGitBranch() if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): From a52d5c7bc027248fca472a402882586a9eaf59bf Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 13:10:20 +0200 Subject: [PATCH 231/260] Fix depot-path determination for git-p4 submit Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 8be0afe828..8b00e350ec 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -494,7 +494,7 @@ class P4Submit(Command): settings = extractSettingsGitLog(extractLogMessageFromGitCommit("p4")) if len(depotPath) == 0 and gitBranchExists("origin"): settings = extractSettingsGitLog(extractLogMessageFromGitCommit("origin")) - depotPaths = settings['depot-paths'] + depotPath = settings['depot-paths'][0] if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" From f7baba8b092bdbc31196de1095c7779b633e28b1 Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Thu, 7 Jun 2007 14:07:01 +0200 Subject: [PATCH 232/260] Ensure that the commit message is Windows formated (CRLF) before invoking the editor. (The default editor on Windows (Notepad) doesn't handle Unix line endings) Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 8b00e350ec..fc4e7d26f0 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -277,6 +277,7 @@ class P4Submit(Command): self.directSubmit = False self.trustMeLikeAFool = False self.verbose = False + self.isWindows = (platform.system() == "Windows") self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -398,6 +399,8 @@ class P4Submit(Command): if not self.directSubmit: logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.replace("\n", "\n\t") + if self.isWindows: + logMessage = logMessage.replace("\n", "\r\n") logMessage = logMessage.strip() template = read_pipe("p4 change -o") @@ -444,6 +447,8 @@ class P4Submit(Command): tmpFile.close() os.remove(fileName) submitTemplate = message[:message.index(separatorLine)] + if self.isWindows: + submitTemplate = submitTemplate.replace("\r\n", "\n") if response == "y" or response == "yes": if self.dryRun: From 98ad4faf95c1e98d6f1aaccda6d96a00a3cddeed Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Thu, 7 Jun 2007 15:08:33 +0200 Subject: [PATCH 233/260] Fix git-p4 clone (defaultDestination) Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index fc4e7d26f0..89581eacaa 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1369,7 +1369,7 @@ class P4Clone(P4Sync): return False if not self.cloneDestination: - self.cloneDestination = self.defaultDestination() + self.cloneDestination = self.defaultDestination(args) print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) os.makedirs(self.cloneDestination) From db775559c2d866de895cc36532ddff6b1ebbeda9 Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Thu, 7 Jun 2007 15:13:59 +0200 Subject: [PATCH 234/260] Fix single branch import into remotes Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 89581eacaa..ad023f203c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1014,7 +1014,7 @@ class P4Sync(Command): if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" else: - self.refPrefix = "refs/heads/" + self.refPrefix = "refs/heads/p4/" if self.syncWithOrigin and self.hasOrigin: if not self.silent: @@ -1022,7 +1022,7 @@ class P4Sync(Command): system("git fetch origin") if len(self.branch) == 0: - self.branch = self.refPrefix + "p4/master" + self.branch = self.refPrefix + "master" if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: system("git update-ref %s refs/heads/p4" % self.branch) system("git branch -D p4"); From c4b33253c221d928f3edde71123a44765495b31a Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Thu, 7 Jun 2007 15:28:04 +0200 Subject: [PATCH 235/260] Exclude the HEAD symbolic ref from the list of known branches Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ad023f203c..965b391cda 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -934,7 +934,7 @@ class P4Sync(Command): line = line.strip() ## only import to p4/ - if not line.startswith('p4/'): + if not line.startswith('p4/') or line == "p4/HEAD": continue branch = line if self.importIntoRemotes: From 5e100b5cd7210f8054fd3464872c0366abc3c1dc Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 21:12:25 +0200 Subject: [PATCH 236/260] Make clone behave like git clone by default again. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 965b391cda..3fe7ae77a8 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1364,6 +1364,11 @@ class P4Clone(P4Sync): sys.exit(1) depotPaths = args + + if not self.cloneDestination and len(depotPaths) > 1: + self.cloneDestination = depotPaths[-1] + depotPaths = depotPaths[:-1] + for p in depotPaths: if not p.startswith("//"): return False From a3fdd57901bfe1014c4a48e13815d80f1f0e8577 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Thu, 7 Jun 2007 22:54:32 +0200 Subject: [PATCH 237/260] Make git-p4 submit detect the correct reference (origin) branch when working with multi-branch imports. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 3fe7ae77a8..efec0be32c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -156,7 +156,8 @@ def extractSettingsGitLog(log): paths = values.get("depot-paths") if not paths: paths = values.get("depot-path") - values['depot-paths'] = paths.split(',') + if paths: + values['depot-paths'] = paths.split(',') return values def gitBranchExists(branch): @@ -494,12 +495,27 @@ class P4Submit(Command): return False depotPath = "" - settings = None - if gitBranchExists("p4"): - settings = extractSettingsGitLog(extractLogMessageFromGitCommit("p4")) - if len(depotPath) == 0 and gitBranchExists("origin"): - settings = extractSettingsGitLog(extractLogMessageFromGitCommit("origin")) - depotPath = settings['depot-paths'][0] + parent = 0 + while parent < 65535: + commit = "HEAD~%s" % parent + log = extractLogMessageFromGitCommit(commit) + settings = extractSettingsGitLog(log) + if not settings.has_key("depot-paths"): + parent = parent + 1 + continue + + depotPath = settings['depot-paths'][0] + + if len(self.origin) == 0: + names = read_pipe_lines("git name-rev '--refs=refs/remotes/p4/*' '%s'" % commit) + if len(names) > 0: + # strip away the beginning of 'HEAD~42 refs/remotes/p4/foo' + self.origin = names[0].strip()[len(commit) + 1:] + + break + + if self.verbose: + print "Origin branch is " + self.origin if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" @@ -530,12 +546,6 @@ class P4Submit(Command): if response == "y" or response == "yes": system("p4 sync ...") - if len(self.origin) == 0: - if gitBranchExists("p4"): - self.origin = "p4" - else: - self.origin = "origin" - if self.reset: self.firstTime = True @@ -969,7 +979,7 @@ class P4Sync(Command): print "creating %s" % remoteHead update = True else: - settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) + settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) if settings.has_key('change') > 0: if settings['depot-paths'] == original['depot-paths']: originP4Change = int(original['change']) From df450923a2a08c50976f2d241a1c4992cf03b3a7 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 8 Jun 2007 08:49:22 +0200 Subject: [PATCH 238/260] Only get the expensive branch mapping from the p4 server when not syncing with the help of an origin remote (which we instead then use to get new branches from). Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index efec0be32c..36fe69a795 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -907,10 +907,6 @@ class P4Sync(Command): return p [p.strip().rfind("/") + 1:] def getBranchMapping(self): - - ## FIXME - what's a P4 projectName ? - self.projectName = self.guessProjectName() - for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 @@ -1141,7 +1137,11 @@ class P4Sync(Command): self.getLabels(); if self.detectBranches: - self.getBranchMapping(); + ## FIXME - what's a P4 projectName ? + self.projectName = self.guessProjectName() + + if not self.hasOrigin: + self.getBranchMapping(); if self.verbose: print "p4-git branches: %s" % self.p4BranchesInGit print "initial parents: %s" % self.initialParents From 1b9a46849a45f2b0f58d6286957316f720a301b6 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 239/260] print error message when p4 print fails (eg. due to permission problems) Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e955ad4f44..35c24c93e3 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -702,6 +702,11 @@ class P4Sync(Command): text += filedata[j]['data'] j += 1 + + if not stat.has_key('depotFile'): + sys.stderr.write("p4 print fails with: %s\n" % repr(stat)) + continue + contents[stat['depotFile']] = text for f in files: From 5265bfcb06f420841e6304b278b552a4654a4ba0 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 23 May 2007 18:49:35 -0300 Subject: [PATCH 240/260] also strip p4/ from local imports. Signed-off-by: Han-Wen Nienhuys --- contrib/fast-import/git-p4 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 28e37fa6ab..88ea12cb37 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -948,9 +948,9 @@ class P4Sync(Command): if not line.startswith('p4/') or line == "p4/HEAD": continue branch = line - if self.importIntoRemotes: - # strip off p4 - branch = re.sub ("^p4/", "", line) + + # strip off p4 + branch = re.sub ("^p4/", "", line) self.p4BranchesInGit.append(branch) self.initialParents[self.refPrefix + branch] = parseRevision(line) From 7aded26ce870ba2998e276604fc3c18dad6133bd Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 10 Jun 2007 00:22:30 +0200 Subject: [PATCH 241/260] Fixed the check to make sure to exclude the HEAD symbolic refs when updating the remotes/p4 branches from origin. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 88ea12cb37..d03ba0b6ad 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -962,7 +962,7 @@ class P4Sync(Command): for line in read_pipe_lines("git rev-parse --symbolic --remotes"): line = line.strip() - if (not line.startswith("origin/")) or line.endswith("HEAD\n"): + if (not line.startswith("origin/")) or line.endswith("HEAD"): continue headName = line[len("origin/"):] From cae7b732d859b06a4f560271099891f15c5700f6 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 10 Jun 2007 10:57:40 +0200 Subject: [PATCH 242/260] Fix updating/creating remotes/p4/* heads from origin/p4/* Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d03ba0b6ad..9d52eada42 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -960,14 +960,16 @@ class P4Sync(Command): print ("Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix) + originPrefix = "origin/p4/" + for line in read_pipe_lines("git rev-parse --symbolic --remotes"): line = line.strip() - if (not line.startswith("origin/")) or line.endswith("HEAD"): + if (not line.startswith(originPrefix)) or line.endswith("HEAD"): continue - headName = line[len("origin/"):] + headName = line[len(originPrefix):] remoteHead = self.refPrefix + headName - originHead = "origin/" + headName + originHead = line original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) if (not original.has_key('depot-paths') @@ -1020,7 +1022,7 @@ class P4Sync(Command): # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} - self.hasOrigin = gitBranchExists("origin") + self.hasOrigin = gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" From 6e5295c4d3e4e79838d1dd7ab4a8b4965e1c7f96 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 08:50:57 +0200 Subject: [PATCH 243/260] Fix project name guessing Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 9d52eada42..551573afc5 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -909,7 +909,12 @@ class P4Sync(Command): def guessProjectName(self): for p in self.depotPaths: - return p [p.strip().rfind("/") + 1:] + if p.endswith("/"): + p = p[:-1] + p = p[p.strip().rfind("/") + 1:] + if not p.endswith("/"): + p += "/" + return p def getBranchMapping(self): for info in p4CmdList("branches"): From 86fda6a327ce9355ae9eab69fc7a3ec33fb5d7d1 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 08:54:45 +0200 Subject: [PATCH 244/260] Fix depot-paths encoding for multi-path imports (don't split up //depot/path/foo) Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 551573afc5..815d8bbb5d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1312,7 +1312,7 @@ class P4Sync(Command): parent = self.initialParents[branch] del self.initialParents[branch] - self.commit(description, filesForCommit, branch, branchPrefix, parent) + self.commit(description, filesForCommit, branch, [branchPrefix], parent) else: files = self.extractFilesFromCommit(description) self.commit(description, files, self.branch, self.depotPaths, From a43ff00c7c2981426d06b3621bbf62476aa5ec0d Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 09:59:27 +0200 Subject: [PATCH 245/260] Fix support for explicit disabling of syncing with the origin Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 815d8bbb5d..ff56181310 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1028,6 +1028,8 @@ class P4Sync(Command): self.knownBranches = {} self.initialParents = {} self.hasOrigin = gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") + if not self.syncWithOrigin: + self.hasOrigin = False if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" From 6581de096e8323385b8ec7d467e91927a80ce3b9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 10:01:58 +0200 Subject: [PATCH 246/260] Write out the options tag in the log message of imports only if we actually have options Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index ff56181310..e380c149b4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -765,12 +765,11 @@ class P4Sync(Command): self.gitStream.write("data < 0: + self.gitStream.write(": options = %s" % details['options']) + self.gitStream.write("]\nEOT\n\n") if len(parent) > 0: if self.verbose: From c3bf3f1301319faab7344e2d8b5ab10a3d648856 Mon Sep 17 00:00:00 2001 From: Kevin Green Date: Mon, 11 Jun 2007 16:48:07 -0400 Subject: [PATCH 247/260] git-p4: check for existence of repo dir before trying to create When using git-p4 in this manner: git-p4 clone //depot/path/project myproject If "myproject" already exists as a dir, but not a valid git repo, it fails to create the directory. Signed-off-by: Kevin Green --- contrib/fast-import/git-p4 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e380c149b4..cababc7fc8 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1400,7 +1400,8 @@ class P4Clone(P4Sync): self.cloneDestination = self.defaultDestination(args) print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) - os.makedirs(self.cloneDestination) + if not os.path.exists(self.cloneDestination): + os.makedirs(self.cloneDestination) os.chdir(self.cloneDestination) system("git init") self.gitdir = os.getcwd() + "/.git" From a9d1a27af1f1c997eeef67aed54136ae83355bb9 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 23:28:03 +0200 Subject: [PATCH 248/260] Provide some information for single branch imports where the commits go Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index cababc7fc8..6c199296d3 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1175,7 +1175,7 @@ class P4Sync(Command): self.gitError = importProcess.stderr if self.revision: - print "Doing initial import of %s from revision %s" % (' '.join(self.depotPaths), self.revision) + print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), self.revision, self.branch) details = { "user" : "git perforce import user", "time" : int(time.time()) } details["desc"] = ("Initial import of %s from the state at revision %s" @@ -1252,6 +1252,9 @@ class P4Sync(Command): print "No changes to import!" return True + if not self.silent and not self.detectBranches: + print "Import destination: %s" % self.branch + self.updatedBranches = set() cnt = 1 From 81b462a629c2feff9ab1dc43148643aad9571271 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 23:30:23 +0200 Subject: [PATCH 249/260] Mention remotes/p4/master also in the documentation. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4.txt b/contrib/fast-import/git-p4.txt index c315158d8d..b16a8384bc 100644 --- a/contrib/fast-import/git-p4.txt +++ b/contrib/fast-import/git-p4.txt @@ -36,8 +36,8 @@ If you want more control you can also use the git-p4 sync command directly: git-p4 sync //path/in/your/perforce/depot This will import the current head revision of the specified depot path into a -"p4" branch of your git repository. You can use the --branch=mybranch option -to use a different branch. +"remotes/p4/master" branch of your git repository. You can use the +--branch=mybranch option to use a different branch. If you want to import the entire history of a given depot path just use @@ -57,7 +57,7 @@ newer changes from the Perforce depot by just calling git-p4 sync -in your git repository. By default the "p4" branch is updated. +in your git repository. By default the "remotes/p4/master" branch is updated. It is recommended to run 'git repack -a -d -f' from time to time when using incremental imports to optimally combine the individual git packs that each From e6b711f00e4578eb4b2127af12f73a5fa438f948 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Mon, 11 Jun 2007 23:40:25 +0200 Subject: [PATCH 250/260] git-p4 submit: Fix missing quotes around p4 commands to make them work with spaces in filenames Noticed by Alex Riesen Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 6c199296d3..21f9ba7e07 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -391,10 +391,10 @@ class P4Submit(Command): system(applyPatchCmd) for f in filesToAdd: - system("p4 add %s" % f) + system("p4 add \"%s\"" % f) for f in filesToDelete: - system("p4 revert %s" % f) - system("p4 delete %s" % f) + system("p4 revert \"%s\"" % f) + system("p4 delete \"%s\"" % f) logMessage = "" if not self.directSubmit: From 27d2d8119bf985a0b59152737316f04f871e03f7 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 12 Jun 2007 14:31:59 +0200 Subject: [PATCH 251/260] Moved the code from git-p4 submit to figure out the upstream branch point into a separate helper method. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 45 ++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 21f9ba7e07..1c7db11529 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -168,6 +168,28 @@ def gitBranchExists(branch): def gitConfig(key): return read_pipe("git config %s" % key, ignore_error=True).strip() +def findUpstreamBranchPoint(): + settings = None + branchPoint = "" + parent = 0 + while parent < 65535: + commit = "HEAD~%s" % parent + log = extractLogMessageFromGitCommit(commit) + settings = extractSettingsGitLog(log) + if not settings.has_key("depot-paths"): + parent = parent + 1 + continue + + names = read_pipe_lines("git name-rev '--refs=refs/remotes/p4/*' '%s'" % commit) + if len(names) <= 0: + continue + + # strip away the beginning of 'HEAD~42 refs/remotes/p4/foo' + branchPoint = names[0].strip()[len(commit) + 1:] + break + + return [branchPoint, settings] + class Command: def __init__(self): self.usage = "usage: %prog [options]" @@ -494,25 +516,10 @@ class P4Submit(Command): else: return False - depotPath = "" - parent = 0 - while parent < 65535: - commit = "HEAD~%s" % parent - log = extractLogMessageFromGitCommit(commit) - settings = extractSettingsGitLog(log) - if not settings.has_key("depot-paths"): - parent = parent + 1 - continue - - depotPath = settings['depot-paths'][0] - - if len(self.origin) == 0: - names = read_pipe_lines("git name-rev '--refs=refs/remotes/p4/*' '%s'" % commit) - if len(names) > 0: - # strip away the beginning of 'HEAD~42 refs/remotes/p4/foo' - self.origin = names[0].strip()[len(commit) + 1:] - - break + [upstream, settings] = findUpstreamBranchPoint() + depotPath = settings['depot-paths'][0] + if len(self.origin) == 0: + self.origin = upstream if self.verbose: print "Origin branch is " + self.origin From d7e3868cdfdc73c3de15296ecf32138a8308c07e Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Tue, 12 Jun 2007 14:34:46 +0200 Subject: [PATCH 252/260] Fix git-p4 rebase to detect the correct upstream branch instead of unconditionally always rebasing on top of remotes/p4/master Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 1c7db11529..1168704be4 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1361,9 +1361,17 @@ class P4Rebase(Command): def run(self, args): sync = P4Sync() sync.run([]) - print "Rebasing the current branch" + + [upstream, settings] = findUpstreamBranchPoint() + if len(upstream) == 0: + die("Cannot find upstream branchpoint for rebase") + + # the branchpoint may be p4/foo~3, so strip off the parent + upstream = re.sub("~[0-9]+$", "", upstream) + + print "Rebasing the current branch onto %s" % upstream oldHead = read_pipe("git rev-parse HEAD").strip() - system("git rebase p4") + system("git rebase %s" % upstream) system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True From cbae7080a7fe0586255e85e1d14f1011260e8eee Mon Sep 17 00:00:00 2001 From: Marius Storm-Olsen Date: Tue, 12 Jun 2007 15:27:52 +0200 Subject: [PATCH 253/260] Only use double quotes on Windows Signed-off-by: Marius Storm-Olsen --- contrib/fast-import/git-p4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 1168704be4..b3f27fe90f 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -180,7 +180,7 @@ def findUpstreamBranchPoint(): parent = parent + 1 continue - names = read_pipe_lines("git name-rev '--refs=refs/remotes/p4/*' '%s'" % commit) + names = read_pipe_lines("git name-rev \"--refs=refs/remotes/p4/*\" \"%s\"" % commit) if len(names) <= 0: continue From 3c699645f589612065b048ecde45a4ea293dc75f Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sat, 16 Jun 2007 13:09:21 +0200 Subject: [PATCH 254/260] Fix initial multi-branch import. The list of existing p4 branches in git wasn't initialized. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b3f27fe90f..e527734be5 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -637,6 +637,7 @@ class P4Sync(Command): self.isWindows = (platform.system() == "Windows") self.keepRepoPath = False self.depotPaths = None + self.p4BranchesInGit = [] if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False From da4a660161cfe9d04c0849d77fa460c6ffc6503c Mon Sep 17 00:00:00 2001 From: Benjamin Sergeant Date: Fri, 8 Jun 2007 11:13:55 -0700 Subject: [PATCH 255/260] git-p4 fails when cloning a p4 depo. A perforce command with all the files in the repo is generated to get all the file content. Here is a patch to break it into multiple successive perforce command who uses 4K of parameter max, and collect the output for later. It works, but not for big depos, because the whole perforce depo content is stored in memory in P4Sync.run(), and it looks like mine is bigger than 2 Gigs, so I had to kill the process. [Simon: I added the bit about using SC_ARG_MAX, as suggested by Han-Wen] Signed-off-by: Benjamin Sergeant Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index e527734be5..d1f8d3b78d 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -711,9 +711,23 @@ class P4Sync(Command): if not files: return - filedata = p4CmdList('print %s' % ' '.join(['"%s#%s"' % (f['path'], - f['rev']) - for f in files])) + # We cannot put all the files on the command line + # OS have limitations on the max lenght of arguments + # POSIX says it's 4096 bytes, default for Linux seems to be 130 K. + # and all OS from the table below seems to be higher than POSIX. + # See http://www.in-ulm.de/~mascheck/various/argmax/ + argmax = min(4000, os.sysconf('SC_ARG_MAX')) + chunk = '' + filedata = [] + for i in xrange(len(files)): + f = files[i] + chunk += '"%s#%s" ' % (f['path'], f['rev']) + if len(chunk) > argmax or i == len(files)-1: + data = p4CmdList('print %s' % chunk) + if "p4ExitCode" in data[0]: + die("Problems executing p4. Error: [%d]." % (data[0]['p4ExitCode'])); + filedata.extend(data) + chunk = '' j = 0; contents = {} From 6555b2ccfe63913b3e5c8b02e117f0f476307ca2 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 17 Jun 2007 11:25:34 +0200 Subject: [PATCH 256/260] Fix the branch mapping detection to be independent from the order of the "p4 branches" output. Collect "unknown" source branches separately and register them at the end. Also added a minor speed up to splitFilesIntoBranches by breaking out of the loop through all branches when it's safe. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index d1f8d3b78d..3b6d8a09d1 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -700,6 +700,7 @@ class P4Sync(Command): if branch not in branches: branches[branch] = [] branches[branch].append(file) + break return branches @@ -938,6 +939,8 @@ class P4Sync(Command): return p def getBranchMapping(self): + lostAndFoundBranches = set() + for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 @@ -953,10 +956,17 @@ class P4Sync(Command): if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]): source = source[len(self.depotPaths[0]):-4] destination = destination[len(self.depotPaths[0]):-4] - if destination not in self.knownBranches: - self.knownBranches[destination] = source + + self.knownBranches[destination] = source + + lostAndFoundBranches.discard(destination) + if source not in self.knownBranches: - self.knownBranches[source] = source + lostAndFoundBranches.add(source) + + + for branch in lostAndFoundBranches: + self.knownBranches[branch] = branch def listExistingP4GitBranches(self): self.p4BranchesInGit = [] From 1a2edf4e8dff05fea66daf82c675cfab673c1242 Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Sun, 17 Jun 2007 15:10:24 +0200 Subject: [PATCH 257/260] Warn about conflicting p4 branch mappings and use the first one found. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 3b6d8a09d1..2040591383 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -957,6 +957,12 @@ class P4Sync(Command): source = source[len(self.depotPaths[0]):-4] destination = destination[len(self.depotPaths[0]):-4] + if destination in self.knownBranches: + if not self.silent: + print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination) + print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination) + continue + self.knownBranches[destination] = source lostAndFoundBranches.discard(destination) From 09d89de2e31ed4d62b6ff344e9ad9ef29550121b Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Wed, 20 Jun 2007 23:10:28 +0200 Subject: [PATCH 258/260] Added git-p4 branches command that shows the mapping of perforce depot paths to imported git branches. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 2040591383..16de15c4ea 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -1465,6 +1465,31 @@ class P4Clone(P4Sync): return True +class P4Branches(Command): + def __init__(self): + Command.__init__(self) + self.options = [ ] + self.description = ("Shows the git branches that hold imports and their " + + "corresponding perforce depot paths") + self.verbose = False + + def run(self, args): + cmdline = "git rev-parse --symbolic " + cmdline += " --remotes" + + for line in read_pipe_lines(cmdline): + line = line.strip() + + if not line.startswith('p4/') or line == "p4/HEAD": + continue + branch = line + + log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch) + settings = extractSettingsGitLog(log) + + print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]) + return True + class HelpFormatter(optparse.IndentedHelpFormatter): def __init__(self): optparse.IndentedHelpFormatter.__init__(self) @@ -1489,7 +1514,8 @@ commands = { "sync" : P4Sync, "rebase" : P4Rebase, "clone" : P4Clone, - "rollback" : P4RollBack + "rollback" : P4RollBack, + "branches" : P4Branches } From 9ceab36375dfd4088b265033e1de9225b3527cab Mon Sep 17 00:00:00 2001 From: Simon Hausmann Date: Fri, 22 Jun 2007 00:01:57 +0200 Subject: [PATCH 259/260] Make it possible to specify the HEAD for the internal findUpstreamBranchPoint function. This isn't used right now in git-p4 but I use it in an external script that loads git-p4 as module. Signed-off-by: Simon Hausmann --- contrib/fast-import/git-p4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 16de15c4ea..54a05eb99c 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -168,12 +168,12 @@ def gitBranchExists(branch): def gitConfig(key): return read_pipe("git config %s" % key, ignore_error=True).strip() -def findUpstreamBranchPoint(): +def findUpstreamBranchPoint(head = "HEAD"): settings = None branchPoint = "" parent = 0 while parent < 65535: - commit = "HEAD~%s" % parent + commit = head + "~%s" % parent log = extractLogMessageFromGitCommit(commit) settings = extractSettingsGitLog(log) if not settings.has_key("depot-paths"): From 92d7c8e37b96244f2c55f5c66ec52c6325acca1d Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 22 Jun 2007 18:44:04 -0400 Subject: [PATCH 260/260] Avoid src:dst syntax as default bash completion for git push Raimund Bauer just discovered that the default bash completion for a local branch name in a git-push line is not the best choice when the branch does not exist on the remote system. In the past we have always completed the local name 'test' as "test:test", indicating that the destination name is the same as the local name. But this fails when "test" does not yet exist on the remote system, as there is no "test" branch for it to match the name against. Fortunately git-push does the right thing when given just the local branch, as it assumes you want to use the same name in the destination repository. So we now offer "test" as the completion in a git-push line, and let git-push assume that is also the remote branch name. We also still support the remote branch completion after the :, but only if the user manually adds the colon before trying to get a completion. Signed-off-by: Shawn O. Pearce --- contrib/completion/git-completion.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 9e72f0f7b1..c7c9963347 100755 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -683,7 +683,7 @@ _git_push () __gitcomp "$(__git_refs "$remote")" "" "${cur#*:}" ;; *) - __gitcomp "$(__git_refs2)" + __gitcomp "$(__git_refs)" ;; esac ;;