undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'yz/p4-py3'

Browse Source 
Update "git p4" to work with Python 3.

* yz/p4-py3:
  ci: use python3 in linux-gcc and osx-gcc and python2 elsewhere
  git-p4: use python3's input() everywhere
  git-p4: simplify regex pattern generation for parsing diff-tree
  git-p4: use dict.items() iteration for python3 compatibility
  git-p4: use functools.reduce instead of reduce
  git-p4: fix freezing while waiting for fast-import progress
  git-p4: use marshal format version 2 when sending to p4
  git-p4: open .gitp4-usercache.txt in text mode
  git-p4: convert path to unicode before processing them
  git-p4: encode/decode communication with git for python3
  git-p4: encode/decode communication with p4 for python3
  git-p4: remove string type aliasing
  git-p4: change the expansion test from basestring to list
  git-p4: make python2.7 the oldest supported version
This commit is contained in:
Junio C Hamano 2020-03-25 13:57:43 -07:00
commit 9a0fa1709c
2 changed files with 146 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ci/lib.sh
View File

@ -162,6 +162,9 @@ linux-clang|linux-gcc)
if [ "$jobname" = linux-gcc ] if [ "$jobname" = linux-gcc ]
then then
export CC=gcc-8 export CC=gcc-8
MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
else
MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
fi fi
export GIT_TEST_HTTPD=true export GIT_TEST_HTTPD=true
@ -182,6 +185,9 @@ osx-clang|osx-gcc)
if [ "$jobname" = osx-gcc ] if [ "$jobname" = osx-gcc ]
then then
export CC=gcc-9 export CC=gcc-9
MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
else
MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
fi fi
# t9810 occasionally fails on Travis CI OS X # t9810 occasionally fails on Travis CI OS X

235
git-p4.py
View File

@ -16,12 +16,12 @@
# pylint: disable=too-many-branches,too-many-nested-blocks # pylint: disable=too-many-branches,too-many-nested-blocks
# #
import sys import sys
if sys.hexversion < 0x02040000: if sys.version_info.major < 3 and sys.version_info.minor < 7:
# The limiter is the subprocess module sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
sys.exit(1) sys.exit(1)
import os import os
import optparse import optparse
import functools
import marshal import marshal
import subprocess import subprocess
import tempfile import tempfile
@ -35,36 +35,15 @@ import zlib
import ctypes import ctypes
import errno import errno
# On python2.7 where raw_input() and input() are both availble,
# we want raw_input's semantics, but aliased to input for python3
# compatibility
# support basestring in python3 # support basestring in python3
try: try:
unicode = unicode if raw_input and input:
except NameError: input = raw_input
# 'unicode' is undefined, must be Python 3 except:
str = str pass
unicode = str
bytes = bytes
basestring = (str,bytes)
else:
# 'unicode' exists, must be Python 2
str = str
unicode = unicode
bytes = str
basestring = basestring
try:
from subprocess import CalledProcessError
except ImportError:
# from python2.7:subprocess.py
# Exception classes used by this module.
class CalledProcessError(Exception):
"""This exception is raised when a process run by check_call() returns
a non-zero exit status. The exit status will be stored in the
returncode attribute."""
def __init__(self, returncode, cmd):
self.returncode = returncode
self.cmd = cmd
def __str__(self):
return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
verbose = False verbose = False
@ -113,7 +92,7 @@ def p4_build_cmd(cmd):
# Provide a way to not pass this option by setting git-p4.retries to 0 # Provide a way to not pass this option by setting git-p4.retries to 0
real_cmd += ["-r", str(retries)] real_cmd += ["-r", str(retries)]
if isinstance(cmd,basestring): if not isinstance(cmd, list):
real_cmd = ' '.join(real_cmd) + ' ' + cmd real_cmd = ' '.join(real_cmd) + ' ' + cmd
else: else:
real_cmd += cmd real_cmd += cmd
@ -186,18 +165,48 @@ def prompt(prompt_text):
""" """
choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text)) choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
while True: while True:
response = raw_input(prompt_text).strip().lower() response = input(prompt_text).strip().lower()
if not response: if not response:
continue continue
response = response[0] response = response[0]
if response in choices: if response in choices:
return response return response
# We need different encoding/decoding strategies for text data being passed
# around in pipes depending on python version
if bytes is not str:
# For python3, always encode and decode as appropriate
def decode_text_stream(s):
return s.decode() if isinstance(s, bytes) else s
def encode_text_stream(s):
return s.encode() if isinstance(s, str) else s
else:
# For python2.7, pass read strings as-is, but also allow writing unicode
def decode_text_stream(s):
return s
def encode_text_stream(s):
return s.encode('utf_8') if isinstance(s, unicode) else s
def decode_path(path):
"""Decode a given string (bytes or otherwise) using configured path encoding options
"""
encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
if bytes is not str:
return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
else:
try:
path.decode('ascii')
except:
path = path.decode(encoding, errors='replace')
if verbose:
print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
return path
def write_pipe(c, stdin): def write_pipe(c, stdin):
if verbose: if verbose:
sys.stderr.write('Writing pipe: %s\n' % str(c)) sys.stderr.write('Writing pipe: %s\n' % str(c))
expand = isinstance(c,basestring) expand = not isinstance(c, list)
p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand) p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
pipe = p.stdin pipe = p.stdin
val = pipe.write(stdin) val = pipe.write(stdin)
@ -209,6 +218,8 @@ def write_pipe(c, stdin):
def p4_write_pipe(c, stdin): def p4_write_pipe(c, stdin):
real_cmd = p4_build_cmd(c) real_cmd = p4_build_cmd(c)
if bytes is not str and isinstance(stdin, str):
stdin = encode_text_stream(stdin)
return write_pipe(real_cmd, stdin) return write_pipe(real_cmd, stdin)
def read_pipe_full(c): def read_pipe_full(c):
@ -219,15 +230,17 @@ def read_pipe_full(c):
if verbose: if verbose:
sys.stderr.write('Reading pipe: %s\n' % str(c)) sys.stderr.write('Reading pipe: %s\n' % str(c))
expand = isinstance(c,basestring) expand = not isinstance(c, list)
p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand) p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
(out, err) = p.communicate() (out, err) = p.communicate()
return (p.returncode, out, err) return (p.returncode, out, decode_text_stream(err))
def read_pipe(c, ignore_error=False): def read_pipe(c, ignore_error=False, raw=False):
""" Read output from command. Returns the output text on """ Read output from command. Returns the output text on
success. On failure, terminates execution, unless success. On failure, terminates execution, unless
ignore_error is True, when it returns an empty string. ignore_error is True, when it returns an empty string.
If raw is True, do not attempt to decode output text.
""" """
(retcode, out, err) = read_pipe_full(c) (retcode, out, err) = read_pipe_full(c)
if retcode != 0: if retcode != 0:
@ -235,6 +248,8 @@ def read_pipe(c, ignore_error=False):
out = "" out = ""
else: else:
die('Command failed: %s\nError: %s' % (str(c), err)) die('Command failed: %s\nError: %s' % (str(c), err))
if not raw:
out = decode_text_stream(out)
return out return out
def read_pipe_text(c): def read_pipe_text(c):
@ -245,23 +260,22 @@ def read_pipe_text(c):
if retcode != 0: if retcode != 0:
return None return None
else: else:
return out.rstrip() return decode_text_stream(out).rstrip()
def p4_read_pipe(c, ignore_error=False): def p4_read_pipe(c, ignore_error=False, raw=False):
real_cmd = p4_build_cmd(c) real_cmd = p4_build_cmd(c)
return read_pipe(real_cmd, ignore_error) return read_pipe(real_cmd, ignore_error, raw=raw)
def read_pipe_lines(c): def read_pipe_lines(c):
if verbose: if verbose:
sys.stderr.write('Reading pipe: %s\n' % str(c)) sys.stderr.write('Reading pipe: %s\n' % str(c))
expand = isinstance(c, basestring) expand = not isinstance(c, list)
p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand) p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
pipe = p.stdout pipe = p.stdout
val = pipe.readlines() val = [decode_text_stream(line) for line in pipe.readlines()]
if pipe.close() or p.wait(): if pipe.close() or p.wait():
die('Command failed: %s' % str(c)) die('Command failed: %s' % str(c))
return val return val
def p4_read_pipe_lines(c): def p4_read_pipe_lines(c):
@ -289,6 +303,7 @@ def p4_has_move_command():
cmd = p4_build_cmd(["move", "-k", "@from", "@to"]) cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = p.communicate() (out, err) = p.communicate()
err = decode_text_stream(err)
# return code will be 1 in either case # return code will be 1 in either case
if err.find("Invalid option") >= 0: if err.find("Invalid option") >= 0:
return False return False
@ -298,7 +313,7 @@ def p4_has_move_command():
return True return True
def system(cmd, ignore_error=False): def system(cmd, ignore_error=False):
expand = isinstance(cmd,basestring) expand = not isinstance(cmd, list)
if verbose: if verbose:
sys.stderr.write("executing %s\n" % str(cmd)) sys.stderr.write("executing %s\n" % str(cmd))
retcode = subprocess.call(cmd, shell=expand) retcode = subprocess.call(cmd, shell=expand)
@ -310,7 +325,7 @@ def system(cmd, ignore_error=False):
def p4_system(cmd): def p4_system(cmd):
"""Specifically invoke p4 as the system command. """ """Specifically invoke p4 as the system command. """
real_cmd = p4_build_cmd(cmd) real_cmd = p4_build_cmd(cmd)
expand = isinstance(real_cmd, basestring) expand = not isinstance(real_cmd, list)
retcode = subprocess.call(real_cmd, shell=expand) retcode = subprocess.call(real_cmd, shell=expand)
if retcode: if retcode:
raise CalledProcessError(retcode, real_cmd) raise CalledProcessError(retcode, real_cmd)
@ -548,7 +563,7 @@ def getP4OpenedType(file):
# Return the set of all p4 labels # Return the set of all p4 labels
def getP4Labels(depotPaths): def getP4Labels(depotPaths):
labels = set() labels = set()
if isinstance(depotPaths,basestring): if not isinstance(depotPaths, list):
depotPaths = [depotPaths] depotPaths = [depotPaths]
for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]): for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
@ -565,12 +580,7 @@ def getGitTags():
gitTags.add(tag) gitTags.add(tag)
return gitTags return gitTags
def diffTreePattern(): _diff_tree_pattern = None
# This is a simple generator for the diff tree regex pattern. This could be
# a class variable if this and parseDiffTreeEntry were a part of a class.
pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
while True:
yield pattern
def parseDiffTreeEntry(entry): def parseDiffTreeEntry(entry):
"""Parses a single diff tree entry into its component elements. """Parses a single diff tree entry into its component elements.
@ -591,7 +601,11 @@ def parseDiffTreeEntry(entry):
If the pattern is not matched, None is returned.""" If the pattern is not matched, None is returned."""
match = diffTreePattern().next().match(entry) global _diff_tree_pattern
if not _diff_tree_pattern:
_diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
match = _diff_tree_pattern.match(entry)
if match: if match:
return { return {
'src_mode': match.group(1), 'src_mode': match.group(1),
@ -643,7 +657,7 @@ def isModeExecChanged(src_mode, dst_mode):
def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False, def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
errors_as_exceptions=False): errors_as_exceptions=False):
if isinstance(cmd,basestring): if not isinstance(cmd, list):
cmd = "-G " + cmd cmd = "-G " + cmd
expand = True expand = True
else: else:
@ -660,11 +674,12 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
stdin_file = None stdin_file = None
if stdin is not None: if stdin is not None:
stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
if isinstance(stdin,basestring): if not isinstance(stdin, list):
stdin_file.write(stdin) stdin_file.write(stdin)
else: else:
for i in stdin: for i in stdin:
stdin_file.write(i + '\n') stdin_file.write(encode_text_stream(i))
stdin_file.write(b'\n')
stdin_file.flush() stdin_file.flush()
stdin_file.seek(0) stdin_file.seek(0)
@ -677,6 +692,20 @@ def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
try: try:
while True: while True:
entry = marshal.load(p4.stdout) entry = marshal.load(p4.stdout)
if bytes is not str:
# Decode unmarshalled dict to use str keys and values, except for:
# - `data` which may contain arbitrary binary data
# - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
decoded_entry = {}
for key, value in entry.items():
key = key.decode()
if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')):
value = value.decode()
decoded_entry[key] = value
# Parse out data if it's an error response
if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
decoded_entry['data'] = decoded_entry['data'].decode()
entry = decoded_entry
if skip_info: if skip_info:
if 'code' in entry and entry['code'] == 'info': if 'code' in entry and entry['code'] == 'info':
continue continue
@ -727,7 +756,8 @@ def p4Where(depotPath):
if "depotFile" in entry: if "depotFile" in entry:
# Search for the base client side depot path, as long as it starts with the branch's P4 path. # Search for the base client side depot path, as long as it starts with the branch's P4 path.
# The base path always ends with "/...". # The base path always ends with "/...".
if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...": entry_path = decode_path(entry['depotFile'])
if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
output = entry output = entry
break break
elif "data" in entry: elif "data" in entry:
@ -742,11 +772,11 @@ def p4Where(depotPath):
return "" return ""
clientPath = "" clientPath = ""
if "path" in output: if "path" in output:
clientPath = output.get("path") clientPath = decode_path(output['path'])
elif "data" in output: elif "data" in output:
data = output.get("data") data = output.get("data")
lastSpace = data.rfind(" ") lastSpace = data.rfind(b" ")
clientPath = data[lastSpace + 1:] clientPath = decode_path(data[lastSpace + 1:])
if clientPath.endswith("..."): if clientPath.endswith("..."):
clientPath = clientPath[:-3] clientPath = clientPath[:-3]
@ -894,6 +924,7 @@ def branch_exists(branch):
cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ] cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _ = p.communicate() out, _ = p.communicate()
out = decode_text_stream(out)
if p.returncode: if p.returncode:
return False return False
# expect exactly one line of output: the branch name # expect exactly one line of output: the branch name
@ -1171,7 +1202,7 @@ class LargeFileSystem(object):
assert False, "Method 'pushFile' required in " + self.__class__.__name__ assert False, "Method 'pushFile' required in " + self.__class__.__name__
def hasLargeFileExtension(self, relPath): def hasLargeFileExtension(self, relPath):
return reduce( return functools.reduce(
lambda a, b: a or b, lambda a, b: a or b,
[relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')], [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
False False
@ -1278,7 +1309,7 @@ class GitLFS(LargeFileSystem):
['git', 'lfs', 'pointer', '--file=' + contentFile], ['git', 'lfs', 'pointer', '--file=' + contentFile],
stdout=subprocess.PIPE stdout=subprocess.PIPE
) )
pointerFile = pointerProcess.stdout.read() pointerFile = decode_text_stream(pointerProcess.stdout.read())
if pointerProcess.wait(): if pointerProcess.wait():
os.remove(contentFile) os.remove(contentFile)
die('git-lfs pointer command failed. Did you install the extension?') die('git-lfs pointer command failed. Did you install the extension?')
@ -1414,14 +1445,14 @@ class P4UserMap:
for (key, val) in self.users.items(): for (key, val) in self.users.items():
s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1)) s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
open(self.getUserCacheFilename(), "wb").write(s) open(self.getUserCacheFilename(), 'w').write(s)
self.userMapFromPerforceServer = True self.userMapFromPerforceServer = True
def loadUserMapFromCache(self): def loadUserMapFromCache(self):
self.users = {} self.users = {}
self.userMapFromPerforceServer = False self.userMapFromPerforceServer = False
try: try:
cache = open(self.getUserCacheFilename(), "rb") cache = open(self.getUserCacheFilename(), 'r')
lines = cache.readlines() lines = cache.readlines()
cache.close() cache.close()
for line in lines: for line in lines:
@ -1698,7 +1729,8 @@ class P4Submit(Command, P4UserMap):
c = changes[0] c = changes[0]
if c['User'] == newUser: return # nothing to do if c['User'] == newUser: return # nothing to do
c['User'] = newUser c['User'] = newUser
input = marshal.dumps(c) # p4 does not understand format version 3 and above
input = marshal.dumps(c, 2)
result = p4CmdList("change -f -i", stdin=input) result = p4CmdList("change -f -i", stdin=input)
for r in result: for r in result:
@ -1762,7 +1794,7 @@ class P4Submit(Command, P4UserMap):
break break
if not change_entry: if not change_entry:
die('Failed to decode output of p4 change -o') die('Failed to decode output of p4 change -o')
for key, value in change_entry.iteritems(): for key, value in change_entry.items():
if key.startswith('File'): if key.startswith('File'):
if 'depot-paths' in settings: if 'depot-paths' in settings:
if not [p for p in settings['depot-paths'] if not [p for p in settings['depot-paths']
@ -2042,7 +2074,7 @@ class P4Submit(Command, P4UserMap):
tmpFile = os.fdopen(handle, "w+b") tmpFile = os.fdopen(handle, "w+b")
if self.isWindows: if self.isWindows:
submitTemplate = submitTemplate.replace("\n", "\r\n") submitTemplate = submitTemplate.replace("\n", "\r\n")
tmpFile.write(submitTemplate) tmpFile.write(encode_text_stream(submitTemplate))
tmpFile.close() tmpFile.close()
if self.prepare_p4_only: if self.prepare_p4_only:
@ -2089,7 +2121,7 @@ class P4Submit(Command, P4UserMap):
if self.edit_template(fileName): if self.edit_template(fileName):
# read the edited message and submit # read the edited message and submit
tmpFile = open(fileName, "rb") tmpFile = open(fileName, "rb")
message = tmpFile.read() message = decode_text_stream(tmpFile.read())
tmpFile.close() tmpFile.close()
if self.isWindows: if self.isWindows:
message = message.replace("\r\n", "\n") message = message.replace("\r\n", "\n")
@ -2509,7 +2541,7 @@ class View(object):
def convert_client_path(self, clientFile): def convert_client_path(self, clientFile):
# chop off //client/ part to make it relative # chop off //client/ part to make it relative
if not clientFile.startswith(self.client_prefix): if not decode_path(clientFile).startswith(self.client_prefix):
die("No prefix '%s' on clientFile '%s'" % die("No prefix '%s' on clientFile '%s'" %
(self.client_prefix, clientFile)) (self.client_prefix, clientFile))
return clientFile[len(self.client_prefix):] return clientFile[len(self.client_prefix):]
@ -2518,7 +2550,7 @@ class View(object):
""" Caching file paths by "p4 where" batch query """ """ Caching file paths by "p4 where" batch query """
# List depot file paths exclude that already cached # List depot file paths exclude that already cached
fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache] fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
if len(fileArgs) == 0: if len(fileArgs) == 0:
return # All files in cache return # All files in cache
@ -2533,16 +2565,18 @@ class View(object):
if "unmap" in res: if "unmap" in res:
# it will list all of them, but only one not unmap-ped # it will list all of them, but only one not unmap-ped
continue continue
depot_path = decode_path(res['depotFile'])
if gitConfigBool("core.ignorecase"): if gitConfigBool("core.ignorecase"):
res['depotFile'] = res['depotFile'].lower() depot_path = depot_path.lower()
self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"]) self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
# not found files or unmap files set to "" # not found files or unmap files set to ""
for depotFile in fileArgs: for depotFile in fileArgs:
depotFile = decode_path(depotFile)
if gitConfigBool("core.ignorecase"): if gitConfigBool("core.ignorecase"):
depotFile = depotFile.lower() depotFile = depotFile.lower()
if depotFile not in self.client_spec_path_cache: if depotFile not in self.client_spec_path_cache:
self.client_spec_path_cache[depotFile] = "" self.client_spec_path_cache[depotFile] = b''
def map_in_client(self, depot_path): def map_in_client(self, depot_path):
"""Return the relative location in the client where this """Return the relative location in the client where this
@ -2647,6 +2681,7 @@ class P4Sync(Command, P4UserMap):
def checkpoint(self): def checkpoint(self):
self.gitStream.write("checkpoint\n\n") self.gitStream.write("checkpoint\n\n")
self.gitStream.write("progress checkpoint\n\n") self.gitStream.write("progress checkpoint\n\n")
self.gitStream.flush()
out = self.gitOutput.readline() out = self.gitOutput.readline()
if self.verbose: if self.verbose:
print("checkpoint finished: " + out) print("checkpoint finished: " + out)
@ -2660,7 +2695,7 @@ class P4Sync(Command, P4UserMap):
elif path.lower() == p.lower(): elif path.lower() == p.lower():
return False return False
for p in self.depotPaths: for p in self.depotPaths:
if p4PathStartsWith(path, p): if p4PathStartsWith(path, decode_path(p)):
return True return True
return False return False
@ -2669,7 +2704,7 @@ class P4Sync(Command, P4UserMap):
fnum = 0 fnum = 0
while "depotFile%s" % fnum in commit: while "depotFile%s" % fnum in commit:
path = commit["depotFile%s" % fnum] path = commit["depotFile%s" % fnum]
found = self.isPathWanted(path) found = self.isPathWanted(decode_path(path))
if not found: if not found:
fnum = fnum + 1 fnum = fnum + 1
continue continue
@ -2703,7 +2738,7 @@ class P4Sync(Command, P4UserMap):
if self.useClientSpec: if self.useClientSpec:
# branch detection moves files up a level (the branch name) # branch detection moves files up a level (the branch name)
# from what client spec interpretation gives # from what client spec interpretation gives
path = self.clientSpecDirs.map_in_client(path) path = decode_path(self.clientSpecDirs.map_in_client(path))
if self.detectBranches: if self.detectBranches:
for b in self.knownBranches: for b in self.knownBranches:
if p4PathStartsWith(path, b + "/"): if p4PathStartsWith(path, b + "/"):
@ -2737,14 +2772,15 @@ class P4Sync(Command, P4UserMap):
branches = {} branches = {}
fnum = 0 fnum = 0
while "depotFile%s" % fnum in commit: while "depotFile%s" % fnum in commit:
path = commit["depotFile%s" % fnum] raw_path = commit["depotFile%s" % fnum]
path = decode_path(raw_path)
found = self.isPathWanted(path) found = self.isPathWanted(path)
if not found: if not found:
fnum = fnum + 1 fnum = fnum + 1
continue continue
file = {} file = {}
file["path"] = path file["path"] = raw_path
file["rev"] = commit["rev%s" % fnum] file["rev"] = commit["rev%s" % fnum]
file["action"] = commit["action%s" % fnum] file["action"] = commit["action%s" % fnum]
file["type"] = commit["type%s" % fnum] file["type"] = commit["type%s" % fnum]
@ -2753,7 +2789,7 @@ class P4Sync(Command, P4UserMap):
# start with the full relative path where this file would # start with the full relative path where this file would
# go in a p4 client # go in a p4 client
if self.useClientSpec: if self.useClientSpec:
relPath = self.clientSpecDirs.map_in_client(path) relPath = decode_path(self.clientSpecDirs.map_in_client(path))
else: else:
relPath = self.stripRepoPath(path, self.depotPaths) relPath = self.stripRepoPath(path, self.depotPaths)
@ -2769,7 +2805,7 @@ class P4Sync(Command, P4UserMap):
return branches return branches
def writeToGitStream(self, gitMode, relPath, contents): def writeToGitStream(self, gitMode, relPath, contents):
self.gitStream.write('M %s inline %s\n' % (gitMode, relPath)) self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
self.gitStream.write('data %d\n' % sum(len(d) for d in contents)) self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
for d in contents: for d in contents:
self.gitStream.write(d) self.gitStream.write(d)
@ -2791,14 +2827,15 @@ class P4Sync(Command, P4UserMap):
# - helper for streamP4Files # - helper for streamP4Files
def streamOneP4File(self, file, contents): def streamOneP4File(self, file, contents):
relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes) file_path = file['depotFile']
relPath = self.encodeWithUTF8(relPath) relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
if verbose: if verbose:
if 'fileSize' in self.stream_file: if 'fileSize' in self.stream_file:
size = int(self.stream_file['fileSize']) size = int(self.stream_file['fileSize'])
else: else:
size = 0 # deleted files don't get a fileSize apparently size = 0 # deleted files don't get a fileSize apparently
sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024)) sys.stdout.write('\r%s --> %s (%i MB)\n' % (file_path, relPath, size/1024/1024))
sys.stdout.flush() sys.stdout.flush()
(type_base, type_mods) = split_p4_type(file["type"]) (type_base, type_mods) = split_p4_type(file["type"])
@ -2810,13 +2847,13 @@ class P4Sync(Command, P4UserMap):
git_mode = "120000" git_mode = "120000"
# p4 print on a symlink sometimes contains "target\n"; # p4 print on a symlink sometimes contains "target\n";
# if it does, remove the newline # if it does, remove the newline
data = ''.join(contents) data = ''.join(decode_text_stream(c) for c in contents)
if not data: if not data:
# Some version of p4 allowed creating a symlink that pointed # Some version of p4 allowed creating a symlink that pointed
# to nothing. This causes p4 errors when checking out such # to nothing. This causes p4 errors when checking out such
# a change, and errors here too. Work around it by ignoring # a change, and errors here too. Work around it by ignoring
# the bad symlink; hopefully a future change fixes it. # the bad symlink; hopefully a future change fixes it.
print("\nIgnoring empty symlink in %s" % file['depotFile']) print("\nIgnoring empty symlink in %s" % file_path)
return return
elif data[-1] == '\n': elif data[-1] == '\n':
contents = [data[:-1]] contents = [data[:-1]]
@ -2835,7 +2872,7 @@ class P4Sync(Command, P4UserMap):
# just the native "NT" type. # just the native "NT" type.
# #
try: try:
text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])]) text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
except Exception as e: except Exception as e:
if 'Translation of file content failed' in str(e): if 'Translation of file content failed' in str(e):
type_base = 'binary' type_base = 'binary'
@ -2843,7 +2880,7 @@ class P4Sync(Command, P4UserMap):
raise e raise e
else: else:
if p4_version_string().find('/NT') >= 0: if p4_version_string().find('/NT') >= 0:
text = text.replace('\r\n', '\n') text = text.replace(b'\r\n', b'\n')
contents = [ text ] contents = [ text ]
if type_base == "apple": if type_base == "apple":
@ -2864,7 +2901,7 @@ class P4Sync(Command, P4UserMap):
pattern = p4_keywords_regexp_for_type(type_base, type_mods) pattern = p4_keywords_regexp_for_type(type_base, type_mods)
if pattern: if pattern:
regexp = re.compile(pattern, re.VERBOSE) regexp = re.compile(pattern, re.VERBOSE)
text = ''.join(contents) text = ''.join(decode_text_stream(c) for c in contents)
text = regexp.sub(r'$\1$', text) text = regexp.sub(r'$\1$', text)
contents = [ text ] contents = [ text ]
@ -2874,12 +2911,11 @@ class P4Sync(Command, P4UserMap):
self.writeToGitStream(git_mode, relPath, contents) self.writeToGitStream(git_mode, relPath, contents)
def streamOneP4Deletion(self, file): def streamOneP4Deletion(self, file):
relPath = self.stripRepoPath(file['path'], self.branchPrefixes) relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
relPath = self.encodeWithUTF8(relPath)
if verbose: if verbose:
sys.stdout.write("delete %s\n" % relPath) sys.stdout.write("delete %s\n" % relPath)
sys.stdout.flush() sys.stdout.flush()
self.gitStream.write("D %s\n" % relPath) self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath): if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
self.largeFileSystem.removeLargeFile(relPath) self.largeFileSystem.removeLargeFile(relPath)
@ -2979,9 +3015,9 @@ class P4Sync(Command, P4UserMap):
if 'shelved_cl' in f: if 'shelved_cl' in f:
# Handle shelved CLs using the "p4 print file@=N" syntax to print # Handle shelved CLs using the "p4 print file@=N" syntax to print
# the contents # the contents
fileArg = '%s@=%d' % (f['path'], f['shelved_cl']) fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
else: else:
fileArg = '%s#%s' % (f['path'], f['rev']) fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
fileArgs.append(fileArg) fileArgs.append(fileArg)
@ -3062,8 +3098,8 @@ class P4Sync(Command, P4UserMap):
if self.clientSpecDirs: if self.clientSpecDirs:
self.clientSpecDirs.update_client_spec_path_cache(files) self.clientSpecDirs.update_client_spec_path_cache(files)
files = [f for f in files files = [f for (f, path) in ((f, decode_path(f['path'])) for f in files)
if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])] if self.inClientSpec(path) and self.hasBranchPrefix(path)]
if gitConfigBool('git-p4.keepEmptyCommits'): if gitConfigBool('git-p4.keepEmptyCommits'):
allow_empty = True allow_empty = True
@ -3635,6 +3671,15 @@ class P4Sync(Command, P4UserMap):
self.gitStream = self.importProcess.stdin self.gitStream = self.importProcess.stdin
self.gitError = self.importProcess.stderr self.gitError = self.importProcess.stderr
if bytes is not str:
# Wrap gitStream.write() so that it can be called using `str` arguments
def make_encoded_write(write):
def encoded_write(s):
return write(s.encode() if isinstance(s, str) else s)
return encoded_write
self.gitStream.write = make_encoded_write(self.gitStream.write)
def closeStreams(self): def closeStreams(self):
if self.gitStream is None: if self.gitStream is None:
return return