diff --git a/Documentation/config.txt b/Documentation/config.txt index a1e36d7e42..76173dbc5d 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -1461,6 +1461,10 @@ remote..tagopt:: Setting this value to \--no-tags disables automatic tag following when fetching from remote +remote..vcs:: + Setting this to a value will cause git to interact with + the remote with the git-remote- helper. + remotes.:: The list of remotes which are fetched by "git remote update ". See linkgit:git-remote[1]. diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt index 8beb42dbb9..5cfdc0cfc5 100644 --- a/Documentation/git-remote-helpers.txt +++ b/Documentation/git-remote-helpers.txt @@ -79,6 +79,17 @@ style string if it contains an LF. + Supported if the helper has the "push" capability. +'import' :: + Produces a fast-import stream which imports the current value + of the named ref. It may additionally import other refs as + needed to construct the history efficiently. The script writes + to a helper-specific private namespace. The value of the named + ref should be written to a location in this namespace derived + by applying the refspecs from the "refspec" capability to the + name of the ref. ++ +Supported if the helper has the "import" capability. + If a fatal error occurs, the program writes the error message to stderr and exits. The caller should expect that a suitable error message has been printed if the child closes the connection without @@ -99,6 +110,19 @@ CAPABILITIES 'push':: This helper supports the 'push' command. +'import':: + This helper supports the 'import' command. + +'refspec' 'spec':: + When using the import command, expect the source ref to have + been written to the destination ref. The earliest applicable + refspec takes precedence. For example + "refs/heads/*:refs/svn/origin/branches/*" means that, after an + "import refs/heads/name", the script has written to + refs/svn/origin/branches/name. If this capability is used at + all, it must cover all refs reported by the list command; if + it is not used, it is effectively "*:*" + REF LIST ATTRIBUTES ------------------- @@ -107,6 +131,10 @@ REF LIST ATTRIBUTES commands. A helper might chose to acquire the ref list by opening a different type of connection to the destination. +'unchanged':: + This ref is unchanged since the last import or fetch, although + the helper cannot necessarily determine what value that produced. + OPTIONS ------- 'option verbosity' :: diff --git a/Makefile b/Makefile index 4a1e5bcc4d..fd4919c0ad 100644 --- a/Makefile +++ b/Makefile @@ -168,6 +168,8 @@ all:: # # Define NO_PERL if you do not want Perl scripts or libraries at all. # +# Define NO_PYTHON if you do not want Python scripts or libraries at all. +# # Define NO_TCLTK if you do not want Tcl/Tk GUI. # # The TCL_PATH variable governs the location of the Tcl interpreter @@ -341,6 +343,7 @@ LIB_H = LIB_OBJS = PROGRAMS = SCRIPT_PERL = +SCRIPT_PYTHON = SCRIPT_SH = TEST_PROGRAMS = @@ -379,6 +382,7 @@ SCRIPT_PERL += git-svn.perl SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) \ $(patsubst %.perl,%,$(SCRIPT_PERL)) \ + $(patsubst %.py,%,$(SCRIPT_PYTHON)) \ git-instaweb # Empty... @@ -434,8 +438,12 @@ endif ifndef PERL_PATH PERL_PATH = /usr/bin/perl endif +ifndef PYTHON_PATH + PYTHON_PATH = /usr/bin/python +endif export PERL_PATH +export PYTHON_PATH LIB_FILE=libgit.a XDIFF_LIB=xdiff/lib.a @@ -1346,6 +1354,10 @@ ifeq ($(PERL_PATH),) NO_PERL=NoThanks endif +ifeq ($(PYTHON_PATH),) +NO_PYTHON=NoThanks +endif + QUIET_SUBDIR0 = +$(MAKE) -C # space to separate -C and subdir QUIET_SUBDIR1 = @@ -1393,6 +1405,7 @@ prefix_SQ = $(subst ','\'',$(prefix)) SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) PERL_PATH_SQ = $(subst ','\'',$(PERL_PATH)) +PYTHON_PATH_SQ = $(subst ','\'',$(PYTHON_PATH)) TCLTK_PATH_SQ = $(subst ','\'',$(TCLTK_PATH)) LIBS = $(GITLIBS) $(EXTLIBS) @@ -1439,6 +1452,9 @@ ifndef NO_TCLTK endif ifndef NO_PERL $(QUIET_SUBDIR0)perl $(QUIET_SUBDIR1) PERL_PATH='$(PERL_PATH_SQ)' prefix='$(prefix_SQ)' all +endif +ifndef NO_PYTHON + $(QUIET_SUBDIR0)git_remote_helpers $(QUIET_SUBDIR1) PYTHON_PATH='$(PYTHON_PATH_SQ)' prefix='$(prefix_SQ)' all endif $(QUIET_SUBDIR0)templates $(QUIET_SUBDIR1) @@ -1566,11 +1582,41 @@ $(patsubst %.perl,%,$(SCRIPT_PERL)) git-instaweb: % : unimplemented.sh mv $@+ $@ endif # NO_PERL + ifdef JSMIN gitweb/gitweb.min.js: gitweb/gitweb.js $(QUIET_GEN)$(JSMIN) <$< >$@ endif # JSMIN +ifndef NO_PYTHON +$(patsubst %.py,%,$(SCRIPT_PYTHON)): GIT-CFLAGS +$(patsubst %.py,%,$(SCRIPT_PYTHON)): % : %.py + $(QUIET_GEN)$(RM) $@ $@+ && \ + INSTLIBDIR=`MAKEFLAGS= $(MAKE) -C git_remote_helpers -s \ + --no-print-directory prefix='$(prefix_SQ)' DESTDIR='$(DESTDIR_SQ)' \ + instlibdir` && \ + sed -e '1{' \ + -e ' s|#!.*python|#!$(PYTHON_PATH_SQ)|' \ + -e '}' \ + -e 's|^import sys.*|&; \\\ + import os; \\\ + sys.path[0] = os.environ.has_key("GITPYTHONLIB") and \\\ + os.environ["GITPYTHONLIB"] or \\\ + "@@INSTLIBDIR@@"|' \ + -e 's|@@INSTLIBDIR@@|'"$$INSTLIBDIR"'|g' \ + $@.py >$@+ && \ + chmod +x $@+ && \ + mv $@+ $@ +else # NO_PYTHON +$(patsubst %.py,%,$(SCRIPT_PYTHON)): % : unimplemented.sh + $(QUIET_GEN)$(RM) $@ $@+ && \ + sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \ + -e 's|@@REASON@@|NO_PYTHON=$(NO_PYTHON)|g' \ + unimplemented.sh >$@+ && \ + chmod +x $@+ && \ + mv $@+ $@ +endif # NO_PYTHON + configure: configure.ac $(QUIET_GEN)$(RM) $@ $<+ && \ sed -e 's/@@GIT_VERSION@@/$(GIT_VERSION)/g' \ @@ -1697,6 +1743,7 @@ GIT-BUILD-OPTIONS: .FORCE-GIT-BUILD-OPTIONS @echo TAR=\''$(subst ','\'',$(subst ','\'',$(TAR)))'\' >>$@ @echo NO_CURL=\''$(subst ','\'',$(subst ','\'',$(NO_CURL)))'\' >>$@ @echo NO_PERL=\''$(subst ','\'',$(subst ','\'',$(NO_PERL)))'\' >>$@ + @echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@ ### Detect Tck/Tk interpreter path changes ifndef NO_TCLTK @@ -1796,6 +1843,9 @@ install: all ifndef NO_PERL $(MAKE) -C perl prefix='$(prefix_SQ)' DESTDIR='$(DESTDIR_SQ)' install endif +ifndef NO_PYTHON + $(MAKE) -C git_remote_helpers prefix='$(prefix_SQ)' DESTDIR='$(DESTDIR_SQ)' install +endif ifndef NO_TCLTK $(MAKE) -C gitk-git install $(MAKE) -C git-gui gitexecdir='$(gitexec_instdir_SQ)' install @@ -1912,6 +1962,9 @@ clean: ifndef NO_PERL $(RM) gitweb/gitweb.cgi $(MAKE) -C perl clean +endif +ifndef NO_PYTHON + $(MAKE) -C git_remote_helpers clean endif $(MAKE) -C templates/ clean $(MAKE) -C t/ clean diff --git a/builtin-clone.c b/builtin-clone.c index caf3025031..5df8b0f72c 100644 --- a/builtin-clone.c +++ b/builtin-clone.c @@ -362,9 +362,10 @@ int cmd_clone(int argc, const char **argv, const char *prefix) const char *repo_name, *repo, *work_tree, *git_dir; char *path, *dir; int dest_exists; - const struct ref *refs, *remote_head, *mapped_refs; + const struct ref *refs, *remote_head; const struct ref *remote_head_points_at; const struct ref *our_head_points_at; + struct ref *mapped_refs; struct strbuf key = STRBUF_INIT, value = STRBUF_INIT; struct strbuf branch_top = STRBUF_INIT, reflog_msg = STRBUF_INIT; struct transport *transport = NULL; diff --git a/builtin-fetch.c b/builtin-fetch.c index 5b7db616dc..8654fa7a2d 100644 --- a/builtin-fetch.c +++ b/builtin-fetch.c @@ -322,7 +322,10 @@ static int store_updated_refs(const char *raw_url, const char *remote_name, if (!fp) return error("cannot open %s: %s\n", filename, strerror(errno)); - url = transport_anonymize_url(raw_url); + if (raw_url) + url = transport_anonymize_url(raw_url); + else + url = xstrdup("foreign"); for (rm = ref_map; rm; rm = rm->next) { struct ref *ref = NULL; @@ -819,7 +822,7 @@ static int fetch_one(struct remote *remote, int argc, const char **argv) if (!remote) die("Where do you want to fetch from today?"); - transport = transport_get(remote, remote->url[0]); + transport = transport_get(remote, NULL); if (verbosity >= 2) transport->verbose = verbosity <= 3 ? verbosity : 3; if (verbosity < 0) diff --git a/builtin-ls-remote.c b/builtin-ls-remote.c index b5bad0c184..70f5622d9d 100644 --- a/builtin-ls-remote.c +++ b/builtin-ls-remote.c @@ -89,7 +89,7 @@ int cmd_ls_remote(int argc, const char **argv, const char *prefix) remote = remote_get(dest); if (!remote->url_nr) die("remote %s has no configured URL", dest); - transport = transport_get(remote, remote->url[0]); + transport = transport_get(remote, NULL); if (uploadpack != NULL) transport_set_option(transport, TRANS_OPT_UPLOADPACK, uploadpack); diff --git a/builtin-push.c b/builtin-push.c index 356d7c1fd3..dcfb53f188 100644 --- a/builtin-push.c +++ b/builtin-push.c @@ -87,6 +87,37 @@ static void setup_default_push_refspecs(void) } } +static int push_with_options(struct transport *transport, int flags) +{ + int err; + int nonfastforward; + if (receivepack) + transport_set_option(transport, + TRANS_OPT_RECEIVEPACK, receivepack); + if (thin) + transport_set_option(transport, TRANS_OPT_THIN, "yes"); + + if (flags & TRANSPORT_PUSH_VERBOSE) + fprintf(stderr, "Pushing to %s\n", transport->url); + err = transport_push(transport, refspec_nr, refspec, flags, + &nonfastforward); + if (err != 0) + error("failed to push some refs to '%s'", transport->url); + + err |= transport_disconnect(transport); + + if (!err) + return 0; + + if (nonfastforward && advice_push_nonfastforward) { + printf("To prevent you from losing history, non-fast-forward updates were rejected\n" + "Merge the remote changes before pushing again. See the 'non-fast-forward'\n" + "section of 'git push --help' for details.\n"); + } + + return 1; +} + static int do_push(const char *repo, int flags) { int i, errs; @@ -135,33 +166,19 @@ static int do_push(const char *repo, int flags) url = remote->url; url_nr = remote->url_nr; } - for (i = 0; i < url_nr; i++) { - struct transport *transport = - transport_get(remote, url[i]); - int err; - int nonfastforward; - if (receivepack) - transport_set_option(transport, - TRANS_OPT_RECEIVEPACK, receivepack); - if (thin) - transport_set_option(transport, TRANS_OPT_THIN, "yes"); - - if (flags & TRANSPORT_PUSH_VERBOSE) - fprintf(stderr, "Pushing to %s\n", url[i]); - err = transport_push(transport, refspec_nr, refspec, flags, - &nonfastforward); - err |= transport_disconnect(transport); - - if (!err) - continue; - - error("failed to push some refs to '%s'", url[i]); - if (nonfastforward && advice_push_nonfastforward) { - printf("To prevent you from losing history, non-fast-forward updates were rejected\n" - "Merge the remote changes before pushing again. See the 'non-fast-forward'\n" - "section of 'git push --help' for details.\n"); + if (url_nr) { + for (i = 0; i < url_nr; i++) { + struct transport *transport = + transport_get(remote, url[i]); + if (push_with_options(transport, flags)) + errs++; } - errs++; + } else { + struct transport *transport = + transport_get(remote, NULL); + + if (push_with_options(transport, flags)) + errs++; } return !!errs; } diff --git a/configure.ac b/configure.ac index 4625b8672b..78345ebb60 100644 --- a/configure.ac +++ b/configure.ac @@ -276,6 +276,9 @@ GIT_ARG_SET_PATH(shell) # Define PERL_PATH to provide path to Perl. GIT_ARG_SET_PATH(perl) # +# Define PYTHON_PATH to provide path to Python. +GIT_ARG_SET_PATH(python) +# # Define ZLIB_PATH to provide path to zlib. GIT_ARG_SET_PATH(zlib) # diff --git a/git_remote_helpers/.gitignore b/git_remote_helpers/.gitignore new file mode 100644 index 0000000000..2247d5f95a --- /dev/null +++ b/git_remote_helpers/.gitignore @@ -0,0 +1,2 @@ +/build +/dist diff --git a/git_remote_helpers/Makefile b/git_remote_helpers/Makefile new file mode 100644 index 0000000000..c62dfd0f4d --- /dev/null +++ b/git_remote_helpers/Makefile @@ -0,0 +1,35 @@ +# +# Makefile for the git_remote_helpers python support modules +# +pysetupfile:=setup.py + +# Shell quote (do not use $(call) to accommodate ancient setups); +DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) + +ifndef PYTHON_PATH + PYTHON_PATH = /usr/bin/python +endif +ifndef prefix + prefix = $(HOME) +endif +ifndef V + QUIET = @ + QUIETSETUP = --quiet +endif + +PYLIBDIR=$(shell $(PYTHON_PATH) -c \ + "import sys; \ + print 'lib/python%i.%i/site-packages' % sys.version_info[:2]") + +all: $(pysetupfile) + $(QUIET)$(PYTHON_PATH) $(pysetupfile) $(QUIETSETUP) build + +install: $(pysetupfile) + $(PYTHON_PATH) $(pysetupfile) install --prefix $(DESTDIR_SQ)$(prefix) + +instlibdir: $(pysetupfile) + @echo "$(DESTDIR_SQ)$(prefix)/$(PYLIBDIR)" + +clean: + $(QUIET)$(PYTHON_PATH) $(pysetupfile) $(QUIETSETUP) clean -a + $(RM) *.pyo *.pyc diff --git a/git_remote_helpers/__init__.py b/git_remote_helpers/__init__.py new file mode 100644 index 0000000000..00f69cbeda --- /dev/null +++ b/git_remote_helpers/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +"""Support library package for git remote helpers. + +Git remote helpers are helper commands that interfaces with a non-git +repository to provide automatic import of non-git history into a Git +repository. + +This package provides the support library needed by these helpers.. +The following modules are included: + +- git.git - Interaction with Git repositories + +- util - General utility functionality use by the other modules in + this package, and also used directly by the helpers. +""" diff --git a/git_remote_helpers/git/__init__.py b/git_remote_helpers/git/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/git_remote_helpers/git/git.py b/git_remote_helpers/git/git.py new file mode 100644 index 0000000000..a383e6c08d --- /dev/null +++ b/git_remote_helpers/git/git.py @@ -0,0 +1,678 @@ +#!/usr/bin/env python + +"""Functionality for interacting with Git repositories. + +This module provides classes for interfacing with a Git repository. +""" + +import os +import re +import time +from binascii import hexlify +from cStringIO import StringIO +import unittest + +from git_remote_helpers.util import debug, error, die, start_command, run_command + + +def get_git_dir (): + """Return the path to the GIT_DIR for this repo.""" + args = ("git", "rev-parse", "--git-dir") + exit_code, output, errors = run_command(args) + if exit_code: + die("Failed to retrieve git dir") + assert not errors + return output.strip() + + +def parse_git_config (): + """Return a dict containing the parsed version of 'git config -l'.""" + exit_code, output, errors = run_command(("git", "config", "-z", "-l")) + if exit_code: + die("Failed to retrieve git configuration") + assert not errors + return dict([e.split('\n', 1) for e in output.split("\0") if e]) + + +def git_config_bool (value): + """Convert the given git config string value to True or False. + + Raise ValueError if the given string was not recognized as a + boolean value. + + """ + norm_value = str(value).strip().lower() + if norm_value in ("true", "1", "yes", "on", ""): + return True + if norm_value in ("false", "0", "no", "off", "none"): + return False + raise ValueError("Failed to parse '%s' into a boolean value" % (value)) + + +def valid_git_ref (ref_name): + """Return True iff the given ref name is a valid git ref name.""" + # The following is a reimplementation of the git check-ref-format + # command. The rules were derived from the git check-ref-format(1) + # manual page. This code should be replaced by a call to + # check_ref_format() in the git library, when such is available. + if ref_name.endswith('/') or \ + ref_name.startswith('.') or \ + ref_name.count('/.') or \ + ref_name.count('..') or \ + ref_name.endswith('.lock'): + return False + for c in ref_name: + if ord(c) < 0x20 or ord(c) == 0x7f or c in " ~^:?*[": + return False + return True + + +class GitObjectFetcher(object): + + """Provide parsed access to 'git cat-file --batch'. + + This provides a read-only interface to the Git object database. + + """ + + def __init__ (self): + """Initiate a 'git cat-file --batch' session.""" + self.queue = [] # List of object names to be submitted + self.in_transit = None # Object name currently in transit + + # 'git cat-file --batch' produces binary output which is likely + # to be corrupted by the default "rU"-mode pipe opened by + # start_command. (Mode == "rU" does universal new-line + # conversion, which mangles carriage returns.) Therefore, we + # open an explicitly binary-safe pipe for transferring the + # output from 'git cat-file --batch'. + pipe_r_fd, pipe_w_fd = os.pipe() + pipe_r = os.fdopen(pipe_r_fd, "rb") + pipe_w = os.fdopen(pipe_w_fd, "wb") + self.proc = start_command(("git", "cat-file", "--batch"), + stdout = pipe_w) + self.f = pipe_r + + def __del__ (self): + """Verify completed communication with 'git cat-file --batch'.""" + assert not self.queue + assert self.in_transit is None + self.proc.stdin.close() + assert self.proc.wait() == 0 # Zero exit code + assert self.f.read() == "" # No remaining output + + def _submit_next_object (self): + """Submit queue items to the 'git cat-file --batch' process. + + If there are items in the queue, and there is currently no item + currently in 'transit', then pop the first item off the queue, + and submit it. + + """ + if self.queue and self.in_transit is None: + self.in_transit = self.queue.pop(0) + print >> self.proc.stdin, self.in_transit[0] + + def push (self, obj, callback): + """Push the given object name onto the queue. + + The given callback function will at some point in the future + be called exactly once with the following arguments: + - self - this GitObjectFetcher instance + - obj - the object name provided to push() + - sha1 - the SHA1 of the object, if 'None' obj is missing + - t - the type of the object (tag/commit/tree/blob) + - size - the size of the object in bytes + - data - the object contents + + """ + self.queue.append((obj, callback)) + self._submit_next_object() # (Re)start queue processing + + def process_next_entry (self): + """Read the next entry off the queue and invoke callback.""" + obj, cb = self.in_transit + self.in_transit = None + header = self.f.readline() + if header == "%s missing\n" % (obj): + cb(self, obj, None, None, None, None) + return + sha1, t, size = header.split(" ") + assert len(sha1) == 40 + assert t in ("tag", "commit", "tree", "blob") + assert size.endswith("\n") + size = int(size.strip()) + data = self.f.read(size) + assert self.f.read(1) == "\n" + cb(self, obj, sha1, t, size, data) + self._submit_next_object() + + def process (self): + """Process the current queue until empty.""" + while self.in_transit is not None: + self.process_next_entry() + + # High-level convenience methods: + + def get_sha1 (self, objspec): + """Return the SHA1 of the object specified by 'objspec'. + + Return None if 'objspec' does not specify an existing object. + + """ + class _ObjHandler(object): + """Helper class for getting the returned SHA1.""" + def __init__ (self, parser): + self.parser = parser + self.sha1 = None + + def __call__ (self, parser, obj, sha1, t, size, data): + # FIXME: Many unused arguments. Could this be cheaper? + assert parser == self.parser + self.sha1 = sha1 + + handler = _ObjHandler(self) + self.push(objspec, handler) + self.process() + return handler.sha1 + + def open_obj (self, objspec): + """Return a file object wrapping the contents of a named object. + + The caller is responsible for calling .close() on the returned + file object. + + Raise KeyError if 'objspec' does not exist in the repo. + + """ + class _ObjHandler(object): + """Helper class for parsing the returned git object.""" + def __init__ (self, parser): + """Set up helper.""" + self.parser = parser + self.contents = StringIO() + self.err = None + + def __call__ (self, parser, obj, sha1, t, size, data): + """Git object callback (see GitObjectFetcher documentation).""" + assert parser == self.parser + if not sha1: # Missing object + self.err = "Missing object '%s'" % obj + else: + assert size == len(data) + self.contents.write(data) + + handler = _ObjHandler(self) + self.push(objspec, handler) + self.process() + if handler.err: + raise KeyError(handler.err) + handler.contents.seek(0) + return handler.contents + + def walk_tree (self, tree_objspec, callback, prefix = ""): + """Recursively walk the given Git tree object. + + Recursively walk all subtrees of the given tree object, and + invoke the given callback passing three arguments: + (path, mode, data) with the path, permission bits, and contents + of all the blobs found in the entire tree structure. + + """ + class _ObjHandler(object): + """Helper class for walking a git tree structure.""" + def __init__ (self, parser, cb, path, mode = None): + """Set up helper.""" + self.parser = parser + self.cb = cb + self.path = path + self.mode = mode + self.err = None + + def parse_tree (self, treedata): + """Parse tree object data, yield tree entries. + + Each tree entry is a 3-tuple (mode, sha1, path) + + self.path is prepended to all paths yielded + from this method. + + """ + while treedata: + mode = int(treedata[:6], 10) + # Turn 100xxx into xxx + if mode > 100000: + mode -= 100000 + assert treedata[6] == " " + i = treedata.find("\0", 7) + assert i > 0 + path = treedata[7:i] + sha1 = hexlify(treedata[i + 1: i + 21]) + yield (mode, sha1, self.path + path) + treedata = treedata[i + 21:] + + def __call__ (self, parser, obj, sha1, t, size, data): + """Git object callback (see GitObjectFetcher documentation).""" + assert parser == self.parser + if not sha1: # Missing object + self.err = "Missing object '%s'" % (obj) + return + assert size == len(data) + if t == "tree": + if self.path: + self.path += "/" + # Recurse into all blobs and subtrees + for m, s, p in self.parse_tree(data): + parser.push(s, + self.__class__(self.parser, self.cb, p, m)) + elif t == "blob": + self.cb(self.path, self.mode, data) + else: + raise ValueError("Unknown object type '%s'" % (t)) + + self.push(tree_objspec, _ObjHandler(self, callback, prefix)) + self.process() + + +class GitRefMap(object): + + """Map Git ref names to the Git object names they currently point to. + + Behaves like a dictionary of Git ref names -> Git object names. + + """ + + def __init__ (self, obj_fetcher): + """Create a new Git ref -> object map.""" + self.obj_fetcher = obj_fetcher + self._cache = {} # dict: refname -> objname + + def _load (self, ref): + """Retrieve the object currently bound to the given ref. + + The name of the object pointed to by the given ref is stored + into this mapping, and also returned. + + """ + if ref not in self._cache: + self._cache[ref] = self.obj_fetcher.get_sha1(ref) + return self._cache[ref] + + def __contains__ (self, refname): + """Return True if the given refname is present in this cache.""" + return bool(self._load(refname)) + + def __getitem__ (self, refname): + """Return the git object name pointed to by the given refname.""" + commit = self._load(refname) + if commit is None: + raise KeyError("Unknown ref '%s'" % (refname)) + return commit + + def get (self, refname, default = None): + """Return the git object name pointed to by the given refname.""" + commit = self._load(refname) + if commit is None: + return default + return commit + + +class GitFICommit(object): + + """Encapsulate the data in a Git fast-import commit command.""" + + SHA1RE = re.compile(r'^[0-9a-f]{40}$') + + @classmethod + def parse_mode (cls, mode): + """Verify the given git file mode, and return it as a string.""" + assert mode in (644, 755, 100644, 100755, 120000) + return "%i" % (mode) + + @classmethod + def parse_objname (cls, objname): + """Return the given object name (or mark number) as a string.""" + if isinstance(objname, int): # Object name is a mark number + assert objname > 0 + return ":%i" % (objname) + + # No existence check is done, only checks for valid format + assert cls.SHA1RE.match(objname) # Object name is valid SHA1 + return objname + + @classmethod + def quote_path (cls, path): + """Return a quoted version of the given path.""" + path = path.replace("\\", "\\\\") + path = path.replace("\n", "\\n") + path = path.replace('"', '\\"') + return '"%s"' % (path) + + @classmethod + def parse_path (cls, path): + """Verify that the given path is valid, and quote it, if needed.""" + assert not isinstance(path, int) # Cannot be a mark number + + # These checks verify the rules on the fast-import man page + assert not path.count("//") + assert not path.endswith("/") + assert not path.startswith("/") + assert not path.count("/./") + assert not path.count("/../") + assert not path.endswith("/.") + assert not path.endswith("/..") + assert not path.startswith("./") + assert not path.startswith("../") + + if path.count('"') + path.count('\n') + path.count('\\'): + return cls.quote_path(path) + return path + + def __init__ (self, name, email, timestamp, timezone, message): + """Create a new Git fast-import commit, with the given metadata.""" + self.name = name + self.email = email + self.timestamp = timestamp + self.timezone = timezone + self.message = message + self.pathops = [] # List of path operations in this commit + + def modify (self, mode, blobname, path): + """Add a file modification to this Git fast-import commit.""" + self.pathops.append(("M", + self.parse_mode(mode), + self.parse_objname(blobname), + self.parse_path(path))) + + def delete (self, path): + """Add a file deletion to this Git fast-import commit.""" + self.pathops.append(("D", self.parse_path(path))) + + def copy (self, path, newpath): + """Add a file copy to this Git fast-import commit.""" + self.pathops.append(("C", + self.parse_path(path), + self.parse_path(newpath))) + + def rename (self, path, newpath): + """Add a file rename to this Git fast-import commit.""" + self.pathops.append(("R", + self.parse_path(path), + self.parse_path(newpath))) + + def note (self, blobname, commit): + """Add a note object to this Git fast-import commit.""" + self.pathops.append(("N", + self.parse_objname(blobname), + self.parse_objname(commit))) + + def deleteall (self): + """Delete all files in this Git fast-import commit.""" + self.pathops.append("deleteall") + + +class TestGitFICommit(unittest.TestCase): + + """GitFICommit selftests.""" + + def test_basic (self): + """GitFICommit basic selftests.""" + + def expect_fail (method, data): + """Verify that the method(data) raises an AssertionError.""" + try: + method(data) + except AssertionError: + return + raise AssertionError("Failed test for invalid data '%s(%s)'" % + (method.__name__, repr(data))) + + def test_parse_mode (self): + """GitFICommit.parse_mode() selftests.""" + self.assertEqual(GitFICommit.parse_mode(644), "644") + self.assertEqual(GitFICommit.parse_mode(755), "755") + self.assertEqual(GitFICommit.parse_mode(100644), "100644") + self.assertEqual(GitFICommit.parse_mode(100755), "100755") + self.assertEqual(GitFICommit.parse_mode(120000), "120000") + self.assertRaises(AssertionError, GitFICommit.parse_mode, 0) + self.assertRaises(AssertionError, GitFICommit.parse_mode, 123) + self.assertRaises(AssertionError, GitFICommit.parse_mode, 600) + self.assertRaises(AssertionError, GitFICommit.parse_mode, "644") + self.assertRaises(AssertionError, GitFICommit.parse_mode, "abc") + + def test_parse_objname (self): + """GitFICommit.parse_objname() selftests.""" + self.assertEqual(GitFICommit.parse_objname(1), ":1") + self.assertRaises(AssertionError, GitFICommit.parse_objname, 0) + self.assertRaises(AssertionError, GitFICommit.parse_objname, -1) + self.assertEqual(GitFICommit.parse_objname("0123456789" * 4), + "0123456789" * 4) + self.assertEqual(GitFICommit.parse_objname("2468abcdef" * 4), + "2468abcdef" * 4) + self.assertRaises(AssertionError, GitFICommit.parse_objname, + "abcdefghij" * 4) + + def test_parse_path (self): + """GitFICommit.parse_path() selftests.""" + self.assertEqual(GitFICommit.parse_path("foo/bar"), "foo/bar") + self.assertEqual(GitFICommit.parse_path("path/with\n and \" in it"), + '"path/with\\n and \\" in it"') + self.assertRaises(AssertionError, GitFICommit.parse_path, 1) + self.assertRaises(AssertionError, GitFICommit.parse_path, 0) + self.assertRaises(AssertionError, GitFICommit.parse_path, -1) + self.assertRaises(AssertionError, GitFICommit.parse_path, "foo//bar") + self.assertRaises(AssertionError, GitFICommit.parse_path, "foo/bar/") + self.assertRaises(AssertionError, GitFICommit.parse_path, "/foo/bar") + self.assertRaises(AssertionError, GitFICommit.parse_path, "foo/./bar") + self.assertRaises(AssertionError, GitFICommit.parse_path, "foo/../bar") + self.assertRaises(AssertionError, GitFICommit.parse_path, "foo/bar/.") + self.assertRaises(AssertionError, GitFICommit.parse_path, "foo/bar/..") + self.assertRaises(AssertionError, GitFICommit.parse_path, "./foo/bar") + self.assertRaises(AssertionError, GitFICommit.parse_path, "../foo/bar") + + +class GitFastImport(object): + + """Encapsulate communication with git fast-import.""" + + def __init__ (self, f, obj_fetcher, last_mark = 0): + """Set up self to communicate with a fast-import process through f.""" + self.f = f # File object where fast-import stream is written + self.obj_fetcher = obj_fetcher # GitObjectFetcher instance + self.next_mark = last_mark + 1 # Next mark number + self.refs = set() # Keep track of the refnames we've seen + + def comment (self, s): + """Write the given comment in the fast-import stream.""" + assert "\n" not in s, "Malformed comment: '%s'" % (s) + self.f.write("# %s\n" % (s)) + + def commit (self, ref, commitdata): + """Make a commit on the given ref, with the given GitFICommit. + + Return the mark number identifying this commit. + + """ + self.f.write("""\ +commit %(ref)s +mark :%(mark)i +committer %(name)s <%(email)s> %(timestamp)i %(timezone)s +data %(msgLength)i +%(msg)s +""" % { + 'ref': ref, + 'mark': self.next_mark, + 'name': commitdata.name, + 'email': commitdata.email, + 'timestamp': commitdata.timestamp, + 'timezone': commitdata.timezone, + 'msgLength': len(commitdata.message), + 'msg': commitdata.message, +}) + + if ref not in self.refs: + self.refs.add(ref) + parent = ref + "^0" + if self.obj_fetcher.get_sha1(parent): + self.f.write("from %s\n" % (parent)) + + for op in commitdata.pathops: + self.f.write(" ".join(op)) + self.f.write("\n") + self.f.write("\n") + retval = self.next_mark + self.next_mark += 1 + return retval + + def blob (self, data): + """Import the given blob. + + Return the mark number identifying this blob. + + """ + self.f.write("blob\nmark :%i\ndata %i\n%s\n" % + (self.next_mark, len(data), data)) + retval = self.next_mark + self.next_mark += 1 + return retval + + def reset (self, ref, objname): + """Reset the given ref to point at the given Git object.""" + self.f.write("reset %s\nfrom %s\n\n" % + (ref, GitFICommit.parse_objname(objname))) + if ref not in self.refs: + self.refs.add(ref) + + +class GitNotes(object): + + """Encapsulate access to Git notes. + + Simulates a dictionary of object name (SHA1) -> Git note mappings. + + """ + + def __init__ (self, notes_ref, obj_fetcher): + """Create a new Git notes interface, bound to the given notes ref.""" + self.notes_ref = notes_ref + self.obj_fetcher = obj_fetcher # Used to get objects from repo + self.imports = [] # list: (objname, note data blob name) tuples + + def __del__ (self): + """Verify that self.commit_notes() was called before destruction.""" + if self.imports: + error("Missing call to self.commit_notes().") + error("%i notes are not committed!", len(self.imports)) + + def _load (self, objname): + """Return the note data associated with the given git object. + + The note data is returned in string form. If no note is found + for the given object, None is returned. + + """ + try: + f = self.obj_fetcher.open_obj("%s:%s" % (self.notes_ref, objname)) + ret = f.read() + f.close() + except KeyError: + ret = None + return ret + + def __getitem__ (self, objname): + """Return the note contents associated with the given object. + + Raise KeyError if given object has no associated note. + + """ + blobdata = self._load(objname) + if blobdata is None: + raise KeyError("Object '%s' has no note" % (objname)) + return blobdata + + def get (self, objname, default = None): + """Return the note contents associated with the given object. + + Return given default if given object has no associated note. + + """ + blobdata = self._load(objname) + if blobdata is None: + return default + return blobdata + + def import_note (self, objname, data, gfi): + """Tell git fast-import to store data as a note for objname. + + This method uses the given GitFastImport object to create a + blob containing the given note data. Also an entry mapping the + given object name to the created blob is stored until + commit_notes() is called. + + Note that this method only works if it is later followed by a + call to self.commit_notes() (which produces the note commit + that refers to the blob produced here). + + """ + if not data.endswith("\n"): + data += "\n" + gfi.comment("Importing note for object %s" % (objname)) + mark = gfi.blob(data) + self.imports.append((objname, mark)) + + def commit_notes (self, gfi, author, message): + """Produce a git fast-import note commit for the imported notes. + + This method uses the given GitFastImport object to create a + commit on the notes ref, introducing the notes previously + submitted to import_note(). + + """ + if not self.imports: + return + commitdata = GitFICommit(author[0], author[1], + time.time(), "0000", message) + for objname, blobname in self.imports: + assert isinstance(objname, int) and objname > 0 + assert isinstance(blobname, int) and blobname > 0 + commitdata.note(blobname, objname) + gfi.commit(self.notes_ref, commitdata) + self.imports = [] + + +class GitCachedNotes(GitNotes): + + """Encapsulate access to Git notes (cached version). + + Only use this class if no caching is done at a higher level. + + Simulates a dictionary of object name (SHA1) -> Git note mappings. + + """ + + def __init__ (self, notes_ref, obj_fetcher): + """Set up a caching wrapper around GitNotes.""" + GitNotes.__init__(self, notes_ref, obj_fetcher) + self._cache = {} # Cache: object name -> note data + + def __del__ (self): + """Verify that GitNotes' destructor is called.""" + GitNotes.__del__(self) + + def _load (self, objname): + """Extend GitNotes._load() with a local objname -> note cache.""" + if objname not in self._cache: + self._cache[objname] = GitNotes._load(self, objname) + return self._cache[objname] + + def import_note (self, objname, data, gfi): + """Extend GitNotes.import_note() with a local objname -> note cache.""" + if not data.endswith("\n"): + data += "\n" + assert objname not in self._cache + self._cache[objname] = data + GitNotes.import_note(self, objname, data, gfi) + + +if __name__ == '__main__': + unittest.main() diff --git a/git_remote_helpers/setup.py b/git_remote_helpers/setup.py new file mode 100644 index 0000000000..4d434b65cb --- /dev/null +++ b/git_remote_helpers/setup.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +"""Distutils build/install script for the git_remote_helpers package.""" + +from distutils.core import setup + +setup( + name = 'git_remote_helpers', + version = '0.1.0', + description = 'Git remote helper program for non-git repositories', + license = 'GPLv2', + author = 'The Git Community', + author_email = 'git@vger.kernel.org', + url = 'http://www.git-scm.com/', + package_dir = {'git_remote_helpers': ''}, + packages = ['git_remote_helpers', 'git_remote_helpers.git'], +) diff --git a/git_remote_helpers/util.py b/git_remote_helpers/util.py new file mode 100644 index 0000000000..dce83e6066 --- /dev/null +++ b/git_remote_helpers/util.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +"""Misc. useful functionality used by the rest of this package. + +This module provides common functionality used by the other modules in +this package. + +""" + +import sys +import os +import subprocess + + +# Whether or not to show debug messages +DEBUG = False + +def notify(msg, *args): + """Print a message to stderr.""" + print >> sys.stderr, msg % args + +def debug (msg, *args): + """Print a debug message to stderr when DEBUG is enabled.""" + if DEBUG: + print >> sys.stderr, msg % args + +def error (msg, *args): + """Print an error message to stderr.""" + print >> sys.stderr, "ERROR:", msg % args + +def warn(msg, *args): + """Print a warning message to stderr.""" + print >> sys.stderr, "warning:", msg % args + +def die (msg, *args): + """Print as error message to stderr and exit the program.""" + error(msg, *args) + sys.exit(1) + + +class ProgressIndicator(object): + + """Simple progress indicator. + + Displayed as a spinning character by default, but can be customized + by passing custom messages that overrides the spinning character. + + """ + + States = ("|", "/", "-", "\\") + + def __init__ (self, prefix = "", f = sys.stdout): + """Create a new ProgressIndicator, bound to the given file object.""" + self.n = 0 # Simple progress counter + self.f = f # Progress is written to this file object + self.prev_len = 0 # Length of previous msg (to be overwritten) + self.prefix = prefix # Prefix prepended to each progress message + self.prefix_lens = [] # Stack of prefix string lengths + + def pushprefix (self, prefix): + """Append the given prefix onto the prefix stack.""" + self.prefix_lens.append(len(self.prefix)) + self.prefix += prefix + + def popprefix (self): + """Remove the last prefix from the prefix stack.""" + prev_len = self.prefix_lens.pop() + self.prefix = self.prefix[:prev_len] + + def __call__ (self, msg = None, lf = False): + """Indicate progress, possibly with a custom message.""" + if msg is None: + msg = self.States[self.n % len(self.States)] + msg = self.prefix + msg + print >> self.f, "\r%-*s" % (self.prev_len, msg), + self.prev_len = len(msg.expandtabs()) + if lf: + print >> self.f + self.prev_len = 0 + self.n += 1 + + def finish (self, msg = "done", noprefix = False): + """Finalize progress indication with the given message.""" + if noprefix: + self.prefix = "" + self(msg, True) + + +def start_command (args, cwd = None, shell = False, add_env = None, + stdin = subprocess.PIPE, stdout = subprocess.PIPE, + stderr = subprocess.PIPE): + """Start the given command, and return a subprocess object. + + This provides a simpler interface to the subprocess module. + + """ + env = None + if add_env is not None: + env = os.environ.copy() + env.update(add_env) + return subprocess.Popen(args, bufsize = 1, stdin = stdin, stdout = stdout, + stderr = stderr, cwd = cwd, shell = shell, + env = env, universal_newlines = True) + + +def run_command (args, cwd = None, shell = False, add_env = None, + flag_error = True): + """Run the given command to completion, and return its results. + + This provides a simpler interface to the subprocess module. + + The results are formatted as a 3-tuple: (exit_code, output, errors) + + If flag_error is enabled, Error messages will be produced if the + subprocess terminated with a non-zero exit code and/or stderr + output. + + The other arguments are passed on to start_command(). + + """ + process = start_command(args, cwd, shell, add_env) + (output, errors) = process.communicate() + exit_code = process.returncode + if flag_error and errors: + error("'%s' returned errors:\n---\n%s---", " ".join(args), errors) + if flag_error and exit_code: + error("'%s' returned exit code %i", " ".join(args), exit_code) + return (exit_code, output, errors) + + +def file_reader_method (missing_ok = False): + """Decorator for simplifying reading of files. + + If missing_ok is True, a failure to open a file for reading will + not raise the usual IOError, but instead the wrapped method will be + called with f == None. The method must in this case properly + handle f == None. + + """ + def _wrap (method): + """Teach given method to handle both filenames and file objects. + + The given method must take a file object as its second argument + (the first argument being 'self', of course). This decorator + will take a filename given as the second argument and promote + it to a file object. + + """ + def _wrapped_method (self, filename, *args, **kwargs): + if isinstance(filename, file): + f = filename + else: + try: + f = open(filename, 'r') + except IOError: + if missing_ok: + f = None + else: + raise + try: + return method(self, f, *args, **kwargs) + finally: + if not isinstance(filename, file) and f: + f.close() + return _wrapped_method + return _wrap + + +def file_writer_method (method): + """Decorator for simplifying writing of files. + + Enables the given method to handle both filenames and file objects. + + The given method must take a file object as its second argument + (the first argument being 'self', of course). This decorator will + take a filename given as the second argument and promote it to a + file object. + + """ + def _new_method (self, filename, *args, **kwargs): + if isinstance(filename, file): + f = filename + else: + # Make sure the containing directory exists + parent_dir = os.path.dirname(filename) + if not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + f = open(filename, 'w') + try: + return method(self, f, *args, **kwargs) + finally: + if not isinstance(filename, file): + f.close() + return _new_method diff --git a/remote.c b/remote.c index b979a9642b..e3afecdb10 100644 --- a/remote.c +++ b/remote.c @@ -53,6 +53,11 @@ static struct rewrites rewrites_push; #define BUF_SIZE (2048) static char buffer[BUF_SIZE]; +static int valid_remote(const struct remote *remote) +{ + return (!!remote->url) || (!!remote->foreign_vcs); +} + static const char *alias_url(const char *url, struct rewrites *r) { int i, j; @@ -441,6 +446,8 @@ static int handle_config(const char *key, const char *value, void *cb) } else if (!strcmp(subkey, ".proxy")) { return git_config_string((const char **)&remote->http_proxy, key, value); + } else if (!strcmp(subkey, ".vcs")) { + return git_config_string(&remote->foreign_vcs, key, value); } return 0; } @@ -668,6 +675,16 @@ static struct refspec *parse_push_refspec(int nr_refspec, const char **refspec) return parse_refspec_internal(nr_refspec, refspec, 0, 0); } +void free_refspec(int nr_refspec, struct refspec *refspec) +{ + int i; + for (i = 0; i < nr_refspec; i++) { + free(refspec[i].src); + free(refspec[i].dst); + } + free(refspec); +} + static int valid_remote_nick(const char *name) { if (!name[0] || is_dot_or_dotdot(name)) @@ -690,14 +707,14 @@ struct remote *remote_get(const char *name) ret = make_remote(name, 0); if (valid_remote_nick(name)) { - if (!ret->url) + if (!valid_remote(ret)) read_remotes_file(ret); - if (!ret->url) + if (!valid_remote(ret)) read_branches_file(ret); } - if (name_given && !ret->url) + if (name_given && !valid_remote(ret)) add_url_alias(ret, name); - if (!ret->url) + if (!valid_remote(ret)) return NULL; ret->fetch = parse_fetch_refspec(ret->fetch_refspec_nr, ret->fetch_refspec); ret->push = parse_push_refspec(ret->push_refspec_nr, ret->push_refspec); @@ -810,6 +827,23 @@ static int match_name_with_pattern(const char *key, const char *name, return ret; } +char *apply_refspecs(struct refspec *refspecs, int nr_refspec, + const char *name) +{ + int i; + char *ret = NULL; + for (i = 0; i < nr_refspec; i++) { + struct refspec *refspec = refspecs + i; + if (refspec->pattern) { + if (match_name_with_pattern(refspec->src, name, + refspec->dst, &ret)) + return ret; + } else if (!strcmp(refspec->src, name)) + return strdup(refspec->dst); + } + return NULL; +} + int remote_find_tracking(struct remote *remote, struct refspec *refspec) { int find_src = refspec->src == NULL; diff --git a/remote.h b/remote.h index d0aba81ead..8b7ecf9197 100644 --- a/remote.h +++ b/remote.h @@ -11,6 +11,8 @@ struct remote { const char *name; int origin; + const char *foreign_vcs; + const char **url; int url_nr; int url_alloc; @@ -89,6 +91,11 @@ void ref_remove_duplicates(struct ref *ref_map); int valid_fetch_refspec(const char *refspec); struct refspec *parse_fetch_refspec(int nr_refspec, const char **refspec); +void free_refspec(int nr_refspec, struct refspec *refspec); + +char *apply_refspecs(struct refspec *refspecs, int nr_refspec, + const char *name); + int match_refs(struct ref *src, struct ref **dst, int nr_refspec, const char **refspec, int all); diff --git a/t/test-lib.sh b/t/test-lib.sh index ec3336aba5..2d523fe0f0 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -632,20 +632,29 @@ GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_NOGLOBAL=1 export PATH GIT_EXEC_PATH GIT_TEMPLATE_DIR GIT_CONFIG_NOSYSTEM GIT_CONFIG_NOGLOBAL +. ../GIT-BUILD-OPTIONS + GITPERLLIB=$(pwd)/../perl/blib/lib:$(pwd)/../perl/blib/arch/auto/Git export GITPERLLIB test -d ../templates/blt || { error "You haven't built things yet, have you?" } +if test -z "$GIT_TEST_INSTALLED" && test -z "$NO_PYTHON" +then + GITPYTHONLIB="$(pwd)/../git_remote_helpers/build/lib" + export GITPYTHONLIB + test -d ../git_remote_helpers/build || { + error "You haven't built git_remote_helpers yet, have you?" + } +fi + if ! test -x ../test-chmtime; then echo >&2 'You need to build test-chmtime:' echo >&2 'Run "make test-chmtime" in the source (toplevel) directory' exit 1 fi -. ../GIT-BUILD-OPTIONS - # Test repository test="trash directory.$(basename "$0" .sh)" test -n "$root" && test="$root/$test" @@ -729,6 +738,7 @@ case $(uname -s) in esac test -z "$NO_PERL" && test_set_prereq PERL +test -z "$NO_PYTHON" && test_set_prereq PYTHON # test whether the filesystem supports symbolic links ln -s x y 2>/dev/null && test -h y 2>/dev/null && test_set_prereq SYMLINKS diff --git a/transport-helper.c b/transport-helper.c index 5078c7100f..11f3d7ec52 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -6,6 +6,7 @@ #include "diff.h" #include "revision.h" #include "quote.h" +#include "remote.h" struct helper_data { @@ -13,8 +14,12 @@ struct helper_data struct child_process *helper; FILE *out; unsigned fetch : 1, + import : 1, option : 1, push : 1; + /* These go from remote name (as in "list") to private name */ + struct refspec *refspecs; + int refspec_nr; }; static struct child_process *get_helper(struct transport *transport) @@ -22,6 +27,9 @@ static struct child_process *get_helper(struct transport *transport) struct helper_data *data = transport->data; struct strbuf buf = STRBUF_INIT; struct child_process *helper; + const char **refspecs = NULL; + int refspec_nr = 0; + int refspec_alloc = 0; if (data->helper) return data->helper; @@ -55,7 +63,25 @@ static struct child_process *get_helper(struct transport *transport) data->option = 1; if (!strcmp(buf.buf, "push")) data->push = 1; + if (!strcmp(buf.buf, "import")) + data->import = 1; + if (!data->refspecs && !prefixcmp(buf.buf, "refspec ")) { + ALLOC_GROW(refspecs, + refspec_nr + 1, + refspec_alloc); + refspecs[refspec_nr++] = strdup(buf.buf + strlen("refspec ")); + } } + if (refspecs) { + int i; + data->refspec_nr = refspec_nr; + data->refspecs = parse_fetch_refspec(refspec_nr, refspecs); + for (i = 0; i < refspec_nr; i++) { + free((char *)refspecs[i]); + } + free(refspecs); + } + strbuf_release(&buf); return data->helper; } @@ -72,7 +98,6 @@ static int disconnect_helper(struct transport *transport) free(data->helper); data->helper = NULL; } - free(data); return 0; } @@ -154,8 +179,18 @@ static void standard_options(struct transport *t) set_helper_option(t, "verbosity", buf); } +static int release_helper(struct transport *transport) +{ + struct helper_data *data = transport->data; + free_refspec(data->refspec_nr, data->refspecs); + data->refspecs = NULL; + disconnect_helper(transport); + free(transport->data); + return 0; +} + static int fetch_with_fetch(struct transport *transport, - int nr_heads, const struct ref **to_fetch) + int nr_heads, struct ref **to_fetch) { struct helper_data *data = transport->data; int i; @@ -197,8 +232,64 @@ static int fetch_with_fetch(struct transport *transport, return 0; } +static int get_importer(struct transport *transport, struct child_process *fastimport) +{ + struct child_process *helper = get_helper(transport); + memset(fastimport, 0, sizeof(*fastimport)); + fastimport->in = helper->out; + fastimport->argv = xcalloc(5, sizeof(*fastimport->argv)); + fastimport->argv[0] = "fast-import"; + fastimport->argv[1] = "--quiet"; + + fastimport->git_cmd = 1; + return start_command(fastimport); +} + +static int fetch_with_import(struct transport *transport, + int nr_heads, struct ref **to_fetch) +{ + struct child_process fastimport; + struct child_process *helper = get_helper(transport); + struct helper_data *data = transport->data; + int i; + struct ref *posn; + struct strbuf buf = STRBUF_INIT; + + if (get_importer(transport, &fastimport)) + die("Couldn't run fast-import"); + + for (i = 0; i < nr_heads; i++) { + posn = to_fetch[i]; + if (posn->status & REF_STATUS_UPTODATE) + continue; + + strbuf_addf(&buf, "import %s\n", posn->name); + write_in_full(helper->in, buf.buf, buf.len); + strbuf_reset(&buf); + } + disconnect_helper(transport); + finish_command(&fastimport); + free(fastimport.argv); + fastimport.argv = NULL; + + for (i = 0; i < nr_heads; i++) { + char *private; + posn = to_fetch[i]; + if (posn->status & REF_STATUS_UPTODATE) + continue; + if (data->refspecs) + private = apply_refspecs(data->refspecs, data->refspec_nr, posn->name); + else + private = strdup(posn->name); + read_ref(private, posn->old_sha1); + free(private); + } + strbuf_release(&buf); + return 0; +} + static int fetch(struct transport *transport, - int nr_heads, const struct ref **to_fetch) + int nr_heads, struct ref **to_fetch) { struct helper_data *data = transport->data; int i, count; @@ -214,6 +305,9 @@ static int fetch(struct transport *transport, if (data->fetch) return fetch_with_fetch(transport, nr_heads, to_fetch); + if (data->import) + return fetch_with_import(transport, nr_heads, to_fetch); + return -1; } @@ -343,6 +437,22 @@ static int push_refs(struct transport *transport, return 0; } +static int has_attribute(const char *attrs, const char *attr) { + int len; + if (!attrs) + return 0; + + len = strlen(attr); + for (;;) { + const char *space = strchrnul(attrs, ' '); + if (len == space - attrs && !strncmp(attrs, attr, len)) + return 1; + if (!*space) + return 0; + attrs = space + 1; + } +} + static struct ref *get_refs_list(struct transport *transport, int for_push) { struct helper_data *data = transport->data; @@ -379,6 +489,12 @@ static struct ref *get_refs_list(struct transport *transport, int for_push) (*tail)->symref = xstrdup(buf.buf + 1); else if (buf.buf[0] != '?') get_sha1_hex(buf.buf, (*tail)->old_sha1); + if (eon) { + if (has_attribute(eon + 1, "unchanged")) { + (*tail)->status |= REF_STATUS_UPTODATE; + read_ref((*tail)->name, (*tail)->old_sha1); + } + } tail = &((*tail)->next); } strbuf_release(&buf); @@ -399,6 +515,6 @@ int transport_helper_init(struct transport *transport, const char *name) transport->get_refs_list = get_refs_list; transport->fetch = fetch; transport->push_refs = push_refs; - transport->disconnect = disconnect_helper; + transport->disconnect = release_helper; return 0; } diff --git a/transport.c b/transport.c index 7362ec09b2..3eea836a33 100644 --- a/transport.c +++ b/transport.c @@ -204,7 +204,7 @@ static struct ref *get_refs_via_rsync(struct transport *transport, int for_push) } static int fetch_objs_via_rsync(struct transport *transport, - int nr_objs, const struct ref **to_fetch) + int nr_objs, struct ref **to_fetch) { struct strbuf buf = STRBUF_INIT; struct child_process rsync; @@ -379,7 +379,7 @@ static struct ref *get_refs_from_bundle(struct transport *transport, int for_pus } static int fetch_refs_from_bundle(struct transport *transport, - int nr_heads, const struct ref **to_fetch) + int nr_heads, struct ref **to_fetch) { struct bundle_transport_data *data = transport->data; return unbundle(&data->header, data->fd); @@ -457,7 +457,7 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus } static int fetch_refs_via_pack(struct transport *transport, - int nr_heads, const struct ref **to_fetch) + int nr_heads, struct ref **to_fetch) { struct git_transport_data *data = transport->data; char **heads = xmalloc(nr_heads * sizeof(*heads)); @@ -788,8 +788,26 @@ struct transport *transport_get(struct remote *remote, const char *url) die("No remote provided to transport_get()"); ret->remote = remote; + + if (!url && remote && remote->url) + url = remote->url[0]; ret->url = url; + /* maybe it is a foreign URL? */ + if (url) { + const char *p = url; + + while (isalnum(*p)) + p++; + if (!prefixcmp(p, "::")) + remote->foreign_vcs = xstrndup(url, p - url); + } + + if (remote && remote->foreign_vcs) { + transport_helper_init(ret, remote->foreign_vcs); + return ret; + } + if (!prefixcmp(url, "rsync:")) { ret->get_refs_list = get_refs_via_rsync; ret->fetch = fetch_objs_via_rsync; @@ -896,16 +914,17 @@ const struct ref *transport_get_remote_refs(struct transport *transport) return transport->remote_refs; } -int transport_fetch_refs(struct transport *transport, const struct ref *refs) +int transport_fetch_refs(struct transport *transport, struct ref *refs) { int rc; int nr_heads = 0, nr_alloc = 0, nr_refs = 0; - const struct ref **heads = NULL; - const struct ref *rm; + struct ref **heads = NULL; + struct ref *rm; for (rm = refs; rm; rm = rm->next) { nr_refs++; if (rm->peer_ref && + !is_null_sha1(rm->old_sha1) && !hashcmp(rm->peer_ref->old_sha1, rm->old_sha1)) continue; ALLOC_GROW(heads, nr_heads + 1, nr_alloc); diff --git a/transport.h b/transport.h index e4e6177e26..9e74406fa2 100644 --- a/transport.h +++ b/transport.h @@ -18,11 +18,48 @@ struct transport { int (*set_option)(struct transport *connection, const char *name, const char *value); + /** + * Returns a list of the remote side's refs. In order to allow + * the transport to try to share connections, for_push is a + * hint as to whether the ultimate operation is a push or a fetch. + * + * If the transport is able to determine the remote hash for + * the ref without a huge amount of effort, it should store it + * in the ref's old_sha1 field; otherwise it should be all 0. + **/ struct ref *(*get_refs_list)(struct transport *transport, int for_push); - int (*fetch)(struct transport *transport, int refs_nr, const struct ref **refs); + + /** + * Fetch the objects for the given refs. Note that this gets + * an array, and should ignore the list structure. + * + * If the transport did not get hashes for refs in + * get_refs_list(), it should set the old_sha1 fields in the + * provided refs now. + **/ + int (*fetch)(struct transport *transport, int refs_nr, struct ref **refs); + + /** + * Push the objects and refs. Send the necessary objects, and + * then, for any refs where peer_ref is set and + * peer_ref->new_sha1 is different from old_sha1, tell the + * remote side to update each ref in the list from old_sha1 to + * peer_ref->new_sha1. + * + * Where possible, set the status for each ref appropriately. + * + * The transport must modify new_sha1 in the ref to the new + * value if the remote accepted the change. Note that this + * could be a different value from peer_ref->new_sha1 if the + * process involved generating new commits. + **/ int (*push_refs)(struct transport *transport, struct ref *refs, int flags); int (*push)(struct transport *connection, int refspec_nr, const char **refspec, int flags); + /** get_refs_list(), fetch(), and push_refs() can keep + * resources (such as a connection) reserved for futher + * use. disconnect() releases these resources. + **/ int (*disconnect)(struct transport *connection); char *pack_lockfile; signed verbose : 3; @@ -74,7 +111,7 @@ int transport_push(struct transport *connection, const struct ref *transport_get_remote_refs(struct transport *transport); -int transport_fetch_refs(struct transport *transport, const struct ref *refs); +int transport_fetch_refs(struct transport *transport, struct ref *refs); void transport_unlock_pack(struct transport *transport); int transport_disconnect(struct transport *transport); char *transport_anonymize_url(const char *url);