Merge branch 'db/vcs-helper'

* db/vcs-helper:
  Makefile: remove remnant of separate http/https/ftp helpers
  Use a clearer style to issue commands to remote helpers
  Make the "traditionally-supported" URLs a special case
  Makefile: install hardlinks for git-remote-<scheme> supported by libcurl if possible
  Makefile: do not link three copies of git-remote-* programs
  Makefile: git-http-fetch does not need expat
  http-fetch: Fix Makefile dependancies
  Add transport native helper executables to .gitignore
  git-http-fetch: not a builtin
  Use an external program to implement fetching with curl
  Add support for external programs for handling native fetches
This commit is contained in:
Junio C Hamano 2009-09-13 01:31:55 -07:00
commit cd03eebbfd
9 changed files with 406 additions and 144 deletions

1
.gitignore vendored
View File

@ -104,6 +104,7 @@ git-receive-pack
git-reflog
git-relink
git-remote
git-remote-curl
git-repack
git-replace
git-repo-config

View File

@ -0,0 +1,71 @@
git-remote-helpers(1)
=====================
NAME
----
git-remote-helpers - Helper programs for interoperation with remote git
SYNOPSIS
--------
'git remote-<transport>' <remote>
DESCRIPTION
-----------
These programs are normally not used directly by end users, but are
invoked by various git programs that interact with remote repositories
when the repository they would operate on will be accessed using
transport code not linked into the main git binary. Various particular
helper programs will behave as documented here.
COMMANDS
--------
Commands are given by the caller on the helper's standard input, one per line.
'capabilities'::
Lists the capabilities of the helper, one per line, ending
with a blank line.
'list'::
Lists the refs, one per line, in the format "<value> <name>
[<attr> ...]". The value may be a hex sha1 hash, "@<dest>" for
a symref, or "?" to indicate that the helper could not get the
value of the ref. A space-separated list of attributes follows
the name; unrecognized attributes are ignored. After the
complete list, outputs a blank line.
'fetch' <sha1> <name>::
Fetches the given object, writing the necessary objects to the
database. Outputs a blank line when the fetch is
complete. Only objects which were reported in the ref list
with a sha1 may be fetched this way.
+
Supported if the helper has the "fetch" capability.
If a fatal error occurs, the program writes the error message to
stderr and exits. The caller should expect that a suitable error
message has been printed if the child closes the connection without
completing a valid response for the current command.
Additional commands may be supported, as may be determined from
capabilities reported by the helper.
CAPABILITIES
------------
'fetch'::
This helper supports the 'fetch' command.
REF LIST ATTRIBUTES
-------------------
None are defined yet, but the caller must accept any which are supplied.
Documentation
-------------
Documentation by Daniel Barkalow.
GIT
---
Part of the linkgit:git[1] suite

View File

@ -380,7 +380,8 @@ BUILT_INS += git-stage$X
BUILT_INS += git-status$X
BUILT_INS += git-whatchanged$X
# what 'all' will build and 'install' will install, in gitexecdir
# what 'all' will build and 'install' will install in gitexecdir,
# excluding programs for built-in commands
ALL_PROGRAMS = $(PROGRAMS) $(SCRIPTS)
# what 'all' will build but not install in gitexecdir
@ -547,6 +548,7 @@ LIB_OBJS += symlinks.o
LIB_OBJS += tag.o
LIB_OBJS += trace.o
LIB_OBJS += transport.o
LIB_OBJS += transport-helper.o
LIB_OBJS += tree-diff.o
LIB_OBJS += tree.o
LIB_OBJS += tree-walk.o
@ -973,9 +975,7 @@ else
else
CURL_LIBCURL = -lcurl
endif
BUILTIN_OBJS += builtin-http-fetch.o
EXTLIBS += $(CURL_LIBCURL)
LIB_OBJS += http.o http-walker.o
PROGRAMS += git-remote-curl$X git-http-fetch$X
curl_check := $(shell (echo 070908; curl-config --vernum) | sort -r | sed -ne 2p)
ifeq "$(curl_check)" "070908"
ifndef NO_EXPAT
@ -1249,6 +1249,7 @@ ifndef V
QUIET_LINK = @echo ' ' LINK $@;
QUIET_BUILT_IN = @echo ' ' BUILTIN $@;
QUIET_GEN = @echo ' ' GEN $@;
QUIET_LNCP = @echo ' ' LN/CP $@;
QUIET_SUBDIR0 = +@subdir=
QUIET_SUBDIR1 = ;$(NO_SUBDIR) echo ' ' SUBDIR $$subdir; \
$(MAKE) $(PRINT_DIR) -C $$subdir
@ -1476,12 +1477,21 @@ git-imap-send$X: imap-send.o $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(LIBS) $(OPENSSL_LINK) $(OPENSSL_LIBSSL)
http.o http-walker.o http-push.o transport.o: http.h
http.o http-walker.o http-push.o: http.h
http.o http-walker.o: $(LIB_H)
git-http-fetch$X: revision.o http.o http-walker.o http-fetch.o $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(LIBS) $(CURL_LIBCURL)
git-http-push$X: revision.o http.o http-push.o $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT)
git-remote-curl$X: remote-curl.o http.o http-walker.o $(GITLIBS)
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
$(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT)
$(LIB_OBJS) $(BUILTIN_OBJS): $(LIB_H)
$(patsubst git-%$X,%.o,$(PROGRAMS)) git.o: $(LIB_H) $(wildcard */*.h)
builtin-revert.o wt-status.o: wt-status.h

3
git.c
View File

@ -312,9 +312,6 @@ static void handle_internal_command(int argc, const char **argv)
{ "get-tar-commit-id", cmd_get_tar_commit_id },
{ "grep", cmd_grep, RUN_SETUP | USE_PAGER },
{ "help", cmd_help },
#ifndef NO_CURL
{ "http-fetch", cmd_http_fetch, RUN_SETUP },
#endif
{ "init", cmd_init_db },
{ "init-db", cmd_init_db },
{ "log", cmd_log, RUN_SETUP | USE_PAGER },

View File

@ -1,8 +1,9 @@
#include "cache.h"
#include "walker.h"
int cmd_http_fetch(int argc, const char **argv, const char *prefix)
int main(int argc, const char **argv)
{
const char *prefix;
struct walker *walker;
int commits_on_stdin = 0;
int commits;
@ -18,6 +19,8 @@ int cmd_http_fetch(int argc, const char **argv, const char *prefix)
int get_verbosely = 0;
int get_recover = 0;
prefix = setup_git_directory();
git_config(git_default_config, NULL);
while (arg < argc && argv[arg][0] == '-') {

139
remote-curl.c Normal file
View File

@ -0,0 +1,139 @@
#include "cache.h"
#include "remote.h"
#include "strbuf.h"
#include "walker.h"
#include "http.h"
static struct ref *get_refs(struct walker *walker, const char *url)
{
struct strbuf buffer = STRBUF_INIT;
char *data, *start, *mid;
char *ref_name;
char *refs_url;
int i = 0;
int http_ret;
struct ref *refs = NULL;
struct ref *ref = NULL;
struct ref *last_ref = NULL;
refs_url = xmalloc(strlen(url) + 11);
sprintf(refs_url, "%s/info/refs", url);
http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE);
switch (http_ret) {
case HTTP_OK:
break;
case HTTP_MISSING_TARGET:
die("%s not found: did you run git update-server-info on the"
" server?", refs_url);
default:
http_error(refs_url, http_ret);
die("HTTP request failed");
}
data = buffer.buf;
start = NULL;
mid = data;
while (i < buffer.len) {
if (!start) {
start = &data[i];
}
if (data[i] == '\t')
mid = &data[i];
if (data[i] == '\n') {
data[i] = 0;
ref_name = mid + 1;
ref = xmalloc(sizeof(struct ref) +
strlen(ref_name) + 1);
memset(ref, 0, sizeof(struct ref));
strcpy(ref->name, ref_name);
get_sha1_hex(start, ref->old_sha1);
if (!refs)
refs = ref;
if (last_ref)
last_ref->next = ref;
last_ref = ref;
start = NULL;
}
i++;
}
strbuf_release(&buffer);
ref = alloc_ref("HEAD");
if (!walker->fetch_ref(walker, ref) &&
!resolve_remote_symref(ref, refs)) {
ref->next = refs;
refs = ref;
} else {
free(ref);
}
strbuf_release(&buffer);
free(refs_url);
return refs;
}
int main(int argc, const char **argv)
{
struct remote *remote;
struct strbuf buf = STRBUF_INIT;
const char *url;
struct walker *walker = NULL;
setup_git_directory();
if (argc < 2) {
fprintf(stderr, "Remote needed\n");
return 1;
}
remote = remote_get(argv[1]);
if (argc > 2) {
url = argv[2];
} else {
url = remote->url[0];
}
do {
if (strbuf_getline(&buf, stdin, '\n') == EOF)
break;
if (!prefixcmp(buf.buf, "fetch ")) {
char *obj = buf.buf + strlen("fetch ");
if (!walker)
walker = get_http_walker(url, remote);
walker->get_all = 1;
walker->get_tree = 1;
walker->get_history = 1;
walker->get_verbosely = 0;
walker->get_recover = 0;
if (walker_fetch(walker, 1, &obj, NULL, NULL))
die("Fetch failed.");
printf("\n");
fflush(stdout);
} else if (!strcmp(buf.buf, "list")) {
struct ref *refs;
struct ref *posn;
if (!walker)
walker = get_http_walker(url, remote);
refs = get_refs(walker, url);
for (posn = refs; posn; posn = posn->next) {
if (posn->symref)
printf("@%s %s\n", posn->symref, posn->name);
else
printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name);
}
printf("\n");
fflush(stdout);
} else if (!strcmp(buf.buf, "capabilities")) {
printf("fetch\n");
printf("\n");
fflush(stdout);
} else {
return 1;
}
strbuf_reset(&buf);
} while (1);
return 0;
}

172
transport-helper.c Normal file
View File

@ -0,0 +1,172 @@
#include "cache.h"
#include "transport.h"
#include "run-command.h"
#include "commit.h"
#include "diff.h"
#include "revision.h"
struct helper_data
{
const char *name;
struct child_process *helper;
unsigned fetch : 1;
};
static struct child_process *get_helper(struct transport *transport)
{
struct helper_data *data = transport->data;
struct strbuf buf = STRBUF_INIT;
struct child_process *helper;
FILE *file;
if (data->helper)
return data->helper;
helper = xcalloc(1, sizeof(*helper));
helper->in = -1;
helper->out = -1;
helper->err = 0;
helper->argv = xcalloc(4, sizeof(*helper->argv));
strbuf_addf(&buf, "remote-%s", data->name);
helper->argv[0] = strbuf_detach(&buf, NULL);
helper->argv[1] = transport->remote->name;
helper->argv[2] = transport->url;
helper->git_cmd = 1;
if (start_command(helper))
die("Unable to run helper: git %s", helper->argv[0]);
data->helper = helper;
strbuf_addstr(&buf, "capabilities\n");
write_in_full(helper->in, buf.buf, buf.len);
strbuf_reset(&buf);
file = fdopen(helper->out, "r");
while (1) {
if (strbuf_getline(&buf, file, '\n') == EOF)
exit(128); /* child died, message supplied already */
if (!*buf.buf)
break;
if (!strcmp(buf.buf, "fetch"))
data->fetch = 1;
}
return data->helper;
}
static int disconnect_helper(struct transport *transport)
{
struct helper_data *data = transport->data;
if (data->helper) {
write_in_full(data->helper->in, "\n", 1);
close(data->helper->in);
finish_command(data->helper);
free((char *)data->helper->argv[0]);
free(data->helper->argv);
free(data->helper);
data->helper = NULL;
}
return 0;
}
static int fetch_with_fetch(struct transport *transport,
int nr_heads, const struct ref **to_fetch)
{
struct child_process *helper = get_helper(transport);
FILE *file = fdopen(helper->out, "r");
int i;
struct strbuf buf = STRBUF_INIT;
for (i = 0; i < nr_heads; i++) {
const struct ref *posn = to_fetch[i];
if (posn->status & REF_STATUS_UPTODATE)
continue;
strbuf_addf(&buf, "fetch %s %s\n",
sha1_to_hex(posn->old_sha1), posn->name);
write_in_full(helper->in, buf.buf, buf.len);
strbuf_reset(&buf);
if (strbuf_getline(&buf, file, '\n') == EOF)
exit(128); /* child died, message supplied already */
}
return 0;
}
static int fetch(struct transport *transport,
int nr_heads, const struct ref **to_fetch)
{
struct helper_data *data = transport->data;
int i, count;
count = 0;
for (i = 0; i < nr_heads; i++)
if (!(to_fetch[i]->status & REF_STATUS_UPTODATE))
count++;
if (!count)
return 0;
if (data->fetch)
return fetch_with_fetch(transport, nr_heads, to_fetch);
return -1;
}
static struct ref *get_refs_list(struct transport *transport, int for_push)
{
struct child_process *helper;
struct ref *ret = NULL;
struct ref **tail = &ret;
struct ref *posn;
struct strbuf buf = STRBUF_INIT;
FILE *file;
helper = get_helper(transport);
strbuf_addstr(&buf, "list\n");
write_in_full(helper->in, buf.buf, buf.len);
strbuf_reset(&buf);
file = fdopen(helper->out, "r");
while (1) {
char *eov, *eon;
if (strbuf_getline(&buf, file, '\n') == EOF)
exit(128); /* child died, message supplied already */
if (!*buf.buf)
break;
eov = strchr(buf.buf, ' ');
if (!eov)
die("Malformed response in ref list: %s", buf.buf);
eon = strchr(eov + 1, ' ');
*eov = '\0';
if (eon)
*eon = '\0';
*tail = alloc_ref(eov + 1);
if (buf.buf[0] == '@')
(*tail)->symref = xstrdup(buf.buf + 1);
else if (buf.buf[0] != '?')
get_sha1_hex(buf.buf, (*tail)->old_sha1);
tail = &((*tail)->next);
}
strbuf_release(&buf);
for (posn = ret; posn; posn = posn->next)
resolve_remote_symref(posn, ret);
return ret;
}
int transport_helper_init(struct transport *transport, const char *name)
{
struct helper_data *data = xcalloc(sizeof(*data), 1);
data->name = name;
transport->data = data;
transport->get_refs_list = get_refs_list;
transport->fetch = fetch;
transport->disconnect = disconnect_helper;
return 0;
}

View File

@ -1,9 +1,6 @@
#include "cache.h"
#include "transport.h"
#include "run-command.h"
#ifndef NO_CURL
#include "http.h"
#endif
#include "pkt-line.h"
#include "fetch-pack.h"
#include "send-pack.h"
@ -352,45 +349,6 @@ static int rsync_transport_push(struct transport *transport,
return result;
}
/* Generic functions for using commit walkers */
#ifndef NO_CURL /* http fetch is the only user */
static int fetch_objs_via_walker(struct transport *transport,
int nr_objs, const struct ref **to_fetch)
{
char *dest = xstrdup(transport->url);
struct walker *walker = transport->data;
char **objs = xmalloc(nr_objs * sizeof(*objs));
int i;
walker->get_all = 1;
walker->get_tree = 1;
walker->get_history = 1;
walker->get_verbosely = transport->verbose >= 0;
walker->get_recover = 0;
for (i = 0; i < nr_objs; i++)
objs[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
if (walker_fetch(walker, nr_objs, objs, NULL, NULL))
die("Fetch failed.");
for (i = 0; i < nr_objs; i++)
free(objs[i]);
free(objs);
free(dest);
return 0;
}
#endif /* NO_CURL */
static int disconnect_walker(struct transport *transport)
{
struct walker *walker = transport->data;
if (walker)
walker_free(walker);
return 0;
}
#ifndef NO_CURL
static int curl_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags)
{
@ -418,96 +376,6 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons
return !!run_command_v_opt(argv, RUN_GIT_CMD);
}
static struct ref *get_refs_via_curl(struct transport *transport, int for_push)
{
struct strbuf buffer = STRBUF_INIT;
char *data, *start, *mid;
char *ref_name;
char *refs_url;
int i = 0;
int http_ret;
struct ref *refs = NULL;
struct ref *ref = NULL;
struct ref *last_ref = NULL;
struct walker *walker;
if (for_push)
return NULL;
if (!transport->data)
transport->data = get_http_walker(transport->url,
transport->remote);
walker = transport->data;
refs_url = xmalloc(strlen(transport->url) + 11);
sprintf(refs_url, "%s/info/refs", transport->url);
http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE);
switch (http_ret) {
case HTTP_OK:
break;
case HTTP_MISSING_TARGET:
die("%s not found: did you run git update-server-info on the"
" server?", refs_url);
default:
http_error(refs_url, http_ret);
die("HTTP request failed");
}
data = buffer.buf;
start = NULL;
mid = data;
while (i < buffer.len) {
if (!start)
start = &data[i];
if (data[i] == '\t')
mid = &data[i];
if (data[i] == '\n') {
data[i] = 0;
ref_name = mid + 1;
ref = xmalloc(sizeof(struct ref) +
strlen(ref_name) + 1);
memset(ref, 0, sizeof(struct ref));
strcpy(ref->name, ref_name);
get_sha1_hex(start, ref->old_sha1);
if (!refs)
refs = ref;
if (last_ref)
last_ref->next = ref;
last_ref = ref;
start = NULL;
}
i++;
}
strbuf_release(&buffer);
ref = alloc_ref("HEAD");
if (!walker->fetch_ref(walker, ref) &&
!resolve_remote_symref(ref, refs)) {
ref->next = refs;
refs = ref;
} else {
free(ref);
}
strbuf_release(&buffer);
free(refs_url);
return refs;
}
static int fetch_objs_via_curl(struct transport *transport,
int nr_objs, const struct ref **to_fetch)
{
if (!transport->data)
transport->data = get_http_walker(transport->url,
transport->remote);
return fetch_objs_via_walker(transport, nr_objs, to_fetch);
}
#endif
struct bundle_transport_data {
@ -955,14 +823,12 @@ struct transport *transport_get(struct remote *remote, const char *url)
} else if (!prefixcmp(url, "http://")
|| !prefixcmp(url, "https://")
|| !prefixcmp(url, "ftp://")) {
transport_helper_init(ret, "curl");
#ifdef NO_CURL
error("git was compiled without libcurl support.");
#else
ret->get_refs_list = get_refs_via_curl;
ret->fetch = fetch_objs_via_curl;
ret->push = curl_transport_push;
#endif
ret->disconnect = disconnect_walker;
} else if (is_local(url) && is_file(url)) {
struct bundle_transport_data *data = xcalloc(1, sizeof(*data));

View File

@ -79,4 +79,7 @@ void transport_unlock_pack(struct transport *transport);
int transport_disconnect(struct transport *transport);
char *transport_anonymize_url(const char *url);
/* Transport methods defined outside transport.c */
int transport_helper_init(struct transport *transport, const char *name);
#endif