undef/git-commit-vandalism
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'ds/bundle-uri-3'

Browse Source 
Define the logical elements of a "bundle list", data structure to
store them in-core, format to transfer them, and code to parse
them.

* ds/bundle-uri-3:
  bundle-uri: suppress stderr from remote-https
  bundle-uri: quiet failed unbundlings
  bundle: add flags to verify_bundle()
  bundle-uri: fetch a list of bundles
  bundle: properly clear all revision flags
  bundle-uri: limit recursion depth for bundle lists
  bundle-uri: parse bundle list in config format
  bundle-uri: unit test "key=value" parsing
  bundle-uri: create "key=value" line parsing
  bundle-uri: create base key-value pair parsing
  bundle-uri: create bundle_list struct and helpers
  bundle-uri: use plain string in find_temp_filename()
This commit is contained in:
Taylor Blau 2022-10-30 21:04:43 -04:00
commit d32dd8add5
17 changed files with 1160 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/config.txt
View File

@ -387,6 +387,8 @@ include::config/branch.txt[]
include::config/browser.txt[]
include::config/bundle.txt[]
include::config/checkout.txt[]
include::config/clean.txt[]

24
Documentation/config/bundle.txt Normal file
View File

@ -0,0 +1,24 @@
bundle.*::
The `bundle.*` keys may appear in a bundle list file found via the
`git clone --bundle-uri` option. These keys currently have no effect
if placed in a repository config file, though this will change in the
future. See link:technical/bundle-uri.html[the bundle URI design
document] for more details.
bundle.version::
This integer value advertises the version of the bundle list format
used by the bundle list. Currently, the only accepted value is `1`.
bundle.mode::
This string value should be either `all` or `any`. This value describes
whether all of the advertised bundles are required to unbundle a
complete understanding of the bundled information (`all`) or if any one
of the listed bundle URIs is sufficient (`any`).
bundle.<id>.*::
The `bundle.<id>.*` keys are used to describe a single item in the
bundle list, grouped under `<id>` for identification purposes.
bundle.<id>.uri::
This string value defines the URI by which Git can reach the contents
of this `<id>`. This URI may be a bundle file or another bundle list.

1
Makefile
View File

@ -722,6 +722,7 @@ PROGRAMS += $(patsubst %.o,git-%$X,$(PROGRAM_OBJS))
TEST_BUILTINS_OBJS += test-advise.o
TEST_BUILTINS_OBJS += test-bitmap.o
TEST_BUILTINS_OBJS += test-bloom.o
TEST_BUILTINS_OBJS += test-bundle-uri.o
TEST_BUILTINS_OBJS += test-chmtime.o
TEST_BUILTINS_OBJS += test-config.o
TEST_BUILTINS_OBJS += test-crontab.o

5
builtin/bundle.c
View File

@ -129,7 +129,8 @@ static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
goto cleanup;
}
close(bundle_fd);
if (verify_bundle(the_repository, &header, !quiet)) {
if (verify_bundle(the_repository, &header,
quiet ? VERIFY_BUNDLE_QUIET : VERIFY_BUNDLE_VERBOSE)) {
ret = 1;
goto cleanup;
}
@ -195,7 +196,7 @@ static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix)
strvec_pushl(&extra_index_pack_args, "-v", "--progress-title",
_("Unbundling objects"), NULL);
ret = !!unbundle(the_repository, &header, bundle_fd,
&extra_index_pack_args) ||
&extra_index_pack_args, 0) ||
list_bundle_refs(&header, argc, argv);
bundle_header_release(&header);
cleanup:

466
bundle-uri.c
View File

@ -4,23 +4,221 @@
#include "object-store.h"
#include "refs.h"
#include "run-command.h"
#include "hashmap.h"
#include "pkt-line.h"
#include "config.h"
static int find_temp_filename(struct strbuf *name)
static int compare_bundles(const void *hashmap_cmp_fn_data,
const struct hashmap_entry *he1,
const struct hashmap_entry *he2,
const void *id)
{
const struct remote_bundle_info *e1 =
container_of(he1, const struct remote_bundle_info, ent);
const struct remote_bundle_info *e2 =
container_of(he2, const struct remote_bundle_info, ent);
return strcmp(e1->id, id ? (const char *)id : e2->id);
}
void init_bundle_list(struct bundle_list *list)
{
memset(list, 0, sizeof(*list));
/* Implied defaults. */
list->mode = BUNDLE_MODE_ALL;
list->version = 1;
hashmap_init(&list->bundles, compare_bundles, NULL, 0);
}
static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
void *data)
{
FREE_AND_NULL(bundle->id);
FREE_AND_NULL(bundle->uri);
FREE_AND_NULL(bundle->file);
bundle->unbundled = 0;
return 0;
}
void clear_bundle_list(struct bundle_list *list)
{
if (!list)
return;
for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
}
int for_all_bundles_in_list(struct bundle_list *list,
bundle_iterator iter,
void *data)
{
struct remote_bundle_info *info;
struct hashmap_iter i;
hashmap_for_each_entry(&list->bundles, &i, info, ent) {
int result = iter(info, data);
if (result)
return result;
}
return 0;
}
static int summarize_bundle(struct remote_bundle_info *info, void *data)
{
FILE *fp = data;
fprintf(fp, "[bundle \"%s\"]\n", info->id);
fprintf(fp, "\turi = %s\n", info->uri);
return 0;
}
void print_bundle_list(FILE *fp, struct bundle_list *list)
{
const char *mode;
switch (list->mode) {
case BUNDLE_MODE_ALL:
mode = "all";
break;
case BUNDLE_MODE_ANY:
mode = "any";
break;
case BUNDLE_MODE_NONE:
default:
mode = "<unknown>";
}
fprintf(fp, "[bundle]\n");
fprintf(fp, "\tversion = %d\n", list->version);
fprintf(fp, "\tmode = %s\n", mode);
for_all_bundles_in_list(list, summarize_bundle, fp);
}
/**
* Given a key-value pair, update the state of the given bundle list.
* Returns 0 if the key-value pair is understood. Returns -1 if the key
* is not understood or the value is malformed.
*/
static int bundle_list_update(const char *key, const char *value,
struct bundle_list *list)
{
struct strbuf id = STRBUF_INIT;
struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
struct remote_bundle_info *bundle;
const char *subsection, *subkey;
size_t subsection_len;
if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
return -1;
if (!subsection_len) {
if (!strcmp(subkey, "version")) {
int version;
if (!git_parse_int(value, &version))
return -1;
if (version != 1)
return -1;
list->version = version;
return 0;
}
if (!strcmp(subkey, "mode")) {
if (!strcmp(value, "all"))
list->mode = BUNDLE_MODE_ALL;
else if (!strcmp(value, "any"))
list->mode = BUNDLE_MODE_ANY;
else
return -1;
return 0;
}
/* Ignore other unknown global keys. */
return 0;
}
strbuf_add(&id, subsection, subsection_len);
/*
* Check for an existing bundle with this <id>, or create one
* if necessary.
*/
lookup.id = id.buf;
hashmap_entry_init(&lookup.ent, strhash(lookup.id));
if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
CALLOC_ARRAY(bundle, 1);
bundle->id = strbuf_detach(&id, NULL);
hashmap_entry_init(&bundle->ent, strhash(bundle->id));
hashmap_add(&list->bundles, &bundle->ent);
}
strbuf_release(&id);
if (!strcmp(subkey, "uri")) {
if (bundle->uri)
return -1;
bundle->uri = xstrdup(value);
return 0;
}
/*
* At this point, we ignore any information that we don't
* understand, assuming it to be hints for a heuristic the client
* does not currently understand.
*/
return 0;
}
static int config_to_bundle_list(const char *key, const char *value, void *data)
{
struct bundle_list *list = data;
return bundle_list_update(key, value, list);
}
int bundle_uri_parse_config_format(const char *uri,
const char *filename,
struct bundle_list *list)
{
int result;
struct config_options opts = {
.error_action = CONFIG_ERROR_ERROR,
};
result = git_config_from_file_with_options(config_to_bundle_list,
filename, list,
&opts);
if (!result && list->mode == BUNDLE_MODE_NONE) {
warning(_("bundle list at '%s' has no mode"), uri);
result = 1;
}
return result;
}
static char *find_temp_filename(void)
{
int fd;
struct strbuf name = STRBUF_INIT;
/*
* Find a temporary filename that is available. This is briefly
* racy, but unlikely to collide.
*/
fd = odb_mkstemp(name, "bundles/tmp_uri_XXXXXX");
fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
if (fd < 0) {
warning(_("failed to create temporary file"));
return -1;
return NULL;
}
close(fd);
unlink(name->buf);
return 0;
unlink(name.buf);
return strbuf_detach(&name, NULL);
}
static int download_https_uri_to_file(const char *file, const char *uri)
@ -32,6 +230,7 @@ static int download_https_uri_to_file(const char *file, const char *uri)
int found_get = 0;
strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
cp.err = -1;
cp.in = -1;
cp.out = -1;
@ -105,7 +304,13 @@ static int unbundle_from_file(struct repository *r, const char *file)
if ((bundle_fd = read_bundle_header(file, &header)) < 0)
return 1;
if ((result = unbundle(r, &header, bundle_fd, NULL)))
/*
* Skip the reachability walk here, since we will be adding
* a reachable ref pointing to the new tips, which will reach
* the prerequisite commits.
*/
if ((result = unbundle(r, &header, bundle_fd, NULL,
VERIFY_BUNDLE_QUIET)))
return 1;
/*
@ -138,31 +343,248 @@ static int unbundle_from_file(struct repository *r, const char *file)
return result;
}
int fetch_bundle_uri(struct repository *r, const char *uri)
struct bundle_list_context {
struct repository *r;
struct bundle_list *list;
enum bundle_list_mode mode;
int count;
int depth;
};
/*
* This early definition is necessary because we use indirect recursion:
*
* While iterating through a bundle list that was downloaded as part
* of fetch_bundle_uri_internal(), iterator methods eventually call it
* again, but with depth + 1.
*/
static int fetch_bundle_uri_internal(struct repository *r,
struct remote_bundle_info *bundle,
int depth,
struct bundle_list *list);
static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
{
int result = 0;
struct strbuf filename = STRBUF_INIT;
int res;
struct bundle_list_context *ctx = data;
if ((result = find_temp_filename(&filename)))
if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
return 0;
res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
/*
* Only increment count if the download succeeded. If our mode is
* BUNDLE_MODE_ANY, then we will want to try other URIs in the
* list in case they work instead.
*/
if (!res)
ctx->count++;
/*
* To be opportunistic as possible, we continue iterating and
* download as many bundles as we can, so we can apply the ones
* that work, even in BUNDLE_MODE_ALL mode.
*/
return 0;
}
static int download_bundle_list(struct repository *r,
struct bundle_list *local_list,
struct bundle_list *global_list,
int depth)
{
struct bundle_list_context ctx = {
.r = r,
.list = global_list,
.depth = depth + 1,
.mode = local_list->mode,
};
return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
}
static int fetch_bundle_list_in_config_format(struct repository *r,
struct bundle_list *global_list,
struct remote_bundle_info *bundle,
int depth)
{
int result;
struct bundle_list list_from_bundle;
init_bundle_list(&list_from_bundle);
if ((result = bundle_uri_parse_config_format(bundle->uri,
bundle->file,
&list_from_bundle)))
goto cleanup;
if ((result = copy_uri_to_file(filename.buf, uri))) {
warning(_("failed to download bundle from URI '%s'"), uri);
if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
warning(_("unrecognized bundle mode from URI '%s'"),
bundle->uri);
result = -1;
goto cleanup;
}
if ((result = !is_bundle(filename.buf, 0))) {
warning(_("file at URI '%s' is not a bundle"), uri);
if ((result = download_bundle_list(r, &list_from_bundle,
global_list, depth)))
goto cleanup;
}
if ((result = unbundle_from_file(r, filename.buf))) {
warning(_("failed to unbundle bundle from URI '%s'"), uri);
goto cleanup;
}
cleanup:
unlink(filename.buf);
strbuf_release(&filename);
clear_bundle_list(&list_from_bundle);
return result;
}
/**
* This limits the recursion on fetch_bundle_uri_internal() when following
* bundle lists.
*/
static int max_bundle_uri_depth = 4;
/**
* Recursively download all bundles advertised at the given URI
* to files. If the file is a bundle, then add it to the given
* 'list'. Otherwise, expect a bundle list and recurse on the
* URIs in that list according to the list mode (ANY or ALL).
*/
static int fetch_bundle_uri_internal(struct repository *r,
struct remote_bundle_info *bundle,
int depth,
struct bundle_list *list)
{
int result = 0;
struct remote_bundle_info *bcopy;
if (depth >= max_bundle_uri_depth) {
warning(_("exceeded bundle URI recursion limit (%d)"),
max_bundle_uri_depth);
return -1;
}
if (!bundle->file &&
!(bundle->file = find_temp_filename())) {
result = -1;
goto cleanup;
}
if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
warning(_("failed to download bundle from URI '%s'"), bundle->uri);
goto cleanup;
}
if ((result = !is_bundle(bundle->file, 1))) {
result = fetch_bundle_list_in_config_format(
r, list, bundle, depth);
if (result)
warning(_("file at URI '%s' is not a bundle or bundle list"),
bundle->uri);
goto cleanup;
}
/* Copy the bundle and insert it into the global list. */
CALLOC_ARRAY(bcopy, 1);
bcopy->id = xstrdup(bundle->id);
bcopy->file = xstrdup(bundle->file);
hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
hashmap_add(&list->bundles, &bcopy->ent);
cleanup:
if (result && bundle->file)
unlink(bundle->file);
return result;
}
/**
* This loop iterator breaks the loop with nonzero return code on the
* first successful unbundling of a bundle.
*/
static int attempt_unbundle(struct remote_bundle_info *info, void *data)
{
struct repository *r = data;
if (!info->file || info->unbundled)
return 0;
if (!unbundle_from_file(r, info->file)) {
info->unbundled = 1;
return 1;
}
return 0;
}
static int unbundle_all_bundles(struct repository *r,
struct bundle_list *list)
{
/*
* Iterate through all bundles looking for ones that can
* successfully unbundle. If any succeed, then perhaps another
* will succeed in the next attempt.
*
* Keep in mind that a non-zero result for the loop here means
* the loop terminated early on a successful unbundling, which
* signals that we can try again.
*/
while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
return 0;
}
static int unlink_bundle(struct remote_bundle_info *info, void *data)
{
if (info->file)
unlink_or_warn(info->file);
return 0;
}
int fetch_bundle_uri(struct repository *r, const char *uri)
{
int result;
struct bundle_list list;
struct remote_bundle_info bundle = {
.uri = xstrdup(uri),
.id = xstrdup(""),
};
init_bundle_list(&list);
/* If a bundle is added to this global list, then it is required. */
list.mode = BUNDLE_MODE_ALL;
if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
goto cleanup;
result = unbundle_all_bundles(r, &list);
cleanup:
for_all_bundles_in_list(&list, unlink_bundle, NULL);
clear_bundle_list(&list);
clear_remote_bundle_info(&bundle, NULL);
return result;
}
/**
* General API for {transport,connect}.c etc.
*/
int bundle_uri_parse_line(struct bundle_list *list, const char *line)
{
int result;
const char *equals;
struct strbuf key = STRBUF_INIT;
if (!strlen(line))
return error(_("bundle-uri: got an empty line"));
equals = strchr(line, '=');
if (!equals)
return error(_("bundle-uri: line is not of the form 'key=value'"));
if (line == equals || !*(equals + 1))
return error(_("bundle-uri: line has empty key or value"));
strbuf_add(&key, line, equals - line);
result = bundle_list_update(key.buf, equals + 1, list);
strbuf_release(&key);
return result;
}

93
bundle-uri.h
View File

@ -1,7 +1,88 @@
#ifndef BUNDLE_URI_H
#define BUNDLE_URI_H
#include "hashmap.h"
#include "strbuf.h"
struct repository;
struct string_list;
/**
* The remote_bundle_info struct contains information for a single bundle
* URI. This may be initialized simply by a given URI or might have
* additional metadata associated with it if the bundle was advertised by
* a bundle list.
*/
struct remote_bundle_info {
struct hashmap_entry ent;
/**
* The 'id' is a name given to the bundle for reference
* by other bundle infos.
*/
char *id;
/**
* The 'uri' is the location of the remote bundle so
* it can be downloaded on-demand. This will be NULL
* if there was no table of contents.
*/
char *uri;
/**
* If the bundle has been downloaded, then 'file' is a
* filename storing its contents. Otherwise, 'file' is
* NULL.
*/
char *file;
/**
* If the bundle has been unbundled successfully, then
* this boolean is true.
*/
unsigned unbundled:1;
};
#define REMOTE_BUNDLE_INFO_INIT { 0 }
enum bundle_list_mode {
BUNDLE_MODE_NONE = 0,
BUNDLE_MODE_ALL,
BUNDLE_MODE_ANY
};
/**
* A bundle_list contains an unordered set of remote_bundle_info structs,
* as well as information about the bundle listing, such as version and
* mode.
*/
struct bundle_list {
int version;
enum bundle_list_mode mode;
struct hashmap bundles;
};
void init_bundle_list(struct bundle_list *list);
void clear_bundle_list(struct bundle_list *list);
typedef int (*bundle_iterator)(struct remote_bundle_info *bundle,
void *data);
int for_all_bundles_in_list(struct bundle_list *list,
bundle_iterator iter,
void *data);
struct FILE;
void print_bundle_list(FILE *fp, struct bundle_list *list);
/**
* A bundle URI may point to a bundle list where the key=value
* pairs are provided in config file format. This method is
* exposed publicly for testing purposes.
*/
int bundle_uri_parse_config_format(const char *uri,
const char *filename,
struct bundle_list *list);
/**
* Fetch data from the given 'uri' and unbundle the bundle data found
@ -11,4 +92,16 @@ struct repository;
*/
int fetch_bundle_uri(struct repository *r, const char *uri);
/**
* General API for {transport,connect}.c etc.
*/
/**
* Parse a "key=value" packet line from the bundle-uri verb.
*
* Returns 0 on success and non-zero on error.
*/
int bundle_uri_parse_line(struct bundle_list *list,
const char *line);
#endif

42
bundle.c
View File

@ -189,7 +189,7 @@ static int list_refs(struct string_list *r, int argc, const char **argv)
int verify_bundle(struct repository *r,
struct bundle_header *header,
int verbose)
enum verify_bundle_flags flags)
{
/*
* Do fast check, then if any prereqs are missing then go line by line
@ -202,10 +202,8 @@ int verify_bundle(struct repository *r,
int i, ret = 0, req_nr;
const char *message = _("Repository lacks these prerequisite commits:");
if (!r || !r->objects || !r->objects->odb) {
ret = error(_("need a repository to verify a bundle"));
goto cleanup;
}
if (!r || !r->objects || !r->objects->odb)
return error(_("need a repository to verify a bundle"));
repo_init_revisions(r, &revs, NULL);
for (i = 0; i < p->nr; i++) {
@ -218,7 +216,10 @@ int verify_bundle(struct repository *r,
add_pending_object(&revs, o, name);
continue;
}
if (++ret == 1)
ret++;
if (flags & VERIFY_BUNDLE_QUIET)
continue;
if (ret == 1)
error("%s", message);
error("%s %s", oid_to_hex(oid), name);
}
@ -245,21 +246,15 @@ int verify_bundle(struct repository *r,
assert(o); /* otherwise we'd have returned early */
if (o->flags & SHOWN)
continue;
if (++ret == 1)
ret++;
if (flags & VERIFY_BUNDLE_QUIET)
continue;
if (ret == 1)
error("%s", message);
error("%s %s", oid_to_hex(oid), name);
}
/* Clean up objects used, as they will be reused. */
for (i = 0; i < p->nr; i++) {
struct string_list_item *e = p->items + i;
struct object_id *oid = e->util;
commit = lookup_commit_reference_gently(r, oid, 1);
if (commit)
clear_commit_marks(commit, ALL_REV_FLAGS);
}
if (verbose) {
if (flags & VERIFY_BUNDLE_VERBOSE) {
struct string_list *r;
r = &header->references;
@ -287,6 +282,14 @@ int verify_bundle(struct repository *r,
list_objects_filter_spec(&header->filter));
}
cleanup:
/* Clean up objects used, as they will be reused. */
for (i = 0; i < p->nr; i++) {
struct string_list_item *e = p->items + i;
struct object_id *oid = e->util;
commit = lookup_commit_reference_gently(r, oid, 1);
if (commit)
clear_commit_marks(commit, ALL_REV_FLAGS | PREREQ_MARK);
}
release_revisions(&revs);
return ret;
}
@ -620,7 +623,8 @@ err:
}
int unbundle(struct repository *r, struct bundle_header *header,
int bundle_fd, struct strvec *extra_index_pack_args)
int bundle_fd, struct strvec *extra_index_pack_args,
enum verify_bundle_flags flags)
{
struct child_process ip = CHILD_PROCESS_INIT;
strvec_pushl(&ip.args, "index-pack", "--fix-thin", "--stdin", NULL);
@ -634,7 +638,7 @@ int unbundle(struct repository *r, struct bundle_header *header,
strvec_clear(extra_index_pack_args);
}
if (verify_bundle(r, header, 0))
if (verify_bundle(r, header, flags))
return -1;
ip.in = bundle_fd;
ip.no_stdout = 1;

15
bundle.h
View File

@ -30,7 +30,14 @@ int read_bundle_header_fd(int fd, struct bundle_header *header,
int create_bundle(struct repository *r, const char *path,
int argc, const char **argv, struct strvec *pack_options,
int version);
int verify_bundle(struct repository *r, struct bundle_header *header, int verbose);
enum verify_bundle_flags {
VERIFY_BUNDLE_VERBOSE = (1 << 0),
VERIFY_BUNDLE_QUIET = (1 << 1),
};
int verify_bundle(struct repository *r, struct bundle_header *header,
enum verify_bundle_flags flags);
/**
* Unbundle after reading the header with read_bundle_header().
@ -41,9 +48,13 @@ int verify_bundle(struct repository *r, struct bundle_header *header, int verbos
* Provide "extra_index_pack_args" to pass any extra arguments
* (e.g. "-v" for verbose/progress), NULL otherwise. The provided
* "extra_index_pack_args" (if any) will be strvec_clear()'d for you.
*
* Before unbundling, this method will call verify_bundle() with the
* given 'flags'.
*/
int unbundle(struct repository *r, struct bundle_header *header,
int bundle_fd, struct strvec *extra_index_pack_args);
int bundle_fd, struct strvec *extra_index_pack_args,
enum verify_bundle_flags flags);
int list_bundle_refs(struct bundle_header *header,
int argc, const char **argv);

2
config.c
View File

@ -1215,7 +1215,7 @@ static int git_parse_unsigned(const char *value, uintmax_t *ret, uintmax_t max)
return 0;
}
static int git_parse_int(const char *value, int *ret)
int git_parse_int(const char *value, int *ret)
{
intmax_t tmp;
if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(int)))

1
config.h
View File

@ -206,6 +206,7 @@ int config_with_options(config_fn_t fn, void *,
int git_parse_ssize_t(const char *, ssize_t *);
int git_parse_ulong(const char *, unsigned long *);
int git_parse_int(const char *value, int *ret);
/**
* Same as `git_config_bool`, except that it returns -1 on error rather

95
t/helper/test-bundle-uri.c Normal file
View File

@ -0,0 +1,95 @@
#include "test-tool.h"
#include "parse-options.h"
#include "bundle-uri.h"
#include "strbuf.h"
#include "string-list.h"
enum input_mode {
KEY_VALUE_PAIRS,
CONFIG_FILE,
};
static int cmd__bundle_uri_parse(int argc, const char **argv, enum input_mode mode)
{
const char *key_value_usage[] = {
"test-tool bundle-uri parse-key-values <input>",
NULL
};
const char *config_usage[] = {
"test-tool bundle-uri parse-config <input>",
NULL
};
const char **usage = key_value_usage;
struct option options[] = {
OPT_END(),
};
struct strbuf sb = STRBUF_INIT;
struct bundle_list list;
int err = 0;
FILE *fp;
if (mode == CONFIG_FILE)
usage = config_usage;
argc = parse_options(argc, argv, NULL, options, usage,
PARSE_OPT_STOP_AT_NON_OPTION);
init_bundle_list(&list);
switch (mode) {
case KEY_VALUE_PAIRS:
if (argc != 1)
goto usage;
fp = fopen(argv[0], "r");
if (!fp)
die("failed to open '%s'", argv[0]);
while (strbuf_getline(&sb, fp) != EOF) {
if (bundle_uri_parse_line(&list, sb.buf))
err = error("bad line: '%s'", sb.buf);
}
fclose(fp);
break;
case CONFIG_FILE:
if (argc != 1)
goto usage;
err = bundle_uri_parse_config_format("<uri>", argv[0], &list);
break;
}
strbuf_release(&sb);
print_bundle_list(stdout, &list);
clear_bundle_list(&list);
return !!err;
usage:
usage_with_options(usage, options);
}
int cmd__bundle_uri(int argc, const char **argv)
{
const char *usage[] = {
"test-tool bundle-uri <subcommand> [<options>]",
NULL
};
struct option options[] = {
OPT_END(),
};
argc = parse_options(argc, argv, NULL, options, usage,
PARSE_OPT_STOP_AT_NON_OPTION |
PARSE_OPT_KEEP_ARGV0);
if (argc == 1)
goto usage;
if (!strcmp(argv[1], "parse-key-values"))
return cmd__bundle_uri_parse(argc - 1, argv + 1, KEY_VALUE_PAIRS);
if (!strcmp(argv[1], "parse-config"))
return cmd__bundle_uri_parse(argc - 1, argv + 1, CONFIG_FILE);
error("there is no test-tool bundle-uri tool '%s'", argv[1]);
usage:
usage_with_options(usage, options);
}

1
t/helper/test-tool.c
View File

@ -13,6 +13,7 @@ static struct test_cmd cmds[] = {
{ "advise", cmd__advise_if_enabled },
{ "bitmap", cmd__bitmap },
{ "bloom", cmd__bloom },
{ "bundle-uri", cmd__bundle_uri },
{ "chmtime", cmd__chmtime },
{ "config", cmd__config },
{ "crontab", cmd__crontab },

1
t/helper/test-tool.h
View File

@ -7,6 +7,7 @@
int cmd__advise_if_enabled(int argc, const char **argv);
int cmd__bitmap(int argc, const char **argv);
int cmd__bloom(int argc, const char **argv);
int cmd__bundle_uri(int argc, const char **argv);
int cmd__chmtime(int argc, const char **argv);
int cmd__config(int argc, const char **argv);
int cmd__crontab(int argc, const char **argv);

275
t/t5558-clone-bundle-uri.sh
View File

@ -41,6 +41,215 @@ test_expect_success 'clone with file:// bundle' '
test_cmp expect actual
'
# To get interesting tests for bundle lists, we need to construct a
# somewhat-interesting commit history.
#
# ---------------- bundle-4
#
# 4
# / \
# ----|---|------- bundle-3
# | |
# | 3
# | |
# ----|---|------- bundle-2
# | |
# 2 |
# | |
# ----|---|------- bundle-1
# \ /
# 1
# |
# (previous commits)
test_expect_success 'construct incremental bundle list' '
(
cd clone-from &&
git checkout -b base &&
test_commit 1 &&
git checkout -b left &&
test_commit 2 &&
git checkout -b right base &&
test_commit 3 &&
git checkout -b merge left &&
git merge right -m "4" &&
git bundle create bundle-1.bundle base &&
git bundle create bundle-2.bundle base..left &&
git bundle create bundle-3.bundle base..right &&
git bundle create bundle-4.bundle merge --not left right
)
'
test_expect_success 'clone bundle list (file, no heuristic)' '
cat >bundle-list <<-EOF &&
[bundle]
version = 1
mode = all
[bundle "bundle-1"]
uri = file://$(pwd)/clone-from/bundle-1.bundle
[bundle "bundle-2"]
uri = file://$(pwd)/clone-from/bundle-2.bundle
[bundle "bundle-3"]
uri = file://$(pwd)/clone-from/bundle-3.bundle
[bundle "bundle-4"]
uri = file://$(pwd)/clone-from/bundle-4.bundle
EOF
git clone --bundle-uri="file://$(pwd)/bundle-list" \
clone-from clone-list-file 2>err &&
! grep "Repository lacks these prerequisite commits" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-list-file cat-file --batch-check <oids &&
git -C clone-list-file for-each-ref --format="%(refname)" >refs &&
grep "refs/bundles/" refs >actual &&
cat >expect <<-\EOF &&
refs/bundles/base
refs/bundles/left
refs/bundles/merge
refs/bundles/right
EOF
test_cmp expect actual
'
test_expect_success 'clone bundle list (file, all mode, some failures)' '
cat >bundle-list <<-EOF &&
[bundle]
version = 1
mode = all
# Does not exist. Should be skipped.
[bundle "bundle-0"]
uri = file://$(pwd)/clone-from/bundle-0.bundle
[bundle "bundle-1"]
uri = file://$(pwd)/clone-from/bundle-1.bundle
[bundle "bundle-2"]
uri = file://$(pwd)/clone-from/bundle-2.bundle
# No bundle-3 means bundle-4 will not apply.
[bundle "bundle-4"]
uri = file://$(pwd)/clone-from/bundle-4.bundle
# Does not exist. Should be skipped.
[bundle "bundle-5"]
uri = file://$(pwd)/clone-from/bundle-5.bundle
EOF
GIT_TRACE2_PERF=1 \
git clone --bundle-uri="file://$(pwd)/bundle-list" \
clone-from clone-all-some 2>err &&
! grep "Repository lacks these prerequisite commits" err &&
! grep "fatal" err &&
grep "warning: failed to download bundle from URI" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-all-some cat-file --batch-check <oids &&
git -C clone-all-some for-each-ref --format="%(refname)" >refs &&
grep "refs/bundles/" refs >actual &&
cat >expect <<-\EOF &&
refs/bundles/base
refs/bundles/left
EOF
test_cmp expect actual
'
test_expect_success 'clone bundle list (file, all mode, all failures)' '
cat >bundle-list <<-EOF &&
[bundle]
version = 1
mode = all
# Does not exist. Should be skipped.
[bundle "bundle-0"]
uri = file://$(pwd)/clone-from/bundle-0.bundle
# Does not exist. Should be skipped.
[bundle "bundle-5"]
uri = file://$(pwd)/clone-from/bundle-5.bundle
EOF
git clone --bundle-uri="file://$(pwd)/bundle-list" \
clone-from clone-all-fail 2>err &&
! grep "Repository lacks these prerequisite commits" err &&
! grep "fatal" err &&
grep "warning: failed to download bundle from URI" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-all-fail cat-file --batch-check <oids &&
git -C clone-all-fail for-each-ref --format="%(refname)" >refs &&
! grep "refs/bundles/" refs
'
test_expect_success 'clone bundle list (file, any mode)' '
cat >bundle-list <<-EOF &&
[bundle]
version = 1
mode = any
# Does not exist. Should be skipped.
[bundle "bundle-0"]
uri = file://$(pwd)/clone-from/bundle-0.bundle
[bundle "bundle-1"]
uri = file://$(pwd)/clone-from/bundle-1.bundle
# Does not exist. Should be skipped.
[bundle "bundle-5"]
uri = file://$(pwd)/clone-from/bundle-5.bundle
EOF
git clone --bundle-uri="file://$(pwd)/bundle-list" \
clone-from clone-any-file 2>err &&
! grep "Repository lacks these prerequisite commits" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-any-file cat-file --batch-check <oids &&
git -C clone-any-file for-each-ref --format="%(refname)" >refs &&
grep "refs/bundles/" refs >actual &&
cat >expect <<-\EOF &&
refs/bundles/base
EOF
test_cmp expect actual
'
test_expect_success 'clone bundle list (file, any mode, all failures)' '
cat >bundle-list <<-EOF &&
[bundle]
version = 1
mode = any
# Does not exist. Should be skipped.
[bundle "bundle-0"]
uri = $HTTPD_URL/bundle-0.bundle
# Does not exist. Should be skipped.
[bundle "bundle-5"]
uri = $HTTPD_URL/bundle-5.bundle
EOF
git clone --bundle-uri="file://$(pwd)/bundle-list" \
clone-from clone-any-fail 2>err &&
! grep "fatal" err &&
grep "warning: failed to download bundle from URI" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-any-fail cat-file --batch-check <oids &&
git -C clone-any-fail for-each-ref --format="%(refname)" >refs &&
! grep "refs/bundles/" refs
'
#########################################################################
# HTTP tests begin here
@ -75,6 +284,72 @@ test_expect_success 'clone HTTP bundle' '
test_config -C clone-http log.excludedecoration refs/bundle/
'
test_expect_success 'clone bundle list (HTTP, no heuristic)' '
cp clone-from/bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/" &&
cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
[bundle]
version = 1
mode = all
[bundle "bundle-1"]
uri = $HTTPD_URL/bundle-1.bundle
[bundle "bundle-2"]
uri = $HTTPD_URL/bundle-2.bundle
[bundle "bundle-3"]
uri = $HTTPD_URL/bundle-3.bundle
[bundle "bundle-4"]
uri = $HTTPD_URL/bundle-4.bundle
EOF
git clone --bundle-uri="$HTTPD_URL/bundle-list" \
clone-from clone-list-http 2>err &&
! grep "Repository lacks these prerequisite commits" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-list-http cat-file --batch-check <oids
'
test_expect_success 'clone bundle list (HTTP, any mode)' '
cp clone-from/bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/" &&
cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
[bundle]
version = 1
mode = any
# Does not exist. Should be skipped.
[bundle "bundle-0"]
uri = $HTTPD_URL/bundle-0.bundle
[bundle "bundle-1"]
uri = $HTTPD_URL/bundle-1.bundle
# Does not exist. Should be skipped.
[bundle "bundle-5"]
uri = $HTTPD_URL/bundle-5.bundle
EOF
git clone --bundle-uri="$HTTPD_URL/bundle-list" \
clone-from clone-any-http 2>err &&
! grep "fatal" err &&
grep "warning: failed to download bundle from URI" err &&
git -C clone-from for-each-ref --format="%(objectname)" >oids &&
git -C clone-any-http cat-file --batch-check <oids &&
git -C clone-list-file for-each-ref --format="%(refname)" >refs &&
grep "refs/bundles/" refs >actual &&
cat >expect <<-\EOF &&
refs/bundles/base
refs/bundles/left
refs/bundles/merge
refs/bundles/right
EOF
test_cmp expect actual
'
# Do not add tests here unless they use the HTTP server, as they will
# not run unless the HTTP dependencies exist.

171
t/t5750-bundle-uri-parse.sh Executable file
View File

@ -0,0 +1,171 @@
#!/bin/sh
test_description="Test bundle-uri bundle_uri_parse_line()"
TEST_NO_CREATE_REPO=1
TEST_PASSES_SANITIZE_LEAK=true
. ./test-lib.sh
test_expect_success 'bundle_uri_parse_line() just URIs' '
cat >in <<-\EOF &&
bundle.one.uri=http://example.com/bundle.bdl
bundle.two.uri=https://example.com/bundle.bdl
bundle.three.uri=file:///usr/share/git/bundle.bdl
EOF
cat >expect <<-\EOF &&
[bundle]
version = 1
mode = all
[bundle "one"]
uri = http://example.com/bundle.bdl
[bundle "two"]
uri = https://example.com/bundle.bdl
[bundle "three"]
uri = file:///usr/share/git/bundle.bdl
EOF
test-tool bundle-uri parse-key-values in >actual 2>err &&
test_must_be_empty err &&
test_cmp_config_output expect actual
'
test_expect_success 'bundle_uri_parse_line() parsing edge cases: empty key or value' '
cat >in <<-\EOF &&
=bogus-value
bogus-key=
EOF
cat >err.expect <<-EOF &&
error: bundle-uri: line has empty key or value
error: bad line: '\''=bogus-value'\''
error: bundle-uri: line has empty key or value
error: bad line: '\''bogus-key='\''
EOF
cat >expect <<-\EOF &&
[bundle]
version = 1
mode = all
EOF
test_must_fail test-tool bundle-uri parse-key-values in >actual 2>err &&
test_cmp err.expect err &&
test_cmp_config_output expect actual
'
test_expect_success 'bundle_uri_parse_line() parsing edge cases: empty lines' '
cat >in <<-\EOF &&
bundle.one.uri=http://example.com/bundle.bdl
bundle.two.uri=https://example.com/bundle.bdl
bundle.three.uri=file:///usr/share/git/bundle.bdl
EOF
cat >err.expect <<-\EOF &&
error: bundle-uri: got an empty line
error: bad line: '\'''\''
error: bundle-uri: got an empty line
error: bad line: '\'''\''
EOF
# We fail, but try to continue parsing regardless
cat >expect <<-\EOF &&
[bundle]
version = 1
mode = all
[bundle "one"]
uri = http://example.com/bundle.bdl
[bundle "two"]
uri = https://example.com/bundle.bdl
[bundle "three"]
uri = file:///usr/share/git/bundle.bdl
EOF
test_must_fail test-tool bundle-uri parse-key-values in >actual 2>err &&
test_cmp err.expect err &&
test_cmp_config_output expect actual
'
test_expect_success 'bundle_uri_parse_line() parsing edge cases: duplicate lines' '
cat >in <<-\EOF &&
bundle.one.uri=http://example.com/bundle.bdl
bundle.two.uri=https://example.com/bundle.bdl
bundle.one.uri=https://example.com/bundle-2.bdl
bundle.three.uri=file:///usr/share/git/bundle.bdl
EOF
cat >err.expect <<-\EOF &&
error: bad line: '\''bundle.one.uri=https://example.com/bundle-2.bdl'\''
EOF
# We fail, but try to continue parsing regardless
cat >expect <<-\EOF &&
[bundle]
version = 1
mode = all
[bundle "one"]
uri = http://example.com/bundle.bdl
[bundle "two"]
uri = https://example.com/bundle.bdl
[bundle "three"]
uri = file:///usr/share/git/bundle.bdl
EOF
test_must_fail test-tool bundle-uri parse-key-values in >actual 2>err &&
test_cmp err.expect err &&
test_cmp_config_output expect actual
'
test_expect_success 'parse config format: just URIs' '
cat >expect <<-\EOF &&
[bundle]
version = 1
mode = all
[bundle "one"]
uri = http://example.com/bundle.bdl
[bundle "two"]
uri = https://example.com/bundle.bdl
[bundle "three"]
uri = file:///usr/share/git/bundle.bdl
EOF
test-tool bundle-uri parse-config expect >actual 2>err &&
test_must_be_empty err &&
test_cmp_config_output expect actual
'
test_expect_success 'parse config format edge cases: empty key or value' '
cat >in1 <<-\EOF &&
= bogus-value
EOF
cat >err1 <<-EOF &&
error: bad config line 1 in file in1
EOF
cat >expect <<-\EOF &&
[bundle]
version = 1
mode = all
EOF
test_must_fail test-tool bundle-uri parse-config in1 >actual 2>err &&
test_cmp err1 err &&
test_cmp_config_output expect actual &&
cat >in2 <<-\EOF &&
bogus-key =
EOF
cat >err2 <<-EOF &&
error: bad config line 1 in file in2
EOF
test_must_fail test-tool bundle-uri parse-config in2 >actual 2>err &&
test_cmp err2 err &&
test_cmp_config_output expect actual
'
test_done

11
t/test-lib-functions.sh
View File

@ -1868,3 +1868,14 @@ test_is_magic_mtime () {
rm -f .git/test-mtime-actual
return $ret
}
# Given two filenames, parse both using 'git config --list --file'
# and compare the sorted output of those commands. Useful when
# wanting to ignore whitespace differences and sorting concerns.
test_cmp_config_output () {
git config --list --file="$1" >config-expect &&
git config --list --file="$2" >config-actual &&
sort config-expect >sorted-expect &&
sort config-actual >sorted-actual &&
test_cmp sorted-expect sorted-actual
}

2
transport.c
View File

@ -178,7 +178,7 @@ static int fetch_refs_from_bundle(struct transport *transport,
if (!data->get_refs_from_bundle_called)
get_refs_from_bundle_inner(transport);
ret = unbundle(the_repository, &data->header, data->fd,
&extra_index_pack_args);
&extra_index_pack_args, 0);
transport->hash_algo = data->header.hash_algo;
return ret;
}