Merge branch 'jk/archive-tar-filter'

* jk/archive-tar-filter:
  upload-archive: allow user to turn off filters
  archive: provide builtin .tar.gz filter
  archive: implement configurable tar filters
  archive: refactor file extension format-guessing
  archive: move file extension format-guessing lower
  archive: pass archiver struct to write_archive callback
  archive: refactor list of archive formats
  archive-tar: don't reload default config options
  archive: reorder option parsing and config reading
This commit is contained in:
Junio C Hamano 2011-07-19 09:45:32 -07:00
commit 765c7e4f31
8 changed files with 375 additions and 73 deletions

View File

@ -101,6 +101,25 @@ tar.umask::
details. If `--remote` is used then only the configuration of
the remote repository takes effect.
tar.<format>.command::
This variable specifies a shell command through which the tar
output generated by `git archive` should be piped. The command
is executed using the shell with the generated tar file on its
standard input, and should produce the final output on its
standard output. Any compression-level options will be passed
to the command (e.g., "-9"). An output file with the same
extension as `<format>` will be use this format if no other
format is given.
+
The "tar.gz" and "tgz" formats are defined automatically and default to
`gzip -cn`. You may override them with custom commands.
tar.<format>.remote::
If true, enable `<format>` for use by remote clients via
linkgit:git-upload-archive[1]. Defaults to false for
user-defined formats, but true for the "tar.gz" and "tgz"
formats.
ATTRIBUTES
----------
@ -133,6 +152,14 @@ git archive --format=tar --prefix=git-1.4.0/ v1.4.0 | gzip >git-1.4.0.tar.gz::
Create a compressed tarball for v1.4.0 release.
git archive --format=tar.gz --prefix=git-1.4.0/ v1.4.0 >git-1.4.0.tar.gz::
Same as above, but using the builtin tar.gz handling.
git archive --prefix=git-1.4.0/ -o git-1.4.0.tar.gz v1.4.0::
Same as above, but the format is inferred from the output file.
git archive --format=tar --prefix=git-1.4.0/ v1.4.0{caret}\{tree\} | gzip >git-1.4.0.tar.gz::
Create a compressed tarball for v1.4.0 release, but without a
@ -149,6 +176,12 @@ git archive -o latest.zip HEAD::
commit on the current branch. Note that the output format is
inferred by the extension of the output file.
git config tar.tar.xz.command "xz -c"::
Configure a "tar.xz" format for making LZMA-compressed tarfiles.
You can use it specifying `--format=tar.xz`, or by creating an
output file like `-o foo.tar.xz`.
SEE ALSO
--------

View File

@ -4,6 +4,7 @@
#include "cache.h"
#include "tar.h"
#include "archive.h"
#include "run-command.h"
#define RECORDSIZE (512)
#define BLOCKSIZE (RECORDSIZE * 20)
@ -13,6 +14,9 @@ static unsigned long offset;
static int tar_umask = 002;
static int write_tar_filter_archive(const struct archiver *ar,
struct archiver_args *args);
/* writes out the whole block, but only if it is full */
static void write_if_needed(void)
{
@ -220,6 +224,67 @@ static int write_global_extended_header(struct archiver_args *args)
return err;
}
static struct archiver **tar_filters;
static int nr_tar_filters;
static int alloc_tar_filters;
static struct archiver *find_tar_filter(const char *name, int len)
{
int i;
for (i = 0; i < nr_tar_filters; i++) {
struct archiver *ar = tar_filters[i];
if (!strncmp(ar->name, name, len) && !ar->name[len])
return ar;
}
return NULL;
}
static int tar_filter_config(const char *var, const char *value, void *data)
{
struct archiver *ar;
const char *dot;
const char *name;
const char *type;
int namelen;
if (prefixcmp(var, "tar."))
return 0;
dot = strrchr(var, '.');
if (dot == var + 9)
return 0;
name = var + 4;
namelen = dot - name;
type = dot + 1;
ar = find_tar_filter(name, namelen);
if (!ar) {
ar = xcalloc(1, sizeof(*ar));
ar->name = xmemdupz(name, namelen);
ar->write_archive = write_tar_filter_archive;
ar->flags = ARCHIVER_WANT_COMPRESSION_LEVELS;
ALLOC_GROW(tar_filters, nr_tar_filters + 1, alloc_tar_filters);
tar_filters[nr_tar_filters++] = ar;
}
if (!strcmp(type, "command")) {
if (!value)
return config_error_nonbool(var);
free(ar->data);
ar->data = xstrdup(value);
return 0;
}
if (!strcmp(type, "remote")) {
if (git_config_bool(var, value))
ar->flags |= ARCHIVER_REMOTE;
else
ar->flags &= ~ARCHIVER_REMOTE;
return 0;
}
return 0;
}
static int git_tar_config(const char *var, const char *value, void *cb)
{
if (!strcmp(var, "tar.umask")) {
@ -231,15 +296,15 @@ static int git_tar_config(const char *var, const char *value, void *cb)
}
return 0;
}
return git_default_config(var, value, cb);
return tar_filter_config(var, value, cb);
}
int write_tar_archive(struct archiver_args *args)
static int write_tar_archive(const struct archiver *ar,
struct archiver_args *args)
{
int err = 0;
git_config(git_tar_config, NULL);
if (args->commit_sha1)
err = write_global_extended_header(args);
if (!err)
@ -248,3 +313,65 @@ int write_tar_archive(struct archiver_args *args)
write_trailer();
return err;
}
static int write_tar_filter_archive(const struct archiver *ar,
struct archiver_args *args)
{
struct strbuf cmd = STRBUF_INIT;
struct child_process filter;
const char *argv[2];
int r;
if (!ar->data)
die("BUG: tar-filter archiver called with no filter defined");
strbuf_addstr(&cmd, ar->data);
if (args->compression_level >= 0)
strbuf_addf(&cmd, " -%d", args->compression_level);
memset(&filter, 0, sizeof(filter));
argv[0] = cmd.buf;
argv[1] = NULL;
filter.argv = argv;
filter.use_shell = 1;
filter.in = -1;
if (start_command(&filter) < 0)
die_errno("unable to start '%s' filter", argv[0]);
close(1);
if (dup2(filter.in, 1) < 0)
die_errno("unable to redirect descriptor");
close(filter.in);
r = write_tar_archive(ar, args);
close(1);
if (finish_command(&filter) != 0)
die("'%s' filter reported error", argv[0]);
strbuf_release(&cmd);
return r;
}
static struct archiver tar_archiver = {
"tar",
write_tar_archive,
ARCHIVER_REMOTE
};
void init_tar_archiver(void)
{
int i;
register_archiver(&tar_archiver);
tar_filter_config("tar.tgz.command", "gzip -cn", NULL);
tar_filter_config("tar.tgz.remote", "true", NULL);
tar_filter_config("tar.tar.gz.command", "gzip -cn", NULL);
tar_filter_config("tar.tar.gz.remote", "true", NULL);
git_config(git_tar_config, NULL);
for (i = 0; i < nr_tar_filters; i++) {
/* omit any filters that never had a command configured */
if (tar_filters[i]->data)
register_archiver(tar_filters[i]);
}
}

View File

@ -261,7 +261,8 @@ static void dos_time(time_t *time, int *dos_date, int *dos_time)
*dos_time = t->tm_sec / 2 + t->tm_min * 32 + t->tm_hour * 2048;
}
int write_zip_archive(struct archiver_args *args)
static int write_zip_archive(const struct archiver *ar,
struct archiver_args *args)
{
int err;
@ -278,3 +279,14 @@ int write_zip_archive(struct archiver_args *args)
return err;
}
static struct archiver zip_archiver = {
"zip",
write_zip_archive,
ARCHIVER_WANT_COMPRESSION_LEVELS|ARCHIVER_REMOTE
};
void init_zip_archiver(void)
{
register_archiver(&zip_archiver);
}

View File

@ -14,16 +14,15 @@ static char const * const archive_usage[] = {
NULL
};
#define USES_ZLIB_COMPRESSION 1
static const struct archiver **archivers;
static int nr_archivers;
static int alloc_archivers;
static const struct archiver {
const char *name;
write_archive_fn_t write_archive;
unsigned int flags;
} archivers[] = {
{ "tar", write_tar_archive },
{ "zip", write_zip_archive, USES_ZLIB_COMPRESSION },
};
void register_archiver(struct archiver *ar)
{
ALLOC_GROW(archivers, nr_archivers + 1, alloc_archivers);
archivers[nr_archivers++] = ar;
}
static void format_subst(const struct commit *commit,
const char *src, size_t len,
@ -208,9 +207,9 @@ static const struct archiver *lookup_archiver(const char *name)
if (!name)
return NULL;
for (i = 0; i < ARRAY_SIZE(archivers); i++) {
if (!strcmp(name, archivers[i].name))
return &archivers[i];
for (i = 0; i < nr_archivers; i++) {
if (!strcmp(name, archivers[i]->name))
return archivers[i];
}
return NULL;
}
@ -299,9 +298,10 @@ static void parse_treeish_arg(const char **argv,
PARSE_OPT_NOARG | PARSE_OPT_NONEG | PARSE_OPT_HIDDEN, NULL, (p) }
static int parse_archive_args(int argc, const char **argv,
const struct archiver **ar, struct archiver_args *args)
const struct archiver **ar, struct archiver_args *args,
const char *name_hint, int is_remote)
{
const char *format = "tar";
const char *format = NULL;
const char *base = NULL;
const char *remote = NULL;
const char *exec = NULL;
@ -355,21 +355,27 @@ static int parse_archive_args(int argc, const char **argv,
base = "";
if (list) {
for (i = 0; i < ARRAY_SIZE(archivers); i++)
printf("%s\n", archivers[i].name);
for (i = 0; i < nr_archivers; i++)
if (!is_remote || archivers[i]->flags & ARCHIVER_REMOTE)
printf("%s\n", archivers[i]->name);
exit(0);
}
if (!format && name_hint)
format = archive_format_from_filename(name_hint);
if (!format)
format = "tar";
/* We need at least one parameter -- tree-ish */
if (argc < 1)
usage_with_options(archive_usage, opts);
*ar = lookup_archiver(format);
if (!*ar)
if (!*ar || (is_remote && !((*ar)->flags & ARCHIVER_REMOTE)))
die("Unknown archive format '%s'", format);
args->compression_level = Z_DEFAULT_COMPRESSION;
if (compression_level != -1) {
if ((*ar)->flags & USES_ZLIB_COMPRESSION)
if ((*ar)->flags & ARCHIVER_WANT_COMPRESSION_LEVELS)
args->compression_level = compression_level;
else {
die("Argument not supported for format '%s': -%d",
@ -385,19 +391,55 @@ static int parse_archive_args(int argc, const char **argv,
}
int write_archive(int argc, const char **argv, const char *prefix,
int setup_prefix)
int setup_prefix, const char *name_hint, int remote)
{
int nongit = 0;
const struct archiver *ar = NULL;
struct archiver_args args;
argc = parse_archive_args(argc, argv, &ar, &args);
if (setup_prefix && prefix == NULL)
prefix = setup_git_directory();
prefix = setup_git_directory_gently(&nongit);
git_config(git_default_config, NULL);
init_tar_archiver();
init_zip_archiver();
argc = parse_archive_args(argc, argv, &ar, &args, name_hint, remote);
if (nongit) {
/*
* We know this will die() with an error, so we could just
* die ourselves; but its error message will be more specific
* than what we could write here.
*/
setup_git_directory();
}
parse_treeish_arg(argv, &args, prefix);
parse_pathspec_arg(argv + 1, &args);
git_config(git_default_config, NULL);
return ar->write_archive(&args);
return ar->write_archive(ar, &args);
}
static int match_extension(const char *filename, const char *ext)
{
int prefixlen = strlen(filename) - strlen(ext);
/*
* We need 1 character for the '.', and 1 character to ensure that the
* prefix is non-empty (k.e., we don't match .tar.gz with no actual
* filename).
*/
if (prefixlen < 2 || filename[prefixlen-1] != '.')
return 0;
return !strcmp(filename + prefixlen, ext);
}
const char *archive_format_from_filename(const char *filename)
{
int i;
for (i = 0; i < nr_archivers; i++)
if (match_extension(filename, archivers[i]->name))
return archivers[i]->name;
return NULL;
}

View File

@ -14,17 +14,24 @@ struct archiver_args {
int compression_level;
};
typedef int (*write_archive_fn_t)(struct archiver_args *);
#define ARCHIVER_WANT_COMPRESSION_LEVELS 1
#define ARCHIVER_REMOTE 2
struct archiver {
const char *name;
int (*write_archive)(const struct archiver *, struct archiver_args *);
unsigned flags;
void *data;
};
extern void register_archiver(struct archiver *);
extern void init_tar_archiver(void);
extern void init_zip_archiver(void);
typedef int (*write_archive_entry_fn_t)(struct archiver_args *args, const unsigned char *sha1, const char *path, size_t pathlen, unsigned int mode, void *buffer, unsigned long size);
/*
* Archive-format specific backends.
*/
extern int write_tar_archive(struct archiver_args *);
extern int write_zip_archive(struct archiver_args *);
extern int write_archive_entries(struct archiver_args *args, write_archive_entry_fn_t write_entry);
extern int write_archive(int argc, const char **argv, const char *prefix, int setup_prefix);
extern int write_archive(int argc, const char **argv, const char *prefix, int setup_prefix, const char *name_hint, int remote);
const char *archive_format_from_filename(const char *filename);
#endif /* ARCHIVE_H */

View File

@ -24,7 +24,8 @@ static void create_output_file(const char *output_file)
}
static int run_remote_archiver(int argc, const char **argv,
const char *remote, const char *exec)
const char *remote, const char *exec,
const char *name_hint)
{
char buf[LARGE_PACKET_MAX];
int fd[2], i, len, rv;
@ -37,6 +38,17 @@ static int run_remote_archiver(int argc, const char **argv,
transport = transport_get(_remote, _remote->url[0]);
transport_connect(transport, "git-upload-archive", exec, fd);
/*
* Inject a fake --format field at the beginning of the
* arguments, with the format inferred from our output
* filename. This way explicit --format options can override
* it.
*/
if (name_hint) {
const char *format = archive_format_from_filename(name_hint);
if (format)
packet_write(fd[1], "argument --format=%s\n", format);
}
for (i = 1; i < argc; i++)
packet_write(fd[1], "argument %s\n", argv[i]);
packet_flush(fd[1]);
@ -63,17 +75,6 @@ static int run_remote_archiver(int argc, const char **argv,
return !!rv;
}
static const char *format_from_name(const char *filename)
{
const char *ext = strrchr(filename, '.');
if (!ext)
return NULL;
ext++;
if (!strcasecmp(ext, "zip"))
return "--format=zip";
return NULL;
}
#define PARSE_OPT_KEEP_ALL ( PARSE_OPT_KEEP_DASHDASH | \
PARSE_OPT_KEEP_ARGV0 | \
PARSE_OPT_KEEP_UNKNOWN | \
@ -84,7 +85,6 @@ int cmd_archive(int argc, const char **argv, const char *prefix)
const char *exec = "git-upload-archive";
const char *output = NULL;
const char *remote = NULL;
const char *format_option = NULL;
struct option local_opts[] = {
OPT_STRING('o', "output", &output, "file",
"write the archive to this file"),
@ -98,32 +98,13 @@ int cmd_archive(int argc, const char **argv, const char *prefix)
argc = parse_options(argc, argv, prefix, local_opts, NULL,
PARSE_OPT_KEEP_ALL);
if (output) {
if (output)
create_output_file(output);
format_option = format_from_name(output);
}
/*
* We have enough room in argv[] to muck it in place, because
* --output must have been given on the original command line
* if we get to this point, and parse_options() must have eaten
* it, i.e. we can add back one element to the array.
*
* We add a fake --format option at the beginning, with the
* format inferred from our output filename. This way explicit
* --format options can override it, and the fake option is
* inserted before any "--" that might have been given.
*/
if (format_option) {
memmove(argv + 2, argv + 1, sizeof(*argv) * argc);
argv[1] = format_option;
argv[++argc] = NULL;
}
if (remote)
return run_remote_archiver(argc, argv, remote, exec);
return run_remote_archiver(argc, argv, remote, exec, output);
setvbuf(stderr, NULL, _IOLBF, BUFSIZ);
return write_archive(argc, argv, prefix, 1);
return write_archive(argc, argv, prefix, 1, output, 0);
}

View File

@ -64,7 +64,7 @@ static int run_upload_archive(int argc, const char **argv, const char *prefix)
sent_argv[sent_argc] = NULL;
/* parse all options sent by the client */
return write_archive(sent_argc, sent_argv, prefix, 0);
return write_archive(sent_argc, sent_argv, prefix, 0, NULL, 1);
}
__attribute__((format (printf, 1, 2)))

View File

@ -26,6 +26,8 @@ commit id embedding:
. ./test-lib.sh
UNZIP=${UNZIP:-unzip}
GZIP=${GZIP:-gzip}
GUNZIP=${GUNZIP:-gzip -d}
SUBSTFORMAT=%H%n
@ -252,4 +254,102 @@ test_expect_success 'git-archive --prefix=olde-' '
test -f h/olde-a/bin/sh
'
test_expect_success 'setup tar filters' '
git config tar.tar.foo.command "tr ab ba" &&
git config tar.bar.command "tr ab ba" &&
git config tar.bar.remote true
'
test_expect_success 'archive --list mentions user filter' '
git archive --list >output &&
grep "^tar\.foo\$" output &&
grep "^bar\$" output
'
test_expect_success 'archive --list shows only enabled remote filters' '
git archive --list --remote=. >output &&
! grep "^tar\.foo\$" output &&
grep "^bar\$" output
'
test_expect_success 'invoke tar filter by format' '
git archive --format=tar.foo HEAD >config.tar.foo &&
tr ab ba <config.tar.foo >config.tar &&
test_cmp b.tar config.tar &&
git archive --format=bar HEAD >config.bar &&
tr ab ba <config.bar >config.tar &&
test_cmp b.tar config.tar
'
test_expect_success 'invoke tar filter by extension' '
git archive -o config-implicit.tar.foo HEAD &&
test_cmp config.tar.foo config-implicit.tar.foo &&
git archive -o config-implicit.bar HEAD &&
test_cmp config.tar.foo config-implicit.bar
'
test_expect_success 'default output format remains tar' '
git archive -o config-implicit.baz HEAD &&
test_cmp b.tar config-implicit.baz
'
test_expect_success 'extension matching requires dot' '
git archive -o config-implicittar.foo HEAD &&
test_cmp b.tar config-implicittar.foo
'
test_expect_success 'only enabled filters are available remotely' '
test_must_fail git archive --remote=. --format=tar.foo HEAD \
>remote.tar.foo &&
git archive --remote=. --format=bar >remote.bar HEAD &&
test_cmp remote.bar config.bar
'
if $GZIP --version >/dev/null 2>&1; then
test_set_prereq GZIP
else
say "Skipping some tar.gz tests because gzip not found"
fi
test_expect_success GZIP 'git archive --format=tgz' '
git archive --format=tgz HEAD >j.tgz
'
test_expect_success GZIP 'git archive --format=tar.gz' '
git archive --format=tar.gz HEAD >j1.tar.gz &&
test_cmp j.tgz j1.tar.gz
'
test_expect_success GZIP 'infer tgz from .tgz filename' '
git archive --output=j2.tgz HEAD &&
test_cmp j.tgz j2.tgz
'
test_expect_success GZIP 'infer tgz from .tar.gz filename' '
git archive --output=j3.tar.gz HEAD &&
test_cmp j.tgz j3.tar.gz
'
if $GUNZIP --version >/dev/null 2>&1; then
test_set_prereq GUNZIP
else
say "Skipping some tar.gz tests because gunzip was not found"
fi
test_expect_success GZIP,GUNZIP 'extract tgz file' '
$GUNZIP -c <j.tgz >j.tar &&
test_cmp b.tar j.tar
'
test_expect_success GZIP 'remote tar.gz is allowed by default' '
git archive --remote=. --format=tar.gz HEAD >remote.tar.gz &&
test_cmp j.tgz remote.tar.gz
'
test_expect_success GZIP 'remote tar.gz can be disabled' '
git config tar.tar.gz.remote false &&
test_must_fail git archive --remote=. --format=tar.gz HEAD \
>remote.tar.gz
'
test_done