Merge branch 'bw/ls-files-recurse-submodules'

"git ls-files" learned "--recurse-submodules" option that can be
used to get a listing of tracked files across submodules (i.e. this
only works with "--cached" option, not for listing untracked or
ignored files).  This would be a useful tool to sit on the upstream
side of a pipe that is read with xargs to work on all working tree
files from the top-level superproject.

* bw/ls-files-recurse-submodules:
  ls-files: add pathspec matching for submodules
  ls-files: pass through safe options for --recurse-submodules
  ls-files: optionally recurse into submodules
  git: make super-prefix option
This commit is contained in:
Junio C Hamano 2016-10-26 13:14:44 -07:00
commit 1c2b1f7018
9 changed files with 453 additions and 45 deletions

View File

@ -18,7 +18,8 @@ SYNOPSIS
[--exclude-per-directory=<file>]
[--exclude-standard]
[--error-unmatch] [--with-tree=<tree-ish>]
[--full-name] [--abbrev] [--] [<file>...]
[--full-name] [--recurse-submodules]
[--abbrev] [--] [<file>...]
DESCRIPTION
-----------
@ -137,6 +138,10 @@ a space) at the start of each line:
option forces paths to be output relative to the project
top directory.
--recurse-submodules::
Recursively calls ls-files on each submodule in the repository.
Currently there is only support for the --cached mode.
--abbrev[=<n>]::
Instead of showing the full 40-byte hexadecimal object
lines, show only a partial prefix.

View File

@ -13,6 +13,7 @@ SYNOPSIS
[--exec-path[=<path>]] [--html-path] [--man-path] [--info-path]
[-p|--paginate|--no-pager] [--no-replace-objects] [--bare]
[--git-dir=<path>] [--work-tree=<path>] [--namespace=<name>]
[--super-prefix=<path>]
<command> [<args>]
DESCRIPTION
@ -602,6 +603,11 @@ foo.bar= ...`) sets `foo.bar` to the empty string.
details. Equivalent to setting the `GIT_NAMESPACE` environment
variable.
--super-prefix=<path>::
Currently for internal use only. Set a prefix which gives a path from
above a repository down to its root. One use is to give submodules
context about the superproject that invoked it.
--bare::
Treat the repository as a bare repository. If GIT_DIR
environment is not set, it is set to the current working

View File

@ -14,6 +14,7 @@
#include "resolve-undo.h"
#include "string-list.h"
#include "pathspec.h"
#include "run-command.h"
static int abbrev;
static int show_deleted;
@ -28,8 +29,11 @@ static int show_valid_bit;
static int line_terminator = '\n';
static int debug_mode;
static int show_eol;
static int recurse_submodules;
static struct argv_array submodules_options = ARGV_ARRAY_INIT;
static const char *prefix;
static const char *super_prefix;
static int max_prefix_len;
static int prefix_len;
static struct pathspec pathspec;
@ -67,12 +71,25 @@ static void write_eolinfo(const struct cache_entry *ce, const char *path)
static void write_name(const char *name)
{
/*
* Prepend the super_prefix to name to construct the full_name to be
* written.
*/
struct strbuf full_name = STRBUF_INIT;
if (super_prefix) {
strbuf_addstr(&full_name, super_prefix);
strbuf_addstr(&full_name, name);
name = full_name.buf;
}
/*
* With "--full-name", prefix_len=0; this caller needs to pass
* an empty string in that case (a NULL is good for "").
*/
write_name_quoted_relative(name, prefix_len ? prefix : NULL,
stdout, line_terminator);
strbuf_release(&full_name);
}
static void show_dir_entry(const char *tag, struct dir_entry *ent)
@ -152,55 +169,117 @@ static void show_killed_files(struct dir_struct *dir)
}
}
/*
* Compile an argv_array with all of the options supported by --recurse_submodules
*/
static void compile_submodule_options(const struct dir_struct *dir, int show_tag)
{
if (line_terminator == '\0')
argv_array_push(&submodules_options, "-z");
if (show_tag)
argv_array_push(&submodules_options, "-t");
if (show_valid_bit)
argv_array_push(&submodules_options, "-v");
if (show_cached)
argv_array_push(&submodules_options, "--cached");
if (show_eol)
argv_array_push(&submodules_options, "--eol");
if (debug_mode)
argv_array_push(&submodules_options, "--debug");
}
/**
* Recursively call ls-files on a submodule
*/
static void show_gitlink(const struct cache_entry *ce)
{
struct child_process cp = CHILD_PROCESS_INIT;
int status;
int i;
argv_array_pushf(&cp.args, "--super-prefix=%s%s/",
super_prefix ? super_prefix : "",
ce->name);
argv_array_push(&cp.args, "ls-files");
argv_array_push(&cp.args, "--recurse-submodules");
/* add supported options */
argv_array_pushv(&cp.args, submodules_options.argv);
/*
* Pass in the original pathspec args. The submodule will be
* responsible for prepending the 'submodule_prefix' prior to comparing
* against the pathspec for matches.
*/
argv_array_push(&cp.args, "--");
for (i = 0; i < pathspec.nr; i++)
argv_array_push(&cp.args, pathspec.items[i].original);
cp.git_cmd = 1;
cp.dir = ce->name;
status = run_command(&cp);
if (status)
exit(status);
}
static void show_ce_entry(const char *tag, const struct cache_entry *ce)
{
struct strbuf name = STRBUF_INIT;
int len = max_prefix_len;
if (super_prefix)
strbuf_addstr(&name, super_prefix);
strbuf_addstr(&name, ce->name);
if (len >= ce_namelen(ce))
die("git ls-files: internal error - cache entry not superset of prefix");
if (!match_pathspec(&pathspec, ce->name, ce_namelen(ce),
len, ps_matched,
S_ISDIR(ce->ce_mode) || S_ISGITLINK(ce->ce_mode)))
return;
if (tag && *tag && show_valid_bit &&
(ce->ce_flags & CE_VALID)) {
static char alttag[4];
memcpy(alttag, tag, 3);
if (isalpha(tag[0]))
alttag[0] = tolower(tag[0]);
else if (tag[0] == '?')
alttag[0] = '!';
else {
alttag[0] = 'v';
alttag[1] = tag[0];
alttag[2] = ' ';
alttag[3] = 0;
if (recurse_submodules && S_ISGITLINK(ce->ce_mode) &&
submodule_path_match(&pathspec, name.buf, ps_matched)) {
show_gitlink(ce);
} else if (match_pathspec(&pathspec, name.buf, name.len,
len, ps_matched,
S_ISDIR(ce->ce_mode) ||
S_ISGITLINK(ce->ce_mode))) {
if (tag && *tag && show_valid_bit &&
(ce->ce_flags & CE_VALID)) {
static char alttag[4];
memcpy(alttag, tag, 3);
if (isalpha(tag[0]))
alttag[0] = tolower(tag[0]);
else if (tag[0] == '?')
alttag[0] = '!';
else {
alttag[0] = 'v';
alttag[1] = tag[0];
alttag[2] = ' ';
alttag[3] = 0;
}
tag = alttag;
}
if (!show_stage) {
fputs(tag, stdout);
} else {
printf("%s%06o %s %d\t",
tag,
ce->ce_mode,
find_unique_abbrev(ce->oid.hash, abbrev),
ce_stage(ce));
}
write_eolinfo(ce, ce->name);
write_name(ce->name);
if (debug_mode) {
const struct stat_data *sd = &ce->ce_stat_data;
printf(" ctime: %d:%d\n", sd->sd_ctime.sec, sd->sd_ctime.nsec);
printf(" mtime: %d:%d\n", sd->sd_mtime.sec, sd->sd_mtime.nsec);
printf(" dev: %d\tino: %d\n", sd->sd_dev, sd->sd_ino);
printf(" uid: %d\tgid: %d\n", sd->sd_uid, sd->sd_gid);
printf(" size: %d\tflags: %x\n", sd->sd_size, ce->ce_flags);
}
tag = alttag;
}
if (!show_stage) {
fputs(tag, stdout);
} else {
printf("%s%06o %s %d\t",
tag,
ce->ce_mode,
find_unique_abbrev(ce->oid.hash,abbrev),
ce_stage(ce));
}
write_eolinfo(ce, ce->name);
write_name(ce->name);
if (debug_mode) {
const struct stat_data *sd = &ce->ce_stat_data;
printf(" ctime: %d:%d\n", sd->sd_ctime.sec, sd->sd_ctime.nsec);
printf(" mtime: %d:%d\n", sd->sd_mtime.sec, sd->sd_mtime.nsec);
printf(" dev: %d\tino: %d\n", sd->sd_dev, sd->sd_ino);
printf(" uid: %d\tgid: %d\n", sd->sd_uid, sd->sd_gid);
printf(" size: %d\tflags: %x\n", sd->sd_size, ce->ce_flags);
}
strbuf_release(&name);
}
static void show_ru_info(void)
@ -468,6 +547,8 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
{ OPTION_SET_INT, 0, "full-name", &prefix_len, NULL,
N_("make the output relative to the project top directory"),
PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL },
OPT_BOOL(0, "recurse-submodules", &recurse_submodules,
N_("recurse through submodules")),
OPT_BOOL(0, "error-unmatch", &error_unmatch,
N_("if any <file> is not in the index, treat this as an error")),
OPT_STRING(0, "with-tree", &with_tree, N_("tree-ish"),
@ -484,6 +565,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
prefix = cmd_prefix;
if (prefix)
prefix_len = strlen(prefix);
super_prefix = get_super_prefix();
git_config(git_default_config, NULL);
if (read_cache() < 0)
@ -519,13 +601,32 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
if (require_work_tree && !is_inside_work_tree())
setup_work_tree();
if (recurse_submodules)
compile_submodule_options(&dir, show_tag);
if (recurse_submodules &&
(show_stage || show_deleted || show_others || show_unmerged ||
show_killed || show_modified || show_resolve_undo || with_tree))
die("ls-files --recurse-submodules unsupported mode");
if (recurse_submodules && error_unmatch)
die("ls-files --recurse-submodules does not support "
"--error-unmatch");
parse_pathspec(&pathspec, 0,
PATHSPEC_PREFER_CWD |
PATHSPEC_STRIP_SUBMODULE_SLASH_CHEAP,
prefix, argv);
/* Find common prefix for all pathspec's */
max_prefix = common_prefix(&pathspec);
/*
* Find common prefix for all pathspec's
* This is used as a performance optimization which unfortunately cannot
* be done when recursing into submodules
*/
if (recurse_submodules)
max_prefix = NULL;
else
max_prefix = common_prefix(&pathspec);
max_prefix_len = max_prefix ? strlen(max_prefix) : 0;
/* Treat unmatching pathspec elements as errors */

View File

@ -409,6 +409,7 @@ static inline enum object_type object_type(unsigned int mode)
#define GIT_NAMESPACE_ENVIRONMENT "GIT_NAMESPACE"
#define GIT_WORK_TREE_ENVIRONMENT "GIT_WORK_TREE"
#define GIT_PREFIX_ENVIRONMENT "GIT_PREFIX"
#define GIT_SUPER_PREFIX_ENVIRONMENT "GIT_INTERNAL_SUPER_PREFIX"
#define DEFAULT_GIT_DIR_ENVIRONMENT ".git"
#define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY"
#define INDEX_ENVIRONMENT "GIT_INDEX_FILE"
@ -476,6 +477,7 @@ extern int get_common_dir_noenv(struct strbuf *sb, const char *gitdir);
extern int get_common_dir(struct strbuf *sb, const char *gitdir);
extern const char *get_git_namespace(void);
extern const char *strip_namespace(const char *namespaced_ref);
extern const char *get_super_prefix(void);
extern const char *get_git_work_tree(void);
/*

46
dir.c
View File

@ -207,8 +207,9 @@ int within_depth(const char *name, int namelen,
return 1;
}
#define DO_MATCH_EXCLUDE 1
#define DO_MATCH_DIRECTORY 2
#define DO_MATCH_EXCLUDE (1<<0)
#define DO_MATCH_DIRECTORY (1<<1)
#define DO_MATCH_SUBMODULE (1<<2)
/*
* Does 'match' match the given name?
@ -283,6 +284,32 @@ static int match_pathspec_item(const struct pathspec_item *item, int prefix,
item->nowildcard_len - prefix))
return MATCHED_FNMATCH;
/* Perform checks to see if "name" is a super set of the pathspec */
if (flags & DO_MATCH_SUBMODULE) {
/* name is a literal prefix of the pathspec */
if ((namelen < matchlen) &&
(match[namelen] == '/') &&
!ps_strncmp(item, match, name, namelen))
return MATCHED_RECURSIVELY;
/* name" doesn't match up to the first wild character */
if (item->nowildcard_len < item->len &&
ps_strncmp(item, match, name,
item->nowildcard_len - prefix))
return 0;
/*
* Here is where we would perform a wildmatch to check if
* "name" can be matched as a directory (or a prefix) against
* the pathspec. Since wildmatch doesn't have this capability
* at the present we have to punt and say that it is a match,
* potentially returning a false positive
* The submodules themselves will be able to perform more
* accurate matching to determine if the pathspec matches.
*/
return MATCHED_RECURSIVELY;
}
return 0;
}
@ -386,6 +413,21 @@ int match_pathspec(const struct pathspec *ps,
return negative ? 0 : positive;
}
/**
* Check if a submodule is a superset of the pathspec
*/
int submodule_path_match(const struct pathspec *ps,
const char *submodule_name,
char *seen)
{
int matched = do_match_pathspec(ps, submodule_name,
strlen(submodule_name),
0, seen,
DO_MATCH_DIRECTORY |
DO_MATCH_SUBMODULE);
return matched;
}
int report_path_error(const char *ps_matched,
const struct pathspec *pathspec,
const char *prefix)

4
dir.h
View File

@ -304,6 +304,10 @@ extern int git_fnmatch(const struct pathspec_item *item,
const char *pattern, const char *string,
int prefix);
extern int submodule_path_match(const struct pathspec *ps,
const char *submodule_name,
char *seen);
static inline int ce_path_match(const struct cache_entry *ce,
const struct pathspec *pathspec,
char *seen)

View File

@ -99,6 +99,8 @@ static char *work_tree;
static const char *namespace;
static size_t namespace_len;
static const char *super_prefix;
static const char *git_dir, *git_common_dir;
static char *git_object_dir, *git_index_file, *git_graft_file;
int git_db_env, git_index_env, git_graft_env, git_common_dir_env;
@ -119,6 +121,7 @@ const char * const local_repo_env[] = {
NO_REPLACE_OBJECTS_ENVIRONMENT,
GIT_REPLACE_REF_BASE_ENVIRONMENT,
GIT_PREFIX_ENVIRONMENT,
GIT_SUPER_PREFIX_ENVIRONMENT,
GIT_SHALLOW_FILE_ENVIRONMENT,
GIT_COMMON_DIR_ENVIRONMENT,
NULL
@ -228,6 +231,16 @@ const char *strip_namespace(const char *namespaced_ref)
return namespaced_ref + namespace_len;
}
const char *get_super_prefix(void)
{
static int initialized;
if (!initialized) {
super_prefix = getenv(GIT_SUPER_PREFIX_ENVIRONMENT);
initialized = 1;
}
return super_prefix;
}
static int git_work_tree_initialized;
/*

27
git.c
View File

@ -164,6 +164,20 @@ static int handle_options(const char ***argv, int *argc, int *envchanged)
setenv(GIT_WORK_TREE_ENVIRONMENT, cmd, 1);
if (envchanged)
*envchanged = 1;
} else if (!strcmp(cmd, "--super-prefix")) {
if (*argc < 2) {
fprintf(stderr, "No prefix given for --super-prefix.\n" );
usage(git_usage_string);
}
setenv(GIT_SUPER_PREFIX_ENVIRONMENT, (*argv)[1], 1);
if (envchanged)
*envchanged = 1;
(*argv)++;
(*argc)--;
} else if (skip_prefix(cmd, "--super-prefix=", &cmd)) {
setenv(GIT_SUPER_PREFIX_ENVIRONMENT, cmd, 1);
if (envchanged)
*envchanged = 1;
} else if (!strcmp(cmd, "--bare")) {
char *cwd = xgetcwd();
is_bare_repository_cfg = 1;
@ -310,6 +324,7 @@ static int handle_alias(int *argcp, const char ***argv)
* RUN_SETUP for reading from the configuration file.
*/
#define NEED_WORK_TREE (1<<3)
#define SUPPORT_SUPER_PREFIX (1<<4)
struct cmd_struct {
const char *cmd;
@ -344,6 +359,13 @@ static int run_builtin(struct cmd_struct *p, int argc, const char **argv)
}
commit_pager_choice();
if (!help && get_super_prefix()) {
if (!(p->option & SUPPORT_SUPER_PREFIX))
die("%s doesn't support --super-prefix", p->cmd);
if (prefix)
die("can't use --super-prefix from a subdirectory");
}
if (!help && p->option & NEED_WORK_TREE)
setup_work_tree();
@ -421,7 +443,7 @@ static struct cmd_struct commands[] = {
{ "init-db", cmd_init_db },
{ "interpret-trailers", cmd_interpret_trailers, RUN_SETUP_GENTLY },
{ "log", cmd_log, RUN_SETUP },
{ "ls-files", cmd_ls_files, RUN_SETUP },
{ "ls-files", cmd_ls_files, RUN_SETUP | SUPPORT_SUPER_PREFIX },
{ "ls-remote", cmd_ls_remote, RUN_SETUP_GENTLY },
{ "ls-tree", cmd_ls_tree, RUN_SETUP },
{ "mailinfo", cmd_mailinfo },
@ -558,6 +580,9 @@ static void execv_dashed_external(const char **argv)
const char *tmp;
int status;
if (get_super_prefix())
die("%s doesn't support --super-prefix", argv[0]);
if (use_pager == -1)
use_pager = check_pager_config(argv[0]);
commit_pager_choice();

View File

@ -0,0 +1,210 @@
#!/bin/sh
test_description='Test ls-files recurse-submodules feature
This test verifies the recurse-submodules feature correctly lists files from
submodules.
'
. ./test-lib.sh
test_expect_success 'setup directory structure and submodules' '
echo a >a &&
mkdir b &&
echo b >b/b &&
git add a b &&
git commit -m "add a and b" &&
git init submodule &&
echo c >submodule/c &&
git -C submodule add c &&
git -C submodule commit -m "add c" &&
git submodule add ./submodule &&
git commit -m "added submodule"
'
test_expect_success 'ls-files correctly outputs files in submodule' '
cat >expect <<-\EOF &&
.gitmodules
a
b/b
submodule/c
EOF
git ls-files --recurse-submodules >actual &&
test_cmp expect actual
'
test_expect_success 'ls-files correctly outputs files in submodule with -z' '
lf_to_nul >expect <<-\EOF &&
.gitmodules
a
b/b
submodule/c
EOF
git ls-files --recurse-submodules -z >actual &&
test_cmp expect actual
'
test_expect_success 'ls-files does not output files not added to a repo' '
cat >expect <<-\EOF &&
.gitmodules
a
b/b
submodule/c
EOF
echo a >not_added &&
echo b >b/not_added &&
echo c >submodule/not_added &&
git ls-files --recurse-submodules >actual &&
test_cmp expect actual
'
test_expect_success 'ls-files recurses more than 1 level' '
cat >expect <<-\EOF &&
.gitmodules
a
b/b
submodule/.gitmodules
submodule/c
submodule/subsub/d
EOF
git init submodule/subsub &&
echo d >submodule/subsub/d &&
git -C submodule/subsub add d &&
git -C submodule/subsub commit -m "add d" &&
git -C submodule submodule add ./subsub &&
git -C submodule commit -m "added subsub" &&
git ls-files --recurse-submodules >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules and pathspecs setup' '
echo e >submodule/subsub/e.txt &&
git -C submodule/subsub add e.txt &&
git -C submodule/subsub commit -m "adding e.txt" &&
echo f >submodule/f.TXT &&
echo g >submodule/g.txt &&
git -C submodule add f.TXT g.txt &&
git -C submodule commit -m "add f and g" &&
echo h >h.txt &&
mkdir sib &&
echo sib >sib/file &&
git add h.txt sib/file &&
git commit -m "add h and sib/file" &&
git init sub &&
echo sub >sub/file &&
git -C sub add file &&
git -C sub commit -m "add file" &&
git submodule add ./sub &&
git commit -m "added sub" &&
cat >expect <<-\EOF &&
.gitmodules
a
b/b
h.txt
sib/file
sub/file
submodule/.gitmodules
submodule/c
submodule/f.TXT
submodule/g.txt
submodule/subsub/d
submodule/subsub/e.txt
EOF
git ls-files --recurse-submodules >actual &&
test_cmp expect actual &&
cat actual &&
git ls-files --recurse-submodules "*" >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules and pathspecs' '
cat >expect <<-\EOF &&
h.txt
submodule/g.txt
submodule/subsub/e.txt
EOF
git ls-files --recurse-submodules "*.txt" >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules and pathspecs' '
cat >expect <<-\EOF &&
h.txt
submodule/f.TXT
submodule/g.txt
submodule/subsub/e.txt
EOF
git ls-files --recurse-submodules ":(icase)*.txt" >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules and pathspecs' '
cat >expect <<-\EOF &&
h.txt
submodule/f.TXT
submodule/g.txt
EOF
git ls-files --recurse-submodules ":(icase)*.txt" ":(exclude)submodule/subsub/*" >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules and pathspecs' '
cat >expect <<-\EOF &&
sub/file
EOF
git ls-files --recurse-submodules "sub" >actual &&
test_cmp expect actual &&
git ls-files --recurse-submodules "sub/" >actual &&
test_cmp expect actual &&
git ls-files --recurse-submodules "sub/file" >actual &&
test_cmp expect actual &&
git ls-files --recurse-submodules "su*/file" >actual &&
test_cmp expect actual &&
git ls-files --recurse-submodules "su?/file" >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules and pathspecs' '
cat >expect <<-\EOF &&
sib/file
sub/file
EOF
git ls-files --recurse-submodules "s??/file" >actual &&
test_cmp expect actual &&
git ls-files --recurse-submodules "s???file" >actual &&
test_cmp expect actual &&
git ls-files --recurse-submodules "s*file" >actual &&
test_cmp expect actual
'
test_expect_success '--recurse-submodules does not support --error-unmatch' '
test_must_fail git ls-files --recurse-submodules --error-unmatch 2>actual &&
test_i18ngrep "does not support --error-unmatch" actual
'
test_incompatible_with_recurse_submodules () {
test_expect_success "--recurse-submodules and $1 are incompatible" "
test_must_fail git ls-files --recurse-submodules $1 2>actual &&
test_i18ngrep 'unsupported mode' actual
"
}
test_incompatible_with_recurse_submodules --deleted
test_incompatible_with_recurse_submodules --modified
test_incompatible_with_recurse_submodules --others
test_incompatible_with_recurse_submodules --stage
test_incompatible_with_recurse_submodules --killed
test_incompatible_with_recurse_submodules --unmerged
test_done