From ad90da73513d7b0055e47c6918e1d75c6165fa69 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 11:23:56 +0000 Subject: [PATCH 1/6] diff: ignore sparse paths in diffstat The diff_populate_filespec() method is used to describe the diff after a merge operation is complete. In order to avoid expanding a sparse index, the reuse_worktree_file() needs to be adapted to ignore files that are outside of the sparse-checkout cone. The file names and OIDs used for this check come from the merged tree in the case of the ORT strategy, not the index, hence the ability to look into these paths without having already expanded the index. The work done by reuse_worktree_file() is only an optimization, and requires the file being on disk for it to be of any value. Thus, it is safe to exit the method early if we do not expect the file on disk. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- diff.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/diff.c b/diff.c index a8113f1707..c8f530ffdb 100644 --- a/diff.c +++ b/diff.c @@ -26,6 +26,7 @@ #include "parse-options.h" #include "help.h" #include "promisor-remote.h" +#include "dir.h" #ifdef NO_FAST_WORKING_DIRECTORY #define FAST_WORKING_DIRECTORY 0 @@ -3907,6 +3908,13 @@ static int reuse_worktree_file(struct index_state *istate, if (!want_file && would_convert_to_git(istate, name)) return 0; + /* + * If this path does not match our sparse-checkout definition, + * then the file will not be in the working directory. + */ + if (!path_in_sparse_checkout(name, istate)) + return 0; + len = strlen(name); pos = index_name_pos(istate, name, len); if (pos < 0) From a33806398a418289388ad992e385a314b4b10225 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 11:23:57 +0000 Subject: [PATCH 2/6] merge: make sparse-aware with ORT Allow 'git merge' to operate without expanding a sparse index, at least not immediately. The index still will be expanded in a few cases: 1. If the merge strategy is 'recursive', then we enable command_requires_full_index at the start of the merge_recursive() method. We expect sparse-index users to also have the 'ort' strategy enabled. 2. With the 'ort' strategy, if the merge results in a conflicted file, then we expand the index before updating the working tree. The loop that iterates over the worktree replaces index entries and tracks 'origintal_cache_nr' which can become completely wrong if the index expands in the middle of the operation. This safety valve is important before that loop starts. A later change will focus this to only expand if we indeed have a conflict outside of the sparse-checkout cone. 3. Other merge strategies are executed as a 'git merge-X' subcommand, and those strategies are currently protected with the 'command_requires_full_index' guard. Some test updates are required, including a mistaken 'git checkout -b' that did not specify the base branch, causing merges to be fast-forward merges. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- builtin/merge.c | 3 +++ merge-ort.c | 8 ++++++++ merge-recursive.c | 3 +++ t/t1092-sparse-checkout-compatibility.sh | 12 ++++++++++-- 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/builtin/merge.c b/builtin/merge.c index 22f23990b3..926de328fb 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -1276,6 +1276,9 @@ int cmd_merge(int argc, const char **argv, const char *prefix) if (argc == 2 && !strcmp(argv[1], "-h")) usage_with_options(builtin_merge_usage, builtin_merge_options); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + /* * Check if we are _not_ on a detached HEAD, i.e. if there is a * current branch. diff --git a/merge-ort.c b/merge-ort.c index 6eb910d6f0..1531b4c94c 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -4058,6 +4058,14 @@ static int record_conflicted_index_entries(struct merge_options *opt) if (strmap_empty(&opt->priv->conflicted)) return 0; + /* + * We are in a conflicted state. These conflicts might be inside + * sparse-directory entries, so expand the index preemptively. + * Also, we set original_cache_nr below, but that might change if + * index_name_pos() calls ask for paths within sparse directories. + */ + ensure_full_index(index); + /* If any entries have skip_worktree set, we'll have to check 'em out */ state.force = 1; state.quiet = 1; diff --git a/merge-recursive.c b/merge-recursive.c index 3355d50e8a..1f563cd687 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -3750,6 +3750,9 @@ int merge_recursive(struct merge_options *opt, assert(opt->ancestor == NULL || !strcmp(opt->ancestor, "constructed merge base")); + prepare_repo_settings(opt->repo); + opt->repo->settings.command_requires_full_index = 1; + if (merge_start(opt, repo_get_commit_tree(opt->repo, h1))) return -1; clean = merge_recursive_internal(opt, h1, h2, merge_bases, result); diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index ddc86bb415..3b331a6bfe 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -47,7 +47,7 @@ test_expect_success 'setup' ' git checkout -b base && for dir in folder1 folder2 deep do - git checkout -b update-$dir && + git checkout -b update-$dir base && echo "updated $dir" >$dir/a && git commit -a -m "update $dir" || return 1 done && @@ -647,7 +647,15 @@ test_expect_success 'sparse-index is not expanded' ' echo >>sparse-index/extra.txt && ensure_not_expanded add extra.txt && echo >>sparse-index/untracked.txt && - ensure_not_expanded add . + ensure_not_expanded add . && + + ensure_not_expanded checkout -f update-deep && + test_config -C sparse-index pull.twohead ort && + ( + sane_unset GIT_TEST_MERGE_ALGORITHM && + ensure_not_expanded merge -m merge update-folder1 && + ensure_not_expanded merge -m merge update-folder2 + ) ' # NEEDSWORK: a sparse-checkout behaves differently from a full checkout From 695763679210420656f4125d9706bba25c76ae4b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 11:23:58 +0000 Subject: [PATCH 3/6] merge-ort: expand only for out-of-cone conflicts Merge conflicts happen often enough to want to avoid expanding a sparse index when they happen, as long as those conflicts are within the sparse-checkout cone. If a conflict exists outside of the sparse-checkout cone, then we still need to expand before iterating over the index entries. This is critical to do in advance because of how the original_cache_nr is tracked to allow inserting and replacing cache entries. Iterate over the conflicted files and check if any paths are outside of the sparse-checkout cone. If so, then expand the full index. Add a test that demonstrates that we do not expand the index, even when we hit a conflict within the sparse-checkout cone. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 13 +++++++--- t/t1092-sparse-checkout-compatibility.sh | 30 ++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index 1531b4c94c..805f7c4139 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -4060,11 +4060,18 @@ static int record_conflicted_index_entries(struct merge_options *opt) /* * We are in a conflicted state. These conflicts might be inside - * sparse-directory entries, so expand the index preemptively. - * Also, we set original_cache_nr below, but that might change if + * sparse-directory entries, so check if any entries are outside + * of the sparse-checkout cone preemptively. + * + * We set original_cache_nr below, but that might change if * index_name_pos() calls ask for paths within sparse directories. */ - ensure_full_index(index); + strmap_for_each_entry(&opt->priv->conflicted, &iter, e) { + if (!path_in_sparse_checkout(e->key, index)) { + ensure_full_index(index); + break; + } + } /* If any entries have skip_worktree set, we'll have to check 'em out */ state.force = 1; diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 3b331a6bfe..36964f52f2 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -617,8 +617,17 @@ test_expect_success 'sparse-index is expanded and converted back' ' ensure_not_expanded () { rm -f trace2.txt && echo >>sparse-index/untracked.txt && - GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ - git -C sparse-index "$@" && + + if test "$1" = "!" + then + shift && + test_must_fail env \ + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index "$@" || return 1 + else + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C sparse-index "$@" || return 1 + fi && test_region ! index ensure_full_index trace2.txt } @@ -658,6 +667,23 @@ test_expect_success 'sparse-index is not expanded' ' ) ' +test_expect_success 'sparse-index is not expanded: merge conflict in cone' ' + init_repos && + + for side in right left + do + git -C sparse-index checkout -b expand-$side base && + echo $side >sparse-index/deep/a && + git -C sparse-index commit -a -m "$side" || return 1 + done && + + ( + sane_unset GIT_TEST_MERGE_ALGORITHM && + git -C sparse-index config pull.twohead ort && + ensure_not_expanded ! merge -m merged expand-right + ) +' + # NEEDSWORK: a sparse-checkout behaves differently from a full checkout # in this scenario, but it shouldn't. test_expect_success 'reset mixed and checkout orphan' ' From c0b99303db317894dc49398cd3e2db4ef02e8dcf Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 11:23:59 +0000 Subject: [PATCH 4/6] t1092: add cherry-pick, rebase tests Add tests to check that cherry-pick and rebase behave the same in the sparse-index case as in the full index cases. These tests are agnostic to GIT_TEST_MERGE_ALGORITHM, so a full CI test suite will check both the 'ort' and 'recursive' strategies on this test. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- t/t1092-sparse-checkout-compatibility.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 36964f52f2..d9424ed642 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -481,14 +481,17 @@ test_expect_success 'checkout and reset (mixed) [sparse]' ' test_sparse_match git reset update-folder2 ' -test_expect_success 'merge' ' +test_expect_success 'merge, cherry-pick, and rebase' ' init_repos && - test_all_match git checkout -b merge update-deep && - test_all_match git merge -m "folder1" update-folder1 && - test_all_match git rev-parse HEAD^{tree} && - test_all_match git merge -m "folder2" update-folder2 && - test_all_match git rev-parse HEAD^{tree} + for OPERATION in "merge -m merge" cherry-pick rebase + do + test_all_match git checkout -B temp update-deep && + test_all_match git $OPERATION update-folder1 && + test_all_match git rev-parse HEAD^{tree} && + test_all_match git $OPERATION update-folder2 && + test_all_match git rev-parse HEAD^{tree} || return 1 + done ' # NEEDSWORK: This test is documenting current behavior, but that From 5d9c9349bd0acda149f37eb1c5edc2a5ef747eea Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 11:24:00 +0000 Subject: [PATCH 5/6] sequencer: ensure full index if not ORT strategy The sequencer is used by 'cherry-pick' and 'rebase' to sequence a list of operations that modify the index. Since we intend to remove the need for 'command_requires_full_index', we need to ensure the sparse index is expanded every time it is written to disk between these steps. That is, unless the merge strategy is 'ort' where the index can remain sparse throughout. There are two main places to be extra careful about a full index: 1. Right before calling merge_trees(), ensure the index is full. This happens within an 'else' where the 'if' block checks if the 'ort' strategy is selected. 2. During read_and_refresh_cache(), the index might be written to disk and converted to sparse in the process. Ensure it expands back to full afterwards by checking if the strategy is _not_ 'ort'. This 'if' statement is the logical negation of the 'if' in item (1). Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- sequencer.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sequencer.c b/sequencer.c index 7f07cd00f3..228bc089d2 100644 --- a/sequencer.c +++ b/sequencer.c @@ -652,6 +652,7 @@ static int do_recursive_merge(struct repository *r, merge_switch_to_result(&o, head_tree, &result, 1, show_output); clean = result.clean; } else { + ensure_full_index(r->index); clean = merge_trees(&o, head_tree, next_tree, base_tree); if (is_rebase_i(opts) && clean <= 0) fputs(o.obuf.buf, stdout); @@ -2346,6 +2347,7 @@ static int read_and_refresh_cache(struct repository *r, _(action_name(opts))); } refresh_index(r->index, REFRESH_QUIET|REFRESH_UNMERGED, NULL, NULL, NULL); + if (index_fd >= 0) { if (write_locked_index(r->index, &index_lock, COMMIT_LOCK | SKIP_IF_UNCHANGED)) { @@ -2353,6 +2355,13 @@ static int read_and_refresh_cache(struct repository *r, _(action_name(opts))); } } + + /* + * If we are resolving merges in any way other than "ort", then + * expand the sparse index. + */ + if (opts->strategy && strcmp(opts->strategy, "ort")) + ensure_full_index(r->index); return 0; } From 516680ba7704c473bb21628aa19cabbd787df4db Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 8 Sep 2021 11:24:01 +0000 Subject: [PATCH 6/6] sparse-index: integrate with cherry-pick and rebase The hard work was already done with 'git merge' and the ORT strategy. Just add extra tests to see that we get the expected results in the non-conflict cases. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- builtin/rebase.c | 6 ++++ builtin/revert.c | 3 ++ t/t1092-sparse-checkout-compatibility.sh | 39 ++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/builtin/rebase.c b/builtin/rebase.c index 33e0961900..27433d7c5a 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -559,6 +559,9 @@ int cmd_rebase__interactive(int argc, const char **argv, const char *prefix) argc = parse_options(argc, argv, prefix, options, builtin_rebase_interactive_usage, PARSE_OPT_KEEP_ARGV0); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + if (!is_null_oid(&squash_onto)) opts.squash_onto = &squash_onto; @@ -1430,6 +1433,9 @@ int cmd_rebase(int argc, const char **argv, const char *prefix) usage_with_options(builtin_rebase_usage, builtin_rebase_options); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + options.allow_empty_message = 1; git_config(rebase_config, &options); /* options.gpg_sign_opt will be either "-S" or NULL */ diff --git a/builtin/revert.c b/builtin/revert.c index 237f2f18d4..6c4c22691b 100644 --- a/builtin/revert.c +++ b/builtin/revert.c @@ -136,6 +136,9 @@ static int run_sequencer(int argc, const char **argv, struct replay_opts *opts) PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN); + prepare_repo_settings(the_repository); + the_repository->settings.command_requires_full_index = 0; + /* implies allow_empty */ if (opts->keep_redundant_commits) opts->allow_empty = 1; diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index d9424ed642..886e78715f 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -527,6 +527,38 @@ test_expect_success 'merge with conflict outside cone' ' test_all_match git rev-parse HEAD^{tree} ' +test_expect_success 'cherry-pick/rebase with conflict outside cone' ' + init_repos && + + for OPERATION in cherry-pick rebase + do + test_all_match git checkout -B tip && + test_all_match git reset --hard merge-left && + test_all_match git status --porcelain=v2 && + test_all_match test_must_fail git $OPERATION merge-right && + test_all_match git status --porcelain=v2 && + + # Resolve the conflict in different ways: + # 1. Revert to the base + test_all_match git checkout base -- deep/deeper2/a && + test_all_match git status --porcelain=v2 && + + # 2. Add the file with conflict markers + test_all_match git add folder1/a && + test_all_match git status --porcelain=v2 && + + # 3. Rename the file to another sparse filename and + # accept conflict markers as resolved content. + run_on_all mv folder2/a folder2/z && + test_all_match git add folder2 && + test_all_match git status --porcelain=v2 && + + test_all_match git $OPERATION --continue && + test_all_match git status --porcelain=v2 && + test_all_match git rev-parse HEAD^{tree} || return 1 + done +' + test_expect_success 'merge with outside renames' ' init_repos && @@ -665,8 +697,11 @@ test_expect_success 'sparse-index is not expanded' ' test_config -C sparse-index pull.twohead ort && ( sane_unset GIT_TEST_MERGE_ALGORITHM && - ensure_not_expanded merge -m merge update-folder1 && - ensure_not_expanded merge -m merge update-folder2 + for OPERATION in "merge -m merge" cherry-pick rebase + do + ensure_not_expanded merge -m merge update-folder1 && + ensure_not_expanded merge -m merge update-folder2 || return 1 + done ) '