From 15f3e1e05683ce710f22d9b1ef0f6b849f1d4b36 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 4 May 2021 02:12:07 +0000 Subject: [PATCH 001/397] t6423: rename file within directory that other side renamed Add a new testcase where one side of history renames: olddir/ -> newdir/ and the other side of history renames: olddir/a -> olddir/alpha When using merge.directoryRenames=true, it seems logical to expect the file to end up at newdir/alpha. Unfortunately, both merge-recursive and merge-ort currently see this as a rename/rename conflict: olddir/a -> newdir/a vs. olddir/a -> newdir/alpha Suggesting that there's some extra logic we probably want to add somewhere to allow this case to run without triggering a conflict. For now simply document this known issue. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- t/t6423-merge-rename-directories.sh | 58 +++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh index 4c3d0b95dc..3037c5c9bf 100755 --- a/t/t6423-merge-rename-directories.sh +++ b/t/t6423-merge-rename-directories.sh @@ -4966,6 +4966,64 @@ test_expect_success '12g: Testcase with two kinds of "relevant" renames' ' ) ' +# Testcase 12h, Testcase with two kinds of "relevant" renames +# Commit O: olddir/{a_1, b} +# Commit A: newdir/{a_2, b} +# Commit B: olddir/{alpha_1, b} +# Expected: newdir/{alpha_2, b} + +test_setup_12h () { + test_create_repo 12h && + ( + cd 12h && + + mkdir olddir && + test_seq 3 8 >olddir/a && + >olddir/b && + git add olddir && + git commit -m orig && + + git branch O && + git branch A && + git branch B && + + git switch A && + test_seq 3 10 >olddir/a && + git add olddir/a && + git mv olddir newdir && + git commit -m A && + + git switch B && + + git mv olddir/a olddir/alpha && + git commit -m B + ) +} + +test_expect_failure '12h: renaming a file within a renamed directory' ' + test_setup_12h && + ( + cd 12h && + + git checkout A^0 && + + test_might_fail git -c merge.directoryRenames=true merge -s recursive B^0 && + + git ls-files >tracked && + test_line_count = 2 tracked && + + test_path_is_missing olddir/a && + test_path_is_file newdir/alpha && + test_path_is_file newdir/b && + + git rev-parse >actual \ + HEAD:newdir/alpha HEAD:newdir/b && + git rev-parse >expect \ + A:newdir/a O:oldir/b && + test_cmp expect actual + ) +' + ########################################################################### # SECTION 13: Checking informational and conflict messages # From 4070c9e09fc4295e5b2f8a7cbb5bae1936fdeef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 5 May 2021 14:21:38 +0200 Subject: [PATCH 002/397] Makefile: don't re-define PERL_DEFINES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 07d90eadb50 (Makefile: add Perl runtime prefix support, 2018-04-10) we have been declaring PERL_DEFINES right after assigning to it, with the effect that the first PERL_DEFINES was ignored. That bug didn't matter in practice since the first line had all the same variables as the second, so we'd correctly re-generate everything. It just made for confusing reading. Let's remove that first assignment, and while we're at it split these across lines to make them more maintainable. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 93664d6714..1d4c02e59d 100644 --- a/Makefile +++ b/Makefile @@ -2270,9 +2270,10 @@ perl_localedir_SQ = $(localedir_SQ) ifndef NO_PERL PERL_HEADER_TEMPLATE = perl/header_templates/fixed_prefix.template.pl -PERL_DEFINES = $(PERL_PATH_SQ):$(PERLLIB_EXTRA_SQ):$(perllibdir_SQ) - -PERL_DEFINES := $(PERL_PATH_SQ) $(PERLLIB_EXTRA_SQ) $(perllibdir_SQ) +PERL_DEFINES := +PERL_DEFINES += $(PERL_PATH_SQ) +PERL_DEFINES += $(PERLLIB_EXTRA_SQ) +PERL_DEFINES += $(perllibdir_SQ) PERL_DEFINES += $(RUNTIME_PREFIX) # Support Perl runtime prefix. In this mode, a different header is installed From 3d49f7220a41147b17c1d826b9c87452b5f68c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 5 May 2021 14:21:39 +0200 Subject: [PATCH 003/397] Makefile: regenerate perl/build/* if GIT-PERL-DEFINES changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the logic to generate perl/build/* to regenerate those files if GIT-PERL-DEFINES changes. This ensures that e.g. changing localedir will result in correctly re-generated files. I don't think that ever worked. The brokenness pre-dates my 20d2a30f8ff (Makefile: replace perl/Makefile.PL with simple make rules, 2017-12-10). Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1d4c02e59d..a15f39e40f 100644 --- a/Makefile +++ b/Makefile @@ -2676,7 +2676,7 @@ endif NO_PERL_CPAN_FALLBACKS_SQ = $(subst ','\'',$(NO_PERL_CPAN_FALLBACKS)) endif -perl/build/lib/%.pm: perl/%.pm +perl/build/lib/%.pm: perl/%.pm GIT-PERL-DEFINES $(QUIET_GEN)mkdir -p $(dir $@) && \ sed -e 's|@@LOCALEDIR@@|$(perl_localedir_SQ)|g' \ -e 's|@@NO_PERL_CPAN_FALLBACKS@@|$(NO_PERL_CPAN_FALLBACKS_SQ)|g' \ From 368a50d9eea16e8e1181ce12d5c3c1d50216b688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 5 May 2021 14:21:40 +0200 Subject: [PATCH 004/397] Makefile: regenerate *.pm on NO_PERL_CPAN_FALLBACKS change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerate the *.pm files in perl/build/* if the NO_PERL_CPAN_FALLBACKS flag added to the *.pm files in 1aca69c0195 (perl Git::LoadCPAN: emit better errors under NO_PERL_CPAN_FALLBACKS, 2018-03-03) is changed. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a15f39e40f..574b25512e 100644 --- a/Makefile +++ b/Makefile @@ -2275,6 +2275,7 @@ PERL_DEFINES += $(PERL_PATH_SQ) PERL_DEFINES += $(PERLLIB_EXTRA_SQ) PERL_DEFINES += $(perllibdir_SQ) PERL_DEFINES += $(RUNTIME_PREFIX) +PERL_DEFINES += $(NO_PERL_CPAN_FALLBACKS) # Support Perl runtime prefix. In this mode, a different header is installed # into Perl scripts. From 256c2dc42c812e4868532b88155f60f0a269c821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 5 May 2021 14:21:41 +0200 Subject: [PATCH 005/397] perl: use mock i18n functions under NO_GETTEXT=Y MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the logic of the i18n functions I added in 5e9637c6297 (i18n: add infrastructure for translating Git with gettext, 2011-11-18) to use pass-through functions when NO_GETTEXT is defined. This speeds up the compilation time of commands that use this library when NO_GETTEXT=Y is in effect. Loading it and POSIX.pm is around 20ms on my machine, whereas it takes 2ms to just instantiate perl itself. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- Makefile | 3 +++ perl/Git/I18N.pm | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/Makefile b/Makefile index 574b25512e..b8d6b31305 100644 --- a/Makefile +++ b/Makefile @@ -1986,6 +1986,7 @@ ETC_GITCONFIG_SQ = $(subst ','\'',$(ETC_GITCONFIG)) ETC_GITATTRIBUTES_SQ = $(subst ','\'',$(ETC_GITATTRIBUTES)) DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) +NO_GETTEXT_SQ = $(subst ','\'',$(NO_GETTEXT)) bindir_SQ = $(subst ','\'',$(bindir)) bindir_relative_SQ = $(subst ','\'',$(bindir_relative)) mandir_SQ = $(subst ','\'',$(mandir)) @@ -2276,6 +2277,7 @@ PERL_DEFINES += $(PERLLIB_EXTRA_SQ) PERL_DEFINES += $(perllibdir_SQ) PERL_DEFINES += $(RUNTIME_PREFIX) PERL_DEFINES += $(NO_PERL_CPAN_FALLBACKS) +PERL_DEFINES += $(NO_GETTEXT) # Support Perl runtime prefix. In this mode, a different header is installed # into Perl scripts. @@ -2680,6 +2682,7 @@ endif perl/build/lib/%.pm: perl/%.pm GIT-PERL-DEFINES $(QUIET_GEN)mkdir -p $(dir $@) && \ sed -e 's|@@LOCALEDIR@@|$(perl_localedir_SQ)|g' \ + -e 's|@@NO_GETTEXT@@|$(NO_GETTEXT_SQ)|g' \ -e 's|@@NO_PERL_CPAN_FALLBACKS@@|$(NO_PERL_CPAN_FALLBACKS_SQ)|g' \ < $< > $@ diff --git a/perl/Git/I18N.pm b/perl/Git/I18N.pm index 2037f387c8..895e759c57 100644 --- a/perl/Git/I18N.pm +++ b/perl/Git/I18N.pm @@ -16,9 +16,19 @@ BEGIN { our @EXPORT = qw(__ __n N__); our @EXPORT_OK = @EXPORT; +# See Git::LoadCPAN's NO_PERL_CPAN_FALLBACKS_STR for a description of +# this "'@@' [...] '@@'" pattern. +use constant NO_GETTEXT_STR => '@@' . 'NO_GETTEXT' . '@@'; +use constant NO_GETTEXT => ( + q[@@NO_GETTEXT@@] ne '' + and + q[@@NO_GETTEXT@@] ne NO_GETTEXT_STR +); + sub __bootstrap_locale_messages { our $TEXTDOMAIN = 'git'; our $TEXTDOMAINDIR ||= $ENV{GIT_TEXTDOMAINDIR} || '@@LOCALEDIR@@'; + die "NO_GETTEXT=" . NO_GETTEXT_STR if NO_GETTEXT; require POSIX; POSIX->import(qw(setlocale)); From edc23840b0baad017d02037f0a833eaa600ee21d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:00 +0200 Subject: [PATCH 006/397] test-lib: bring $remove_trash out of retirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's no point in creating a repository or directory only to decide right afterwards that we're skipping all the tests. We can save ourselves the redundant "git init" or "mkdir" and "rm -rf" in this case. We carry around the "$remove_trash" variable because if the directory is unexpectedly gone at test_done time we'll still want to hit the "trash directory already removed" error, but not if we never created the trash directory. See df4c0d1a792 (test-lib: abort when can't remove trash directory, 2017-04-20) for the addition of that error. So let's partially revert 06478dab4c (test-lib: retire $remove_trash variable, 2017-04-23) and move the decision about whether to skip all tests earlier. Let's also fix a bug that was with us since abc5d372ec (Enable parallel tests, 2008-08-08): we would leak $remove_trash from the environment. We don't want this to error out, so let's reset it to the empty string first: remove_trash=t GIT_SKIP_TESTS=t0001 ./t0001-init.sh I tested this with --debug, see 4d0912a206 (test-lib.sh: do not barf under --debug at the end of the test, 2017-04-24) for a bug we don't want to re-introduce. While I'm at it, let's move the HOME assignment to just before test_create_repo, it could be lower, but it seems better to set it before calling anything in test-lib-functions.sh Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/test-lib.sh | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index d3f6af6a65..b81d57bc0d 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1167,7 +1167,7 @@ test_done () { esac fi - if test -z "$debug" + if test -z "$debug" && test -n "$remove_trash" then test -d "$TRASH_DIRECTORY" || error "Tests passed but trash directory already removed before test cleanup; aborting" @@ -1332,6 +1332,22 @@ then exit 1 fi +# Are we running this test at all? +remove_trash= +this_test=${0##*/} +this_test=${this_test%%-*} +if match_pattern_list "$this_test" $GIT_SKIP_TESTS +then + say_color info >&3 "skipping test $this_test altogether" + skip_all="skip all tests in $this_test" + test_done +fi + +# Last-minute variable setup +HOME="$TRASH_DIRECTORY" +GNUPGHOME="$HOME/gnupg-home-not-used" +export HOME GNUPGHOME + # Test repository rm -fr "$TRASH_DIRECTORY" || { GIT_EXIT_OK=t @@ -1339,10 +1355,7 @@ rm -fr "$TRASH_DIRECTORY" || { exit 1 } -HOME="$TRASH_DIRECTORY" -GNUPGHOME="$HOME/gnupg-home-not-used" -export HOME GNUPGHOME - +remove_trash=t if test -z "$TEST_NO_CREATE_REPO" then test_create_repo "$TRASH_DIRECTORY" @@ -1354,15 +1367,6 @@ fi # in subprocesses like git equals our $PWD (for pathname comparisons). cd -P "$TRASH_DIRECTORY" || exit 1 -this_test=${0##*/} -this_test=${this_test%%-*} -if match_pattern_list "$this_test" $GIT_SKIP_TESTS -then - say_color info >&3 "skipping test $this_test altogether" - skip_all="skip all tests in $this_test" - test_done -fi - if test -n "$write_junit_xml" then junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out" From b57913f205ce73f8bc2e8bab3719a50f2f3a7199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:01 +0200 Subject: [PATCH 007/397] test-lib tests: remove dead GIT_TEST_FRAMEWORK_SELFTEST variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop setting the GIT_TEST_FRAMEWORK_SELFTEST variable. This was originally needed back in 4231d1ba99 (t0000: do not get self-test disrupted by environment warnings, 2018-09-20). It hasn't been needed since I deleted the relevant code in test-lib.sh in c0eedbc009 (test-lib: remove check_var_migration, 2021-02-09), I just didn't notice that it was set here. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t0000-basic.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh index 705d62cc27..2c6e34b947 100755 --- a/t/t0000-basic.sh +++ b/t/t0000-basic.sh @@ -84,10 +84,6 @@ _run_sub_test_lib_test_common () { passing metrics ' - # Tell the framework that we are self-testing to make sure - # it yields a stable result. - GIT_TEST_FRAMEWORK_SELFTEST=t && - # Point to the t/test-lib.sh, which isn't in ../ as usual . "\$TEST_DIRECTORY"/test-lib.sh EOF From cb8fb7f861a8178bd78a1e5b1f2cbe5a20de6eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:02 +0200 Subject: [PATCH 008/397] test-lib-functions: reword "test_commit --append" docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reword the documentation for "test_commit --append" added in my 3373518cc8 (test-lib functions: add an --append option to test_commit, 2021-01-12). A follow-up commit will make the "echo" part of this configurable, and in any case saying "echo >>" rather than ">>" was redundant. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/test-lib-functions.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 6348e8d733..d169fb2f59 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -172,8 +172,7 @@ debug () { # --notick # Do not call test_tick before making a commit # --append -# Use "echo >>" instead of "echo >" when writing "" to -# "" +# Use ">>" instead of ">" when writing "" to "" # --signoff # Invoke "git commit" with --signoff # --author From 5144219b7d7aeac9290cbee5a7425763bd253667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:03 +0200 Subject: [PATCH 009/397] test-lib-functions: document test_commit --no-tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 76b8b8d05c (test-lib functions: document arguments to test_commit, 2021-01-12) I added missing documentation to test_commit, but in less than a month later in 3803a3a099 (t: add --no-tag option to test_commit, 2021-02-09) we got another undocumented option. Let's fix that. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/test-lib-functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index d169fb2f59..d0f4f3885d 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -177,6 +177,8 @@ debug () { # Invoke "git commit" with --signoff # --author # Invoke "git commit" with --author +# --no-tag +# Do not tag the resulting commit # # This will commit a file with the given contents and the given commit # message, and tag the resulting commit with the given tag name. From 6cf8d96fa28668a62f05be5b276739c03c514581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:04 +0200 Subject: [PATCH 010/397] test-lib functions: add an --annotated option to "test_commit" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an --annotated option to test_commit to create annotated tags. The tag will share the same message as the commit, and we'll call test_tick before creating it (unless --notick) is provided. There's quite a few tests that could be simplified with this construct. I've picked one to convert in this change as a demonstration. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1403-show-ref.sh | 6 ++---- t/test-lib-functions.sh | 27 ++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/t/t1403-show-ref.sh b/t/t1403-show-ref.sh index 6ce62f878c..17d3cc1405 100755 --- a/t/t1403-show-ref.sh +++ b/t/t1403-show-ref.sh @@ -7,11 +7,9 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh test_expect_success setup ' - test_commit A && - git tag -f -a -m "annotated A" A && + test_commit --annotate A && git checkout -b side && - test_commit B && - git tag -f -a -m "annotated B" B && + test_commit --annotate B && git checkout main && test_commit C && git branch B A^0 diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index d0f4f3885d..6e2332a324 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -179,6 +179,10 @@ debug () { # Invoke "git commit" with --author # --no-tag # Do not tag the resulting commit +# --annotate +# Create an annotated tag with "--annotate -m ". Calls +# test_tick between making the commit and tag, unless --notick +# is given. # # This will commit a file with the given contents and the given commit # message, and tag the resulting commit with the given tag name. @@ -191,7 +195,7 @@ test_commit () { author= && signoff= && indir= && - no_tag= && + tag=light && while test $# != 0 do case "$1" in @@ -219,7 +223,10 @@ test_commit () { shift ;; --no-tag) - no_tag=yes + tag=none + ;; + --annotate) + tag=annotate ;; *) break @@ -243,10 +250,20 @@ test_commit () { git ${indir:+ -C "$indir"} commit \ ${author:+ --author "$author"} \ $signoff -m "$1" && - if test -z "$no_tag" - then + case "$tag" in + none) + ;; + light) git ${indir:+ -C "$indir"} tag "${4:-$1}" - fi + ;; + annotate) + if test -z "$notick" + then + test_tick + fi && + git ${indir:+ -C "$indir"} tag -a -m "$1" "${4:-$1}" + ;; + esac } # Call test_merge with the arguments " ", where From 8cfe386b78c15eff38388479aa2f9fae00a9cf53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:05 +0200 Subject: [PATCH 011/397] describe tests: convert setup to use test_commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the setup of the describe tests to use test_commit when possible. This makes use of the new --annotate option to test_commit. Some of the setup here could simply be removed since the data being created wasn't important to any of the subsequent tests, so I've done so. E.g. assigning to the "one" variable was always useless, and just checking that we can describe HEAD after the first commit wasn't useful. In the case of the "two" variable we could instead use the tag we just created. See 5312ab11fbf (Add describe test., 2007-01-13) for the initial version of this code. There's other cases here like redundant "test_tick" invocations, or the simplification of not echoing "X" to a file we're about to tag as "x", now we just use "x" in both cases. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t6120-describe.sh | 58 ++++++++++----------------------------------- 1 file changed, 13 insertions(+), 45 deletions(-) diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index e89b6747be..88fddc9142 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -31,64 +31,32 @@ check_describe () { } test_expect_success setup ' + test_commit initial file one && + test_commit second file two && + test_commit third file three && + test_commit --annotate A file A && + test_commit c file c && - test_tick && - echo one >file && git add file && git commit -m initial && - one=$(git rev-parse HEAD) && - - git describe --always HEAD && - - test_tick && - echo two >file && git add file && git commit -m second && - two=$(git rev-parse HEAD) && - - test_tick && - echo three >file && git add file && git commit -m third && - - test_tick && - echo A >file && git add file && git commit -m A && - test_tick && - git tag -a -m A A && - - test_tick && - echo c >file && git add file && git commit -m c && - test_tick && - git tag c && - - git reset --hard $two && - test_tick && - echo B >side && git add side && git commit -m B && - test_tick && - git tag -a -m B B && + git reset --hard second && + test_commit --annotate B side B && test_tick && git merge -m Merged c && merged=$(git rev-parse HEAD) && - git reset --hard $two && - test_tick && - echo D >another && git add another && git commit -m D && - test_tick && - git tag -a -m D D && + git reset --hard second && + test_commit --no-tag D another D && + test_tick && git tag -a -m R R && - test_tick && - echo DD >another && git commit -a -m another && - - test_tick && - git tag e && - - test_tick && - echo DDD >another && git commit -a -m "yet another" && + test_commit e another DD && + test_commit --no-tag "yet another" another DDD && test_tick && git merge -m Merged $merged && - test_tick && - echo X >file && echo X >side && git add file side && - git commit -m x - + test_commit --no-tag x file ' check_describe A-* HEAD From 47c88d16ba6c5c0237238ac600ee8b74a522e41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:06 +0200 Subject: [PATCH 012/397] test-lib functions: add --printf option to test_commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a --printf option to test_commit to allow writing to the file with "printf" instead of "echo". This is useful for writing "\n", "\0" etc., in particular in combination with the --append option added in 3373518cc8 (test-lib functions: add an --append option to test_commit, 2021-01-12). I'm converting a few tests to use the new option rather than a manual printf/add/commit combination to demonstrate its usefulness. While I'm at it use "test_create_repo" where appropriate, and give the first/second commit a meaningful/more conventional log message in cases where no test cared about that message. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1307-config-blob.sh | 4 +--- t/t2030-unresolve-info.sh | 3 +-- t/t4006-diff-mode.sh | 6 ++---- t/t4030-diff-textconv.sh | 8 ++------ t/t5520-pull.sh | 10 ++-------- t/test-lib-functions.sh | 14 ++++++++++++-- 6 files changed, 20 insertions(+), 25 deletions(-) diff --git a/t/t1307-config-blob.sh b/t/t1307-config-blob.sh index 002e6d3388..930dce06f0 100755 --- a/t/t1307-config-blob.sh +++ b/t/t1307-config-blob.sh @@ -65,9 +65,7 @@ test_expect_success 'parse errors in blobs are properly attributed' ' ' test_expect_success 'can parse blob ending with CR' ' - printf "[some]key = value\\r" >config && - git add config && - git commit -m CR && + test_commit --printf CR config "[some]key = value\\r" && echo value >expect && git config --blob=HEAD:config some.key >actual && test_cmp expect actual diff --git a/t/t2030-unresolve-info.sh b/t/t2030-unresolve-info.sh index be6c84c52a..f691e6d903 100755 --- a/t/t2030-unresolve-info.sh +++ b/t/t2030-unresolve-info.sh @@ -179,8 +179,7 @@ test_expect_success 'rerere and rerere forget (subdirectory)' ' test_expect_success 'rerere forget (binary)' ' git checkout -f side && - printf "a\0c" >binary && - git commit -a -m binary && + test_commit --printf binary binary "a\0c" && test_must_fail git merge second && git rerere forget binary ' diff --git a/t/t4006-diff-mode.sh b/t/t4006-diff-mode.sh index 275ce5fa15..6cdee2a216 100755 --- a/t/t4006-diff-mode.sh +++ b/t/t4006-diff-mode.sh @@ -26,10 +26,8 @@ test_expect_success 'chmod' ' ' test_expect_success 'prepare binary file' ' - git commit -m rezrov && - printf "\00\01\02\03\04\05\06" >binbin && - git add binbin && - git commit -m binbin + git commit -m one && + test_commit --printf two binbin "\00\01\02\03\04\05\06" ' test_expect_success '--stat output after text chmod' ' diff --git a/t/t4030-diff-textconv.sh b/t/t4030-diff-textconv.sh index c906320b60..a39a626664 100755 --- a/t/t4030-diff-textconv.sh +++ b/t/t4030-diff-textconv.sh @@ -26,12 +26,8 @@ EOF chmod +x hexdump test_expect_success 'setup binary file with history' ' - printf "\\0\\n" >file && - git add file && - git commit -m one && - printf "\\01\\n" >>file && - git add file && - git commit -m two + test_commit --printf one file "\\0\\n" && + test_commit --printf --append two file "\\01\\n" ' test_expect_success 'file is considered binary by porcelain' ' diff --git a/t/t5520-pull.sh b/t/t5520-pull.sh index a09411327f..e2c0c51022 100755 --- a/t/t5520-pull.sh +++ b/t/t5520-pull.sh @@ -746,14 +746,8 @@ test_expect_success 'pull --rebase fails on corrupt HEAD' ' ' test_expect_success 'setup for detecting upstreamed changes' ' - mkdir src && - ( - cd src && - git init && - printf "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" > stuff && - git add stuff && - git commit -m "Initial revision" - ) && + test_create_repo src && + test_commit -C src --printf one stuff "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" && git clone src dst && ( cd src && diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 6e2332a324..6f9199a65b 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -173,6 +173,12 @@ debug () { # Do not call test_tick before making a commit # --append # Use ">>" instead of ">" when writing "" to "" +# --printf +# Use "printf" instead of "echo" when writing "" to +# "", use this to write escape sequences such as "\0", a +# trailing "\n" won't be added automatically. This option +# supports nothing but the FORMAT of printf(1), i.e. no custom +# ARGUMENT(s). # --signoff # Invoke "git commit" with --signoff # --author @@ -191,6 +197,7 @@ debug () { test_commit () { notick= && + echo=echo && append= && author= && signoff= && @@ -202,6 +209,9 @@ test_commit () { --notick) notick=yes ;; + --printf) + echo=printf + ;; --append) append=yes ;; @@ -238,9 +248,9 @@ test_commit () { file=${2:-"$1.t"} && if test -n "$append" then - echo "${3-$1}" >>"$indir$file" + $echo "${3-$1}" >>"$indir$file" else - echo "${3-$1}" >"$indir$file" + $echo "${3-$1}" >"$indir$file" fi && git ${indir:+ -C "$indir"} add "$file" && if test -z "$notick" From ba7d318504a4f69e2b3ce664d6a3f2c413f5bf61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:07 +0200 Subject: [PATCH 013/397] submodule tests: use symbolic-ref --short to discover branch name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change a use of $GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME added in 704fed9ea22 (tests: start moving to a different default main branch name, 2020-10-23) to simply discover the initial branch name of a repository set up in this function with "symbolic-ref --short". That's something done in another test in 704fed9ea22, so doing it like this seems like an omission, or rather an overly eager search/replacement instead of fixing the test logic. There are only three uses of the GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME variable in the test suite, this gets rid of one of those. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/lib-submodule-update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/lib-submodule-update.sh b/t/lib-submodule-update.sh index 4b714e9308..f7c7df0ca4 100644 --- a/t/lib-submodule-update.sh +++ b/t/lib-submodule-update.sh @@ -63,6 +63,7 @@ create_lib_submodule_repo () { git init submodule_update_repo && ( cd submodule_update_repo && + branch=$(git symbolic-ref --short HEAD) && echo "expect" >>.gitignore && echo "actual" >>.gitignore && echo "x" >file1 && @@ -144,7 +145,7 @@ create_lib_submodule_repo () { git checkout -b valid_sub1 && git revert HEAD && - git checkout "${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME-master}" + git checkout "$branch" ) } From 04d12d6590d316778f1b1f180a97fa94f9990928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:08 +0200 Subject: [PATCH 014/397] test-lib: reformat argument list in test_create_repo() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reformat an argument list changed in 675704c74dd (init: provide useful advice about init.defaultBranch, 2020-12-11) to have the "-c" on the same line as the argument it sets. This whitespace-only change makes it easier to review a subsequent commit. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/test-lib-functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 6f9199a65b..bcb187b632 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1259,8 +1259,8 @@ test_create_repo () { mkdir -p "$repo" ( cd "$repo" || error "Cannot setup test environment" - "${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" -c \ - init.defaultBranch="${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME-master}" \ + "${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" \ + -c init.defaultBranch="${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME-master}" \ init \ "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 || error "cannot run git init -- have you built things yet?" From 97c8aac9c5fd54b5b6092d7ae5d3289ce5d9afe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:09 +0200 Subject: [PATCH 015/397] test-lib: do not show advice about init.defaultBranch under --verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arrange for the advice about naming the initial branch not to be shown in the --verbose output of the test suite. Since 675704c74dd (init: provide useful advice about init.defaultBranch, 2020-12-11) some tests have been very chatty with repeated occurrences of this multi-line advice. Having it be this verbose isn't helpful for anyone in the context of git's own test suite, and it makes debugging tests that use their own "git init" invocations needlessly distracting. By setting the GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME variable early in test-lib.sh itself we'll squash the warning not only for test_create_repo(), as 675704c74dd explicitly intended, but also for other "git init" invocations. And once we'd like to have this configuration set for all "git init" invocations in the test suite we can get rid of the init.defaultBranch configuration setting in test_create_repo(), as repo_default_branch_name() in refs.c will take the GIT_TEST_* variable over it being set. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/test-lib-functions.sh | 1 - t/test-lib.sh | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index bcb187b632..ed7299db3d 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1260,7 +1260,6 @@ test_create_repo () { ( cd "$repo" || error "Cannot setup test environment" "${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" \ - -c init.defaultBranch="${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME-master}" \ init \ "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 || error "cannot run git init -- have you built things yet?" diff --git a/t/test-lib.sh b/t/test-lib.sh index b81d57bc0d..bc696e29b9 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -64,6 +64,11 @@ then export GIT_TEST_DISALLOW_ABBREVIATED_OPTIONS fi +# Explicitly set the default branch name for testing, to avoid the +# transitory "git init" warning under --verbose. +: ${GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME:=master} +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + ################################################################ # It appears that people try to run tests without building... "${GIT_TEST_INSTALLED:-$GIT_BUILD_DIR}/git$X" >/dev/null From f0d4d398e281009bc5e34d830b37c0c1df2fb8a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 10 May 2021 16:19:10 +0200 Subject: [PATCH 016/397] test-lib: split up and deprecate test_create_repo() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove various redundant or obsolete code from the test_create_repo() function, and split up its use in test-lib.sh from what tests need from it. This leave us with a pass-through wrapper for "git init" in test-lib-functions.sh, in test-lib.sh we have the same, except for needing to redirect stdout/stderr, and emitting an error ourselves if it fails. We don't need to error() ourselves when test_create_repo() is invoked, as the invocation will be a part of a test's "&&"-chain. Everything below this paragraph is a detailed summary of the history of test_create_repo() explaining why it's safe to remove the various things it was doing: 1. "mkdir -p" isn't needed because "git init" itself will create leading directories if needed. 2. Since we're now a simple wrapper for "git init" we don't need to check that we have only one argument. If someone wants to run "test_create_repo --bare x" that's OK. 3. We won't ever hit that "Cannot setup test environment" error. Checking the test environment sanity when doing "git init" dates back to eea420693be (t0000: catch trivial pilot errors., 2005-12-10) and 2ccd2027b01 (trivial: check, if t/trash directory was successfully created, 2006-01-05). We can also see it in another form a bit later in my own 0d314ce834d (test-lib: use subshell instead of cd $new && .. && cd $old, 2010-08-30). But since 2006f0adaee (t/test-lib: make sure Git has already been built, 2012-09-17) we already check if we have a built git earlier. The one thing this was testing after that 2012 change was that we'd just built "git", but not "git-init", but since 3af4c7156c4 (tests: respect GIT_TEST_INSTALLED when initializing repositories, 2018-11-12) we invoke "git", not "git-init". So all of that's been checked already, and we don't need to re-check it here. 4. We don't need to move .git/hooks out of the way. That dates back to c09a69a83e3 (Disable hooks during tests., 2005-10-16), since then hooks became disabled by default in f98f8cbac01 (Ship sample hooks with .sample suffix, 2008-06-24). So the hooks were already disabled by default, but as can be seen from "mkdir .git/hooks" changes various tests needed to re-setup that directory. Now they no longer do. This makes us implicitly depend on the default hooks being disabled, which is a good thing. If and when we'd have any on-by-default hooks (I see no reason we ever would) we'd want to see the subtle and not so subtle ways that would break the test suite. 5. We don't need to "cd" to the "$repo" directory at all anymore. In the code being removed here we both "cd"'d to the repository before calling "init", and did so in a subshell. It's not important to do either, so both of those can be removed. We cd'd because this code grew from test-lib.sh code where we'd have done so already, see eedf8f97e58 (Abstract test_create_repo out for use in tests., 2006-02-17), and later "cd"'d inside a subshell since 0d314ce834d to avoid having to keep track of an "old pwd" variable to cd back after the setup. Being in the repository directory made moving the hooks around easier (we wouldn't have to fully qualify the path). Since we're not moving the hooks per #4 above we don't need to "cd" for that reason either. 6. We can drop the --template argument and instead rely on the GIT_TEMPLATE_DIR set to the same path earlier in test-lib.sh. See 8683a45d669 (Introduce GIT_TEMPLATE_DIR, 2006-12-19) 7. We only needed that ">&3 2>&4" redirection when invoked from test-lib.sh. We could still invoke test_create_repo() there, but as the invocation is now trivial and we don't have a good reason to use test_create_repo() elsewhere let's call "git init" there ourselves. 8. We didn't need to resolve "git" as "${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" in test_create_repo(), even for the use of test-lib.sh PATH is already set up in test-lib.sh to start with GIT_TEST_INSTALLED and/or GIT_EXEC_PATH before test_create_repo() (now "git init") is called.. So we can simply run "git" and rely on the PATH lookup choosing the right executable. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t5406-remote-rejects.sh | 1 - t/t5407-post-rewrite-hook.sh | 2 -- t/t5409-colorize-remote-messages.sh | 1 - t/test-lib-functions.sh | 15 ++------------- t/test-lib.sh | 3 ++- 5 files changed, 4 insertions(+), 18 deletions(-) diff --git a/t/t5406-remote-rejects.sh b/t/t5406-remote-rejects.sh index ff06f99649..5c509db6fc 100755 --- a/t/t5406-remote-rejects.sh +++ b/t/t5406-remote-rejects.sh @@ -5,7 +5,6 @@ test_description='remote push rejects are reported by client' . ./test-lib.sh test_expect_success 'setup' ' - mkdir .git/hooks && write_script .git/hooks/update <<-\EOF && exit 1 EOF diff --git a/t/t5407-post-rewrite-hook.sh b/t/t5407-post-rewrite-hook.sh index 5bb23cc3a4..6da8d760e2 100755 --- a/t/t5407-post-rewrite-hook.sh +++ b/t/t5407-post-rewrite-hook.sh @@ -20,8 +20,6 @@ test_expect_success 'setup' ' git checkout main ' -mkdir .git/hooks - cat >.git/hooks/post-rewrite < "$TRASH_DIRECTORY"/post-rewrite.args diff --git a/t/t5409-colorize-remote-messages.sh b/t/t5409-colorize-remote-messages.sh index 5d8f401d8e..9f1a483f42 100755 --- a/t/t5409-colorize-remote-messages.sh +++ b/t/t5409-colorize-remote-messages.sh @@ -5,7 +5,6 @@ test_description='remote messages are colorized on the client' . ./test-lib.sh test_expect_success 'setup' ' - mkdir .git/hooks && write_script .git/hooks/update <<-\EOF && echo error: error echo ERROR: also highlighted diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index ed7299db3d..93a3fca16d 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1250,21 +1250,10 @@ test_atexit () { } && (exit \"\$eval_ret\"); eval_ret=\$?; $test_atexit_cleanup" } -# Most tests can use the created repository, but some may need to create more. +# Deprecated wrapper for "git init", use "git init" directly instead # Usage: test_create_repo test_create_repo () { - test "$#" = 1 || - BUG "not 1 parameter to test-create-repo" - repo="$1" - mkdir -p "$repo" - ( - cd "$repo" || error "Cannot setup test environment" - "${GIT_TEST_INSTALLED:-$GIT_EXEC_PATH}/git$X" \ - init \ - "--template=$GIT_BUILD_DIR/templates/blt/" >&3 2>&4 || - error "cannot run git init -- have you built things yet?" - mv .git/hooks .git/hooks-disabled - ) || exit + git init "$@" } # This function helps on symlink challenged file systems when it is not diff --git a/t/test-lib.sh b/t/test-lib.sh index bc696e29b9..e986c5839e 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1363,7 +1363,8 @@ rm -fr "$TRASH_DIRECTORY" || { remove_trash=t if test -z "$TEST_NO_CREATE_REPO" then - test_create_repo "$TRASH_DIRECTORY" + git init "$TRASH_DIRECTORY" >&3 2>&4 || + error "cannot run git init" else mkdir -p "$TRASH_DIRECTORY" fi From ecbff141a1c6e4a32a43f90db70cde2b2199b241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:08 +0200 Subject: [PATCH 017/397] grep/pcre2 tests: reword comments referring to kwset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kwset optimization has not been used by grep since 48de2a768cf (grep: remove the kwset optimization, 2019-07-01). Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t7816-grep-binary-pattern.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/t7816-grep-binary-pattern.sh b/t/t7816-grep-binary-pattern.sh index 60bab291e4..9d67a5fc4c 100755 --- a/t/t7816-grep-binary-pattern.sh +++ b/t/t7816-grep-binary-pattern.sh @@ -59,7 +59,7 @@ test_expect_success 'setup' " git commit -m. " -# Simple fixed-string matching that can use kwset (no -i && non-ASCII) +# Simple fixed-string matching nul_match P P P '-F' 'yQf' nul_match P P P '-F' 'yQx' nul_match P P P '-Fi' 'YQf' @@ -78,7 +78,7 @@ nul_match P P P '-Fi' '[Y]QF' nul_match P P P '-F' 'æQ[ð]' nul_match P P P '-F' '[æ]Qð' -# The -F kwset codepath can't handle -i && non-ASCII... +# Matching pattern and subject case with -i nul_match P 1 1 '-i' '[æ]Qð' # ...PCRE v2 only matches non-ASCII with -i casefolding under UTF-8 From 6d0a40166e87a78ca17fddeee25228ded92a169e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:09 +0200 Subject: [PATCH 018/397] pickaxe tests: refactor to use test_commit --append --printf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor the existing tests added in e0e7cb8080c (log -G: ignore binary files, 2018-12-14) to use the --append option I added in 3373518cc8b (test-lib functions: add an --append option to test_commit, 2021-01-12) and the --printf option added as part of an in-flight topic of mine this commit depends on. While I'm at it change some of the setup of the test to use a more sensible pattern, e.g. setting up a temporary repo instead of creating an orphan branch. Since the -G and -S options will behave the same way with truncated and removed content also change the "git rm" to emptying data.bin, that's just catering to how test_commit works. The resulting test is shorter. See also f5d79bf7dd6 (tests: refactor a few tests to use "test_commit --append", 2021-01-12) for prior similar refactoring. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t4209-log-pickaxe.sh | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 5d06f5f45e..ad45d8cfd0 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -107,37 +107,35 @@ test_expect_success 'log -S --no-textconv (missing textconv tool)' ' ' test_expect_success 'setup log -[GS] binary & --text' ' - git checkout --orphan GS-binary-and-text && - git read-tree --empty && - printf "a\na\0a\n" >data.bin && - git add data.bin && - git commit -m "create binary file" data.bin && - printf "a\na\0a\n" >>data.bin && - git commit -m "modify binary file" data.bin && - git rm data.bin && - git commit -m "delete binary file" data.bin && - git log >full-log + test_create_repo GS-bin-txt && + test_commit -C GS-bin-txt --printf A data.bin "a\na\0a\n" && + test_commit -C GS-bin-txt --append --printf B data.bin "a\na\0a\n" && + test_commit -C GS-bin-txt C data.bin "" && + git -C GS-bin-txt log >full-log ' test_expect_success 'log -G ignores binary files' ' - git log -Ga >log && + git -C GS-bin-txt log -Ga >log && test_must_be_empty log ' test_expect_success 'log -G looks into binary files with -a' ' - git log -a -Ga >log && + git -C GS-bin-txt log -a -Ga >log && test_cmp log full-log ' test_expect_success 'log -G looks into binary files with textconv filter' ' - test_when_finished "rm .gitattributes" && - echo "* diff=bin" >.gitattributes && - git -c diff.bin.textconv=cat log -Ga >log && + test_when_finished "rm GS-bin-txt/.gitattributes" && + ( + cd GS-bin-txt && + echo "* diff=bin" >.gitattributes && + git -c diff.bin.textconv=cat log -Ga >../log + ) && test_cmp log full-log ' test_expect_success 'log -S looks into binary files' ' - git log -Sa >log && + git -C GS-bin-txt log -Sa >log && test_cmp log full-log ' From c9609398580518dffaadaace17188c901a4a4522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:10 +0200 Subject: [PATCH 019/397] pickaxe tests: add test for diffgrep_consume() internals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In diffgrep_consume() we generate a diff, and then advance past the "+" or "-" at the start of the line for matching. This has been done ever since the code was added in f506b8e8b5f (git log/diff: add -G that greps in the patch text, 2010-08-23). If we match "line" instead of "line + 1" no tests fail, i.e. we've got zero coverage for whether any of our searches match the beginning of the line or not. Let's add a test for this. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t4209-log-pickaxe.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index ad45d8cfd0..eacb9f0a1b 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -106,6 +106,21 @@ test_expect_success 'log -S --no-textconv (missing textconv tool)' ' rm .gitattributes ' +test_expect_success 'setup log -[GS] plain' ' + test_create_repo GS-plain && + test_commit -C GS-plain --append A data.txt "a" && + test_commit -C GS-plain --append B data.txt "a a" && + test_commit -C GS-plain C data.txt "" && + git -C GS-plain log >full-log +' + +test_expect_success 'log -G trims diff new/old [-+]' ' + git -C GS-plain log -G"[+-]a" >log && + test_must_be_empty log && + git -C GS-plain log -G"^a" >log && + test_cmp log full-log +' + test_expect_success 'setup log -[GS] binary & --text' ' test_create_repo GS-bin-txt && test_commit -C GS-bin-txt --printf A data.bin "a\na\0a\n" && From 69ae93089cd892d70429f7f06371860d65cb1f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:11 +0200 Subject: [PATCH 020/397] pickaxe tests: add test for "log -S" not being a regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No test in our test suite checked for "log -S" being a fixed string, as opposed to "log -S --pickaxe-regex". Let's test for it. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t4209-log-pickaxe.sh | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index eacb9f0a1b..9fa770b5fb 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -106,11 +106,18 @@ test_expect_success 'log -S --no-textconv (missing textconv tool)' ' rm .gitattributes ' -test_expect_success 'setup log -[GS] plain' ' +test_expect_success 'setup log -[GS] plain & regex' ' test_create_repo GS-plain && test_commit -C GS-plain --append A data.txt "a" && test_commit -C GS-plain --append B data.txt "a a" && - test_commit -C GS-plain C data.txt "" && + test_commit -C GS-plain --append C data.txt "b" && + test_commit -C GS-plain --append D data.txt "[b]" && + test_commit -C GS-plain E data.txt "" && + + # We also include E, the deletion commit + git -C GS-plain log --grep="[ABE]" >A-to-B-then-E-log && + git -C GS-plain log --grep="[CDE]" >C-to-D-then-E-log && + git -C GS-plain log --grep="[DE]" >D-then-E-log && git -C GS-plain log >full-log ' @@ -118,7 +125,24 @@ test_expect_success 'log -G trims diff new/old [-+]' ' git -C GS-plain log -G"[+-]a" >log && test_must_be_empty log && git -C GS-plain log -G"^a" >log && - test_cmp log full-log + test_cmp log A-to-B-then-E-log +' + +test_expect_success 'log -S is not a regex, but -S --pickaxe-regex is' ' + git -C GS-plain log -S"a" >log && + test_cmp log A-to-B-then-E-log && + + git -C GS-plain log -S"[a]" >log && + test_must_be_empty log && + + git -C GS-plain log -S"[a]" --pickaxe-regex >log && + test_cmp log A-to-B-then-E-log && + + git -C GS-plain log -S"[b]" >log && + test_cmp log D-then-E-log && + + git -C GS-plain log -S"[b]" --pickaxe-regex >log && + test_cmp log C-to-D-then-E-log ' test_expect_success 'setup log -[GS] binary & --text' ' From 064952fc34b9765282fec057b3af260eae7c75c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:12 +0200 Subject: [PATCH 021/397] pickaxe tests: test for -G, -S and --find-object incompatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a test for the options sanity check added in 5e505257f2 (diff: properly error out when combining multiple pickaxe options, 2018-01-04). Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t4209-log-pickaxe.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 9fa770b5fb..21e22af1e7 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -55,6 +55,17 @@ test_expect_success setup ' git rev-parse --verify HEAD >expect_second ' +test_expect_success 'usage' ' + test_expect_code 128 git log -Gregex -Sstring 2>err && + grep "mutually exclusive" err && + + test_expect_code 128 git log -Gregex --find-object=HEAD 2>err && + grep "mutually exclusive" err && + + test_expect_code 128 git log -Sstring --find-object=HEAD 2>err && + grep "mutually exclusive" err +' + test_log expect_initial --grep initial test_log expect_nomatch --grep InItial test_log_icase expect_initial --grep InItial From 7cd5d5b299497fb874897595c642144d7d894ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:13 +0200 Subject: [PATCH 022/397] pickaxe tests: add missing test for --no-pickaxe-regex being an error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a missing test for --no-pickaxe-regex. This has been an error ever since before the -S or -G options were added, or since 7ae0b0cb65f (git-log (internal): more options., 2006-03-01). The reason for adding this test is that Junio suggested in [1] in response to a later test addition in this series that it might be good to support --no-pickaxe-regex in combination with -G. This would allow for fixed-string searching with -G, similr to grep's --fixed-strings mode. I agree that that would make sense if anyone would like to implement it, but since it dies right now let's first add this test to assert the existing long-standing behavior. We can always add support for --[no-]pickaxe-regex in combination with -G at some later date. 1. http://lore.kernel.org/git/xmqqwnto9pt7.fsf@gitster.g Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t4209-log-pickaxe.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 21e22af1e7..532bb875f0 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -66,6 +66,18 @@ test_expect_success 'usage' ' grep "mutually exclusive" err ' +test_expect_success 'usage: --no-pickaxe-regex' ' + cat >expect <<-\EOF && + fatal: unrecognized argument: --no-pickaxe-regex + EOF + + test_expect_code 128 git log -Sstring --no-pickaxe-regex 2>actual && + test_cmp expect actual && + + test_expect_code 128 git log -Gstring --no-pickaxe-regex 2>err && + test_cmp expect actual +' + test_log expect_initial --grep initial test_log expect_nomatch --grep InItial test_log_icase expect_initial --grep InItial From 188e9e28c5287f3f160b5f14e3e551b1c55c7301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:14 +0200 Subject: [PATCH 023/397] pickaxe: die when -G and --pickaxe-regex are combined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the -G and --pickaxe-regex options are combined we simply ignore the --pickaxe-regex option. Let's die instead as suggested by our documentation, since -G is always a regex. When --pickaxe-regex was added in d01d8c6782 (Support for pickaxe matching regular expressions, 2006-03-29) only the -S option existed. Then when -G was added in f506b8e8b5 (git log/diff: add -G that greps in the patch text, 2010-08-23) neither the documentation for --pickaxe-regex was updated accordingly, nor was something like this assertion added. Since 5bc3f0b567 (diffcore-pickaxe doc: document -S and -G properly, 2013-05-31) we've claimed that --pickaxe-regex should only be used with -S, but have silently tolerated combining it with -G, let's die instead. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diff.c | 3 +++ diff.h | 2 ++ t/t4209-log-pickaxe.sh | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/diff.c b/diff.c index 4acccd9d7e..f9e86bca04 100644 --- a/diff.c +++ b/diff.c @@ -4628,6 +4628,9 @@ void diff_setup_done(struct diff_options *options) if (HAS_MULTI_BITS(options->pickaxe_opts & DIFF_PICKAXE_KINDS_MASK)) die(_("-G, -S and --find-object are mutually exclusive")); + if (HAS_MULTI_BITS(options->pickaxe_opts & DIFF_PICKAXE_KINDS_G_REGEX_MASK)) + die(_("-G and --pickaxe-regex are mutually exclusive, use --pickaxe-regex with -S")); + /* * Most of the time we can say "there are changes" * only by checking if there are changed paths, but diff --git a/diff.h b/diff.h index c8f3faea8a..5e110d349b 100644 --- a/diff.h +++ b/diff.h @@ -556,6 +556,8 @@ int git_config_rename(const char *var, const char *value); #define DIFF_PICKAXE_KINDS_MASK (DIFF_PICKAXE_KIND_S | \ DIFF_PICKAXE_KIND_G | \ DIFF_PICKAXE_KIND_OBJFIND) +#define DIFF_PICKAXE_KINDS_G_REGEX_MASK (DIFF_PICKAXE_KIND_G | \ + DIFF_PICKAXE_REGEX) #define DIFF_PICKAXE_IGNORE_CASE 32 diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 532bb875f0..772c6c1a7c 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -66,6 +66,11 @@ test_expect_success 'usage' ' grep "mutually exclusive" err ' +test_expect_success 'usage: --pickaxe-regex' ' + test_expect_code 128 git log -Gregex --pickaxe-regex 2>err && + grep "mutually exclusive" err +' + test_expect_success 'usage: --no-pickaxe-regex' ' cat >expect <<-\EOF && fatal: unrecognized argument: --no-pickaxe-regex From d26ec8800969ea1b692e0c87100dc4235cfa12e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:15 +0200 Subject: [PATCH 024/397] pickaxe: die when --find-object and --pickaxe-all are combined MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither the --pickaxe-all documentation nor --find-object's has ever suggested that you can combine the two. See f506b8e8b5 (git log/diff: add -G that greps in the patch text, 2010-08-23) and 15af58c1ad (diffcore: add a pickaxe option to find a specific blob, 2018-01-04). But we've silently tolerated it, which makes the logic in diffcore_pickaxe() harder to reason about. Let's assert that we won't have the two combined. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diff.c | 3 +++ diff.h | 2 ++ t/t4209-log-pickaxe.sh | 3 +++ 3 files changed, 8 insertions(+) diff --git a/diff.c b/diff.c index f9e86bca04..c1f47a7f01 100644 --- a/diff.c +++ b/diff.c @@ -4631,6 +4631,9 @@ void diff_setup_done(struct diff_options *options) if (HAS_MULTI_BITS(options->pickaxe_opts & DIFF_PICKAXE_KINDS_G_REGEX_MASK)) die(_("-G and --pickaxe-regex are mutually exclusive, use --pickaxe-regex with -S")); + if (HAS_MULTI_BITS(options->pickaxe_opts & DIFF_PICKAXE_KINDS_ALL_OBJFIND_MASK)) + die(_("---pickaxe-all and --find-object are mutually exclusive, use --pickaxe-all with -G and -S")); + /* * Most of the time we can say "there are changes" * only by checking if there are changed paths, but diff --git a/diff.h b/diff.h index 5e110d349b..82254396f9 100644 --- a/diff.h +++ b/diff.h @@ -558,6 +558,8 @@ int git_config_rename(const char *var, const char *value); DIFF_PICKAXE_KIND_OBJFIND) #define DIFF_PICKAXE_KINDS_G_REGEX_MASK (DIFF_PICKAXE_KIND_G | \ DIFF_PICKAXE_REGEX) +#define DIFF_PICKAXE_KINDS_ALL_OBJFIND_MASK (DIFF_PICKAXE_ALL | \ + DIFF_PICKAXE_KIND_OBJFIND) #define DIFF_PICKAXE_IGNORE_CASE 32 diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 772c6c1a7c..16166ffd3e 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -63,6 +63,9 @@ test_expect_success 'usage' ' grep "mutually exclusive" err && test_expect_code 128 git log -Sstring --find-object=HEAD 2>err && + grep "mutually exclusive" err && + + test_expect_code 128 git log --pickaxe-all --find-object=HEAD 2>err && grep "mutually exclusive" err ' From a47fcbe6e412df295f1f5ffb8eca6dbe86d2b7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:16 +0200 Subject: [PATCH 025/397] diff.h: move pickaxe fields together again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the pickaxe and pickaxe_opts fields next to each other again. In a past life they'd been on adjacent lines, but when they got moved from a global variable to the diff_options struct in 6b5ee137e5 (Diff clean-up., 2005-09-21) they got split apart. That split made sense at the time, the "char*" and "int" (flags) options were being grouped, but we've long since abandoned that pattern in the diff_options struct, and now it makes more sense to group these together again. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diff.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/diff.h b/diff.h index 82254396f9..8ba85c5e60 100644 --- a/diff.h +++ b/diff.h @@ -265,6 +265,7 @@ struct diff_options { * postimage of the diff_queue. */ const char *pickaxe; + unsigned pickaxe_opts; /* -I */ regex_t **ignore_regex; @@ -304,8 +305,6 @@ struct diff_options { /* The output format used when `diff_flush()` is run. */ int output_format; - unsigned pickaxe_opts; - /* Affects the way detection logic for complete rewrites, renames and * copies. */ From 102fdd2e07f72919060224b3eb3560524d6cf1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:17 +0200 Subject: [PATCH 026/397] pickaxe/style: consolidate declarations and assignments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor contains() to do its assignments at the same time that it does its declarations. This code could have been refactored in ef90ab66e8e (pickaxe: use textconv for -S counting, 2012-10-28) when a function call between the declarations and assignments was removed. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index a9c6d60df2..a278b9b71d 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -70,13 +70,9 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws) { - unsigned int cnt; - unsigned long sz; - const char *data; - - sz = mf->size; - data = mf->ptr; - cnt = 0; + unsigned int cnt = 0; + unsigned long sz = mf->size; + const char *data = mf->ptr; if (regexp) { regmatch_t regmatch; From d90d441c336cc120b47e025d14c3879864ce60c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:18 +0200 Subject: [PATCH 027/397] perf: add performance test for pickaxe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a test for the -G and -S pickaxe options and related options. This test supports being run with GIT_TEST_LONG=1 to adjust the limit on the number of commits from 1k to 10k. The 1k limit seems to hit a good spot on git.git Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/perf/p4209-pickaxe.sh | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 t/perf/p4209-pickaxe.sh diff --git a/t/perf/p4209-pickaxe.sh b/t/perf/p4209-pickaxe.sh new file mode 100755 index 0000000000..f585a4465a --- /dev/null +++ b/t/perf/p4209-pickaxe.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +test_description="Test pickaxe performance" + +. ./perf-lib.sh + +test_perf_default_repo + +# Not --max-count, as that's the number of matching commit, so it's +# unbounded. We want to limit our revision walk here. +from_rev_desc= +from_rev= +max_count=1000 +if test_have_prereq EXPENSIVE +then + max_count=10000 +fi +from_rev=" $(git rev-list HEAD | head -n $max_count | tail -n 1).." +from_rev_desc=" .." + +for icase in \ + '' \ + '-i ' +do + # -S (no regex) + for pattern in \ + 'int main' \ + 'æ' + do + for opts in \ + '-S' + do + test_perf "git log $icase$opts'$pattern'$from_rev_desc" " + git log --pretty=format:%H $icase$opts'$pattern'$from_rev + " + done + done + + # -S (regex) + for pattern in \ + '(int|void|null)' \ + 'if *\([^ ]+ & ' \ + '[àáâãäåæñøùúûüýþ]' + do + for opts in \ + '--pickaxe-regex -S' + do + test_perf "git log $icase$opts'$pattern'$from_rev_desc" " + git log --pretty=format:%H $icase$opts'$pattern'$from_rev + " + done + done + + # -G + for pattern in \ + '(int|void|null)' \ + 'if *\([^ ]+ & ' \ + '[àáâãäåæñøùúûüýþ]' + do + for opts in \ + '-G' + do + test_perf "git log $icase$opts'$pattern'$from_rev_desc" " + git log --pretty=format:%H $icase$opts'$pattern'$from_rev + " + done + done +done + +test_done From 03c1f14acf5169d8e6c06a60c28ee5a6cfb9fd54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:19 +0200 Subject: [PATCH 028/397] pickaxe: refactor function selection in diffcore-pickaxe() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's hard to read this codepath at a glance and reason about exactly what combination of -G and -S will compile either regexes or kwset, and whether we'll then dispatch to "diff_grep" or "has_changes". Then in the "--find-object" case we aren't using the callback function, but were previously passing down "has_changes". Refactor this code to exhaustively check "opts", it's now more obvious what callback function (or none) we want under what mode. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index a278b9b71d..953b6ec1b4 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -228,6 +228,7 @@ void diffcore_pickaxe(struct diff_options *o) int opts = o->pickaxe_opts; regex_t regex, *regexp = NULL; kwset_t kws = NULL; + pickaxe_fn fn; if (opts & (DIFF_PICKAXE_REGEX | DIFF_PICKAXE_KIND_G)) { int cflags = REG_EXTENDED | REG_NEWLINE; @@ -235,6 +236,20 @@ void diffcore_pickaxe(struct diff_options *o) cflags |= REG_ICASE; regcomp_or_die(®ex, needle, cflags); regexp = ®ex; + + if (opts & DIFF_PICKAXE_KIND_G) + fn = diff_grep; + else if (opts & DIFF_PICKAXE_REGEX) + fn = has_changes; + else + /* + * We don't need to check the combination of + * -G and --pickaxe-regex, by the time we get + * here diff.c has already died if they're + * combined. See the usage tests in + * t4209-log-pickaxe.sh. + */ + BUG("unreachable"); } else if (opts & DIFF_PICKAXE_KIND_S) { if (o->pickaxe_opts & DIFF_PICKAXE_IGNORE_CASE && has_non_ascii(needle)) { @@ -251,10 +266,14 @@ void diffcore_pickaxe(struct diff_options *o) kwsincr(kws, needle, strlen(needle)); kwsprep(kws); } + fn = has_changes; + } else if (opts & DIFF_PICKAXE_KIND_OBJFIND) { + fn = NULL; + } else { + BUG("unknown pickaxe_opts flag"); } - pickaxe(&diff_queued_diff, o, regexp, kws, - (opts & DIFF_PICKAXE_KIND_G) ? diff_grep : has_changes); + pickaxe(&diff_queued_diff, o, regexp, kws, fn); if (regexp) regfree(regexp); From 2e197a759221921d42c5e94ae3f4897cd456ebb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:20 +0200 Subject: [PATCH 029/397] pickaxe: assert that we must have a needle under -G or -S MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assert early in diffcore_pickaxe() that we've got a needle to work with under -G and -S. This code is redundant to the check -G and -S get from parse-options.c's get_arg(), which I'm adding a test for. This check dates back to e1b161161d (diffcore-pickaxe: fix infinite loop on zero-length needle, 2007-01-25) when "git log -S" could send this code into an infinite loop. It was then later refactored in 8fa4b09fb1 (pickaxe: hoist empty needle check, 2012-10-28) into its current form, but it seemingly wasn't noticed that in the meantime a move to the parse-options.c API in dea007fb4c (diff: parse separate options like -S foo, 2010-08-05) had made it redundant. Let's retain some of the paranoia here with a BUG(), but there's no need to be checking this in the pickaxe_match() inner loop. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 6 +++--- t/t4209-log-pickaxe.sh | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index 953b6ec1b4..88b6ca840f 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -132,9 +132,6 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, oidset_contains(o->objfind, &p->two->oid)); } - if (!o->pickaxe[0]) - return 0; - if (o->flags.allow_textconv) { textconv_one = get_textconv(o->repo, p->one); textconv_two = get_textconv(o->repo, p->two); @@ -230,6 +227,9 @@ void diffcore_pickaxe(struct diff_options *o) kwset_t kws = NULL; pickaxe_fn fn; + if (opts & ~DIFF_PICKAXE_KIND_OBJFIND && + (!needle || !*needle)) + BUG("should have needle under -G or -S"); if (opts & (DIFF_PICKAXE_REGEX | DIFF_PICKAXE_KIND_G)) { int cflags = REG_EXTENDED | REG_NEWLINE; if (o->pickaxe_opts & DIFF_PICKAXE_IGNORE_CASE) diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 16166ffd3e..3f9aad0fdb 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -56,6 +56,12 @@ test_expect_success setup ' ' test_expect_success 'usage' ' + test_expect_code 129 git log -S 2>err && + test_i18ngrep "switch.*requires a value" err && + + test_expect_code 129 git log -G 2>err && + test_i18ngrep "switch.*requires a value" err && + test_expect_code 128 git log -Gregex -Sstring 2>err && grep "mutually exclusive" err && From 52e011cd2b13e00fe40b1d59986948330b61e030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:21 +0200 Subject: [PATCH 030/397] pickaxe -S: support content with NULs under --pickaxe-regex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a bug in the matching routine powering -S --pickaxe-regex so that we won't abort early on content that has NULs in it. We've had a hard requirement on REG_STARTEND since 2f8952250a8 (regex: add regexec_buf() that can work on a non NUL-terminated string, 2016-09-21), but this sanity check dates back to d01d8c67828 (Support for pickaxe matching regular expressions, 2006-03-29). It wasn't needed anymore, and as the now-passing test shows, actively getting in our way. Since we always require REG_STARTEND support we do not need to stop at NULs. If we are dealing with a haystack with NUL in it. The needle may be behind that NUL. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 4 ++-- t/t4209-log-pickaxe.sh | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index 88b6ca840f..be0dd683b6 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -78,12 +78,12 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws) regmatch_t regmatch; int flags = 0; - while (sz && *data && + while (sz && !regexec_buf(regexp, data, sz, 1, ®match, flags)) { flags |= REG_NOTBOL; data += regmatch.rm_eo; sz -= regmatch.rm_eo; - if (sz && *data && regmatch.rm_so == regmatch.rm_eo) { + if (sz && regmatch.rm_so == regmatch.rm_eo) { data++; sz--; } diff --git a/t/t4209-log-pickaxe.sh b/t/t4209-log-pickaxe.sh index 3f9aad0fdb..75795d0b49 100755 --- a/t/t4209-log-pickaxe.sh +++ b/t/t4209-log-pickaxe.sh @@ -215,4 +215,12 @@ test_expect_success 'log -S looks into binary files' ' test_cmp log full-log ' +test_expect_success 'log -S --pickaxe-regex looks into binary files' ' + git -C GS-bin-txt log --pickaxe-regex -Sa >log && + test_cmp log full-log && + + git -C GS-bin-txt log --pickaxe-regex -S"[a]" >log && + test_cmp log full-log +' + test_done From 5d35a9531cc9af717383bc11c2dfa4edc020ac56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:22 +0200 Subject: [PATCH 031/397] pickaxe: rename variables in has_changes() for brevity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the {one,two}_contains variables to c{1,2}. This will make a follow-up change easier to read. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index be0dd683b6..23362a2359 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -108,9 +108,9 @@ static int has_changes(mmfile_t *one, mmfile_t *two, struct diff_options *o, regex_t *regexp, kwset_t kws) { - unsigned int one_contains = one ? contains(one, regexp, kws) : 0; - unsigned int two_contains = two ? contains(two, regexp, kws) : 0; - return one_contains != two_contains; + unsigned int c1 = one ? contains(one, regexp, kws) : 0; + unsigned int c2 = two ? contains(two, regexp, kws) : 0; + return c1 != c2; } static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, From 5b0672a26e51387bc18adad89eaa3ebb131b2e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:23 +0200 Subject: [PATCH 032/397] pickaxe -S: slightly optimize contains() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the "log -S" switch counts occurrences of on the pre-image and post-image of a change. As soon as we know we had e.g. 1 before and 2 now we can stop, we don't need to keep counting past 2. With this change a diff between A and B may have different performance characteristics than between B and A. That's OK in this case, since we'll emit the same output, and the effect is to make one of them better. I'm picking a check of "one" first on the assumption that it's a more common case to have files grow over time than not. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index 23362a2359..b7494fdf89 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -68,7 +68,8 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, return ecbdata.hit; } -static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws) +static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws, + unsigned int limit) { unsigned int cnt = 0; unsigned long sz = mf->size; @@ -88,6 +89,9 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws) sz--; } cnt++; + + if (limit && cnt == limit) + return cnt; } } else { /* Classic exact string match */ @@ -99,6 +103,9 @@ static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws) sz -= offset + kwsm.size[0]; data += offset + kwsm.size[0]; cnt++; + + if (limit && cnt == limit) + return cnt; } } return cnt; @@ -108,8 +115,8 @@ static int has_changes(mmfile_t *one, mmfile_t *two, struct diff_options *o, regex_t *regexp, kwset_t kws) { - unsigned int c1 = one ? contains(one, regexp, kws) : 0; - unsigned int c2 = two ? contains(two, regexp, kws) : 0; + unsigned int c1 = one ? contains(one, regexp, kws, 0) : 0; + unsigned int c2 = two ? contains(two, regexp, kws, c1 + 1) : 0; return c1 != c2; } From a8d5eb6dc0d61625667b0d8155c425d3629baa12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:24 +0200 Subject: [PATCH 033/397] xdiff-interface: prepare for allowing early return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the function prototype of xdiff_emit_line_fn to return an "int" instead of "void". Change all of those functions to "return 0", nothing checks those return values yet, and no behavior is being changed. In subsequent commits the interface will be changed to allow early return via this new return value. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- combine-diff.c | 5 +++-- diff.c | 26 +++++++++++++++----------- diffcore-pickaxe.c | 7 ++++--- range-diff.c | 3 ++- xdiff-interface.c | 3 ++- xdiff-interface.h | 2 +- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/combine-diff.c b/combine-diff.c index 06635f91bc..a12d3bc0d9 100644 --- a/combine-diff.c +++ b/combine-diff.c @@ -403,11 +403,11 @@ static void consume_hunk(void *state_, state->sline[state->nb-1].p_lno[state->n] = state->ob; } -static void consume_line(void *state_, char *line, unsigned long len) +static int consume_line(void *state_, char *line, unsigned long len) { struct combine_diff_state *state = state_; if (!state->lost_bucket) - return; /* not in any hunk yet */ + return 0; /* not in any hunk yet */ switch (line[0]) { case '-': append_lost(state->lost_bucket, state->n, line+1, len-1); @@ -417,6 +417,7 @@ static void consume_line(void *state_, char *line, unsigned long len) state->lno++; break; } + return 0; } static void combine_diff(struct repository *r, diff --git a/diff.c b/diff.c index c1f47a7f01..7a03c581c7 100644 --- a/diff.c +++ b/diff.c @@ -2336,7 +2336,7 @@ static void find_lno(const char *line, struct emit_callback *ecbdata) ecbdata->lno_in_postimage = strtol(p + 1, NULL, 10); } -static void fn_out_consume(void *priv, char *line, unsigned long len) +static int fn_out_consume(void *priv, char *line, unsigned long len) { struct emit_callback *ecbdata = priv; struct diff_options *o = ecbdata->opt; @@ -2372,7 +2372,7 @@ static void fn_out_consume(void *priv, char *line, unsigned long len) len = sane_truncate_line(line, len); find_lno(line, ecbdata); emit_hunk_header(ecbdata, line, len); - return; + return 0; } if (ecbdata->diff_words) { @@ -2382,11 +2382,11 @@ static void fn_out_consume(void *priv, char *line, unsigned long len) if (line[0] == '-') { diff_words_append(line, len, &ecbdata->diff_words->minus); - return; + return 0; } else if (line[0] == '+') { diff_words_append(line, len, &ecbdata->diff_words->plus); - return; + return 0; } else if (starts_with(line, "\\ ")) { /* * Eat the "no newline at eof" marker as if we @@ -2395,11 +2395,11 @@ static void fn_out_consume(void *priv, char *line, unsigned long len) * defer processing. If this is the end of * preimage, more "+" lines may come after it. */ - return; + return 0; } diff_words_flush(ecbdata); emit_diff_symbol(o, s, line, len, 0); - return; + return 0; } switch (line[0]) { @@ -2423,6 +2423,7 @@ static void fn_out_consume(void *priv, char *line, unsigned long len) line, len, 0); break; } + return 0; } static void pprint_rename(struct strbuf *name, const char *a, const char *b) @@ -2522,7 +2523,7 @@ static struct diffstat_file *diffstat_add(struct diffstat_t *diffstat, return x; } -static void diffstat_consume(void *priv, char *line, unsigned long len) +static int diffstat_consume(void *priv, char *line, unsigned long len) { struct diffstat_t *diffstat = priv; struct diffstat_file *x = diffstat->files[diffstat->nr - 1]; @@ -2531,6 +2532,7 @@ static void diffstat_consume(void *priv, char *line, unsigned long len) x->added++; else if (line[0] == '-') x->deleted++; + return 0; } const char mime_boundary_leader[] = "------------"; @@ -3208,7 +3210,7 @@ static void checkdiff_consume_hunk(void *priv, data->lineno = nb - 1; } -static void checkdiff_consume(void *priv, char *line, unsigned long len) +static int checkdiff_consume(void *priv, char *line, unsigned long len) { struct checkdiff_t *data = priv; int marker_size = data->conflict_marker_size; @@ -3232,7 +3234,7 @@ static void checkdiff_consume(void *priv, char *line, unsigned long len) } bad = ws_check(line + 1, len - 1, data->ws_rule); if (!bad) - return; + return 0; data->status |= bad; err = whitespace_error_string(bad); fprintf(data->o->file, "%s%s:%d: %s.\n", @@ -3244,6 +3246,7 @@ static void checkdiff_consume(void *priv, char *line, unsigned long len) } else if (line[0] == ' ') { data->lineno++; } + return 0; } static unsigned char *deflate_it(char *data, @@ -6121,17 +6124,18 @@ void flush_one_hunk(struct object_id *result, git_hash_ctx *ctx) } } -static void patch_id_consume(void *priv, char *line, unsigned long len) +static int patch_id_consume(void *priv, char *line, unsigned long len) { struct patch_id_t *data = priv; int new_len; if (len > 12 && starts_with(line, "\\ ")) - return; + return 0; new_len = remove_space(line, len); the_hash_algo->update_fn(data->ctx, line, new_len); data->patchlen += new_len; + return 0; } static void patch_id_add_string(git_hash_ctx *ctx, const char *str) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index b7494fdf89..27aa20be35 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -19,21 +19,22 @@ struct diffgrep_cb { int hit; }; -static void diffgrep_consume(void *priv, char *line, unsigned long len) +static int diffgrep_consume(void *priv, char *line, unsigned long len) { struct diffgrep_cb *data = priv; regmatch_t regmatch; if (line[0] != '+' && line[0] != '-') - return; + return 0; if (data->hit) /* * NEEDSWORK: we should have a way to terminate the * caller early. */ - return; + return 0; data->hit = !regexec_buf(data->regexp, line + 1, len - 1, 1, ®match, 0); + return 0; } static int diff_grep(mmfile_t *one, mmfile_t *two, diff --git a/range-diff.c b/range-diff.c index 116fb0735c..83c90f946e 100644 --- a/range-diff.c +++ b/range-diff.c @@ -274,9 +274,10 @@ static void find_exact_matches(struct string_list *a, struct string_list *b) hashmap_clear(&map); } -static void diffsize_consume(void *data, char *line, unsigned long len) +static int diffsize_consume(void *data, char *line, unsigned long len) { (*(int *)data)++; + return 0; } static void diffsize_hunk(void *data, long ob, long on, long nb, long nn, diff --git a/xdiff-interface.c b/xdiff-interface.c index 4d20069302..5d8c8c67dc 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -31,7 +31,7 @@ static int xdiff_out_hunk(void *priv_, return 0; } -static void consume_one(void *priv_, char *s, unsigned long size) +static int consume_one(void *priv_, char *s, unsigned long size) { struct xdiff_emit_state *priv = priv_; char *ep; @@ -43,6 +43,7 @@ static void consume_one(void *priv_, char *s, unsigned long size) size -= this_size; s += this_size; } + return 0; } static int xdiff_outf(void *priv_, mmbuffer_t *mb, int nbuf) diff --git a/xdiff-interface.h b/xdiff-interface.h index 93df26900c..0198f9632f 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -11,7 +11,7 @@ */ #define MAX_XDIFF_SIZE (1024UL * 1024 * 1023) -typedef void (*xdiff_emit_line_fn)(void *, char *, unsigned long); +typedef int (*xdiff_emit_line_fn)(void *, char *, unsigned long); typedef void (*xdiff_emit_hunk_fn)(void *data, long old_begin, long old_nr, long new_begin, long new_nr, From 9e204422985a518ac700889d1ca4d521b3a7bfb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:25 +0200 Subject: [PATCH 034/397] xdiff-interface: allow early return from xdiff_emit_line_fn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish the change started in the preceding commit and allow an early return from "xdiff_emit_line_fn" callbacks, this will allows diffcore-pickaxe.c to save itself redundant work. Our xdiff interface also had the limitation of not being able to abort early since the beginning, see d9ea73e0564 (combine-diff: refactor built-in xdiff interface., 2006-04-05). Although at that time "xdiff_emit_line_fn" was called "xdiff_emit_consume_fn", and "xdiff_emit_hunk_fn" didn't exist yet. There was some work in this area of xdiff-interface.[ch] recently with 3b40a090fd4 (diff: avoid generating unused hunk header lines, 2018-11-02) and 7c61e25fbf1 (diff: use hunk callback for word-diff, 2018-11-02). In combination those two changes allow us to not do any work on the hunks and diff at all, but didn't change the status quo with regards to consumers that e.g. want the diff lines, but might want to abort early. Whereas now we can abort e.g. on the first "-line" of a 1000 line diff if that's all we needed. This interface is rather scary as noted in the comment to xdiff-interface.h being added here, as noted there a future change could add more exit codes, and hack xdl_emit_diff() and friends to ignore or skip things more selectively as a result. I did not see an inherent reason for why xdl_emit_{diffrec,record}() could not be changed to ferry the "xdiff_emit_line_fn" error code upwards instead of returning -1 on all "ret < 0". But doing so would require corresponding changes in xdl_emit_diff(), xdl_diff(). I didn't see any issue with narrowly doing that to accomplish what I needed here, but it would leave xdiff's own return values in an inconsistent state. Instead I've left it at returning a more conventional (for git's own codebase) 1 for an early return, and translating it (or rather, all non-zero) to -1 for xdiff's consumption. The reason for most of the "stop" complexity in xdiff_outf() is because we want to be able to abort early, but do so in a way that doesn't skip the appropriate strbuf_reset() invocations. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- xdiff-interface.c | 18 ++++++++++++++---- xdiff-interface.h | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/xdiff-interface.c b/xdiff-interface.c index 5d8c8c67dc..50c0ef759d 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -37,9 +37,12 @@ static int consume_one(void *priv_, char *s, unsigned long size) char *ep; while (size) { unsigned long this_size; + int ret; ep = memchr(s, '\n', size); this_size = (ep == NULL) ? size : (ep - s + 1); - priv->line_fn(priv->consume_callback_data, s, this_size); + ret = priv->line_fn(priv->consume_callback_data, s, this_size); + if (ret) + return ret; size -= this_size; s += this_size; } @@ -50,11 +53,14 @@ static int xdiff_outf(void *priv_, mmbuffer_t *mb, int nbuf) { struct xdiff_emit_state *priv = priv_; int i; + int stop = 0; if (!priv->line_fn) return 0; for (i = 0; i < nbuf; i++) { + if (stop) + return 1; if (mb[i].ptr[mb[i].size-1] != '\n') { /* Incomplete line */ strbuf_add(&priv->remainder, mb[i].ptr, mb[i].size); @@ -63,17 +69,21 @@ static int xdiff_outf(void *priv_, mmbuffer_t *mb, int nbuf) /* we have a complete line */ if (!priv->remainder.len) { - consume_one(priv, mb[i].ptr, mb[i].size); + stop = consume_one(priv, mb[i].ptr, mb[i].size); continue; } strbuf_add(&priv->remainder, mb[i].ptr, mb[i].size); - consume_one(priv, priv->remainder.buf, priv->remainder.len); + stop = consume_one(priv, priv->remainder.buf, priv->remainder.len); strbuf_reset(&priv->remainder); } + if (stop) + return -1; if (priv->remainder.len) { - consume_one(priv, priv->remainder.buf, priv->remainder.len); + stop = consume_one(priv, priv->remainder.buf, priv->remainder.len); strbuf_reset(&priv->remainder); } + if (stop) + return -1; return 0; } diff --git a/xdiff-interface.h b/xdiff-interface.h index 0198f9632f..7d1724abb6 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -11,6 +11,23 @@ */ #define MAX_XDIFF_SIZE (1024UL * 1024 * 1023) +/** + * The `xdiff_emit_line_fn` function can return 1 to abort early, or 0 + * to continue processing. Note that doing so is an all-or-nothing + * affair, as returning 1 will return all the way to the top-level, + * e.g. the xdi_diff_outf() call to generate the diff. + * + * Thus returning 1 means you won't be getting any more diff lines. If + * you need something in-between those two options you'll to use + * `xdl_emit_hunk_consume_func_t` and implement your own version of + * xdl_emit_diff(). + * + * We may extend the interface in the future to understand other more + * granular return values. While you should return 1 to exit early, + * doing so will currently make your early return indistinguishable + * from an error internal to xdiff, xdiff itself will see that + * non-zero return and translate it to -1. + */ typedef int (*xdiff_emit_line_fn)(void *, char *, unsigned long); typedef void (*xdiff_emit_hunk_fn)(void *data, long old_begin, long old_nr, From fa59e7beb2a61d9cd1312b212075edf12935fdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:26 +0200 Subject: [PATCH 035/397] pickaxe -G: terminate early on matching lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Solve a long-standing item for "git log -Grx" of us e.g. finding "+ str" in the diff context and noting that we had a "hit", but xdiff diligently continuing to generate and spew the rest of the diff at us. This makes use of a new "early return" xdiff interface added by preceding commits. The TODO item (or, the NEEDSWORK comment) has been there since "git log -G" was implemented. See f506b8e8b5f (git log/diff: add -G that greps in the patch text, 2010-08-23). But now with the support added in the preceding changes to the xdiff-interface we can return early. Let's assert the behavior of that new early-return xdiff-interface by having a BUG() call here to die if it ever starts handing us needless work again. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 30 +++++++++++++++++++----------- xdiff-interface.h | 4 ++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index 27aa20be35..2147afef72 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -27,13 +27,12 @@ static int diffgrep_consume(void *priv, char *line, unsigned long len) if (line[0] != '+' && line[0] != '-') return 0; if (data->hit) - /* - * NEEDSWORK: we should have a way to terminate the - * caller early. - */ - return 0; - data->hit = !regexec_buf(data->regexp, line + 1, len - 1, 1, - ®match, 0); + BUG("Already matched in diffgrep_consume! Broken xdiff_emit_line_fn?"); + if (!regexec_buf(data->regexp, line + 1, len - 1, 1, + ®match, 0)) { + data->hit = 1; + return 1; + } return 0; } @@ -45,6 +44,7 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, struct diffgrep_cb ecbdata; xpparam_t xpp; xdemitconf_t xecfg; + int ret; if (!one) return !regexec_buf(regexp, two->ptr, two->size, @@ -63,10 +63,18 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, ecbdata.hit = 0; xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; - if (xdi_diff_outf(one, two, discard_hunk_line, diffgrep_consume, - &ecbdata, &xpp, &xecfg)) - return 0; - return ecbdata.hit; + + /* + * An xdiff error might be our "data->hit" from above. See the + * comment for xdiff_emit_line_fn in xdiff-interface.h + */ + ret = xdi_diff_outf(one, two, discard_hunk_line, diffgrep_consume, + &ecbdata, &xpp, &xecfg); + if (ecbdata.hit) + return 1; + if (ret) + return ret; + return 0; } static unsigned int contains(mmfile_t *mf, regex_t *regexp, kwset_t kws, diff --git a/xdiff-interface.h b/xdiff-interface.h index 7d1724abb6..3b6819586d 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -27,6 +27,10 @@ * doing so will currently make your early return indistinguishable * from an error internal to xdiff, xdiff itself will see that * non-zero return and translate it to -1. + * + * See "diff_grep" in diffcore-pickaxe.c for a trick to work around + * this, i.e. using the "consume_callback_data" to note the desired + * early return. */ typedef int (*xdiff_emit_line_fn)(void *, char *, unsigned long); typedef void (*xdiff_emit_hunk_fn)(void *data, From f97fe358576c81d995843644ab58de1ebeb7fee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:27 +0200 Subject: [PATCH 036/397] pickaxe -G: don't special-case create/delete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of special-casing creations and deletions let's just generate a diff for them. This logic of not running a diff under -G if we don't have both sides dates back to the original implementation of -S in 52e9578985f ([PATCH] Introducing software archaeologist's tool "pickaxe"., 2005-05-21). In the case of -S we were not working with the xdiff interface and needed to do this, but when -G was implemented in f506b8e8b5f (git log/diff: add -G that greps in the patch text, 2010-08-23) this logic was diligently copied over. But as the performance test added earlier in this series shows, this does not make much of a difference. With: time GIT_TEST_LONG= GIT_PERF_REPEAT_COUNT=10 GIT_PERF_MAKE_OPTS='-j8 CFLAGS=-O3' ./run origin/next HEAD~ HEAD -- p4209-pickaxe.sh With the HEAD~ commit being the preceding "pickaxe -G: terminate early on matching lines" we get these results. Note that it's only the -G codepaths that are relevant to this change: Test origin/next HEAD~ HEAD ----------------------------------------------------------------------------------------------------------------------------------------- 4209.1: git log -S'int main' .. 0.35(0.32+0.03) 0.35(0.33+0.02) +0.0% 0.35(0.30+0.05) +0.0% 4209.2: git log -S'æ' .. 0.46(0.42+0.04) 0.46(0.41+0.05) +0.0% 0.46(0.42+0.04) +0.0% 4209.3: git log --pickaxe-regex -S'(int|void|null)' .. 0.65(0.62+0.02) 0.64(0.61+0.02) -1.5% 0.64(0.60+0.04) -1.5% 4209.4: git log --pickaxe-regex -S'if *\([^ ]+ & ' .. 0.52(0.45+0.06) 0.52(0.50+0.01) +0.0% 0.54(0.47+0.04) +3.8% 4209.5: git log --pickaxe-regex -S'[àáâãäåæñøùúûüýþ]' .. 0.39(0.34+0.05) 0.39(0.34+0.04) +0.0% 0.39(0.36+0.03) +0.0% 4209.6: git log -G'(int|void|null)' .. 0.60(0.55+0.04) 0.58(0.54+0.03) -3.3% 0.58(0.49+0.08) -3.3% 4209.7: git log -G'if *\([^ ]+ & ' .. 0.61(0.52+0.06) 0.59(0.53+0.05) -3.3% 0.59(0.54+0.05) -3.3% 4209.8: git log -G'[àáâãäåæñøùúûüýþ]' .. 0.61(0.51+0.07) 0.58(0.54+0.04) -4.9% 0.57(0.51+0.06) -6.6% 4209.9: git log -i -S'int main' .. 0.36(0.31+0.04) 0.36(0.34+0.02) +0.0% 0.35(0.32+0.03) -2.8% 4209.10: git log -i -S'æ' .. 0.36(0.33+0.03) 0.39(0.34+0.01) +8.3% 0.36(0.32+0.03) +0.0% 4209.11: git log -i --pickaxe-regex -S'(int|void|null)' .. 0.83(0.77+0.05) 0.82(0.77+0.05) -1.2% 0.80(0.75+0.04) -3.6% 4209.12: git log -i --pickaxe-regex -S'if *\([^ ]+ & ' .. 0.67(0.61+0.03) 0.64(0.61+0.03) -4.5% 0.63(0.61+0.02) -6.0% 4209.13: git log -i --pickaxe-regex -S'[àáâãäåæñøùúûüýþ]' .. 0.40(0.37+0.02) 0.40(0.37+0.03) +0.0% 0.40(0.36+0.04) +0.0% 4209.14: git log -i -G'(int|void|null)' .. 0.58(0.51+0.07) 0.59(0.52+0.06) +1.7% 0.58(0.52+0.05) +0.0% 4209.15: git log -i -G'if *\([^ ]+ & ' .. 0.60(0.54+0.05) 0.60(0.54+0.06) +0.0% 0.60(0.56+0.03) +0.0% 4209.16: git log -i -G'[àáâãäåæñøùúûüýþ]' .. 0.58(0.51+0.06) 0.57(0.52+0.05) -1.7% 0.60(0.48+0.09) +3.4% This small simplification really doesn't buy us much now, but I've got plans to both convert the pickaxe code to using a PCREv2 backend[1] and to implement additional pickaxe modes to do custom searches through the diff[2]. Always having the diff available under -G is going to help to simplify both of those changes. 1. https://lore.kernel.org/git/20210203032811.14979-22-avarab@gmail.com/ 2. https://lore.kernel.org/git/20190424152215.16251-3-avarab@gmail.com/ Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diffcore-pickaxe.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index 2147afef72..96183f4cfa 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -40,19 +40,11 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, struct diff_options *o, regex_t *regexp, kwset_t kws) { - regmatch_t regmatch; struct diffgrep_cb ecbdata; xpparam_t xpp; xdemitconf_t xecfg; int ret; - if (!one) - return !regexec_buf(regexp, two->ptr, two->size, - 1, ®match, 0); - if (!two) - return !regexec_buf(regexp, one->ptr, one->size, - 1, ®match, 0); - /* * We have both sides; need to run textual diff and see if * the pattern appears on added/deleted lines. @@ -172,9 +164,7 @@ static int pickaxe_match(struct diff_filepair *p, struct diff_options *o, mf1.size = fill_textconv(o->repo, textconv_one, p->one, &mf1.ptr); mf2.size = fill_textconv(o->repo, textconv_two, p->two, &mf2.ptr); - ret = fn(DIFF_FILE_VALID(p->one) ? &mf1 : NULL, - DIFF_FILE_VALID(p->two) ? &mf2 : NULL, - o, regexp, kws); + ret = fn(&mf1, &mf2, o, regexp, kws); if (textconv_one) free(mf1.ptr); From 22233d43eb5479d1076e034eb5cdba973fa2b02a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:28 +0200 Subject: [PATCH 037/397] xdiff users: use designated initializers for out_line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Amend the code added in 611e42a5980 (xdiff: provide a separate emit callback for hunks, 2018-11-02) to be more readable by using designated initializers. This changes "priv" in rerere.c to be initialized to NULL as we did in merge-tree.c. That's not needed as we'll only use it if the callback is defined, but being consistent here is better and less verbose. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- builtin/merge-tree.c | 5 +---- builtin/rerere.c | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/builtin/merge-tree.c b/builtin/merge-tree.c index de8520778d..5dc94d6f88 100644 --- a/builtin/merge-tree.c +++ b/builtin/merge-tree.c @@ -107,15 +107,12 @@ static void show_diff(struct merge_list *entry) mmfile_t src, dst; xpparam_t xpp; xdemitconf_t xecfg; - xdemitcb_t ecb; + xdemitcb_t ecb = { .out_line = show_outf }; memset(&xpp, 0, sizeof(xpp)); xpp.flags = 0; memset(&xecfg, 0, sizeof(xecfg)); xecfg.ctxlen = 3; - ecb.out_hunk = NULL; - ecb.out_line = show_outf; - ecb.priv = NULL; src.ptr = origin(entry, &size); if (!src.ptr) diff --git a/builtin/rerere.c b/builtin/rerere.c index fd3be17b97..83d7a778e3 100644 --- a/builtin/rerere.c +++ b/builtin/rerere.c @@ -28,7 +28,7 @@ static int diff_two(const char *file1, const char *label1, { xpparam_t xpp; xdemitconf_t xecfg; - xdemitcb_t ecb; + xdemitcb_t ecb = { .out_line = outf }; mmfile_t minus, plus; int ret; @@ -41,8 +41,6 @@ static int diff_two(const char *file1, const char *label1, xpp.flags = 0; memset(&xecfg, 0, sizeof(xecfg)); xecfg.ctxlen = 3; - ecb.out_hunk = NULL; - ecb.out_line = outf; ret = xdi_diff(&minus, &plus, &xpp, &xecfg, &ecb); free(minus.ptr); From 5d93460024541909337d6b08a8bec10b71caaf73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 19:15:29 +0200 Subject: [PATCH 038/397] xdiff-interface: replace discard_hunk_line() with a flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the dummy discard_hunk_line() function added in 3b40a090fd4 (diff: avoid generating unused hunk header lines, 2018-11-02) in favor of having a new XDL_EMIT_NO_HUNK_HDR flag, for use along with the two existing and similar XDL_EMIT_* flags. Unlike the recently amended xdiff_emit_line_fn interface which'll be called in a loop in xdl_emit_diff(), the hunk header is only emitted once. It makes more sense to pass this as a flag than provide a dummy callback because that function may be able to skip doing certain work if it knows the caller is doing nothing with the hunk header. It would be possible to do so in the case of -U0 now, but the benefit of doing so is so small that I haven't bothered. But this leaves the door open to that, and more importantly makes the API use more intuitive. The reason we're putting a flag in the gap between 1<<0 and 1<<2 is that the old 1<<1 flag was removed in 907681e940d (xdiff: drop XDL_EMIT_COMMON, 2016-02-23) without re-ordering the remaining flags. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- diff.c | 7 ++++--- diffcore-pickaxe.c | 3 ++- xdiff-interface.c | 6 ------ xdiff-interface.h | 8 -------- xdiff/xdiff.h | 1 + xdiff/xemit.c | 3 ++- 6 files changed, 9 insertions(+), 19 deletions(-) diff --git a/diff.c b/diff.c index 7a03c581c7..fe3abac79f 100644 --- a/diff.c +++ b/diff.c @@ -3725,7 +3725,8 @@ static void builtin_diffstat(const char *name_a, const char *name_b, xpp.anchors_nr = o->anchors_nr; xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; - if (xdi_diff_outf(&mf1, &mf2, discard_hunk_line, + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; + if (xdi_diff_outf(&mf1, &mf2, NULL, diffstat_consume, diffstat, &xpp, &xecfg)) die("unable to generate diffstat for %s", one->path); @@ -6233,8 +6234,8 @@ static int diff_get_patch_id(struct diff_options *options, struct object_id *oid xpp.flags = 0; xecfg.ctxlen = 3; - xecfg.flags = 0; - if (xdi_diff_outf(&mf1, &mf2, discard_hunk_line, + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; + if (xdi_diff_outf(&mf1, &mf2, NULL, patch_id_consume, &data, &xpp, &xecfg)) return error("unable to generate patch-id diff for %s", p->one->path); diff --git a/diffcore-pickaxe.c b/diffcore-pickaxe.c index 96183f4cfa..c88e50c632 100644 --- a/diffcore-pickaxe.c +++ b/diffcore-pickaxe.c @@ -53,6 +53,7 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, memset(&xecfg, 0, sizeof(xecfg)); ecbdata.regexp = regexp; ecbdata.hit = 0; + xecfg.flags = XDL_EMIT_NO_HUNK_HDR; xecfg.ctxlen = o->context; xecfg.interhunkctxlen = o->interhunkcontext; @@ -60,7 +61,7 @@ static int diff_grep(mmfile_t *one, mmfile_t *two, * An xdiff error might be our "data->hit" from above. See the * comment for xdiff_emit_line_fn in xdiff-interface.h */ - ret = xdi_diff_outf(one, two, discard_hunk_line, diffgrep_consume, + ret = xdi_diff_outf(one, two, NULL, diffgrep_consume, &ecbdata, &xpp, &xecfg); if (ecbdata.hit) return 1; diff --git a/xdiff-interface.c b/xdiff-interface.c index 50c0ef759d..95f13a93ff 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -126,12 +126,6 @@ int xdi_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp, xdemitconf_t co return xdl_diff(&a, &b, xpp, xecfg, xecb); } -void discard_hunk_line(void *priv, - long ob, long on, long nb, long nn, - const char *func, long funclen) -{ -} - int xdi_diff_outf(mmfile_t *mf1, mmfile_t *mf2, xdiff_emit_hunk_fn hunk_fn, xdiff_emit_line_fn line_fn, diff --git a/xdiff-interface.h b/xdiff-interface.h index 3b6819586d..4301a7eef2 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -53,14 +53,6 @@ void xdiff_clear_find_func(xdemitconf_t *xecfg); int git_xmerge_config(const char *var, const char *value, void *cb); extern int git_xmerge_style; -/* - * Can be used as a no-op hunk_fn for xdi_diff_outf(), since a NULL - * one just sends the hunk line to the line_fn callback). - */ -void discard_hunk_line(void *priv, - long ob, long on, long nb, long nn, - const char *func, long funclen); - /* * Compare the strings l1 with l2 which are of size s1 and s2 respectively. * Returns 1 if the strings are deemed equal, 0 otherwise. diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index 7a04605146..b29deca5de 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -50,6 +50,7 @@ extern "C" { /* xdemitconf_t.flags */ #define XDL_EMIT_FUNCNAMES (1 << 0) +#define XDL_EMIT_NO_HUNK_HDR (1 << 1) #define XDL_EMIT_FUNCCONTEXT (1 << 2) /* merge simplification levels */ diff --git a/xdiff/xemit.c b/xdiff/xemit.c index 9d7d6c5087..1cbf2b9829 100644 --- a/xdiff/xemit.c +++ b/xdiff/xemit.c @@ -278,7 +278,8 @@ pre_context_calculation: s1 - 1, funclineprev); funclineprev = s1 - 1; } - if (xdl_emit_hunk_hdr(s1 + 1, e1 - s1, s2 + 1, e2 - s2, + if (!(xecfg->flags & XDL_EMIT_NO_HUNK_HDR) && + xdl_emit_hunk_hdr(s1 + 1, e1 - s1, s2 + 1, e2 - s2, func_line.buf, func_line.len, ecb) < 0) return -1; From 2df546e17fa1c28ac44c886f59ba39ea2f00124a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 13:21:43 +0200 Subject: [PATCH 039/397] describe tests: improve test for --work-tree & --dirty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve tests added in 9f67d2e8279 (Teach "git describe" --dirty option, 2009-10-21) and 2ed5c8e174d (describe: setup working tree for --dirty, 2019-02-03) so that they make sense in combination with each other. The "check_describe" being removed here was the earlier test, we then later added these --work-tree tests which really just wanted to check if we got the exact same output from "describe", but the test wasn't structured to test for that. Let's change it to do that, which both improves test coverage and makes it more obvious what's going on here. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t6120-describe.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index 88fddc9142..a83ea15faa 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -151,24 +151,24 @@ test_expect_success 'set-up dirty work tree' ' echo >>file ' -check_describe "A-*[0-9a-f]-dirty" --dirty - test_expect_success 'describe --dirty with --work-tree (dirty)' ' + git describe --dirty >expected && ( cd "$TEST_DIRECTORY" && git --git-dir "$TRASH_DIRECTORY/.git" --work-tree "$TRASH_DIRECTORY" describe --dirty >"$TRASH_DIRECTORY/out" ) && - grep -E "^A-[1-9][0-9]?-g[0-9a-f]+-dirty$" out + grep -E "^A-[1-9][0-9]?-g[0-9a-f]+-dirty$" out && + test_cmp expected out ' -check_describe "A-*[0-9a-f].mod" --dirty=.mod - test_expect_success 'describe --dirty=.mod with --work-tree (dirty)' ' + git describe --dirty=.mod >expected && ( cd "$TEST_DIRECTORY" && git --git-dir "$TRASH_DIRECTORY/.git" --work-tree "$TRASH_DIRECTORY" describe --dirty=.mod >"$TRASH_DIRECTORY/out" ) && - grep -E "^A-[1-9][0-9]?-g[0-9a-f]+.mod$" out + grep -E "^A-[1-9][0-9]?-g[0-9a-f]+.mod$" out && + test_cmp expected out ' test_expect_success 'describe --dirty HEAD' ' From a46a84829650fbe6077e5a33f4ac1c9d2041d1df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 13:21:44 +0200 Subject: [PATCH 040/397] describe tests: refactor away from glob matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the glob matching via a "case" statement to a "test_cmp" after we've stripped out the hash-specific g suffix. 5312ab11fbf (Add describe test., 2007-01-13). This means that we can use test_cmp to compare the output. I could omit the "-8" change of e.g. "A-*" to "A-8-gHASH", but I think it makes sense to test that here explicitly. It means you need to add new tests to the bottom of the file, but that's not a burden in this case. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t6120-describe.sh | 78 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index a83ea15faa..13117bbcfb 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -21,12 +21,10 @@ check_describe () { shift describe_opts="$@" test_expect_success "describe $describe_opts" ' - R=$(git describe $describe_opts 2>err.actual) && - case "$R" in - $expect) echo happy ;; - *) echo "Oops - $R is not $expect" && - false ;; - esac + git describe $describe_opts 2>err.actual >raw && + sed -e "s/-g[0-9a-f]*\$/-gHASH/" actual && + echo "$expect" >expect && + test_cmp expect actual ' } @@ -59,29 +57,29 @@ test_expect_success setup ' test_commit --no-tag x file ' -check_describe A-* HEAD -check_describe A-* HEAD^ -check_describe R-* HEAD^^ -check_describe A-* HEAD^^2 +check_describe A-8-gHASH HEAD +check_describe A-7-gHASH HEAD^ +check_describe R-2-gHASH HEAD^^ +check_describe A-3-gHASH HEAD^^2 check_describe B HEAD^^2^ -check_describe R-* HEAD^^^ +check_describe R-1-gHASH HEAD^^^ -check_describe c-* --tags HEAD -check_describe c-* --tags HEAD^ -check_describe e-* --tags HEAD^^ -check_describe c-* --tags HEAD^^2 +check_describe c-7-gHASH --tags HEAD +check_describe c-6-gHASH --tags HEAD^ +check_describe e-1-gHASH --tags HEAD^^ +check_describe c-2-gHASH --tags HEAD^^2 check_describe B --tags HEAD^^2^ check_describe e --tags HEAD^^^ check_describe heads/main --all HEAD -check_describe tags/c-* --all HEAD^ +check_describe tags/c-6-gHASH --all HEAD^ check_describe tags/e --all HEAD^^^ -check_describe B-0-* --long HEAD^^2^ -check_describe A-3-* --long HEAD^^2 +check_describe B-0-gHASH --long HEAD^^2^ +check_describe A-3-gHASH --long HEAD^^2 -check_describe c-7-* --tags -check_describe e-3-* --first-parent --tags +check_describe c-7-gHASH --tags +check_describe e-3-gHASH --first-parent --tags test_expect_success 'describe --contains defaults to HEAD without commit-ish' ' echo "A^0" >expect && @@ -102,7 +100,7 @@ test_expect_success 'rename tag A to Q locally' ' cat - >err.expect <out && - grep -E "^A-[1-9][0-9]?-g[0-9a-f]+$" out + grep -E "^A-8-g[0-9a-f]+$" out ' -check_describe "A-*[0-9a-f]" --dirty +check_describe "A-8-gHASH" --dirty test_expect_success 'describe --dirty with --work-tree' ' ( cd "$TEST_DIRECTORY" && git --git-dir "$TRASH_DIRECTORY/.git" --work-tree "$TRASH_DIRECTORY" describe --dirty >"$TRASH_DIRECTORY/out" ) && - grep -E "^A-[1-9][0-9]?-g[0-9a-f]+$" out + grep -E "^A-8-g[0-9a-f]+$" out ' test_expect_success 'set-up dirty work tree' ' @@ -157,7 +155,7 @@ test_expect_success 'describe --dirty with --work-tree (dirty)' ' cd "$TEST_DIRECTORY" && git --git-dir "$TRASH_DIRECTORY/.git" --work-tree "$TRASH_DIRECTORY" describe --dirty >"$TRASH_DIRECTORY/out" ) && - grep -E "^A-[1-9][0-9]?-g[0-9a-f]+-dirty$" out && + grep -E "^A-8-g[0-9a-f]+-dirty$" out && test_cmp expected out ' @@ -167,7 +165,7 @@ test_expect_success 'describe --dirty=.mod with --work-tree (dirty)' ' cd "$TEST_DIRECTORY" && git --git-dir "$TRASH_DIRECTORY/.git" --work-tree "$TRASH_DIRECTORY" describe --dirty=.mod >"$TRASH_DIRECTORY/out" ) && - grep -E "^A-[1-9][0-9]?-g[0-9a-f]+.mod$" out && + grep -E "^A-8-g[0-9a-f]+.mod$" out && test_cmp expected out ' @@ -191,21 +189,21 @@ test_expect_success 'set-up matching pattern tests' ' ' -check_describe "test-annotated-*" --match="test-*" +check_describe "test-annotated-3-gHASH" --match="test-*" -check_describe "test1-lightweight-*" --tags --match="test1-*" +check_describe "test1-lightweight-2-gHASH" --tags --match="test1-*" -check_describe "test2-lightweight-*" --tags --match="test2-*" +check_describe "test2-lightweight-1-gHASH" --tags --match="test2-*" -check_describe "test2-lightweight-*" --long --tags --match="test2-*" HEAD^ +check_describe "test2-lightweight-0-gHASH" --long --tags --match="test2-*" HEAD^ -check_describe "test2-lightweight-*" --long --tags --match="test1-*" --match="test2-*" HEAD^ +check_describe "test2-lightweight-0-gHASH" --long --tags --match="test1-*" --match="test2-*" HEAD^ -check_describe "test2-lightweight-*" --long --tags --match="test1-*" --no-match --match="test2-*" HEAD^ +check_describe "test2-lightweight-0-gHASH" --long --tags --match="test1-*" --no-match --match="test2-*" HEAD^ -check_describe "test1-lightweight-*" --long --tags --match="test1-*" --match="test3-*" HEAD +check_describe "test1-lightweight-2-gHASH" --long --tags --match="test1-*" --match="test3-*" HEAD -check_describe "test1-lightweight-*" --long --tags --match="test3-*" --match="test1-*" HEAD +check_describe "test1-lightweight-2-gHASH" --long --tags --match="test3-*" --match="test1-*" HEAD test_expect_success 'set-up branches' ' git branch branch_A A && @@ -215,11 +213,11 @@ test_expect_success 'set-up branches' ' git update-ref refs/original/original_branch_A test-annotated~2 ' -check_describe "heads/branch_A*" --all --match="branch_*" --exclude="branch_C" HEAD +check_describe "heads/branch_A-11-gHASH" --all --match="branch_*" --exclude="branch_C" HEAD -check_describe "remotes/origin/remote_branch_A*" --all --match="origin/remote_branch_*" --exclude="origin/remote_branch_C" HEAD +check_describe "remotes/origin/remote_branch_A-11-gHASH" --all --match="origin/remote_branch_*" --exclude="origin/remote_branch_C" HEAD -check_describe "original/original_branch_A*" --all test-annotated~1 +check_describe "original/original_branch_A-6-gHASH" --all test-annotated~1 test_expect_success '--match does not work for other types' ' test_must_fail git describe --all --match="*original_branch_*" test-annotated~1 @@ -489,7 +487,7 @@ test_expect_success 'describe commits with disjoint bases' ' git tag B -a -m B && git merge --no-ff --allow-unrelated-histories main -m x && - check_describe "A-3-*" HEAD + check_describe "A-3-gHASH" HEAD ) ' @@ -515,7 +513,7 @@ test_expect_success 'describe commits with disjoint bases 2' ' git tag B -a -m B && git merge --no-ff --allow-unrelated-histories main -m x && - check_describe "B-3-*" HEAD + check_describe "B-3-gHASH" HEAD ) ' From 7f07c1cdb7f46d2b0ec4468a90da7a4d697d2978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 13:21:45 +0200 Subject: [PATCH 041/397] describe tests: don't rely on err.actual from "check_describe" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the one test that relied on the "err.actual" file produced by check_describe() to instead do its own check of "git describe" output. This means that the two tests won't have an inter-dependency (e.g. if the earlier test is skipped). An earlier version of this patch instead asserted that no other test had any output on stderr. We're not doing that here out of fear that "gc --auto" or another future change to "git describe" will cause it to legitimately emit output on stderr unexpectedly[1]. I'd think that inverting the test added in 3291fe4072e (Add git-describe test for "verify annotated tag names on output", 2008-03-03) to make checking that we don't have warnings the rule rather than the exception would be the sort of thing the describe tests should be catching, but for now let's leave it as it is. 1. http://lore.kernel.org/git/xmqqwnuqo8ze.fsf@gitster.c.googlers.com Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t6120-describe.sh | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index 13117bbcfb..911b192805 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -21,7 +21,7 @@ check_describe () { shift describe_opts="$@" test_expect_success "describe $describe_opts" ' - git describe $describe_opts 2>err.actual >raw && + git describe $describe_opts >raw && sed -e "s/-g[0-9a-f]*\$/-gHASH/" actual && echo "$expect" >expect && test_cmp expect actual @@ -90,20 +90,17 @@ test_expect_success 'describe --contains defaults to HEAD without commit-ish' ' ' check_describe tags/A --all A^0 -test_expect_success 'no warning was displayed for A' ' - test_must_be_empty err.actual -' -test_expect_success 'rename tag A to Q locally' ' - mv .git/refs/tags/A .git/refs/tags/Q -' -cat - >err.expect <err >out && + cat >expected <<-\EOF && + warning: tag 'Q' is externally known as 'A' + EOF + test_cmp expected err && + grep -E '^A-8-g[0-9a-f]+$' out +" + test_expect_success 'misnamed annotated tag forces long output' ' description=$(git describe --no-long Q^0) && expr "$description" : "A-0-g[0-9a-f]*$" && From 33b4ae11146ba0c468176d238d231e1101ba19cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 13:21:46 +0200 Subject: [PATCH 042/397] describe tests: fix nested "test_expect_success" call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a nested invocation of "test_expect_success", the "check_describe()" function is a wrapper for calling test_expect_success, and therefore needs to be called outside the body of another "test_expect_success". The two tests added in 30b1c7ad9d6 (describe: don't abort too early when searching tags, 2020-02-26) were not testing for anything due to this logic error. Without this fix reverting the C code changes in that commit still has all tests passing, with this fix we're actually testing the "describe" output. This is because "test_expect_success" calls "test_finish_", whose last statement happens to be true. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t6120-describe.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index 911b192805..9dc07782ea 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -469,7 +469,7 @@ test_expect_success 'name-rev covers all conditions while looking at parents' ' # o-----o---o----x # A # -test_expect_success 'describe commits with disjoint bases' ' +test_expect_success 'setup: describe commits with disjoint bases' ' git init disjoint1 && ( cd disjoint1 && @@ -482,19 +482,22 @@ test_expect_success 'describe commits with disjoint bases' ' git checkout --orphan branch && rm file && echo B > file2 && git add file2 && git commit -m B && git tag B -a -m B && - git merge --no-ff --allow-unrelated-histories main -m x && - - check_describe "A-3-gHASH" HEAD + git merge --no-ff --allow-unrelated-histories main -m x ) ' +( + cd disjoint1 && + check_describe "A-3-gHASH" HEAD +) + # B # o---o---o------------. # \ # o---o---x # A # -test_expect_success 'describe commits with disjoint bases 2' ' +test_expect_success 'setup: describe commits with disjoint bases 2' ' git init disjoint2 && ( cd disjoint2 && @@ -508,10 +511,13 @@ test_expect_success 'describe commits with disjoint bases 2' ' echo o >> file2 && git add file2 && GIT_COMMITTER_DATE="2020-01-01 15:01" git commit -m o && echo B >> file2 && git add file2 && GIT_COMMITTER_DATE="2020-01-01 15:02" git commit -m B && git tag B -a -m B && - git merge --no-ff --allow-unrelated-histories main -m x && - - check_describe "B-3-gHASH" HEAD + git merge --no-ff --allow-unrelated-histories main -m x ) ' +( + cd disjoint2 && + check_describe "B-3-gHASH" HEAD +) + test_done From 64568c717159b31b5a37d2c1bb284c71e6cd011f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 12 Apr 2021 13:21:47 +0200 Subject: [PATCH 043/397] describe tests: support -C in "check_describe" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change a subshell added in a preceding commit to instead use a new "-C" option to "check_describe". The idiom for this is copied as-is from the "test_commit" function in test-lib-functions.sh Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t6120-describe.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/t/t6120-describe.sh b/t/t6120-describe.sh index 9dc07782ea..1a501ee09e 100755 --- a/t/t6120-describe.sh +++ b/t/t6120-describe.sh @@ -17,11 +17,26 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh check_describe () { + indir= && + while test $# != 0 + do + case "$1" in + -C) + indir="$2" + shift + ;; + *) + break + ;; + esac + shift + done && + indir=${indir:+"$indir"/} && expect="$1" shift describe_opts="$@" test_expect_success "describe $describe_opts" ' - git describe $describe_opts >raw && + git ${indir:+ -C "$indir"} describe $describe_opts >raw && sed -e "s/-g[0-9a-f]*\$/-gHASH/" actual && echo "$expect" >expect && test_cmp expect actual @@ -486,10 +501,7 @@ test_expect_success 'setup: describe commits with disjoint bases' ' ) ' -( - cd disjoint1 && - check_describe "A-3-gHASH" HEAD -) +check_describe -C disjoint1 "A-3-gHASH" HEAD # B # o---o---o------------. @@ -515,9 +527,6 @@ test_expect_success 'setup: describe commits with disjoint bases 2' ' ) ' -( - cd disjoint2 && - check_describe "B-3-gHASH" HEAD -) +check_describe -C disjoint2 "B-3-gHASH" HEAD test_done From 1e1c4c5eac972516db47394ae94a52a3b97cb41a Mon Sep 17 00:00:00 2001 From: ZheNing Hu Date: Tue, 11 May 2021 15:35:20 +0000 Subject: [PATCH 044/397] ref-filter: fix read invalid union member bug used_atom.u is an union, and it has different members depending on what atom the auxiliary data the union part of the "struct used_atom" wants to record. At most only one of the members can be valid at any one time. Since the code checks u.remote_ref without even making sure if the atom is "push" or "push:" (which are only two cases that u.remote_ref.push becomes valid), but u.remote_ref shares the same storage for other members of the union, the check was reading from an invalid member, which was the bug. Modify the condition here to check whether the atom name equals to "push" or starts with "push:", to avoid reading the value of invalid member of the union. Signed-off-by: ZheNing Hu [jc: further test fixes] Signed-off-by: Junio C Hamano --- ref-filter.c | 2 +- t/t6302-for-each-ref-filter.sh | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ref-filter.c b/ref-filter.c index bf7b70299b..a6c0804f40 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -1664,7 +1664,7 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) else v->s = xstrdup(""); continue; - } else if (atom->u.remote_ref.push) { + } else if (!strcmp(atom->name, "push") || starts_with(atom->name, "push:")) { const char *branch_name; v->s = xstrdup(""); if (!skip_prefix(ref->refname, "refs/heads/", diff --git a/t/t6302-for-each-ref-filter.sh b/t/t6302-for-each-ref-filter.sh index 35408d53fd..529ca556c8 100755 --- a/t/t6302-for-each-ref-filter.sh +++ b/t/t6302-for-each-ref-filter.sh @@ -113,6 +113,25 @@ test_expect_success '%(color) must fail' ' test_must_fail git for-each-ref --format="%(color)%(refname)" ' +test_expect_success '%(color:#aa22ac) must succeed' ' + test_when_finished rm -rf test && + git init test && + ( + cd test && + test_commit initial && + git branch -M main && + cat >expect <<-\EOF && + refs/heads/main + refs/tags/initial + EOF + git remote add origin nowhere && + git config branch.main.remote origin && + git config branch.main.merge refs/heads/main && + git for-each-ref --format="%(color:#aa22ac)%(refname)" >actual && + test_cmp expect actual + ) +' + test_expect_success 'left alignment is default' ' cat >expect <<-\EOF && refname is refs/heads/master |refs/heads/master From 00bc8390d8cd764c52f0308b303a270e9bf57d84 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 11 May 2021 10:37:30 +0000 Subject: [PATCH 045/397] remote-curl: fix clone on sha256 repos The remote-https process needs to update it's own instance of `the_repository' when it sees an HTTP(S) remote is using sha256. Without this, parse_oid_hex() fails to handle sha256 OIDs when it's eventually called by parse_fetch(). Tested with: git clone https://yhbt.net/sha256test.git GIT_SMART_HTTP=0 git clone https://yhbt.net/sha256test.git (plain http:// also works) Cloning the URL via git:// required no changes Signed-off-by: Eric Wong Acked-by: brian m. carlson Signed-off-by: Junio C Hamano --- remote-curl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/remote-curl.c b/remote-curl.c index 0290b04891..9d432c299a 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -555,6 +555,8 @@ static void output_refs(struct ref *refs) struct ref *posn; if (options.object_format && options.hash_algo) { printf(":object-format %s\n", options.hash_algo->name); + repo_set_hash_algo(the_repository, + hash_algo_by_ptr(options.hash_algo)); } for (posn = refs; posn; posn = posn->next) { if (posn->symref) From 8c55753c68e9c3e56f7a0d96301d2211d9e4ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 12 May 2021 11:49:44 +0200 Subject: [PATCH 046/397] Makefile: make PERL_DEFINES recursively expanded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since 07d90eadb50 (Makefile: add Perl runtime prefix support, 2018-04-10) PERL_DEFINES has been a simply-expanded variable, let's make it recursively expanded instead. This change doesn't matter for the correctness of the logic. Whether we used simply-expanded or recursively expanded didn't change what we wrote out in GIT-PERL-DEFINES, but being consistent with other rules makes this easier to understand. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b8d6b31305..0705fc2d3f 100644 --- a/Makefile +++ b/Makefile @@ -2271,7 +2271,7 @@ perl_localedir_SQ = $(localedir_SQ) ifndef NO_PERL PERL_HEADER_TEMPLATE = perl/header_templates/fixed_prefix.template.pl -PERL_DEFINES := +PERL_DEFINES = PERL_DEFINES += $(PERL_PATH_SQ) PERL_DEFINES += $(PERLLIB_EXTRA_SQ) PERL_DEFINES += $(perllibdir_SQ) From 7f9dd87922c9065c8adcea1469c3caf3b3af2afa Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:14 +0000 Subject: [PATCH 047/397] dir: convert trace calls to trace2 equivalents Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- dir.c | 50 ++++++-- t/t7063-status-untracked-cache.sh | 205 ++++++++++++++++++------------ t/t7519-status-fsmonitor.sh | 8 +- 3 files changed, 162 insertions(+), 101 deletions(-) diff --git a/dir.c b/dir.c index 19c2fa239b..b8dfe2e413 100644 --- a/dir.c +++ b/dir.c @@ -2751,15 +2751,46 @@ static struct untracked_cache_dir *validate_untracked_cache(struct dir_struct *d return root; } +static void emit_traversal_statistics(struct dir_struct *dir, + struct repository *repo, + const char *path, + int path_len) +{ + if (!trace2_is_enabled()) + return; + + if (!path_len) { + trace2_data_string("read_directory", repo, "path", ""); + } else { + struct strbuf tmp = STRBUF_INIT; + strbuf_add(&tmp, path, path_len); + trace2_data_string("read_directory", repo, "path", tmp.buf); + strbuf_release(&tmp); + } + + if (!dir->untracked) + return; + trace2_data_intmax("read_directory", repo, + "node-creation", dir->untracked->dir_created); + trace2_data_intmax("read_directory", repo, + "gitignore-invalidation", + dir->untracked->gitignore_invalidated); + trace2_data_intmax("read_directory", repo, + "directory-invalidation", + dir->untracked->dir_invalidated); + trace2_data_intmax("read_directory", repo, + "opendir", dir->untracked->dir_opened); +} + int read_directory(struct dir_struct *dir, struct index_state *istate, const char *path, int len, const struct pathspec *pathspec) { struct untracked_cache_dir *untracked; - trace_performance_enter(); + trace2_region_enter("dir", "read_directory", istate->repo); if (has_symlink_leading_path(path, len)) { - trace_performance_leave("read directory %.*s", len, path); + trace2_region_leave("dir", "read_directory", istate->repo); return dir->nr; } @@ -2775,23 +2806,15 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, QSORT(dir->entries, dir->nr, cmp_dir_entry); QSORT(dir->ignored, dir->ignored_nr, cmp_dir_entry); - trace_performance_leave("read directory %.*s", len, path); + emit_traversal_statistics(dir, istate->repo, path, len); + + trace2_region_leave("dir", "read_directory", istate->repo); if (dir->untracked) { static int force_untracked_cache = -1; - static struct trace_key trace_untracked_stats = TRACE_KEY_INIT(UNTRACKED_STATS); if (force_untracked_cache < 0) force_untracked_cache = git_env_bool("GIT_FORCE_UNTRACKED_CACHE", 0); - trace_printf_key(&trace_untracked_stats, - "node creation: %u\n" - "gitignore invalidation: %u\n" - "directory invalidation: %u\n" - "opendir: %u\n", - dir->untracked->dir_created, - dir->untracked->gitignore_invalidated, - dir->untracked->dir_invalidated, - dir->untracked->dir_opened); if (force_untracked_cache && dir->untracked == istate->untracked && (dir->untracked->dir_opened || @@ -2802,6 +2825,7 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, FREE_AND_NULL(dir->untracked); } } + return dir->nr; } diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index accefde72f..9710d33b3c 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -57,6 +57,19 @@ iuc () { return $ret } +get_relevant_traces () { + # From the GIT_TRACE2_PERF data of the form + # $TIME $FILE:$LINE | d0 | main | data | r1 | ? | ? | read_directo | $RELEVANT_STAT + # extract the $RELEVANT_STAT fields. We don't care about region_enter + # or region_leave, or stats for things outside read_directory. + INPUT_FILE=$1 + OUTPUT_FILE=$2 + grep data.*read_directo $INPUT_FILE | + cut -d "|" -f 9 \ + >"$OUTPUT_FILE" +} + + test_lazy_prereq UNTRACKED_CACHE ' { git update-index --test-untracked-cache; ret=$?; } && test $ret -ne 1 @@ -129,19 +142,21 @@ EOF test_expect_success 'status first time (empty cache)' ' avoid_racy && - : >../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && test_cmp ../status.expect ../status.iuc && test_cmp ../status.expect ../actual && + get_relevant_traces ../trace.output ../trace.relevant && cat >../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && test_cmp ../status.expect ../status.iuc && test_cmp ../status.expect ../actual && + get_relevant_traces ../trace.output ../trace.relevant && cat >../trace.expect <four && - : >../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <.gitignore && - : >../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <>.git/info/exclude && - : >../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && + : >../trace.output && avoid_racy && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../status.actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../status.actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../status.actual && iuc status --porcelain >../status.iuc && cat >../status.expect <../trace.expect <../trace && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace" \ + : >../trace.output && + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ git status --porcelain >../status.actual && iuc status --porcelain >../status.iuc && test_cmp ../status.expect ../status.iuc && test_cmp ../status.expect ../status.actual && + get_relevant_traces ../trace.output ../trace.relevant && cat >../trace.expect <../before ) && @@ -346,12 +346,12 @@ test_expect_success UNTRACKED_CACHE 'ignore .git changes when invalidating UNTR' EOF ( cd dot-git && - GIT_TRACE_UNTRACKED_STATS="$TRASH_DIRECTORY/trace-after" \ + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace-after" \ git status && test-tool dump-untracked-cache >../after ) && - grep "directory invalidation" trace-before >>before && - grep "directory invalidation" trace-after >>after && + grep "directory-invalidation" trace-before | cut -d"|" -f 9 >>before && + grep "directory-invalidation" trace-after | cut -d"|" -f 9 >>after && # UNTR extension unchanged, dir invalidation count unchanged test_cmp before after ' From 7fe1ffdafa56b8453a47a40b866d029f24a56d76 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:15 +0000 Subject: [PATCH 048/397] dir: report number of visited directories and paths with trace2 Provide more statistics in trace2 output that include the number of directories and total paths visited by the directory traversal logic. Subsequent patches will take advantage of this to ensure we do not unnecessarily traverse into ignored directories. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- dir.c | 9 +++++++++ dir.h | 4 ++++ t/t7063-status-untracked-cache.sh | 3 ++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dir.c b/dir.c index b8dfe2e413..151cc37e23 100644 --- a/dir.c +++ b/dir.c @@ -2431,6 +2431,7 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, if (open_cached_dir(&cdir, dir, untracked, istate, &path, check_only)) goto out; + dir->visited_directories++; if (untracked) untracked->check_only = !!check_only; @@ -2439,6 +2440,7 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir, /* check how the file or directory should be treated */ state = treat_path(dir, untracked, &cdir, istate, &path, baselen, pathspec); + dir->visited_paths++; if (state > dir_state) dir_state = state; @@ -2768,6 +2770,11 @@ static void emit_traversal_statistics(struct dir_struct *dir, strbuf_release(&tmp); } + trace2_data_intmax("read_directory", repo, + "directories-visited", dir->visited_directories); + trace2_data_intmax("read_directory", repo, + "paths-visited", dir->visited_paths); + if (!dir->untracked) return; trace2_data_intmax("read_directory", repo, @@ -2788,6 +2795,8 @@ int read_directory(struct dir_struct *dir, struct index_state *istate, struct untracked_cache_dir *untracked; trace2_region_enter("dir", "read_directory", istate->repo); + dir->visited_paths = 0; + dir->visited_directories = 0; if (has_symlink_leading_path(path, len)) { trace2_region_leave("dir", "read_directory", istate->repo); diff --git a/dir.h b/dir.h index facfae4740..70c750e305 100644 --- a/dir.h +++ b/dir.h @@ -336,6 +336,10 @@ struct dir_struct { struct oid_stat ss_info_exclude; struct oid_stat ss_excludes_file; unsigned unmanaged_exclude_files; + + /* Stats about the traversal */ + unsigned visited_paths; + unsigned visited_directories; }; /*Count the number of slashes for string s*/ diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 9710d33b3c..a0c123b0a7 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -65,7 +65,8 @@ get_relevant_traces () { INPUT_FILE=$1 OUTPUT_FILE=$2 grep data.*read_directo $INPUT_FILE | - cut -d "|" -f 9 \ + cut -d "|" -f 9 | + grep -v visited \ >"$OUTPUT_FILE" } From b338e9f668737e08201c990450b8c3d744f63162 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:16 +0000 Subject: [PATCH 049/397] ls-files: error out on -i unless -o or -c are specified ls-files --ignored can be used together with either --others or --cached. After being perplexed for a bit and digging in to the code, I assumed that ls-files -i was just broken and not printing anything and I had a nice patch ready to submit when I finally realized that -i can be used with --cached to find tracked ignores. While that was a mistake on my part, and a careful reading of the documentation could have made this more clear, I suspect this is an error others are likely to make as well. In fact, of two uses in our testsuite, I believe one of the two did make this error. In t1306.13, there are NO tracked files, and all the excludes built up and used in that test and in previous tests thus have to be about untracked files. However, since they were looking for an empty result, the mistake went unnoticed as their erroneous command also just happened to give an empty answer. -i will most the time be used with -o, which would suggest we could just make -i imply -o in the absence of either a -o or -c, but that would be a backward incompatible break. Instead, let's just flag -i without either a -o or -c as an error, and update the two relevant testcases to specify their intent. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- builtin/ls-files.c | 3 +++ t/t1306-xdg-files.sh | 2 +- t/t3003-ls-files-exclude.sh | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/builtin/ls-files.c b/builtin/ls-files.c index f6f9e483b2..cd01404417 100644 --- a/builtin/ls-files.c +++ b/builtin/ls-files.c @@ -675,6 +675,9 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix) if (pathspec.nr && error_unmatch) ps_matched = xcalloc(pathspec.nr, 1); + if ((dir.flags & DIR_SHOW_IGNORED) && !show_others && !show_cached) + die("ls-files -i must be used with either -o or -c"); + if ((dir.flags & DIR_SHOW_IGNORED) && !exc_given) die("ls-files --ignored needs some exclude pattern"); diff --git a/t/t1306-xdg-files.sh b/t/t1306-xdg-files.sh index dd87b43be1..40d3c42618 100755 --- a/t/t1306-xdg-files.sh +++ b/t/t1306-xdg-files.sh @@ -116,7 +116,7 @@ test_expect_success 'Exclusion in a non-XDG global ignore file' ' test_expect_success 'Checking XDG ignore file when HOME is unset' ' (sane_unset HOME && git config --unset core.excludesfile && - git ls-files --exclude-standard --ignored >actual) && + git ls-files --exclude-standard --ignored --others >actual) && test_must_be_empty actual ' diff --git a/t/t3003-ls-files-exclude.sh b/t/t3003-ls-files-exclude.sh index d5ec333131..c41c4f046a 100755 --- a/t/t3003-ls-files-exclude.sh +++ b/t/t3003-ls-files-exclude.sh @@ -29,11 +29,11 @@ test_expect_success 'add file to gitignore' ' ' check_all_output -test_expect_success 'ls-files -i lists only tracked-but-ignored files' ' +test_expect_success 'ls-files -i -c lists only tracked-but-ignored files' ' echo content >other-file && git add other-file && echo file >expect && - git ls-files -i --exclude-standard >output && + git ls-files -i -c --exclude-standard >output && test_cmp expect output ' From 2e4e43a6910393d681d095f515d41232c2372966 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:17 +0000 Subject: [PATCH 050/397] t7300: add testcase showing unnecessary traversal into ignored directory The PNPM package manager is apparently creating deeply nested (but ignored) directory structures; traversing them is costly performance-wise, unnecessary, and in some cases is even throwing warnings/errors because the paths are too long to handle on various platforms. Add a testcase that checks for such unnecessary directory traversal. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- t/t7300-clean.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index a74816ca8b..07e8ba2d4b 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -746,4 +746,27 @@ test_expect_success 'clean untracked paths by pathspec' ' test_must_be_empty actual ' +test_expect_failure 'avoid traversing into ignored directories' ' + test_when_finished rm -f output error trace.* && + test_create_repo avoid-traversing-deep-hierarchy && + ( + cd avoid-traversing-deep-hierarchy && + + mkdir -p untracked/subdir/with/a && + >untracked/subdir/with/a/random-file.txt && + + GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.output" \ + git clean -ffdxn -e untracked + ) && + + # Make sure we only visited into the top-level directory, and did + # not traverse into the "untracked" subdirectory since it was excluded + grep data.*read_directo.*directories-visited trace.output | + cut -d "|" -f 9 >trace.relevant && + cat >trace.expect <<-EOF && + ..directories-visited:1 + EOF + test_cmp trace.expect trace.relevant +' + test_done From a97c7a8bc4f2068b2e8c23ce8d17f7db8333ded0 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:18 +0000 Subject: [PATCH 051/397] t3001, t7300: add testcase showcasing missed directory traversal In the last commit, we added a testcase showing that the directory traversal machinery sometimes traverses into directories unnecessarily. Here we show that there are cases where it does the opposite: it does not traverse into directories, despite those directories having important files that need to be flagged. Add a testcase showing that `git ls-files -o -i --directory` can omit some of the files it should be listing, and another showing that `git clean -fX` can fail to clean out some of the expected files. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- t/t3001-ls-files-others-exclude.sh | 5 +++++ t/t7300-clean.sh | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh index 1ec7cb57c7..ac05d1a179 100755 --- a/t/t3001-ls-files-others-exclude.sh +++ b/t/t3001-ls-files-others-exclude.sh @@ -292,6 +292,11 @@ EOF test_cmp expect actual ' +test_expect_failure 'ls-files with "**" patterns and --directory' ' + # Expectation same as previous test + git ls-files --directory -o -i --exclude "**/a.1" >actual && + test_cmp expect actual +' test_expect_success 'ls-files with "**" patterns and no slashes' ' git ls-files -o -i --exclude "one**a.1" >actual && diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index 07e8ba2d4b..34c08c3254 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -769,4 +769,23 @@ test_expect_failure 'avoid traversing into ignored directories' ' test_cmp trace.expect trace.relevant ' +test_expect_failure 'traverse into directories that may have ignored entries' ' + test_when_finished rm -f output && + test_create_repo need-to-traverse-into-hierarchy && + ( + cd need-to-traverse-into-hierarchy && + mkdir -p modules/foobar/src/generated && + > modules/foobar/src/generated/code.c && + > modules/foobar/Makefile && + echo "/modules/**/src/generated/" >.gitignore && + + git clean -fX modules/foobar >../output && + + grep Removing ../output && + + test_path_is_missing modules/foobar/src/generated/code.c && + test_path_is_file modules/foobar/Makefile + ) +' + test_done From aa6e1b21e5de8fae7121b8c6543e1482144c1ed3 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:19 +0000 Subject: [PATCH 052/397] dir: avoid unnecessary traversal into ignored directory The show_other_directories case in treat_directory() tried to handle both excludes and untracked files with the same logic, and mishandled both the excludes and the untracked files in the process, in different ways. Split that logic apart, and then focus on the logic for the excludes; a subsequent commit will address the logic for untracked files. For show_other_directories, an excluded directory means that every path underneath that directory will also be excluded. Given that the calling code requested to just show directories when everything under a directory had the same state (that's what the "DIR_SHOW_OTHER_DIRECTORIES" flag means), we generally do not need to traverse into such directories and can just immediately mark them as ignored (i.e. as path_excluded). The only reason we cannot just immediately return path_excluded is the DIR_HIDE_EMPTY_DIRECTORIES flag and the possibility that the ignored directory is an empty directory. The code previously treated DIR_SHOW_IGNORED_TOO in most cases as an exception as well, which was wrong. It can sometimes reduce the number of cases where we need to recurse (namely if DIR_SHOW_IGNORED_TOO_MODE_MATCHING is also set), but should not be able to increase the number of cases where we need to recurse. Fix the logic accordingly. Some sidenotes about possible confusion with dir.c: * "ignored" often refers to an untracked ignore", i.e. a file which is not tracked which matches one of the ignore/exclusion rules. But you can also have a "tracked ignore", a tracked file that happens to match one of the ignore/exclusion rules and which dir.c has to worry about since "git ls-files -c -i" is supposed to list them. * The dir code often uses "ignored" and "excluded" interchangeably, which you need to keep in mind while reading the code. * "exclude" is used multiple ways in the code: * As noted above, "exclude" is often a synonym for "ignored". * The logic for parsing .gitignore files was re-used in .git/info/sparse-checkout, except there it is used to mark paths that the user wants to *keep*. This was mostly addressed by commit 65edd96aec ("treewide: rename 'exclude' methods to 'pattern'", 2019-09-03), but every once in a while you'll find a comment about "exclude" referring to these patterns that might in fact be in use by the sparse-checkout machinery for inclusion rules. * The word "EXCLUDE" is also used for pathspec negation, as in (pathspec->items[3].magic & PATHSPEC_EXCLUDE) Thus if a user had a .gitignore file containing *~ *.log !settings.log And then ran git add -- 'settings.*' ':^settings.log' Then :^settings.log is a pathspec negation making settings.log not be requested to be added even though all other settings.* files are being added. Also, !settings.log in the gitignore file is a negative exclude pattern meaning that settings.log is normally a file we want to track even though all other *.log files are ignored. Sometimes it feels like dir.c needs its own glossary with its many definitions, including the multiply-defined terms. Reported-by: Jason Gore Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- dir.c | 44 +++++++++++++++++++++++++++++--------------- t/t7300-clean.sh | 2 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/dir.c b/dir.c index 151cc37e23..66261cf98b 100644 --- a/dir.c +++ b/dir.c @@ -1835,6 +1835,7 @@ static enum path_treatment treat_directory(struct dir_struct *dir, } /* This is the "show_other_directories" case */ + assert(dir->flags & DIR_SHOW_OTHER_DIRECTORIES); /* * If we have a pathspec which could match something _below_ this @@ -1845,27 +1846,40 @@ static enum path_treatment treat_directory(struct dir_struct *dir, if (matches_how == MATCHED_RECURSIVELY_LEADING_PATHSPEC) return path_recurse; + /* Special cases for where this directory is excluded/ignored */ + if (excluded) { + /* + * In the show_other_directories case, if we're not + * hiding empty directories, there is no need to + * recurse into an ignored directory. + */ + if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES)) + return path_excluded; + + /* + * Even if we are hiding empty directories, we can still avoid + * recursing into ignored directories for DIR_SHOW_IGNORED_TOO + * if DIR_SHOW_IGNORED_TOO_MODE_MATCHING is also set. + */ + if ((dir->flags & DIR_SHOW_IGNORED_TOO) && + (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING)) + return path_excluded; + } + /* - * Other than the path_recurse case immediately above, we only need - * to recurse into untracked/ignored directories if either of the - * following bits is set: + * Other than the path_recurse case above, we only need to + * recurse into untracked directories if either of the following + * bits is set: * - DIR_SHOW_IGNORED_TOO (because then we need to determine if * there are ignored entries below) * - DIR_HIDE_EMPTY_DIRECTORIES (because we have to determine if * the directory is empty) */ - if (!(dir->flags & (DIR_SHOW_IGNORED_TOO | DIR_HIDE_EMPTY_DIRECTORIES))) - return excluded ? path_excluded : path_untracked; - - /* - * ...and even if DIR_SHOW_IGNORED_TOO is set, we can still avoid - * recursing into ignored directories if the path is excluded and - * DIR_SHOW_IGNORED_TOO_MODE_MATCHING is also set. - */ - if (excluded && - (dir->flags & DIR_SHOW_IGNORED_TOO) && - (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING)) - return path_excluded; + if (!excluded && + !(dir->flags & (DIR_SHOW_IGNORED_TOO | + DIR_HIDE_EMPTY_DIRECTORIES))) { + return path_untracked; + } /* * Even if we don't want to know all the paths under an untracked or diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index 34c08c3254..21e48b3ba5 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -746,7 +746,7 @@ test_expect_success 'clean untracked paths by pathspec' ' test_must_be_empty actual ' -test_expect_failure 'avoid traversing into ignored directories' ' +test_expect_success 'avoid traversing into ignored directories' ' test_when_finished rm -f output error trace.* && test_create_repo avoid-traversing-deep-hierarchy && ( From dd55fc0df15c338a8dbec6dec6ca6b58edc8acef Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:20 +0000 Subject: [PATCH 053/397] dir: traverse into untracked directories if they may have ignored subfiles A directory that is untracked does not imply that all files under it should be categorized as untracked; in particular, if the caller is interested in ignored files, many files or directories underneath the untracked directory may be ignored. We previously partially handled this right with DIR_SHOW_IGNORED_TOO, but missed DIR_SHOW_IGNORED. It was not obvious, though, because the logic for untracked and excluded files had been fused together making it harder to reason about. The previous commit split that logic out, making it easier to notice that DIR_SHOW_IGNORED was missing. Add it. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- dir.c | 10 ++++++---- t/t3001-ls-files-others-exclude.sh | 2 +- t/t7300-clean.sh | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dir.c b/dir.c index 66261cf98b..ed68b7e641 100644 --- a/dir.c +++ b/dir.c @@ -1868,15 +1868,17 @@ static enum path_treatment treat_directory(struct dir_struct *dir, /* * Other than the path_recurse case above, we only need to - * recurse into untracked directories if either of the following + * recurse into untracked directories if any of the following * bits is set: - * - DIR_SHOW_IGNORED_TOO (because then we need to determine if - * there are ignored entries below) + * - DIR_SHOW_IGNORED (because then we need to determine if + * there are ignored entries below) + * - DIR_SHOW_IGNORED_TOO (same as above) * - DIR_HIDE_EMPTY_DIRECTORIES (because we have to determine if * the directory is empty) */ if (!excluded && - !(dir->flags & (DIR_SHOW_IGNORED_TOO | + !(dir->flags & (DIR_SHOW_IGNORED | + DIR_SHOW_IGNORED_TOO | DIR_HIDE_EMPTY_DIRECTORIES))) { return path_untracked; } diff --git a/t/t3001-ls-files-others-exclude.sh b/t/t3001-ls-files-others-exclude.sh index ac05d1a179..516c95ea0e 100755 --- a/t/t3001-ls-files-others-exclude.sh +++ b/t/t3001-ls-files-others-exclude.sh @@ -292,7 +292,7 @@ EOF test_cmp expect actual ' -test_expect_failure 'ls-files with "**" patterns and --directory' ' +test_expect_success 'ls-files with "**" patterns and --directory' ' # Expectation same as previous test git ls-files --directory -o -i --exclude "**/a.1" >actual && test_cmp expect actual diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index 21e48b3ba5..0399701e62 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -769,7 +769,7 @@ test_expect_success 'avoid traversing into ignored directories' ' test_cmp trace.expect trace.relevant ' -test_expect_failure 'traverse into directories that may have ignored entries' ' +test_expect_success 'traverse into directories that may have ignored entries' ' test_when_finished rm -f output && test_create_repo need-to-traverse-into-hierarchy && ( From 4e689d81718eb6e939cace317ea3e33cb994dcbb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 May 2021 17:28:21 +0000 Subject: [PATCH 054/397] dir: update stale description of treat_directory() The documentation comment for treat_directory() was originally written in 095952 (Teach directory traversal about subprojects, 2007-04-11) which was before the 'struct dir_struct' split its bitfield of named options into a 'flags' enum in 7c4c97c0 (Turn the flags in struct dir_struct into a single variable, 2009-02-16). When those flags changed, the comment became stale, since members like 'show_other_directories' transitioned into flags like DIR_SHOW_OTHER_DIRECTORIES. Update the comments for treat_directory() to use these flag names rather than the old member names. Signed-off-by: Derrick Stolee Reviewed-by: Elijah Newren Signed-off-by: Junio C Hamano --- dir.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dir.c b/dir.c index ed68b7e641..ff004b298b 100644 --- a/dir.c +++ b/dir.c @@ -1740,13 +1740,13 @@ static enum exist_status directory_exists_in_index(struct index_state *istate, * Case 3: if we didn't have it in the index previously, we * have a few sub-cases: * - * (a) if "show_other_directories" is true, we show it as - * just a directory, unless "hide_empty_directories" is + * (a) if DIR_SHOW_OTHER_DIRECTORIES flag is set, we show it as + * just a directory, unless DIR_HIDE_EMPTY_DIRECTORIES is * also true, in which case we need to check if it contains any * untracked and / or ignored files. - * (b) if it looks like a git directory, and we don't have - * 'no_gitlinks' set we treat it as a gitlink, and show it - * as a directory. + * (b) if it looks like a git directory and we don't have the + * DIR_NO_GITLINKS flag, then we treat it as a gitlink, and + * show it as a directory. * (c) otherwise, we recurse into it. */ static enum path_treatment treat_directory(struct dir_struct *dir, @@ -1834,7 +1834,6 @@ static enum path_treatment treat_directory(struct dir_struct *dir, return path_recurse; } - /* This is the "show_other_directories" case */ assert(dir->flags & DIR_SHOW_OTHER_DIRECTORIES); /* @@ -1849,7 +1848,7 @@ static enum path_treatment treat_directory(struct dir_struct *dir, /* Special cases for where this directory is excluded/ignored */ if (excluded) { /* - * In the show_other_directories case, if we're not + * If DIR_SHOW_OTHER_DIRECTORIES is set and we're not * hiding empty directories, there is no need to * recurse into an ignored directory. */ From b548f0f1568f6b01e55ca69c24d3cb19489f92aa Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Wed, 12 May 2021 17:28:22 +0000 Subject: [PATCH 055/397] dir: introduce readdir_skip_dot_and_dotdot() helper Many places in the code were doing while ((d = readdir(dir)) != NULL) { if (is_dot_or_dotdot(d->d_name)) continue; ...process d... } Introduce a readdir_skip_dot_and_dotdot() helper to make that a one-liner: while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { ...process d... } This helper particularly simplifies checks for empty directories. Also use this helper in read_cached_dir() so that our statistics are consistent across platforms. (In other words, read_cached_dir() should have been using is_dot_or_dotdot() and skipping such entries, but did not and left it to treat_path() to detect and mark such entries as path_none.) Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- builtin/clean.c | 4 +--- builtin/worktree.c | 4 +--- diff-no-index.c | 5 ++--- dir.c | 26 +++++++++++++++++--------- dir.h | 2 ++ entry.c | 5 +---- notes-merge.c | 5 +---- object-file.c | 4 +--- packfile.c | 5 +---- rerere.c | 4 +--- worktree.c | 12 +++--------- 11 files changed, 31 insertions(+), 45 deletions(-) diff --git a/builtin/clean.c b/builtin/clean.c index 995053b791..a1a5747615 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -189,10 +189,8 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, strbuf_complete(path, '/'); len = path->len; - while ((e = readdir(dir)) != NULL) { + while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; - if (is_dot_or_dotdot(e->d_name)) - continue; strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); diff --git a/builtin/worktree.c b/builtin/worktree.c index 1cd5c2016e..e081ca9bef 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -118,10 +118,8 @@ static void prune_worktrees(void) struct dirent *d; if (!dir) return; - while ((d = readdir(dir)) != NULL) { + while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { char *path; - if (is_dot_or_dotdot(d->d_name)) - continue; strbuf_reset(&reason); if (should_prune_worktree(d->d_name, &reason, &path, expire)) prune_worktree(d->d_name, reason.buf); diff --git a/diff-no-index.c b/diff-no-index.c index 7814eabfe0..e5cc878371 100644 --- a/diff-no-index.c +++ b/diff-no-index.c @@ -26,9 +26,8 @@ static int read_directory_contents(const char *path, struct string_list *list) if (!(dir = opendir(path))) return error("Could not open directory %s", path); - while ((e = readdir(dir))) - if (!is_dot_or_dotdot(e->d_name)) - string_list_insert(list, e->d_name); + while ((e = readdir_skip_dot_and_dotdot(dir))) + string_list_insert(list, e->d_name); closedir(dir); return 0; diff --git a/dir.c b/dir.c index ff004b298b..e47b4c507f 100644 --- a/dir.c +++ b/dir.c @@ -59,6 +59,18 @@ void dir_init(struct dir_struct *dir) memset(dir, 0, sizeof(*dir)); } +struct dirent * +readdir_skip_dot_and_dotdot(DIR *dirp) +{ + struct dirent *e; + + while ((e = readdir(dirp)) != NULL) { + if (!is_dot_or_dotdot(e->d_name)) + break; + } + return e; +} + int count_slashes(const char *s) { int cnt = 0; @@ -2332,7 +2344,7 @@ static int read_cached_dir(struct cached_dir *cdir) struct dirent *de; if (cdir->fdir) { - de = readdir(cdir->fdir); + de = readdir_skip_dot_and_dotdot(cdir->fdir); if (!de) { cdir->d_name = NULL; cdir->d_type = DT_UNKNOWN; @@ -2931,11 +2943,9 @@ int is_empty_dir(const char *path) if (!dir) return 0; - while ((e = readdir(dir)) != NULL) - if (!is_dot_or_dotdot(e->d_name)) { - ret = 0; - break; - } + e = readdir_skip_dot_and_dotdot(dir); + if (e) + ret = 0; closedir(dir); return ret; @@ -2975,10 +2985,8 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) strbuf_complete(path, '/'); len = path->len; - while ((e = readdir(dir)) != NULL) { + while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; - if (is_dot_or_dotdot(e->d_name)) - continue; strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); diff --git a/dir.h b/dir.h index 70c750e305..6b3fac0829 100644 --- a/dir.h +++ b/dir.h @@ -342,6 +342,8 @@ struct dir_struct { unsigned visited_directories; }; +struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp); + /*Count the number of slashes for string s*/ int count_slashes(const char *s); diff --git a/entry.c b/entry.c index 7b9f43716f..e3d3add300 100644 --- a/entry.c +++ b/entry.c @@ -56,12 +56,9 @@ static void remove_subtree(struct strbuf *path) if (!dir) die_errno("cannot opendir '%s'", path->buf); - while ((de = readdir(dir)) != NULL) { + while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; - if (is_dot_or_dotdot(de->d_name)) - continue; - strbuf_addch(path, '/'); strbuf_addstr(path, de->d_name); if (lstat(path->buf, &st)) diff --git a/notes-merge.c b/notes-merge.c index d2771fa3d4..e9d6f86d34 100644 --- a/notes-merge.c +++ b/notes-merge.c @@ -695,13 +695,10 @@ int notes_merge_commit(struct notes_merge_options *o, strbuf_addch(&path, '/'); baselen = path.len; - while ((e = readdir(dir)) != NULL) { + while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; struct object_id obj_oid, blob_oid; - if (is_dot_or_dotdot(e->d_name)) - continue; - if (get_oid_hex(e->d_name, &obj_oid)) { if (o->verbosity >= 3) printf("Skipping non-SHA1 entry '%s%s'\n", diff --git a/object-file.c b/object-file.c index 624af408cd..77bdcfd21b 100644 --- a/object-file.c +++ b/object-file.c @@ -2304,10 +2304,8 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr, strbuf_addch(path, '/'); baselen = path->len; - while ((de = readdir(dir))) { + while ((de = readdir_skip_dot_and_dotdot(dir))) { size_t namelen; - if (is_dot_or_dotdot(de->d_name)) - continue; namelen = strlen(de->d_name); strbuf_setlen(path, baselen); diff --git a/packfile.c b/packfile.c index ea29f4ba77..463d61c877 100644 --- a/packfile.c +++ b/packfile.c @@ -813,10 +813,7 @@ void for_each_file_in_pack_dir(const char *objdir, } strbuf_addch(&path, '/'); dirnamelen = path.len; - while ((de = readdir(dir)) != NULL) { - if (is_dot_or_dotdot(de->d_name)) - continue; - + while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) { strbuf_setlen(&path, dirnamelen); strbuf_addstr(&path, de->d_name); diff --git a/rerere.c b/rerere.c index dee60dc6df..d83d58df4f 100644 --- a/rerere.c +++ b/rerere.c @@ -1190,13 +1190,11 @@ void rerere_gc(struct repository *r, struct string_list *rr) if (!dir) die_errno(_("unable to open rr-cache directory")); /* Collect stale conflict IDs ... */ - while ((e = readdir(dir))) { + while ((e = readdir_skip_dot_and_dotdot(dir))) { struct rerere_dir *rr_dir; struct rerere_id id; int now_empty; - if (is_dot_or_dotdot(e->d_name)) - continue; if (!is_rr_cache_dirname(e->d_name)) continue; /* or should we remove e->d_name? */ diff --git a/worktree.c b/worktree.c index f35ac40a84..237517baee 100644 --- a/worktree.c +++ b/worktree.c @@ -128,10 +128,8 @@ struct worktree **get_worktrees(void) dir = opendir(path.buf); strbuf_release(&path); if (dir) { - while ((d = readdir(dir)) != NULL) { + while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct worktree *linked = NULL; - if (is_dot_or_dotdot(d->d_name)) - continue; if ((linked = get_linked_worktree(d->d_name))) { ALLOC_GROW(list, counter + 1, alloc); @@ -486,13 +484,9 @@ int submodule_uses_worktrees(const char *path) if (!dir) return 0; - while ((d = readdir(dir)) != NULL) { - if (is_dot_or_dotdot(d->d_name)) - continue; - + d = readdir_skip_dot_and_dotdot(dir); + if (d != NULL) ret = 1; - break; - } closedir(dir); return ret; } From 5c0cbdb107c5e1c1474618535fda438f25d260ca Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 13 May 2021 06:22:36 +0000 Subject: [PATCH 056/397] git-prompt: work under set -u Commit afda36dbf3 ("git-prompt: include sparsity state as well", 2020-06-21) added the use of some variables to control how to show sparsity state in the git prompt, but implicitly assumed that undefined variables would be treated as the empty string. This breaks users who run under 'set -u'; fix the code to be more explicit. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- contrib/completion/git-prompt.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/completion/git-prompt.sh b/contrib/completion/git-prompt.sh index 4640a1535d..db7c0068fb 100644 --- a/contrib/completion/git-prompt.sh +++ b/contrib/completion/git-prompt.sh @@ -433,8 +433,8 @@ __git_ps1 () fi local sparse="" - if [ -z "${GIT_PS1_COMPRESSSPARSESTATE}" ] && - [ -z "${GIT_PS1_OMITSPARSESTATE}" ] && + if [ -z "${GIT_PS1_COMPRESSSPARSESTATE-}" ] && + [ -z "${GIT_PS1_OMITSPARSESTATE-}" ] && [ "$(git config --bool core.sparseCheckout)" = "true" ]; then sparse="|SPARSE" fi @@ -543,7 +543,7 @@ __git_ps1 () u="%${ZSH_VERSION+%}" fi - if [ -n "${GIT_PS1_COMPRESSSPARSESTATE}" ] && + if [ -n "${GIT_PS1_COMPRESSSPARSESTATE-}" ] && [ "$(git config --bool core.sparseCheckout)" = "true" ]; then h="?" fi From 2d86a962203fe0faf6ae441cbe21b92a4bb03c20 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 13 May 2021 02:25:53 -0400 Subject: [PATCH 057/397] t: avoid sed-based chain-linting in some expensive cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 878f988350 (t/test-lib: teach --chain-lint to detect broken &&-chains in subshells, 2018-07-11) introduced additional chain-lint tests which add an extra "sed" pipeline to each test we run. This has a measurable impact on runtime. Here are timings with and without a new environment variable (added by this patch) that lets you disable just the additional sed-based chain-lint tests: Benchmark #1: GIT_TEST_CHAIN_LINT_HARDER=1 make test Time (mean ± σ): 64.202 s ± 1.030 s [User: 622.469 s, System: 301.402 s] Range (min … max): 61.571 s … 65.662 s 10 runs Benchmark #2: GIT_TEST_CHAIN_LINT_HARDER=0 make test Time (mean ± σ): 57.591 s ± 0.333 s [User: 529.368 s, System: 270.618 s] Range (min … max): 57.143 s … 58.309 s 10 runs Summary 'GIT_TEST_CHAIN_LINT_HARDER=0 make test' ran 1.11 ± 0.02 times faster than 'GIT_TEST_CHAIN_LINT_HARDER=1 make test' Of course those extra lint checks are doing something useful, so paying a few extra seconds (at least on Linux) isn't so bad (though note the CPU time; we're bounded in our parallel run here by the slowest test, so it really is ~120s of CPU improvement). But we can observe that there are some test scripts where they produce a much stronger effect, and provide less value. In t0027 and t3070 we run a very large number of small tests, all driven by a series of functions/loops which are filling in the test bodies. There we get much less bang for our buck in terms of bug-finding versus CPU cost. This patch introduces a mechanism for controlling when those extra lint checks are run, at two levels: - a user can ask to disable or to force-enable the checks by setting GIT_TEST_CHAIN_LINT_HARDER - if the user hasn't specified a preference, individual scripts can disable the checks by setting GIT_TEST_CHAIN_LINT_HARDER_DEFAULT; scripts which don't set that get the current behavior of enabling them. In addition, this patch flips the default for t0027 and t3070's mass-generated sections to disable the extra checks. Here are the timing results for t0027: Benchmark #1: GIT_TEST_CHAIN_LINT_HARDER=1 ./t0027-auto-crlf.sh Time (mean ± σ): 17.078 s ± 0.848 s [User: 14.878 s, System: 7.075 s] Range (min … max): 15.952 s … 18.421 s 10 runs Benchmark #2: GIT_TEST_CHAIN_LINT_HARDER=0 ./t0027-auto-crlf.sh Time (mean ± σ): 9.063 s ± 0.759 s [User: 7.890 s, System: 3.362 s] Range (min … max): 7.747 s … 10.619 s 10 runs Benchmark #3: ./t0027-auto-crlf.sh Time (mean ± σ): 9.186 s ± 0.881 s [User: 7.957 s, System: 3.427 s] Range (min … max): 7.796 s … 10.498 s 10 runs Summary 'GIT_TEST_CHAIN_LINT_HARDER=0 ./t0027-auto-crlf.sh' ran 1.01 ± 0.13 times faster than './t0027-auto-crlf.sh' 1.88 ± 0.18 times faster than 'GIT_TEST_CHAIN_LINT_HARDER=1 ./t0027-auto-crlf.sh' We can see that disabling the checks for the whole script buys us an almost 2x speedup. But the new default behavior, disabling them only for the mass-generated part, gets us most of that speedup (but still leaves the checks on for further manual tests people might write). As a side note, I'd caution about comparing runtimes and CPU seconds between this timing and the earlier "make test" one. In "make test", we're running a lot of scripts in parallel, so the CPU is throttling down (and thus a CPU second saved here would count for more during a parallel run; the same work takes more CPU seconds there). We get similar results for t3070: Benchmark #1: GIT_TEST_CHAIN_LINT_HARDER=1 ./t3070-wildmatch.sh Time (mean ± σ): 20.054 s ± 3.967 s [User: 16.003 s, System: 8.286 s] Range (min … max): 11.891 s … 23.671 s 10 runs Benchmark #2: GIT_TEST_CHAIN_LINT_HARDER=0 ./t3070-wildmatch.sh Time (mean ± σ): 12.399 s ± 2.256 s [User: 7.542 s, System: 5.342 s] Range (min … max): 9.606 s … 15.727 s 10 runs Benchmark #3: ./t3070-wildmatch.sh Time (mean ± σ): 10.726 s ± 3.476 s [User: 6.790 s, System: 4.365 s] Range (min … max): 5.444 s … 15.376 s 10 runs Summary './t3070-wildmatch.sh' ran 1.16 ± 0.43 times faster than 'GIT_TEST_CHAIN_LINT_HARDER=0 ./t3070-wildmatch.sh' 1.87 ± 0.71 times faster than 'GIT_TEST_CHAIN_LINT_HARDER=1 ./t3070-wildmatch.sh' Again, we get almost a 2x speedup disabling these. In this case, there are no tests not covered by the script's "default to disable" behavior, so the second two benchmarks should be the same (and while they do differ, you can see the variance is quite high but they're within one standard deviation). So it seems like for these two scripts, at least, disabling the extra checks is a reasonable tradeoff. Sadly, the overall runtime of "make test" on my system doesn't get much faster. But that's because we're mostly limited by the cost of the single biggest test. Here are the top-5 tests by wall-clock time from a parallel run, before my patch: 57.9192368984222 t9001-send-email.sh 45.6329638957977 t0027-auto-crlf.sh 32.5278220176697 t3070-wildmatch.sh 22.2701289653778 t7610-mergetool.sh 20.8635759353638 t1701-racy-split-index.sh And after: 57.1476998329163 t9001-send-email.sh 33.776211977005 t0027-auto-crlf.sh 21.3116669654846 t7610-mergetool.sh 20.7748689651489 t1701-racy-split-index.sh 19.6957249641418 t7112-reset-submodule.sh We dropped 12s from t0027, and t3070 dropped off our list entirely at around 16s. In both cases we're bound by t9001, but its slowness is due to the actual tests, so we'll have to deal with it in a different way. But this reduces overall CPU, and means that dealing with t9001 (by improving the speed of send-email or splitting it apart) will let us reduce our overall runtime even on multi-core machines. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- t/README | 5 +++++ t/t0027-auto-crlf.sh | 7 ++++++- t/t3070-wildmatch.sh | 5 +++++ t/test-lib.sh | 7 +++++-- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/t/README b/t/README index 593d4a4e27..63f9e07605 100644 --- a/t/README +++ b/t/README @@ -196,6 +196,11 @@ appropriately before running "make". Short options can be bundled, i.e. this feature by setting the GIT_TEST_CHAIN_LINT environment variable to "1" or "0", respectively. + A few test scripts disable some of the more advanced + chain-linting detection in the name of efficiency. You can + override this by setting the GIT_TEST_CHAIN_LINT_HARDER + environment variable to "1". + --stress:: Run the test script repeatedly in multiple parallel jobs until one of them fails. Useful for reproducing rare failures in diff --git a/t/t0027-auto-crlf.sh b/t/t0027-auto-crlf.sh index d24d5acfbc..4a5c5c602c 100755 --- a/t/t0027-auto-crlf.sh +++ b/t/t0027-auto-crlf.sh @@ -386,7 +386,9 @@ test_expect_success 'setup main' ' test_tick ' - +# Disable extra chain-linting for the next set of tests. There are many +# auto-generated ones that are not worth checking over and over. +GIT_TEST_CHAIN_LINT_HARDER_DEFAULT=0 warn_LF_CRLF="LF will be replaced by CRLF" warn_CRLF_LF="CRLF will be replaced by LF" @@ -597,6 +599,9 @@ do checkout_files auto "$id" "" false native $NL CRLF CRLF_mix_LF LF_mix_CR LF_nul done +# The rest of the tests are unique; do the usual linting. +unset GIT_TEST_CHAIN_LINT_HARDER_DEFAULT + # Should be the last test case: remove some files from the worktree test_expect_success 'ls-files --eol -d -z' ' rm crlf_false_attr__CRLF.txt crlf_false_attr__CRLF_mix_LF.txt crlf_false_attr__LF.txt .gitattributes && diff --git a/t/t3070-wildmatch.sh b/t/t3070-wildmatch.sh index 891d4d7cb9..56ea4bda13 100755 --- a/t/t3070-wildmatch.sh +++ b/t/t3070-wildmatch.sh @@ -4,6 +4,11 @@ test_description='wildmatch tests' . ./test-lib.sh +# Disable expensive chain-lint tests; all of the tests in this script +# are variants of a few trivial test-tool invocations, and there are a lot of +# them. +GIT_TEST_CHAIN_LINT_HARDER_DEFAULT=0 + should_create_test_file() { file=$1 diff --git a/t/test-lib.sh b/t/test-lib.sh index d3f6af6a65..fac144adce 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -945,8 +945,11 @@ test_run_ () { trace= # 117 is magic because it is unlikely to match the exit # code of other programs - if $(printf '%s\n' "$1" | sed -f "$GIT_BUILD_DIR/t/chainlint.sed" | grep -q '?![A-Z][A-Z]*?!') || - test "OK-117" != "$(test_eval_ "(exit 117) && $1${LF}${LF}echo OK-\$?" 3>&1)" + if test "OK-117" != "$(test_eval_ "(exit 117) && $1${LF}${LF}echo OK-\$?" 3>&1)" || + { + test "${GIT_TEST_CHAIN_LINT_HARDER:-${GIT_TEST_CHAIN_LINT_HARDER_DEFAULT:-1}}" != 0 && + $(printf '%s\n' "$1" | sed -f "$GIT_BUILD_DIR/t/chainlint.sed" | grep -q '?![A-Z][A-Z]*?!') + } then BUG "broken &&-chain or run-away HERE-DOC: $1" fi From 0caf20f22806dfd478ab3ac26b5f3682f383d3b4 Mon Sep 17 00:00:00 2001 From: ZheNing Hu Date: Thu, 13 May 2021 15:15:37 +0000 Subject: [PATCH 058/397] ref-filter: add objectsize to used_atom When the support for "objectsize:disk" was bolted onto the existing support for "objectsize", it didn't follow the usual pattern for handling "atomtype:modifier", which reads the part just once while parsing the format string, and store the parsed result in the union in the used_atom structure, so that the string form of it does not have to be parsed over and over at runtime (e.g. in grab_common_values()). Add a new member `objectsize` to the union `used_atom.u`, so that we can separate the check of from the check of , this will bring scalability to atom `%(objectsize)`. Signed-off-by: ZheNing Hu Signed-off-by: Junio C Hamano --- ref-filter.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ref-filter.c b/ref-filter.c index e2eac50d95..8cb622b95d 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -146,6 +146,9 @@ static struct used_atom { enum { O_FULL, O_LENGTH, O_SHORT } option; unsigned int length; } oid; + struct { + enum { O_SIZE, O_SIZE_DISK } option; + } objectsize; struct email_option { enum { EO_RAW, EO_TRIM, EO_LOCALPART } option; } email_option; @@ -269,11 +272,13 @@ static int objectsize_atom_parser(const struct ref_format *format, struct used_a const char *arg, struct strbuf *err) { if (!arg) { + atom->u.objectsize.option = O_SIZE; if (*atom->name == '*') oi_deref.info.sizep = &oi_deref.size; else oi.info.sizep = &oi.size; } else if (!strcmp(arg, "disk")) { + atom->u.objectsize.option = O_SIZE_DISK; if (*atom->name == '*') oi_deref.info.disk_sizep = &oi_deref.disk_size; else @@ -967,12 +972,14 @@ static void grab_common_values(struct atom_value *val, int deref, struct expand_ name++; if (!strcmp(name, "objecttype")) v->s = xstrdup(type_name(oi->type)); - else if (!strcmp(name, "objectsize:disk")) { - v->value = oi->disk_size; - v->s = xstrfmt("%"PRIuMAX, (uintmax_t)oi->disk_size); - } else if (!strcmp(name, "objectsize")) { - v->value = oi->size; - v->s = xstrfmt("%"PRIuMAX , (uintmax_t)oi->size); + else if (starts_with(name, "objectsize")) { + if (used_atom[i].u.objectsize.option == O_SIZE_DISK) { + v->value = oi->disk_size; + v->s = xstrfmt("%"PRIuMAX, (uintmax_t)oi->disk_size); + } else if (used_atom[i].u.objectsize.option == O_SIZE) { + v->value = oi->size; + v->s = xstrfmt("%"PRIuMAX , (uintmax_t)oi->size); + } } else if (!strcmp(name, "deltabase")) v->s = xstrdup(oid_to_hex(&oi->delta_base_oid)); else if (deref) From 1197f1a46360d3ae96bd9c15908a3a6f8e562207 Mon Sep 17 00:00:00 2001 From: ZheNing Hu Date: Thu, 13 May 2021 15:15:38 +0000 Subject: [PATCH 059/397] ref-filter: introduce enum atom_type In the original ref-filter design, it will copy the parsed atom's name and attributes to `used_atom[i].name` in the atom's parsing step, and use it again for string matching in the later specific ref attributes filling step. It use a lot of string matching to determine which atom we need. Introduce the enum "atom_type", each enum value is named as `ATOM_*`, which is the index of each corresponding valid_atom entry. In the first step of the atom parsing, `used_atom.atom_type` will record corresponding enum value from valid_atom entry index, and then in specific reference attribute filling step, only need to compare the value of the `used_atom[i].atom_type` to check the atom type. Helped-by: Junio C Hamano Helped-by: Christian Couder Signed-off-by: ZheNing Hu Signed-off-by: Junio C Hamano --- ref-filter.c | 197 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 128 insertions(+), 69 deletions(-) diff --git a/ref-filter.c b/ref-filter.c index 8cb622b95d..4db0e40ff4 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -108,6 +108,56 @@ static struct ref_to_worktree_map { struct worktree **worktrees; } ref_to_worktree_map; +/* + * The enum atom_type is used as the index of valid_atom array. + * In the atom parsing stage, it will be passed to used_atom.atom_type + * as the identifier of the atom type. We can check the type of used_atom + * entry by `if (used_atom[i].atom_type == ATOM_*)`. + */ +enum atom_type { + ATOM_REFNAME, + ATOM_OBJECTTYPE, + ATOM_OBJECTSIZE, + ATOM_OBJECTNAME, + ATOM_DELTABASE, + ATOM_TREE, + ATOM_PARENT, + ATOM_NUMPARENT, + ATOM_OBJECT, + ATOM_TYPE, + ATOM_TAG, + ATOM_AUTHOR, + ATOM_AUTHORNAME, + ATOM_AUTHOREMAIL, + ATOM_AUTHORDATE, + ATOM_COMMITTER, + ATOM_COMMITTERNAME, + ATOM_COMMITTEREMAIL, + ATOM_COMMITTERDATE, + ATOM_TAGGER, + ATOM_TAGGERNAME, + ATOM_TAGGEREMAIL, + ATOM_TAGGERDATE, + ATOM_CREATOR, + ATOM_CREATORDATE, + ATOM_SUBJECT, + ATOM_BODY, + ATOM_TRAILERS, + ATOM_CONTENTS, + ATOM_UPSTREAM, + ATOM_PUSH, + ATOM_SYMREF, + ATOM_FLAG, + ATOM_HEAD, + ATOM_COLOR, + ATOM_WORKTREEPATH, + ATOM_ALIGN, + ATOM_END, + ATOM_IF, + ATOM_THEN, + ATOM_ELSE, +}; + /* * An atom is a valid field atom listed below, possibly prefixed with * a "*" to denote deref_tag(). @@ -119,6 +169,7 @@ static struct ref_to_worktree_map { * array. */ static struct used_atom { + enum atom_type atom_type; const char *name; cmp_type type; info_source source; @@ -506,47 +557,47 @@ static struct { int (*parser)(const struct ref_format *format, struct used_atom *atom, const char *arg, struct strbuf *err); } valid_atom[] = { - { "refname", SOURCE_NONE, FIELD_STR, refname_atom_parser }, - { "objecttype", SOURCE_OTHER, FIELD_STR, objecttype_atom_parser }, - { "objectsize", SOURCE_OTHER, FIELD_ULONG, objectsize_atom_parser }, - { "objectname", SOURCE_OTHER, FIELD_STR, oid_atom_parser }, - { "deltabase", SOURCE_OTHER, FIELD_STR, deltabase_atom_parser }, - { "tree", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, - { "parent", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, - { "numparent", SOURCE_OBJ, FIELD_ULONG }, - { "object", SOURCE_OBJ }, - { "type", SOURCE_OBJ }, - { "tag", SOURCE_OBJ }, - { "author", SOURCE_OBJ }, - { "authorname", SOURCE_OBJ }, - { "authoremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, - { "authordate", SOURCE_OBJ, FIELD_TIME }, - { "committer", SOURCE_OBJ }, - { "committername", SOURCE_OBJ }, - { "committeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, - { "committerdate", SOURCE_OBJ, FIELD_TIME }, - { "tagger", SOURCE_OBJ }, - { "taggername", SOURCE_OBJ }, - { "taggeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, - { "taggerdate", SOURCE_OBJ, FIELD_TIME }, - { "creator", SOURCE_OBJ }, - { "creatordate", SOURCE_OBJ, FIELD_TIME }, - { "subject", SOURCE_OBJ, FIELD_STR, subject_atom_parser }, - { "body", SOURCE_OBJ, FIELD_STR, body_atom_parser }, - { "trailers", SOURCE_OBJ, FIELD_STR, trailers_atom_parser }, - { "contents", SOURCE_OBJ, FIELD_STR, contents_atom_parser }, - { "upstream", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, - { "push", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, - { "symref", SOURCE_NONE, FIELD_STR, refname_atom_parser }, - { "flag", SOURCE_NONE }, - { "HEAD", SOURCE_NONE, FIELD_STR, head_atom_parser }, - { "color", SOURCE_NONE, FIELD_STR, color_atom_parser }, - { "worktreepath", SOURCE_NONE }, - { "align", SOURCE_NONE, FIELD_STR, align_atom_parser }, - { "end", SOURCE_NONE }, - { "if", SOURCE_NONE, FIELD_STR, if_atom_parser }, - { "then", SOURCE_NONE }, - { "else", SOURCE_NONE }, + [ATOM_REFNAME] = { "refname", SOURCE_NONE, FIELD_STR, refname_atom_parser }, + [ATOM_OBJECTTYPE] = { "objecttype", SOURCE_OTHER, FIELD_STR, objecttype_atom_parser }, + [ATOM_OBJECTSIZE] = { "objectsize", SOURCE_OTHER, FIELD_ULONG, objectsize_atom_parser }, + [ATOM_OBJECTNAME] = { "objectname", SOURCE_OTHER, FIELD_STR, oid_atom_parser }, + [ATOM_DELTABASE] = { "deltabase", SOURCE_OTHER, FIELD_STR, deltabase_atom_parser }, + [ATOM_TREE] = { "tree", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, + [ATOM_PARENT] = { "parent", SOURCE_OBJ, FIELD_STR, oid_atom_parser }, + [ATOM_NUMPARENT] = { "numparent", SOURCE_OBJ, FIELD_ULONG }, + [ATOM_OBJECT] = { "object", SOURCE_OBJ }, + [ATOM_TYPE] = { "type", SOURCE_OBJ }, + [ATOM_TAG] = { "tag", SOURCE_OBJ }, + [ATOM_AUTHOR] = { "author", SOURCE_OBJ }, + [ATOM_AUTHORNAME] = { "authorname", SOURCE_OBJ }, + [ATOM_AUTHOREMAIL] = { "authoremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, + [ATOM_AUTHORDATE] = { "authordate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_COMMITTER] = { "committer", SOURCE_OBJ }, + [ATOM_COMMITTERNAME] = { "committername", SOURCE_OBJ }, + [ATOM_COMMITTEREMAIL] = { "committeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, + [ATOM_COMMITTERDATE] = { "committerdate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_TAGGER] = { "tagger", SOURCE_OBJ }, + [ATOM_TAGGERNAME] = { "taggername", SOURCE_OBJ }, + [ATOM_TAGGEREMAIL] = { "taggeremail", SOURCE_OBJ, FIELD_STR, person_email_atom_parser }, + [ATOM_TAGGERDATE] = { "taggerdate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_CREATOR] = { "creator", SOURCE_OBJ }, + [ATOM_CREATORDATE] = { "creatordate", SOURCE_OBJ, FIELD_TIME }, + [ATOM_SUBJECT] = { "subject", SOURCE_OBJ, FIELD_STR, subject_atom_parser }, + [ATOM_BODY] = { "body", SOURCE_OBJ, FIELD_STR, body_atom_parser }, + [ATOM_TRAILERS] = { "trailers", SOURCE_OBJ, FIELD_STR, trailers_atom_parser }, + [ATOM_CONTENTS] = { "contents", SOURCE_OBJ, FIELD_STR, contents_atom_parser }, + [ATOM_UPSTREAM] = { "upstream", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, + [ATOM_PUSH] = { "push", SOURCE_NONE, FIELD_STR, remote_ref_atom_parser }, + [ATOM_SYMREF] = { "symref", SOURCE_NONE, FIELD_STR, refname_atom_parser }, + [ATOM_FLAG] = { "flag", SOURCE_NONE }, + [ATOM_HEAD] = { "HEAD", SOURCE_NONE, FIELD_STR, head_atom_parser }, + [ATOM_COLOR] = { "color", SOURCE_NONE, FIELD_STR, color_atom_parser }, + [ATOM_WORKTREEPATH] = { "worktreepath", SOURCE_NONE }, + [ATOM_ALIGN] = { "align", SOURCE_NONE, FIELD_STR, align_atom_parser }, + [ATOM_END] = { "end", SOURCE_NONE }, + [ATOM_IF] = { "if", SOURCE_NONE, FIELD_STR, if_atom_parser }, + [ATOM_THEN] = { "then", SOURCE_NONE }, + [ATOM_ELSE] = { "else", SOURCE_NONE }, /* * Please update $__git_ref_fieldlist in git-completion.bash * when you add new atoms @@ -628,6 +679,7 @@ static int parse_ref_filter_atom(const struct ref_format *format, at = used_atom_cnt; used_atom_cnt++; REALLOC_ARRAY(used_atom, used_atom_cnt); + used_atom[at].atom_type = i; used_atom[at].name = xmemdupz(atom, ep - atom); used_atom[at].type = valid_atom[i].cmp_type; used_atom[at].source = valid_atom[i].source; @@ -652,7 +704,7 @@ static int parse_ref_filter_atom(const struct ref_format *format, return -1; if (*atom == '*') need_tagged = 1; - if (!strcmp(valid_atom[i].name, "symref")) + if (i == ATOM_SYMREF) need_symref = 1; return at; } @@ -965,14 +1017,15 @@ static void grab_common_values(struct atom_value *val, int deref, struct expand_ for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (!strcmp(name, "objecttype")) + if (atom_type == ATOM_OBJECTTYPE) v->s = xstrdup(type_name(oi->type)); - else if (starts_with(name, "objectsize")) { + else if (atom_type == ATOM_OBJECTSIZE) { if (used_atom[i].u.objectsize.option == O_SIZE_DISK) { v->value = oi->disk_size; v->s = xstrfmt("%"PRIuMAX, (uintmax_t)oi->disk_size); @@ -980,9 +1033,9 @@ static void grab_common_values(struct atom_value *val, int deref, struct expand_ v->value = oi->size; v->s = xstrfmt("%"PRIuMAX , (uintmax_t)oi->size); } - } else if (!strcmp(name, "deltabase")) + } else if (atom_type == ATOM_DELTABASE) v->s = xstrdup(oid_to_hex(&oi->delta_base_oid)); - else if (deref) + else if (atom_type == ATOM_OBJECTNAME && deref) grab_oid(name, "objectname", &oi->oid, v, &used_atom[i]); } } @@ -995,16 +1048,17 @@ static void grab_tag_values(struct atom_value *val, int deref, struct object *ob for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (!strcmp(name, "tag")) + if (atom_type == ATOM_TAG) v->s = xstrdup(tag->tag); - else if (!strcmp(name, "type") && tag->tagged) + else if (atom_type == ATOM_TYPE && tag->tagged) v->s = xstrdup(type_name(tag->tagged->type)); - else if (!strcmp(name, "object") && tag->tagged) + else if (atom_type == ATOM_OBJECT && tag->tagged) v->s = xstrdup(oid_to_hex(&tag->tagged->oid)); } } @@ -1017,18 +1071,20 @@ static void grab_commit_values(struct atom_value *val, int deref, struct object for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (grab_oid(name, "tree", get_commit_tree_oid(commit), v, &used_atom[i])) + if (atom_type == ATOM_TREE && + grab_oid(name, "tree", get_commit_tree_oid(commit), v, &used_atom[i])) continue; - if (!strcmp(name, "numparent")) { + if (atom_type == ATOM_NUMPARENT) { v->value = commit_list_count(commit->parents); v->s = xstrfmt("%lu", (unsigned long)v->value); } - else if (starts_with(name, "parent")) { + else if (atom_type == ATOM_PARENT) { struct commit_list *parents; struct strbuf s = STRBUF_INIT; for (parents = commit->parents; parents; parents = parents->next) { @@ -1208,15 +1264,16 @@ static void grab_person(const char *who, struct atom_value *val, int deref, void return; for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; + enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; if (deref) name++; - if (starts_with(name, "creatordate")) + if (atom_type == ATOM_CREATORDATE) grab_date(wholine, v, name); - else if (!strcmp(name, "creator")) + else if (atom_type == ATOM_CREATOR) v->s = copy_line(wholine); } } @@ -1696,6 +1753,7 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) /* Fill in specials first */ for (i = 0; i < used_atom_cnt; i++) { struct used_atom *atom = &used_atom[i]; + enum atom_type atom_type = atom->atom_type; const char *name = used_atom[i].name; struct atom_value *v = &ref->value[i]; int deref = 0; @@ -1710,18 +1768,18 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) name++; } - if (starts_with(name, "refname")) + if (atom_type == ATOM_REFNAME) refname = get_refname(atom, ref); - else if (!strcmp(name, "worktreepath")) { + else if (atom_type == ATOM_WORKTREEPATH) { if (ref->kind == FILTER_REFS_BRANCHES) v->s = get_worktree_path(atom, ref); else v->s = xstrdup(""); continue; } - else if (starts_with(name, "symref")) + else if (atom_type == ATOM_SYMREF) refname = get_symref(atom, ref); - else if (starts_with(name, "upstream")) { + else if (atom_type == ATOM_UPSTREAM) { const char *branch_name; /* only local branches may have an upstream */ if (!skip_prefix(ref->refname, "refs/heads/", @@ -1737,7 +1795,7 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) else v->s = xstrdup(""); continue; - } else if (atom->u.remote_ref.push) { + } else if (atom_type == ATOM_PUSH && atom->u.remote_ref.push) { const char *branch_name; v->s = xstrdup(""); if (!skip_prefix(ref->refname, "refs/heads/", @@ -1756,10 +1814,10 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) free((char *)v->s); fill_remote_ref_details(atom, refname, branch, &v->s); continue; - } else if (starts_with(name, "color:")) { + } else if (atom_type == ATOM_COLOR) { v->s = xstrdup(atom->u.color); continue; - } else if (!strcmp(name, "flag")) { + } else if (atom_type == ATOM_FLAG) { char buf[256], *cp = buf; if (ref->flag & REF_ISSYMREF) cp = copy_advance(cp, ",symref"); @@ -1772,23 +1830,24 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) v->s = xstrdup(buf + 1); } continue; - } else if (!deref && grab_oid(name, "objectname", &ref->objectname, v, atom)) { - continue; - } else if (!strcmp(name, "HEAD")) { + } else if (!deref && atom_type == ATOM_OBJECTNAME && + grab_oid(name, "objectname", &ref->objectname, v, atom)) { + continue; + } else if (atom_type == ATOM_HEAD) { if (atom->u.head && !strcmp(ref->refname, atom->u.head)) v->s = xstrdup("*"); else v->s = xstrdup(" "); continue; - } else if (starts_with(name, "align")) { + } else if (atom_type == ATOM_ALIGN) { v->handler = align_atom_handler; v->s = xstrdup(""); continue; - } else if (!strcmp(name, "end")) { + } else if (atom_type == ATOM_END) { v->handler = end_atom_handler; v->s = xstrdup(""); continue; - } else if (starts_with(name, "if")) { + } else if (atom_type == ATOM_IF) { const char *s; if (skip_prefix(name, "if:", &s)) v->s = xstrdup(s); @@ -1796,11 +1855,11 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) v->s = xstrdup(""); v->handler = if_atom_handler; continue; - } else if (!strcmp(name, "then")) { + } else if (atom_type == ATOM_THEN) { v->handler = then_atom_handler; v->s = xstrdup(""); continue; - } else if (!strcmp(name, "else")) { + } else if (atom_type == ATOM_ELSE) { v->handler = else_atom_handler; v->s = xstrdup(""); continue; From a30e43f61a1e614309875ab7775f2274b4e40742 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Sat, 15 May 2021 14:01:11 -0600 Subject: [PATCH 060/397] merge: don't translate literal commands These strings have not been modified in any translation, nor should they be. Signed-off-by: Alex Henrie Signed-off-by: Junio C Hamano --- builtin/merge.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/merge.c b/builtin/merge.c index 388619536a..2d3424cecc 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -56,8 +56,8 @@ struct strategy { static const char * const builtin_merge_usage[] = { N_("git merge [] [...]"), - N_("git merge --abort"), - N_("git merge --continue"), + "git merge --abort", + "git merge --continue", NULL }; From cd5b33fbdc0299765278d0b607b64202603f0882 Mon Sep 17 00:00:00 2001 From: Gregory Anders Date: Fri, 14 May 2021 09:15:53 -0600 Subject: [PATCH 061/397] git-send-email: add option to specify sendmail command The sendemail.smtpServer configuration option and --smtp-server command line option both support using a sendmail-like program to send emails by specifying an absolute file path. However, this is not ideal for the following reasons: 1. It overloads the meaning of smtpServer (now a program is being used for the server?) 2. It doesn't allow for non-absolute paths, arguments, or arbitrary scripting Requiring an absolute path is bad for portability, as the same program may be in different locations on different systems. If a user wishes to pass arguments to their program, they have to use the smtpServerOption option, which is cumbersome (as it must be repeated for each option) and doesn't adhere to normal git conventions. Introduce a new configuration option sendemail.sendmailCmd as well as a command line option --sendmail-cmd that can be used to specify a command (with or without arguments) or shell expression to run to send email. The name of this option is consistent with --to-cmd and --cc-cmd. This invocation honors the user's $PATH so that absolute paths are not necessary. Arbitrary shell expressions are also supported, allowing users to do basic scripting. Give this option a higher precedence over --smtp-server and sendemail.smtpServer, as the new interface is more flexible. For backward compatibility, continue to support absolute paths in --smtp-server and sendemail.smtpServer. Signed-off-by: Gregory Anders Signed-off-by: Junio C Hamano --- Documentation/git-send-email.txt | 25 ++++++++++++++++------- git-send-email.perl | 34 +++++++++++++++++++++++++------- t/t9001-send-email.sh | 31 +++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/Documentation/git-send-email.txt b/Documentation/git-send-email.txt index 93708aefea..3db4eab4ba 100644 --- a/Documentation/git-send-email.txt +++ b/Documentation/git-send-email.txt @@ -167,6 +167,14 @@ Sending `sendemail.envelopeSender` configuration variable; if that is unspecified, choosing the envelope sender is left to your MTA. +--sendmail-cmd=:: + Specify a command to run to send the email. The command should + be sendmail-like; specifically, it must support the `-i` option. + The command will be executed in the shell if necessary. Default + is the value of `sendemail.sendmailcmd`. If unspecified, and if + --smtp-server is also unspecified, git-send-email will search + for `sendmail` in `/usr/sbin`, `/usr/lib` and $PATH. + --smtp-encryption=:: Specify the encryption to use, either 'ssl' or 'tls'. Any other value reverts to plain SMTP. Default is the value of @@ -211,13 +219,16 @@ a password is obtained using 'git-credential'. --smtp-server=:: If set, specifies the outgoing SMTP server to use (e.g. - `smtp.example.com` or a raw IP address). Alternatively it can - specify a full pathname of a sendmail-like program instead; - the program must support the `-i` option. Default value can - be specified by the `sendemail.smtpServer` configuration - option; the built-in default is to search for `sendmail` in - `/usr/sbin`, `/usr/lib` and $PATH if such program is - available, falling back to `localhost` otherwise. + `smtp.example.com` or a raw IP address). If unspecified, and if + `--sendmail-cmd` is also unspecified, the default is to search + for `sendmail` in `/usr/sbin`, `/usr/lib` and $PATH if such a + program is available, falling back to `localhost` otherwise. ++ +For backward compatibility, this option can also specify a full pathname +of a sendmail-like program instead; the program must support the `-i` +option. This method does not support passing arguments or using plain +command names. For those use cases, consider using `--sendmail-cmd` +instead. --smtp-server-port=:: Specifies a port different from the default port (SMTP diff --git a/git-send-email.perl b/git-send-email.perl index 1f425c0809..d9cff04067 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -70,6 +70,7 @@ git send-email --dump-aliases Sending: --envelope-sender * Email envelope sender. + --sendmail-cmd * Command to run to send email. --smtp-server * Outgoing SMTP server to use. The port is optional. Default 'localhost'. --smtp-server-option * Outgoing SMTP server option to use. @@ -243,6 +244,7 @@ my ($confirm); my (@suppress_cc); my ($auto_8bit_encoding); my ($compose_encoding); +my ($sendmail_cmd); # Variables with corresponding config settings & hardcoded defaults my ($debug_net_smtp) = 0; # Net::SMTP, see send_message() my $thread = 1; @@ -290,6 +292,7 @@ my %config_settings = ( "assume8bitencoding" => \$auto_8bit_encoding, "composeencoding" => \$compose_encoding, "transferencoding" => \$target_xfer_encoding, + "sendmailcmd" => \$sendmail_cmd, ); my %config_path_settings = ( @@ -423,6 +426,7 @@ $rc = GetOptions( "no-bcc" => \$no_bcc, "chain-reply-to!" => \$chain_reply_to, "no-chain-reply-to" => sub {$chain_reply_to = 0}, + "sendmail-cmd=s" => \$sendmail_cmd, "smtp-server=s" => \$smtp_server, "smtp-server-option=s" => \@smtp_server_options, "smtp-server-port=s" => \$smtp_server_port, @@ -996,16 +1000,19 @@ if (defined $reply_to) { $reply_to = sanitize_address($reply_to); } -if (!defined $smtp_server) { +if (!defined $sendmail_cmd && !defined $smtp_server) { my @sendmail_paths = qw( /usr/sbin/sendmail /usr/lib/sendmail ); push @sendmail_paths, map {"$_/sendmail"} split /:/, $ENV{PATH}; foreach (@sendmail_paths) { if (-x $_) { - $smtp_server = $_; + $sendmail_cmd = $_; last; } } - $smtp_server ||= 'localhost'; # could be 127.0.0.1, too... *shrug* + + if (!defined $sendmail_cmd) { + $smtp_server = 'localhost'; # could be 127.0.0.1, too... *shrug* + } } if ($compose && $compose > 0) { @@ -1485,11 +1492,17 @@ EOF if ($dry_run) { # We don't want to send the email. - } elsif (file_name_is_absolute($smtp_server)) { + } elsif (defined $sendmail_cmd || file_name_is_absolute($smtp_server)) { my $pid = open my $sm, '|-'; defined $pid or die $!; if (!$pid) { - exec($smtp_server, @sendmail_parameters) or die $!; + if (defined $sendmail_cmd) { + exec ("sh", "-c", "$sendmail_cmd \"\$@\"", "-", @sendmail_parameters) + or die $!; + } else { + exec ($smtp_server, @sendmail_parameters) + or die $!; + } } print $sm "$header\n$message"; close $sm or die $!; @@ -1585,14 +1598,21 @@ EOF printf($dry_run ? __("Dry-Sent %s\n") : __("Sent %s\n"), $subject); } else { print($dry_run ? __("Dry-OK. Log says:\n") : __("OK. Log says:\n")); - if (!file_name_is_absolute($smtp_server)) { + if (!defined $sendmail_cmd && !file_name_is_absolute($smtp_server)) { print "Server: $smtp_server\n"; print "MAIL FROM:<$raw_from>\n"; foreach my $entry (@recipients) { print "RCPT TO:<$entry>\n"; } } else { - print "Sendmail: $smtp_server ".join(' ',@sendmail_parameters)."\n"; + my $sm; + if (defined $sendmail_cmd) { + $sm = $sendmail_cmd; + } else { + $sm = $smtp_server; + } + + print "Sendmail: $sm ".join(' ',@sendmail_parameters)."\n"; } print $header, "\n"; if ($smtp) { diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 4eee9c3dcb..35289d9135 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -2097,6 +2097,37 @@ test_expect_success $PREREQ 'leading and trailing whitespaces are removed' ' test_cmp expected-list actual-list ' +test_expect_success $PREREQ 'test using command name with --sendmail-cmd' ' + clean_fake_sendmail && + PATH="$(pwd):$PATH" \ + git send-email \ + --from="Example " \ + --to=nobody@example.com \ + --sendmail-cmd="fake.sendmail" \ + HEAD^ && + test_path_is_file commandline1 +' + +test_expect_success $PREREQ 'test using arguments with --sendmail-cmd' ' + clean_fake_sendmail && + git send-email \ + --from="Example " \ + --to=nobody@example.com \ + --sendmail-cmd='\''"$(pwd)/fake.sendmail" -f nobody@example.com'\'' \ + HEAD^ && + test_path_is_file commandline1 +' + +test_expect_success $PREREQ 'test shell expression with --sendmail-cmd' ' + clean_fake_sendmail && + git send-email \ + --from="Example " \ + --to=nobody@example.com \ + --sendmail-cmd='\''f() { "$(pwd)/fake.sendmail" "$@"; };f'\'' \ + HEAD^ && + test_path_is_file commandline1 +' + test_expect_success $PREREQ 'invoke hook' ' mkdir -p .git/hooks && From 4901884a23c6eaabb0738bddc440f4155f81b973 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Sun, 16 May 2021 15:57:04 -0600 Subject: [PATCH 062/397] stash: don't translate literal commands Signed-off-by: Alex Henrie Signed-off-by: Junio C Hamano --- builtin/stash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/stash.c b/builtin/stash.c index ba774cce67..df451b7483 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -24,7 +24,7 @@ static const char * const git_stash_usage[] = { N_("git stash drop [-q|--quiet] []"), N_("git stash ( pop | apply ) [--index] [-q|--quiet] []"), N_("git stash branch []"), - N_("git stash clear"), + "git stash clear", N_("git stash [push [-p|--patch] [-k|--[no-]keep-index] [-q|--quiet]\n" " [-u|--include-untracked] [-a|--all] [-m|--message ]\n" " [--pathspec-from-file= [--pathspec-file-nul]]\n" @@ -65,7 +65,7 @@ static const char * const git_stash_branch_usage[] = { }; static const char * const git_stash_clear_usage[] = { - N_("git stash clear"), + "git stash clear", NULL }; From f5f5a61d5a1d54200972e104ffb3640dfa80bfb6 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Sun, 16 May 2021 15:59:57 -0600 Subject: [PATCH 063/397] submodule: use the imperative mood to describe the --files option Signed-off-by: Alex Henrie Signed-off-by: Junio C Hamano --- builtin/submodule--helper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 9d505a6329..da321a7565 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1299,7 +1299,7 @@ static int module_summary(int argc, const char **argv, const char *prefix) OPT_BOOL(0, "cached", &cached, N_("use the commit stored in the index instead of the submodule HEAD")), OPT_BOOL(0, "files", &files, - N_("to compare the commit in the index with that in the submodule HEAD")), + N_("compare the commit in the index with that in the submodule HEAD")), OPT_BOOL(0, "for-status", &for_status, N_("skip submodules with 'ignore_config' value set to 'all'")), OPT_INTEGER('n', "summary-limit", &summary_limit, From 72ee47ceebc7d3ddbd31942b28f9fe47f00b0540 Mon Sep 17 00:00:00 2001 From: edef Date: Sun, 16 May 2021 15:07:19 +0000 Subject: [PATCH 064/397] mailinfo: don't discard names under 3 characters I sometimes receive patches from people with short mononyms, and in my cultural environment these are not uncommon. To my dismay, git-am currently discards their names, and replaces them with their email addresses. Link: https://www.kalzumeus.com/2010/06/17/falsehoods-programmers-believe-about-names/ Signed-off-by: edef Signed-off-by: Junio C Hamano --- mailinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mailinfo.c b/mailinfo.c index 5681d9130d..3161a3940f 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -19,7 +19,7 @@ static void cleanup_space(struct strbuf *sb) static void get_sane_name(struct strbuf *out, struct strbuf *name, struct strbuf *email) { struct strbuf *src = name; - if (name->len < 3 || 60 < name->len || strpbrk(name->buf, "@<>")) + if (!name->len || 60 < name->len || strpbrk(name->buf, "@<>")) src = email; else if (name == out) return; From bfe35a61653c5789c9038b3fe9925941cf10f623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20H=C3=B6ckersten?= Date: Mon, 17 May 2021 05:53:50 +0000 Subject: [PATCH 065/397] describe-doc: clarify default length of abbreviation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarify the default length used for the abbreviated form used for commits in git describe. The behavior was modified in Git 2.11.0, but the documentation was not updated to clarify the new behavior. Signed-off-by: Anders Höckersten Signed-off-by: Junio C Hamano --- Documentation/git-describe.txt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Documentation/git-describe.txt b/Documentation/git-describe.txt index a88f6ae2c6..c6a79c2a0f 100644 --- a/Documentation/git-describe.txt +++ b/Documentation/git-describe.txt @@ -63,9 +63,10 @@ OPTIONS Automatically implies --tags. --abbrev=:: - Instead of using the default 7 hexadecimal digits as the - abbreviated object name, use digits, or as many digits - as needed to form a unique object name. An of 0 + Instead of using the default number of hexadecimal digits (which + will vary according to the number of objects in the repository with + a default of 7) of the abbreviated object name, use digits, or + as many digits as needed to form a unique object name. An of 0 will suppress long format, only showing the closest tag. --candidates=:: @@ -139,8 +140,11 @@ at the end. The number of additional commits is the number of commits which would be displayed by "git log v1.0.4..parent". -The hash suffix is "-g" + unambiguous abbreviation for the tip commit -of parent (which was `2414721b194453f058079d897d13c4e377f92dc6`). +The hash suffix is "-g" + an unambigous abbreviation for the tip commit +of parent (which was `2414721b194453f058079d897d13c4e377f92dc6`). The +length of the abbreviation scales as the repository grows, using the +approximate number of objects in the repository and a bit of math +around the birthday paradox, and defaults to a minimum of 7. The "g" prefix stands for "git" and is used to allow describing the version of a software depending on the SCM the software is managed with. This is useful in an environment where people may use different SCMs. From 99fc555188681caeedc983b9fc982d6dc8ee2a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20M=C3=BCller?= Date: Mon, 17 May 2021 10:02:42 +0200 Subject: [PATCH 066/397] rev-parse: fix segfault with missing --path-format argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calling "git rev-parse --path-format" without an argument segfaults instead of giving an error message. Commit fac60b8925 (rev-parse: add option for absolute or relative path formatting, 2020-12-13) added the argument parsing code but forgot to handle NULL. Returning an error makes sense here because there is no default value we could use. Add a test case to verify. Signed-off-by: Wolfgang Müller Signed-off-by: Junio C Hamano --- builtin/rev-parse.c | 2 ++ t/t1500-rev-parse.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/builtin/rev-parse.c b/builtin/rev-parse.c index 85bad9052e..7af8dab8bc 100644 --- a/builtin/rev-parse.c +++ b/builtin/rev-parse.c @@ -759,6 +759,8 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) continue; } if (opt_with_value(arg, "--path-format", &arg)) { + if (!arg) + die("--path-format requires an argument"); if (!strcmp(arg, "absolute")) { format = FORMAT_CANONICAL; } else if (!strcmp(arg, "relative")) { diff --git a/t/t1500-rev-parse.sh b/t/t1500-rev-parse.sh index deae916707..1c2df08333 100755 --- a/t/t1500-rev-parse.sh +++ b/t/t1500-rev-parse.sh @@ -146,6 +146,10 @@ test_expect_success '--path-format can change in the middle of the command line' test_cmp expect actual ' +test_expect_success '--path-format does not segfault without an argument' ' + test_must_fail git rev-parse --path-format +' + test_expect_success 'git-common-dir from worktree root' ' echo .git >expect && git rev-parse --git-common-dir >actual && From e2c5993744f2c802a907cba1cfb226dc688b527e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolfgang=20M=C3=BCller?= Date: Mon, 17 May 2021 10:02:43 +0200 Subject: [PATCH 067/397] rev-parse: mark die() messages for translation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These error messages are intended for the user. Let's touch them up since we're here from the previous commit. Signed-off-by: Wolfgang Müller Signed-off-by: Junio C Hamano --- builtin/rev-parse.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/builtin/rev-parse.c b/builtin/rev-parse.c index 7af8dab8bc..22c4e1a4ff 100644 --- a/builtin/rev-parse.c +++ b/builtin/rev-parse.c @@ -435,11 +435,11 @@ static int cmd_parseopt(int argc, const char **argv, const char *prefix) /* get the usage up to the first line with a -- on it */ for (;;) { if (strbuf_getline(&sb, stdin) == EOF) - die("premature end of input"); + die(_("premature end of input")); ALLOC_GROW(usage, unb + 1, usz); if (!strcmp("--", sb.buf)) { if (unb < 1) - die("no usage string given before the `--' separator"); + die(_("no usage string given before the `--' separator")); usage[unb] = NULL; break; } @@ -545,7 +545,7 @@ static void die_no_single_rev(int quiet) if (quiet) exit(1); else - die("Needed a single revision"); + die(_("Needed a single revision")); } static const char builtin_rev_parse_usage[] = @@ -709,10 +709,10 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (!strcmp(arg, "--resolve-git-dir")) { const char *gitdir = argv[++i]; if (!gitdir) - die("--resolve-git-dir requires an argument"); + die(_("--resolve-git-dir requires an argument")); gitdir = resolve_gitdir(gitdir); if (!gitdir) - die("not a gitdir '%s'", argv[i]); + die(_("not a gitdir '%s'"), argv[i]); puts(gitdir); continue; } @@ -736,7 +736,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (!seen_end_of_options && *arg == '-') { if (!strcmp(arg, "--git-path")) { if (!argv[i + 1]) - die("--git-path requires an argument"); + die(_("--git-path requires an argument")); strbuf_reset(&buf); print_path(git_path("%s", argv[i + 1]), prefix, format, @@ -746,7 +746,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) } if (!strcmp(arg,"-n")) { if (++i >= argc) - die("-n requires an argument"); + die(_("-n requires an argument")); if ((filter & DO_FLAGS) && (filter & DO_REVS)) { show(arg); show(argv[i]); @@ -760,26 +760,26 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) } if (opt_with_value(arg, "--path-format", &arg)) { if (!arg) - die("--path-format requires an argument"); + die(_("--path-format requires an argument")); if (!strcmp(arg, "absolute")) { format = FORMAT_CANONICAL; } else if (!strcmp(arg, "relative")) { format = FORMAT_RELATIVE; } else { - die("unknown argument to --path-format: %s", arg); + die(_("unknown argument to --path-format: %s"), arg); } continue; } if (!strcmp(arg, "--default")) { def = argv[++i]; if (!def) - die("--default requires an argument"); + die(_("--default requires an argument")); continue; } if (!strcmp(arg, "--prefix")) { prefix = argv[++i]; if (!prefix) - die("--prefix requires an argument"); + die(_("--prefix requires an argument")); startup_info->prefix = prefix; output_prefix = 1; continue; @@ -848,7 +848,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) else if (!strcmp(arg, "loose")) abbrev_ref_strict = 0; else - die("unknown mode for --abbrev-ref: %s", + die(_("unknown mode for --abbrev-ref: %s"), arg); } continue; @@ -892,7 +892,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (work_tree) print_path(work_tree, prefix, format, DEFAULT_UNMODIFIED); else - die("this operation must be run in a work tree"); + die(_("this operation must be run in a work tree")); continue; } if (!strcmp(arg, "--show-superproject-working-tree")) { @@ -1020,7 +1020,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (strcmp(val, "storage") && strcmp(val, "input") && strcmp(val, "output")) - die("unknown mode for --show-object-format: %s", + die(_("unknown mode for --show-object-format: %s"), arg); puts(the_hash_algo->name); continue; @@ -1058,7 +1058,7 @@ int cmd_rev_parse(int argc, const char **argv, const char *prefix) if (verify) die_no_single_rev(quiet); if (has_dashdash) - die("bad revision '%s'", arg); + die(_("bad revision '%s'"), arg); as_is = 1; if (!show_file(arg, output_prefix)) continue; From 58cf6056c9289e145c8f3de79ad0385e0976dbc0 Mon Sep 17 00:00:00 2001 From: Todd Zullinger Date: Mon, 17 May 2021 11:12:22 -0400 Subject: [PATCH 068/397] t7500: remove non-existant C_LOCALE_OUTPUT prereq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C_LOCALE_OUTPUT prerequisite was removed in b1e079807b (tests: remove last uses of C_LOCALE_OUTPUT, 2021-02-11), where Ævar noted: I'm not leaving the prerequisite itself in place for in-flight changes as there currently are none that introduce new tests that rely on it, and because C_LOCALE_OUTPUT is currently a noop on the master branch we likely won't have any new submissions that use it. One more use of C_LOCALE_OUTPUT did creep in with 3d1bda6b5b (t7500: add tests for --fixup=[amend|reword] options, 2021-03-15). This causes a number of the tests to be skipped by default: ok 35 # SKIP --fixup=reword: incompatible with --all (missing C_LOCALE_OUTPUT) ok 36 # SKIP --fixup=reword: incompatible with --include (missing C_LOCALE_OUTPUT) ok 37 # SKIP --fixup=reword: incompatible with --only (missing C_LOCALE_OUTPUT) ok 38 # SKIP --fixup=reword: incompatible with --interactive (missing C_LOCALE_OUTPUT) ok 39 # SKIP --fixup=reword: incompatible with --patch (missing C_LOCALE_OUTPUT) Remove the C_LOCALE_OUTPUT prerequisite from these tests so they are not skipped. Signed-off-by: Todd Zullinger Signed-off-by: Junio C Hamano --- t/t7500-commit-template-squash-signoff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7500-commit-template-squash-signoff.sh b/t/t7500-commit-template-squash-signoff.sh index 9092db5fdc..7d02f79c0d 100755 --- a/t/t7500-commit-template-squash-signoff.sh +++ b/t/t7500-commit-template-squash-signoff.sh @@ -413,7 +413,7 @@ test_expect_success 'amend! commit allows empty commit msg body with --allow-emp ' test_fixup_reword_opt () { - test_expect_success C_LOCALE_OUTPUT "--fixup=reword: incompatible with $1" " + test_expect_success "--fixup=reword: incompatible with $1" " echo 'fatal: reword option of --fixup is mutually exclusive with'\ '--patch/--interactive/--all/--include/--only' >expect && test_must_fail git commit --fixup=reword:HEAD~ $1 2>actual && From 3d20ed27b8bdac5c82f4f78af802f9afa499f651 Mon Sep 17 00:00:00 2001 From: Matheus Tavares Date: Mon, 17 May 2021 16:49:03 -0300 Subject: [PATCH 069/397] parallel-checkout: send the new object_id algo field to the workers An object_id storing a SHA-1 name has some unused bytes at the end of the hash array. Since these bytes are not used, they are usually not initialized to any value either. However, at parallel_checkout.c:send_one_item() the object_id of a cache entry is copied into a buffer which is later sent to a checkout worker through a pipe write(). This makes Valgrind complain about passing uninitialized bytes to a syscall. The worker won't use these uninitialized bytes either, but the warning could confuse someone trying to debug this code; So instead of using oidcpy(), send_one_item() uses hashcpy() to only copy the used/initialized bytes of the object_id, and leave the remaining part with zeros. However, since cf0983213c ("hash: add an algo member to struct object_id", 2021-04-26), using hashcpy() is no longer sufficient here as it won't copy the new algo field from the object_id. Let's add and use a new function which meets both our requirements of copying all the important object_id data while still avoiding the uninitialized bytes, by padding the end of the hash array in the destination object_id. With this change, we also no longer need the destination buffer from send_one_item() to be initialized with zeros, so let's switch from xcalloc() to xmalloc() to make this clear. Signed-off-by: Matheus Tavares Signed-off-by: Junio C Hamano --- hash.h | 16 ++++++++++++++++ parallel-checkout.c | 13 ++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hash.h b/hash.h index 2986f991c6..9c6df4d952 100644 --- a/hash.h +++ b/hash.h @@ -263,6 +263,22 @@ static inline void oidcpy(struct object_id *dst, const struct object_id *src) dst->algo = src->algo; } +/* Like oidcpy() but zero-pads the unused bytes in dst's hash array. */ +static inline void oidcpy_with_padding(struct object_id *dst, + struct object_id *src) +{ + size_t hashsz; + + if (!src->algo) + hashsz = the_hash_algo->rawsz; + else + hashsz = hash_algos[src->algo].rawsz; + + memcpy(dst->hash, src->hash, hashsz); + memset(dst->hash + hashsz, 0, GIT_MAX_RAWSZ - hashsz); + dst->algo = src->algo; +} + static inline struct object_id *oiddup(const struct object_id *src) { struct object_id *dst = xmalloc(sizeof(struct object_id)); diff --git a/parallel-checkout.c b/parallel-checkout.c index 6b1af32bb3..ddc0ff3c06 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -411,7 +411,7 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) len_data = sizeof(struct pc_item_fixed_portion) + name_len + working_tree_encoding_len; - data = xcalloc(1, len_data); + data = xmalloc(len_data); fixed_portion = (struct pc_item_fixed_portion *)data; fixed_portion->id = pc_item->id; @@ -421,13 +421,12 @@ static void send_one_item(int fd, struct parallel_checkout_item *pc_item) fixed_portion->name_len = name_len; fixed_portion->working_tree_encoding_len = working_tree_encoding_len; /* - * We use hashcpy() instead of oidcpy() because the hash[] positions - * after `the_hash_algo->rawsz` might not be initialized. And Valgrind - * would complain about passing uninitialized bytes to a syscall - * (write(2)). There is no real harm in this case, but the warning could - * hinder the detection of actual errors. + * We pad the unused bytes in the hash array because, otherwise, + * Valgrind would complain about passing uninitialized bytes to a + * write() syscall. The warning doesn't represent any real risk here, + * but it could hinder the detection of actual errors. */ - hashcpy(fixed_portion->oid.hash, pc_item->ce->oid.hash); + oidcpy_with_padding(&fixed_portion->oid, &pc_item->ce->oid); variant = data + sizeof(*fixed_portion); if (working_tree_encoding_len) { From 4279cb1c6e4e5b60b099833d8e3debba4ac35eba Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 17 May 2021 12:22:17 +0000 Subject: [PATCH 070/397] sparse-index: fix uninitialized jump While testing the sparse-index, I verified a test with --valgrind and it complained about an uninitialized value being used in a jump in the path_matches_pattern_list() method. The line was this one: if (*dtype == DT_UNKNOWN) In the call stack, the culprit was the initialization of the dtype variable in convert_to_sparse_rec(). Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- sparse-index.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparse-index.c b/sparse-index.c index 6f21397e2e..ca839e4381 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -34,7 +34,7 @@ static int convert_to_sparse_rec(struct index_state *istate, int i, can_convert = 1; int start_converted = num_converted; enum pattern_match_result match; - int dtype; + int dtype = DT_UNKNOWN; struct strbuf child_path = STRBUF_INIT; struct pattern_list *pl = istate->sparse_checkout_patterns; From 68142e117c080b7a563d681dc34c7df2ab945df5 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 17 May 2021 12:24:49 +0000 Subject: [PATCH 071/397] hashfile: use write_in_full() The flush() logic in csum-file.c was introduced originally by c38138c (git-pack-objects: write the pack files with a SHA1 csum, 2005-06-26) and a portion of the logic performs similar utility to write_in_full() in wrapper.c. The history of write_in_full() is full of moves and renames, but was originally introduced by 7230e6d (Add write_or_die(), a helper function, 2006-08-21). The point of these sections of code are to flush a write buffer using xwrite() and report errors in the case of disk space issues or other generic input/output errors. The logic in flush() can interpret the output of write_in_full() to provide the correct error messages to users. The logic in the hashfile API has an additional set of logic to augment the progress indicator between calls to xwrite(). This was introduced by 2a128d6 (add throughput display to git-push, 2007-10-30). It seems that since the hashfile's buffer is only 8KB, these additional progress indicators might not be incredibly necessary. Instead, update the progress only when write_in_full() complete. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- csum-file.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/csum-file.c b/csum-file.c index 7510950fa3..3c26389d49 100644 --- a/csum-file.c +++ b/csum-file.c @@ -25,21 +25,14 @@ static void flush(struct hashfile *f, const void *buf, unsigned int count) die("sha1 file '%s' validation error", f->name); } - for (;;) { - int ret = xwrite(f->fd, buf, count); - if (ret > 0) { - f->total += ret; - display_throughput(f->tp, f->total); - buf = (char *) buf + ret; - count -= ret; - if (count) - continue; - return; - } - if (!ret) + if (write_in_full(f->fd, buf, count) < 0) { + if (errno == ENOSPC) die("sha1 file '%s' write error. Out of diskspace", f->name); die_errno("sha1 file '%s' write error", f->name); } + + f->total += count; + display_throughput(f->tp, f->total); } void hashflush(struct hashfile *f) From f302c1e4aa09ca6120968256d65dba291990929d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Tue, 18 May 2021 09:53:37 +0900 Subject: [PATCH 072/397] revisions(7): clarify that most commands take a single revision range Sometimes new people are confused by how a revision "range" works, in that it is not a random collection of commits but a set of commits that are all connected to each other, and most Git commands work on a single such "range". Give an example to clarify it. Signed-off-by: Junio C Hamano --- Documentation/revisions.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Documentation/revisions.txt b/Documentation/revisions.txt index d9169c062e..f5f17b65a1 100644 --- a/Documentation/revisions.txt +++ b/Documentation/revisions.txt @@ -260,6 +260,9 @@ any of the given commits. A commit's reachable set is the commit itself and the commits in its ancestry chain. +There are several notations to specify a set of connected commits +(called a "revision range"), illustrated below. + Commit Exclusions ~~~~~~~~~~~~~~~~~ @@ -294,6 +297,26 @@ is a shorthand for 'HEAD..origin' and asks "What did the origin do since I forked from them?" Note that '..' would mean 'HEAD..HEAD' which is an empty range that is both reachable and unreachable from HEAD. +Commands that are specifically designed to take two distinct ranges +(e.g. "git range-diff R1 R2" to compare two ranges) do exist, but +they are exceptions. Unless otherwise noted, all "git" commands +that operate on a set of commits work on a single revision range. +In other words, writing two "two-dot range notation" next to each +other, e.g. + + $ git log A..B C..D + +does *not* specify two revision ranges for most commands. Instead +it will name a single connected set of commits, i.e. those that are +reachable from either B or D but are reachable from neither A or C. +In a linear history like this: + + ---A---B---o---o---C---D + +because A and B are reachable from C, the revision range specified +by these two dotted ranges is a single commit D. + + Other {caret} Parent Shorthand Notations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Three other shorthands exist, particularly useful for merge commits, From b694f1e49ec39f45f61ca3b6d9df5945cbf43b71 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Tue, 18 May 2021 02:27:36 -0400 Subject: [PATCH 073/397] t5551: test http interaction with credential helpers We test authentication with http, and we independently test that credential helpers work, but we don't have any tests that cover the two features working together. Let's add two: 1. Make sure that a successful request asks the helper to save the credential. This works as expected. 2. Make sure that a failed request asks the helper to forget the credential. This is marked as expect_failure, as it was recently regressed by 1b0d9545bb (remote-curl: fall back to basic auth if Negotiate fails, 2021-03-22). The symptom here is that the second request should prompt the user, but doesn't. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- t/t5551-http-fetch-smart.sh | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index 984dba22af..1de87e4ffe 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -517,4 +517,45 @@ test_expect_success 'server-side error detected' ' test_i18ngrep "server-side error" actual ' +test_expect_success 'http auth remembers successful credentials' ' + rm -f .git-credentials && + test_config credential.helper store && + + # the first request prompts the user... + set_askpass user@host pass@host && + git ls-remote "$HTTPD_URL/auth/smart/repo.git" >/dev/null && + expect_askpass both user@host && + + # ...and the second one uses the stored value rather than + # prompting the user. + set_askpass bogus-user bogus-pass && + git ls-remote "$HTTPD_URL/auth/smart/repo.git" >/dev/null && + expect_askpass none +' + +test_expect_failure 'http auth forgets bogus credentials' ' + # seed credential store with bogus values. In real life, + # this would probably come from a password which worked + # for a previous request. + rm -f .git-credentials && + test_config credential.helper store && + { + echo "url=$HTTPD_URL" && + echo "username=bogus" && + echo "password=bogus" + } | git credential approve && + + # we expect this to use the bogus values and fail, never even + # prompting the user... + set_askpass user@host pass@host && + test_must_fail git ls-remote "$HTTPD_URL/auth/smart/repo.git" >/dev/null && + expect_askpass none && + + # ...but now we should have forgotten the bad value, causing + # us to prompt the user again. + set_askpass user@host pass@host && + git ls-remote "$HTTPD_URL/auth/smart/repo.git" >/dev/null && + expect_askpass both user@host +' + test_done From ecf7b129fa8619bfe16c5f7e470717ad79186e07 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Tue, 18 May 2021 02:27:42 -0400 Subject: [PATCH 074/397] Revert "remote-curl: fall back to basic auth if Negotiate fails" This reverts commit 1b0d9545bb85912a16b367229d414f55d140d3be. That commit does fix the situation it intended to (avoiding Negotiate even when the credentials were provided in the URL), but it creates a more serious regression: we now never hit the conditional for "we had a username and password, tried them, but the server still gave us a 401". That has two bad effects: 1. we never call credential_reject(), and thus a bogus credential stored by a helper will live on forever 2. we never return HTTP_NOAUTH, so the error message the user gets is "The requested URL returned error: 401", instead of "Authentication failed". Doing this correctly seems non-trivial, as we don't know whether the Negotiate auth was a problem. Since this is a regression in the upcoming v2.23.0 release (for which we're in -rc0), let's revert for now and work on a fix separately. (Note that this isn't a pure revert; the previous commit added a test showing the regression, so we can now flip it to expect_success). Reported-by: Ben Humphreys Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- http.c | 15 +++++++-------- t/t5551-http-fetch-smart.sh | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/http.c b/http.c index 19c203d0ca..0e31fc21bc 100644 --- a/http.c +++ b/http.c @@ -1641,18 +1641,17 @@ static int handle_curl_result(struct slot_results *results) } else if (missing_target(results)) return HTTP_MISSING_TARGET; else if (results->http_code == 401) { -#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY - http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE; - if (results->auth_avail) { - http_auth_methods &= results->auth_avail; - http_auth_methods_restricted = 1; - return HTTP_REAUTH; - } -#endif if (http_auth.username && http_auth.password) { credential_reject(&http_auth); return HTTP_NOAUTH; } else { +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY + http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE; + if (results->auth_avail) { + http_auth_methods &= results->auth_avail; + http_auth_methods_restricted = 1; + } +#endif return HTTP_REAUTH; } } else { diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index 1de87e4ffe..4f87d90c5b 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -533,7 +533,7 @@ test_expect_success 'http auth remembers successful credentials' ' expect_askpass none ' -test_expect_failure 'http auth forgets bogus credentials' ' +test_expect_success 'http auth forgets bogus credentials' ' # seed credential store with bogus values. In real life, # this would probably come from a password which worked # for a previous request. From aafa5df0df39036c6500846acd3db5b75d264a3b Mon Sep 17 00:00:00 2001 From: Jonathan Nieder Date: Tue, 18 May 2021 18:52:56 -0700 Subject: [PATCH 075/397] xsize_t: avoid implementation defined behavior when len < 0 The xsize_t helper aims to safely convert an off_t to a size_t, erroring out when a file offset is too large to fit into a memory address. It does this by using two casts: size_t size = (size_t) len; if (len != (off_t) size) ... error out ... On a platform with sizeof(size_t) < sizeof(off_t), this check is safe and correct. The first cast truncates to a size_t by finding the remainder modulo SIZE_MAX+1 (see C99 section 6.3.1.3 Signed and unsigned integers) and the second promotes to an off_t, meaning the result is true if and only if len is representable as a size_t. On other platforms, this two-casts strategy still works well (always succeeds) for len >= 0. But for len < 0, when the first cast succeeds and produces SIZE_MAX + 1 + len, the resulting value is too large to be represented as an off_t, so the second cast produces implementation defined behavior. In practice, it is likely to produce a result of true despite len not being representable as size_t. Simplify by replacing with a more straightforward check: compare len to the relevant bounds and then cast it. (To avoid a -Wsign-compare warning, after checking that len >= 0, we explicitly convert to a sufficiently-large unsigned type before comparing to SIZE_MAX.) In practice, this is not likely to come up since typical callers use nonnegative len. Still, it's helpful to handle this case to make the behavior easy to reason about. Historical note: the original bounds-checking in 46be82dfd0 (xsize_t: check whether we lose bits, 2010-07-28) did not produce this implementation-defined behavior, though it still did not handle negative offsets. It was not until 73560c793a (git-compat-util.h: xsize_t() - avoid -Wsign-compare warnings, 2017-09-21) introduced the double cast that the implementation-defined behavior was triggered. Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- git-compat-util.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/git-compat-util.h b/git-compat-util.h index dcc786edaa..acef7f6bcb 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -981,11 +981,9 @@ static inline char *xstrdup_or_null(const char *str) static inline size_t xsize_t(off_t len) { - size_t size = (size_t) len; - - if (len != (off_t) size) + if (len < 0 || (uintmax_t) len > SIZE_MAX) die("Cannot handle files this big"); - return size; + return (size_t) len; } __attribute__((format (printf, 3, 4))) From 2ca245f8be56e3269d02076b658e825b91236e5d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 18 May 2021 18:32:46 +0000 Subject: [PATCH 076/397] csum-file.h: increase hashfile buffer size The hashfile API uses a hard-coded buffer size of 8KB and has ever since it was introduced in c38138c (git-pack-objects: write the pack files with a SHA1 csum, 2005-06-26). It performs a similar function to the hashing buffers in read-cache.c, but that code was updated from 8KB to 128KB in f279894 (read-cache: make the index write buffer size 128K, 2021-02-18). The justification there was that do_write_index() improves from 1.02s to 0.72s. Since our end goal is to have the index writing code use the hashfile API, we need to unify this buffer size to avoid a performance regression. There is a buffer, 'check_buffer', that is used to verify the check_fd file descriptor. When this buffer increases to 128K to fit the data being flushed, it causes the stack to overflow the limits placed in the test suite. To avoid issues with stack size, move both 'buffer' and 'check_buffer' to be heap pointers within 'struct hashfile'. The 'check_buffer' member is left as NULL unless check_fd is set in hashfd_check(). Both buffers are cleared as part of finalize_hashfile() which also frees the full structure. Since these buffers are now on the heap, we can adjust their size based on the needs of the consumer. In particular, callers to hashfd_throughput() are expecting to report progress indicators as the buffer flushes. These callers would prefer the smaller 8k buffer to avoid large delays between updates, especially for users with slower networks. When the progress indicator is not used, the larger buffer is preferrable. By adding a new trace2 region in the chunk-format API, we can see that the writing portion of 'git multi-pack-index write' lowers from ~1.49s to ~1.47s on a Linux machine. These effects may be more pronounced or diminished on other filesystems. The end-to-end timing is too noisy to have a definitive change either way. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- chunk-format.c | 12 +++++--- csum-file.c | 77 +++++++++++++++++++++++++++++++++++++------------- csum-file.h | 4 ++- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/chunk-format.c b/chunk-format.c index da191e59a2..1c3dca62e2 100644 --- a/chunk-format.c +++ b/chunk-format.c @@ -58,9 +58,11 @@ void add_chunk(struct chunkfile *cf, int write_chunkfile(struct chunkfile *cf, void *data) { - int i; + int i, result = 0; uint64_t cur_offset = hashfile_total(cf->f); + trace2_region_enter("chunkfile", "write", the_repository); + /* Add the table of contents to the current offset */ cur_offset += (cf->chunks_nr + 1) * CHUNK_TOC_ENTRY_SIZE; @@ -77,10 +79,10 @@ int write_chunkfile(struct chunkfile *cf, void *data) for (i = 0; i < cf->chunks_nr; i++) { off_t start_offset = hashfile_total(cf->f); - int result = cf->chunks[i].write_fn(cf->f, data); + result = cf->chunks[i].write_fn(cf->f, data); if (result) - return result; + goto cleanup; if (hashfile_total(cf->f) - start_offset != cf->chunks[i].size) BUG("expected to write %"PRId64" bytes to chunk %"PRIx32", but wrote %"PRId64" instead", @@ -88,7 +90,9 @@ int write_chunkfile(struct chunkfile *cf, void *data) hashfile_total(cf->f) - start_offset); } - return 0; +cleanup: + trace2_region_leave("chunkfile", "write", the_repository); + return result; } int read_table_of_contents(struct chunkfile *cf, diff --git a/csum-file.c b/csum-file.c index 3c26389d49..3487d28ed7 100644 --- a/csum-file.c +++ b/csum-file.c @@ -11,19 +11,24 @@ #include "progress.h" #include "csum-file.h" +static void verify_buffer_or_die(struct hashfile *f, + const void *buf, + unsigned int count) +{ + ssize_t ret = read_in_full(f->check_fd, f->check_buffer, count); + + if (ret < 0) + die_errno("%s: sha1 file read error", f->name); + if (ret != count) + die("%s: sha1 file truncated", f->name); + if (memcmp(buf, f->check_buffer, count)) + die("sha1 file '%s' validation error", f->name); +} + static void flush(struct hashfile *f, const void *buf, unsigned int count) { - if (0 <= f->check_fd && count) { - unsigned char check_buffer[8192]; - ssize_t ret = read_in_full(f->check_fd, check_buffer, count); - - if (ret < 0) - die_errno("%s: sha1 file read error", f->name); - if (ret != count) - die("%s: sha1 file truncated", f->name); - if (memcmp(buf, check_buffer, count)) - die("sha1 file '%s' validation error", f->name); - } + if (0 <= f->check_fd && count) + verify_buffer_or_die(f, buf, count); if (write_in_full(f->fd, buf, count) < 0) { if (errno == ENOSPC) @@ -46,6 +51,13 @@ void hashflush(struct hashfile *f) } } +static void free_hashfile(struct hashfile *f) +{ + free(f->buffer); + free(f->check_buffer); + free(f); +} + int finalize_hashfile(struct hashfile *f, unsigned char *result, unsigned int flags) { int fd; @@ -75,20 +87,20 @@ int finalize_hashfile(struct hashfile *f, unsigned char *result, unsigned int fl if (close(f->check_fd)) die_errno("%s: sha1 file error on close", f->name); } - free(f); + free_hashfile(f); return fd; } void hashwrite(struct hashfile *f, const void *buf, unsigned int count) { while (count) { - unsigned left = sizeof(f->buffer) - f->offset; + unsigned left = f->buffer_len - f->offset; unsigned nr = count > left ? left : count; if (f->do_crc) f->crc32 = crc32(f->crc32, buf, nr); - if (nr == sizeof(f->buffer)) { + if (nr == f->buffer_len) { /* * Flush a full batch worth of data directly * from the input, skipping the memcpy() to @@ -114,11 +126,6 @@ void hashwrite(struct hashfile *f, const void *buf, unsigned int count) } } -struct hashfile *hashfd(int fd, const char *name) -{ - return hashfd_throughput(fd, name, NULL); -} - struct hashfile *hashfd_check(const char *name) { int sink, check; @@ -132,10 +139,14 @@ struct hashfile *hashfd_check(const char *name) die_errno("unable to open '%s'", name); f = hashfd(sink, name); f->check_fd = check; + f->check_buffer = xmalloc(f->buffer_len); + return f; } -struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp) +static struct hashfile *hashfd_internal(int fd, const char *name, + struct progress *tp, + size_t buffer_len) { struct hashfile *f = xmalloc(sizeof(*f)); f->fd = fd; @@ -146,9 +157,35 @@ struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp f->name = name; f->do_crc = 0; the_hash_algo->init_fn(&f->ctx); + + f->buffer_len = buffer_len; + f->buffer = xmalloc(buffer_len); + f->check_buffer = NULL; + return f; } +struct hashfile *hashfd(int fd, const char *name) +{ + /* + * Since we are not going to use a progress meter to + * measure the rate of data passing through this hashfile, + * use a larger buffer size to reduce fsync() calls. + */ + return hashfd_internal(fd, name, NULL, 128 * 1024); +} + +struct hashfile *hashfd_throughput(int fd, const char *name, struct progress *tp) +{ + /* + * Since we are expecting to report progress of the + * write into this hashfile, use a smaller buffer + * size so the progress indicators arrive at a more + * frequent rate. + */ + return hashfd_internal(fd, name, tp, 8 * 1024); +} + void hashfile_checkpoint(struct hashfile *f, struct hashfile_checkpoint *checkpoint) { hashflush(f); diff --git a/csum-file.h b/csum-file.h index e54d53d1d0..3044bd19ab 100644 --- a/csum-file.h +++ b/csum-file.h @@ -16,7 +16,9 @@ struct hashfile { const char *name; int do_crc; uint32_t crc32; - unsigned char buffer[8192]; + size_t buffer_len; + unsigned char *buffer; + unsigned char *check_buffer; }; /* Checkpoint */ From 410334ed5212979a3ff6c8ca119ca14290ee7790 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 18 May 2021 18:32:47 +0000 Subject: [PATCH 077/397] read-cache: use hashfile instead of git_hash_ctx The do_write_index() method in read-cache.c has its own hashing logic and buffering mechanism. Specifically, the ce_write() method was introduced by 4990aadc (Speed up index file writing by chunking it nicely, 2005-04-20) and similar mechanisms were introduced a few months later in c38138cd (git-pack-objects: write the pack files with a SHA1 csum, 2005-06-26). Based on the timing, in the early days of the Git codebase, I figured that these roughly equivalent code paths were never unified only because it got lost in the shuffle. The hashfile API has since been used extensively in other file formats, such as pack-indexes, multi-pack-indexes, and commit-graphs. Therefore, it seems prudent to unify the index writing code to use the same mechanism. I discovered this disparity while trying to create a new index format that uses the chunk-format API. That API uses a hashfile as its base, so it is incompatible with the custom code in read-cache.c. This rewrite is rather straightforward. It replaces all writes to the temporary file with writes to the hashfile struct. This takes care of many of the direct interactions with the_hash_algo. There are still some git_hash_ctx uses remaining: the extension headers are hashed for use in the End of Index Entries (EOIE) extension. This use of the git_hash_ctx is left as-is. There are multiple reasons to not use a hashfile here, including the fact that the data is not actually writing to a file, just a hash computation. These hashes do not block our adoption of the chunk-format API in a future change to the index, so leave it as-is. The internals of the algorithms are mostly identical. Previously, the hashfile API used a smaller 8KB buffer instead of the 128KB buffer from read-cache.c. The previous change already unified these sizes. There is one subtle point: we do not pass the CSUM_FSYNC to the finalize_hashfile() method, which differs from most consumers of the hashfile API. The extra fsync() call indicated by this flag causes a significant peformance degradation that is noticeable for quick commands that write the index, such as "git add". Other consumers can absorb this cost with their more complicated data structure organization, and further writing structures such as pack-files and commit-graphs is rarely in the critical path for common user interactions. Some static methods become orphaned in this diff, so I marked them as MAYBE_UNUSED. The diff is much harder to read if they are deleted during this change. Instead, they will be deleted in the following change. In addition to the test suite passing, I computed indexes using the previous binaries and the binaries compiled after this change, and found the index data to be exactly equal. Finally, I did extensive performance testing of "git update-index --force-write" on repos of various sizes, including one with over 2 million paths at HEAD. These tests demonstrated less than 1% difference in behavior. As expected, the performance should be considered unchanged. The previous changes to increase the hashfile buffer size from 8K to 128K ensured this change would not create a peformance regression. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- read-cache.c | 137 +++++++++++++++++++++++++-------------------------- 1 file changed, 66 insertions(+), 71 deletions(-) diff --git a/read-cache.c b/read-cache.c index 1b3c2eb408..e3c9ab8033 100644 --- a/read-cache.c +++ b/read-cache.c @@ -26,6 +26,7 @@ #include "thread-utils.h" #include "progress.h" #include "sparse-index.h" +#include "csum-file.h" /* Mask for the name length in ce_flags in the on-disk index */ @@ -2525,6 +2526,7 @@ int repo_index_has_changes(struct repository *repo, static unsigned char write_buffer[WRITE_BUFFER_SIZE]; static unsigned long write_buffer_len; +MAYBE_UNUSED static int ce_write_flush(git_hash_ctx *context, int fd) { unsigned int buffered = write_buffer_len; @@ -2537,6 +2539,7 @@ static int ce_write_flush(git_hash_ctx *context, int fd) return 0; } +MAYBE_UNUSED static int ce_write(git_hash_ctx *context, int fd, void *data, unsigned int len) { while (len) { @@ -2559,19 +2562,24 @@ static int ce_write(git_hash_ctx *context, int fd, void *data, unsigned int len) return 0; } -static int write_index_ext_header(git_hash_ctx *context, git_hash_ctx *eoie_context, - int fd, unsigned int ext, unsigned int sz) +static int write_index_ext_header(struct hashfile *f, + git_hash_ctx *eoie_f, + unsigned int ext, + unsigned int sz) { - ext = htonl(ext); - sz = htonl(sz); - if (eoie_context) { - the_hash_algo->update_fn(eoie_context, &ext, 4); - the_hash_algo->update_fn(eoie_context, &sz, 4); + hashwrite_be32(f, ext); + hashwrite_be32(f, sz); + + if (eoie_f) { + ext = htonl(ext); + sz = htonl(sz); + the_hash_algo->update_fn(eoie_f, &ext, sizeof(ext)); + the_hash_algo->update_fn(eoie_f, &sz, sizeof(sz)); } - return ((ce_write(context, fd, &ext, 4) < 0) || - (ce_write(context, fd, &sz, 4) < 0)) ? -1 : 0; + return 0; } +MAYBE_UNUSED static int ce_flush(git_hash_ctx *context, int fd, unsigned char *hash) { unsigned int left = write_buffer_len; @@ -2673,11 +2681,10 @@ static void copy_cache_entry_to_ondisk(struct ondisk_cache_entry *ondisk, } } -static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, +static int ce_write_entry(struct hashfile *f, struct cache_entry *ce, struct strbuf *previous_name, struct ondisk_cache_entry *ondisk) { int size; - int result; unsigned int saved_namelen; int stripped_name = 0; static unsigned char padding[8] = { 0x00 }; @@ -2693,11 +2700,9 @@ static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, if (!previous_name) { int len = ce_namelen(ce); copy_cache_entry_to_ondisk(ondisk, ce); - result = ce_write(c, fd, ondisk, size); - if (!result) - result = ce_write(c, fd, ce->name, len); - if (!result) - result = ce_write(c, fd, padding, align_padding_size(size, len)); + hashwrite(f, ondisk, size); + hashwrite(f, ce->name, len); + hashwrite(f, padding, align_padding_size(size, len)); } else { int common, to_remove, prefix_size; unsigned char to_remove_vi[16]; @@ -2711,13 +2716,10 @@ static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, prefix_size = encode_varint(to_remove, to_remove_vi); copy_cache_entry_to_ondisk(ondisk, ce); - result = ce_write(c, fd, ondisk, size); - if (!result) - result = ce_write(c, fd, to_remove_vi, prefix_size); - if (!result) - result = ce_write(c, fd, ce->name + common, ce_namelen(ce) - common); - if (!result) - result = ce_write(c, fd, padding, 1); + hashwrite(f, ondisk, size); + hashwrite(f, to_remove_vi, prefix_size); + hashwrite(f, ce->name + common, ce_namelen(ce) - common); + hashwrite(f, padding, 1); strbuf_splice(previous_name, common, to_remove, ce->name + common, ce_namelen(ce) - common); @@ -2727,7 +2729,7 @@ static int ce_write_entry(git_hash_ctx *c, int fd, struct cache_entry *ce, ce->ce_flags &= ~CE_STRIP_NAME; } - return result; + return 0; } /* @@ -2839,8 +2841,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, int strip_extensions) { uint64_t start = getnanotime(); - int newfd = tempfile->fd; - git_hash_ctx c, eoie_c; + struct hashfile *f; + git_hash_ctx *eoie_c = NULL; struct cache_header hdr; int i, err = 0, removed, extended, hdr_version; struct cache_entry **cache = istate->cache; @@ -2854,6 +2856,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct index_entry_offset_table *ieot = NULL; int nr, nr_threads; + f = hashfd(tempfile->fd, tempfile->filename.buf); + for (i = removed = extended = 0; i < entries; i++) { if (cache[i]->ce_flags & CE_REMOVE) removed++; @@ -2882,9 +2886,7 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, hdr.hdr_version = htonl(hdr_version); hdr.hdr_entries = htonl(entries - removed); - the_hash_algo->init_fn(&c); - if (ce_write(&c, newfd, &hdr, sizeof(hdr)) < 0) - return -1; + hashwrite(f, &hdr, sizeof(hdr)); if (!HAVE_THREADS || git_config_get_index_threads(&nr_threads)) nr_threads = 1; @@ -2919,12 +2921,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, } } - offset = lseek(newfd, 0, SEEK_CUR); - if (offset < 0) { - free(ieot); - return -1; - } - offset += write_buffer_len; + offset = hashfile_total(f); + nr = 0; previous_name = (hdr_version == 4) ? &previous_name_buf : NULL; @@ -2959,14 +2957,10 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, if (previous_name) previous_name->buf[0] = 0; nr = 0; - offset = lseek(newfd, 0, SEEK_CUR); - if (offset < 0) { - free(ieot); - return -1; - } - offset += write_buffer_len; + + offset = hashfile_total(f); } - if (ce_write_entry(&c, newfd, ce, previous_name, (struct ondisk_cache_entry *)&ondisk) < 0) + if (ce_write_entry(f, ce, previous_name, (struct ondisk_cache_entry *)&ondisk) < 0) err = -1; if (err) @@ -2985,14 +2979,16 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, return err; } - /* Write extension data here */ - offset = lseek(newfd, 0, SEEK_CUR); - if (offset < 0) { - free(ieot); - return -1; + offset = hashfile_total(f); + + /* + * The extension headers must be hashed on their own for the + * EOIE extension. Create a hashfile here to compute that hash. + */ + if (offset && record_eoie()) { + CALLOC_ARRAY(eoie_c, 1); + the_hash_algo->init_fn(eoie_c); } - offset += write_buffer_len; - the_hash_algo->init_fn(&eoie_c); /* * Lets write out CACHE_EXT_INDEXENTRYOFFSETTABLE first so that we @@ -3005,8 +3001,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; write_ieot_extension(&sb, ieot); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_INDEXENTRYOFFSETTABLE, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); free(ieot); if (err) @@ -3018,9 +3014,9 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; err = write_link_extension(&sb, istate) < 0 || - write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_LINK, - sb.len) < 0 || - ce_write(&c, newfd, sb.buf, sb.len) < 0; + write_index_ext_header(f, eoie_c, CACHE_EXT_LINK, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3029,8 +3025,8 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; cache_tree_write(&sb, istate->cache_tree); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_TREE, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_TREE, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3039,9 +3035,9 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; resolve_undo_write(&sb, istate->resolve_undo); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_RESOLVE_UNDO, - sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_RESOLVE_UNDO, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3050,9 +3046,9 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; write_untracked_extension(&sb, istate->untracked); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_UNTRACKED, - sb.len) < 0 || - ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_UNTRACKED, + sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; @@ -3061,14 +3057,14 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, struct strbuf sb = STRBUF_INIT; write_fsmonitor_extension(&sb, istate); - err = write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_FSMONITOR, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + err = write_index_ext_header(f, eoie_c, CACHE_EXT_FSMONITOR, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; } if (istate->sparse_index) { - if (write_index_ext_header(&c, &eoie_c, newfd, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) + if (write_index_ext_header(f, eoie_c, CACHE_EXT_SPARSE_DIRECTORIES, 0) < 0) return -1; } @@ -3078,19 +3074,18 @@ static int do_write_index(struct index_state *istate, struct tempfile *tempfile, * read. Write it out regardless of the strip_extensions parameter as we need it * when loading the shared index. */ - if (offset && record_eoie()) { + if (eoie_c) { struct strbuf sb = STRBUF_INIT; - write_eoie_extension(&sb, &eoie_c, offset); - err = write_index_ext_header(&c, NULL, newfd, CACHE_EXT_ENDOFINDEXENTRIES, sb.len) < 0 - || ce_write(&c, newfd, sb.buf, sb.len) < 0; + write_eoie_extension(&sb, eoie_c, offset); + err = write_index_ext_header(f, NULL, CACHE_EXT_ENDOFINDEXENTRIES, sb.len) < 0; + hashwrite(f, sb.buf, sb.len); strbuf_release(&sb); if (err) return -1; } - if (ce_flush(&c, newfd, istate->oid.hash)) - return -1; + finalize_hashfile(f, istate->oid.hash, CSUM_HASH_IN_STREAM); if (close_tempfile_gently(tempfile)) { error(_("could not close '%s'"), get_tempfile_path(tempfile)); return -1; From f6e2cd0625c8f15505a1bc11828b21490dff59e6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 18 May 2021 18:32:48 +0000 Subject: [PATCH 078/397] read-cache: delete unused hashing methods These methods were marked as MAYBE_UNUSED in the previous change to avoid a complicated diff. Delete them entirely, since we now use the hashfile API instead of this custom hashing code. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- read-cache.c | 64 ---------------------------------------------------- 1 file changed, 64 deletions(-) diff --git a/read-cache.c b/read-cache.c index e3c9ab8033..77961a3885 100644 --- a/read-cache.c +++ b/read-cache.c @@ -2522,46 +2522,6 @@ int repo_index_has_changes(struct repository *repo, } } -#define WRITE_BUFFER_SIZE (128 * 1024) -static unsigned char write_buffer[WRITE_BUFFER_SIZE]; -static unsigned long write_buffer_len; - -MAYBE_UNUSED -static int ce_write_flush(git_hash_ctx *context, int fd) -{ - unsigned int buffered = write_buffer_len; - if (buffered) { - the_hash_algo->update_fn(context, write_buffer, buffered); - if (write_in_full(fd, write_buffer, buffered) < 0) - return -1; - write_buffer_len = 0; - } - return 0; -} - -MAYBE_UNUSED -static int ce_write(git_hash_ctx *context, int fd, void *data, unsigned int len) -{ - while (len) { - unsigned int buffered = write_buffer_len; - unsigned int partial = WRITE_BUFFER_SIZE - buffered; - if (partial > len) - partial = len; - memcpy(write_buffer + buffered, data, partial); - buffered += partial; - if (buffered == WRITE_BUFFER_SIZE) { - write_buffer_len = buffered; - if (ce_write_flush(context, fd)) - return -1; - buffered = 0; - } - write_buffer_len = buffered; - len -= partial; - data = (char *) data + partial; - } - return 0; -} - static int write_index_ext_header(struct hashfile *f, git_hash_ctx *eoie_f, unsigned int ext, @@ -2579,30 +2539,6 @@ static int write_index_ext_header(struct hashfile *f, return 0; } -MAYBE_UNUSED -static int ce_flush(git_hash_ctx *context, int fd, unsigned char *hash) -{ - unsigned int left = write_buffer_len; - - if (left) { - write_buffer_len = 0; - the_hash_algo->update_fn(context, write_buffer, left); - } - - /* Flush first if not enough space for hash signature */ - if (left + the_hash_algo->rawsz > WRITE_BUFFER_SIZE) { - if (write_in_full(fd, write_buffer, left) < 0) - return -1; - left = 0; - } - - /* Append the hash signature at the end */ - the_hash_algo->final_fn(write_buffer + left, context); - hashcpy(hash, write_buffer + left); - left += the_hash_algo->rawsz; - return (write_in_full(fd, write_buffer, left) < 0) ? -1 : 0; -} - static void ce_smudge_racily_clean_entry(struct index_state *istate, struct cache_entry *ce) { From e22f2daed0fde4595696e09c6802a1ecd5e0c527 Mon Sep 17 00:00:00 2001 From: Reuven Y Date: Wed, 19 May 2021 09:28:49 +0000 Subject: [PATCH 079/397] docs: improve fast-forward in glossary content The text was somewhat confusing between the revision itself and the author. Signed-off-by: Reuven Yagel Signed-off-by: Junio C Hamano --- Documentation/glossary-content.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/glossary-content.txt b/Documentation/glossary-content.txt index 67c7a50b96..c077971335 100644 --- a/Documentation/glossary-content.txt +++ b/Documentation/glossary-content.txt @@ -146,8 +146,8 @@ current branch integrates with) obviously do not work, as there is no <> and you are "merging" another <>'s changes that happen to be a descendant of what you have. In such a case, you do not make a new <> - <> but instead just update to his - revision. This will happen frequently on a + <> but instead just update your branch to point at the same + revision as the branch you are merging. This will happen frequently on a <> of a remote <>. From 6aacb7d8619c50b3c92fe3e99b93ffa9b6065dfc Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 19 May 2021 07:17:15 -0400 Subject: [PATCH 080/397] clone: clean up directory after transport_fetch_refs() failure git-clone started respecting errors from the transport subsystem in aab179d937 (builtin/clone.c: don't ignore transport_fetch_refs() errors, 2020-12-03). However, that commit didn't handle the cleanup of the filesystem quite right. The cleanup of the directory that cmd_clone() creates is done by an atexit() handler, which we control with a flag. It starts as JUNK_LEAVE_NONE ("clean up everything"), then progresses to JUNK_LEAVE_REPO when we know we have a valid repo but not working tree, and then finally JUNK_LEAVE_ALL when we have a successful checkout. Most errors cause us to die(), which then triggers the handler to do the right thing based on how far into cmd_clone() we got. But the checks added by aab179d937 instead set the "err" variable and then jump to a new "cleanup" label, which then returns our non-zero status. However, the code after the cleanup label includes setting the flag to JUNK_LEAVE_ALL, and so we accidentally leave the repository and working tree in place. One obvious option to fix this is to reorder the end of the function to set the flag first, before cleanup code, and put the label between them. But we can observe another small bug: the error return from transport_fetch_refs() is generally "-1", and we propagate that to the return value of cmd_clone(), which ultimately becomes the exit code of the process. And we try to avoid transmitting negative values via exit codes (only the low 8 bits are passed along as an unsigned value, though in practice for "-1" this at least retains the property that it's non-zero). Instead, let's just die(). That makes us consistent with rest of the code in the function. It does add a new "fatal:" line to the output, but I'd argue that's a good thing: - in the rare case that the transport code didn't say anything, now the user gets _some_ error message - even if the transport code said something like "error: ssh died of signal 9", it's nice to also say "fatal" to indicate that we considered that to be a show-stopper. Triggering this in the test suite turns out to be surprisingly difficult. Almost every error we'd encounter, including ones deep inside the transport code, cause us to just die() right there! However, one way is to put a fake wrapper around git-upload-pack that sends the complete packfile but exits with a failure code. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/clone.c | 11 ++++------- t/t5600-clone-fail-cleanup.sh | 7 +++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/builtin/clone.c b/builtin/clone.c index e335734b4c..5888d2ae60 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1294,9 +1294,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) } if (!is_local && !complete_refs_before_fetch) { - err = transport_fetch_refs(transport, mapped_refs); - if (err) - goto cleanup; + if (transport_fetch_refs(transport, mapped_refs)) + die(_("remote transport reported error")); } remote_head = find_ref_by_name(refs, "HEAD"); @@ -1343,9 +1342,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix) if (is_local) clone_local(path, git_dir); else if (refs && complete_refs_before_fetch) { - err = transport_fetch_refs(transport, mapped_refs); - if (err) - goto cleanup; + if (transport_fetch_refs(transport, mapped_refs)) + die(_("remote transport reported error")); } update_remote_refs(refs, mapped_refs, remote_head_points_at, @@ -1373,7 +1371,6 @@ int cmd_clone(int argc, const char **argv, const char *prefix) junk_mode = JUNK_LEAVE_REPO; err = checkout(submodule_progress); -cleanup: free(remote_name); strbuf_release(&reflog_msg); strbuf_release(&branch_top); diff --git a/t/t5600-clone-fail-cleanup.sh b/t/t5600-clone-fail-cleanup.sh index 4a1a912e03..5bf10261d3 100755 --- a/t/t5600-clone-fail-cleanup.sh +++ b/t/t5600-clone-fail-cleanup.sh @@ -97,4 +97,11 @@ test_expect_success 'failed clone into empty leaves directory (separate, wt)' ' test_dir_is_empty empty-wt ' +test_expect_success 'transport failure cleans up directory' ' + test_must_fail git clone --no-local \ + -u "f() { git-upload-pack \"\$@\"; return 1; }; f" \ + foo broken-clone && + test_path_is_missing broken-clone +' + test_done From ae1a7eefffe60425e6bf6a2065e042ae051cfb6c Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 19 May 2021 12:11:05 -0400 Subject: [PATCH 081/397] fetch-pack: signal v2 server that we are done making requests When fetching with the v0 protocol over ssh (or a local upload-pack with pipes), the server closes the connection as soon as it is finished sending the pack. So even though the client may still be operating on the data via index-pack (e.g., resolving deltas, checking connectivity, etc), the server has released all resources. With the v2 protocol, however, the server considers the ssh session only as a transport, with individual requests coming over it. After sending the pack, it goes back to its main loop, waiting for another request to come from the client. As a result, the ssh session hangs around until the client process ends, which may be much later (because resolving deltas, etc, may consume a lot of CPU). This is bad for two reasons: - it's consuming resources on the server to leave open a connection that won't see any more use - if something bad happens to the ssh connection in the meantime (say, it gets killed by the network because it's idle, as happened in a real-world report), then ssh will exit non-zero, and we'll propagate the error up the stack. The server is correct here not to hang up after serving the pack. The v2 protocol's design is meant to allow multiple requests like this, and hanging up would be the wrong thing for a hypothetical client which was planning to make more requests (though in practice, the git.git client never would, and I doubt any other implementations would either). The right thing is instead for the client to signal to the server that it's not interested in making more requests. We can do that by closing the pipe descriptor we use to write to ssh. This will propagate to the server upload-pack as an EOF when it tries to read the next request (and then it will close its half, and the whole connection will go away). It's important to do this "half duplex" shutdown, because we have to do it _before_ we actually receive the pack. This is an artifact of the way fetch-pack and index-pack (or unpack-objects) interact. We hand the connection off to index-pack (really, a sideband demuxer which feeds it), and then wait until it returns. And it doesn't do that until it has resolved all of the deltas in the pack, even though it was done reading from the server long before. So just closing the connection fully after index-pack returns would be too late; we'd have held it open much longer than was necessary. And teaching index-pack to close the connection is awkward. It's not even seeing the whole conversation (the sideband demuxer is, but it doesn't actually know what's in the packets, or when the end comes). Note that this close() is happening deep within the transport code. It's possible that a caller would want to perform other operations over the same ssh transport after receiving the pack. But as of the current code, none of the callers do, and there haven't been discussions of any plans to change this. If we need to support that later, we can probably do so by passing down a flag for "you're the last request on the transport; it's OK to close" instead of the code just assuming that's true. The description above all discusses v2 ssh, so it's worth thinking about how this interacts with other protocols: - in v0 protocols, we could do the same half-duplex shutdown (it just goes into the v0 do_fetch_pack() instead). This does work, but since it doesn't have the same persistence problem in the first place, there's little reason to change it at this point. - local fetches against git-upload-pack on the same machine will behave the same as ssh (they are talking over two pipes, and see EOF on their input pipe) - fetches against git-daemon will run this same code, and close one of the descriptors. In practice, this won't do anything, since there our two descriptors are dups of each other, and not part of a half-duplex pair. The right thing would probably be to call shutdown(SHUT_WR) on it. I didn't bother with that here. It doesn't face the same error-code problem (since it's just a TCP connection), so it's really only an optimization problem. And git:// is not that widely used these days, and has less impact on server resources than an ssh termination. - v2 http doesn't suffer from this problem in the first place, as our pipes terminate at a local git-remote-https, which is passing data along as individual requests via curl. Probably curl is keeping the TCP/TLS connection open for more requests, and we might be able to tell it manually "hey, we are done making requests now". But I think that's much less important. It again doesn't suffer from the error-code problem, and HTTP keepalive is pretty well understood (importantly, the timeouts can be set low, because clients like curl know how to reconnect for subsequent requests if necessary). So it's probably not worth figuring out how to tell curl that we're done (though if we do, this patch is probably the first step anyway; fetch-pack closes the pipe back to remote-https, which would be the signal that it should tell curl we're done). The code is pretty straightforward. We close the pipe at the right moment, and set it to -1 to mark it as invalid. I modified the later cleanup code to avoid calling close(-1). That's not strictly necessary, since close(-1) is a noop, but hopefully makes things a bit more obvious to a reader. I suspect that trying to call more transport functions after the close() (e.g., calling transport_fetch_refs() again) would fail, as it's not smart enough to realize we need to re-open the ssh connection. But that's already true when v0 is in use. And no current callers want to do that (and again, the solution is probably a flag in the transport code to keep things open, which can be added later). There's no test here, as the situation it covers is inherently racy (the question is when upload-pack exits, compared to when index-pack finishes resolving deltas and exits). The rather gross shell snippet below does recreate the problematic situation; when run on a sufficiently-large repository (git.git works fine), it kills an "idle" upload-pack while the client is resolving deltas, leading to a failed clone. ( git clone --no-local --progress . foo.git 2>&1 echo >&2 "clone exit code=$?" ) | tr '\r' '\n' | while read line do case "$done,$line" in ,Resolving*) echo "hit resolving deltas; killing upload-pack" killall -9 git-upload-pack done=t ;; esac done Reported-by: Greg Pflaum Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- fetch-pack.c | 9 +++++++++ transport.c | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/fetch-pack.c b/fetch-pack.c index c135635e34..b0c7be717c 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1645,6 +1645,15 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, if (process_section_header(&reader, "packfile-uris", 1)) receive_packfile_uris(&reader, &packfile_uris); process_section_header(&reader, "packfile", 0); + + /* + * this is the final request we'll make of the server; + * do a half-duplex shutdown to indicate that they can + * hang up as soon as the pack is sent. + */ + close(fd[1]); + fd[1] = -1; + if (get_pack(args, fd, pack_lockfiles, packfile_uris.nr ? &index_pack_args : NULL, sought, nr_sought, &fsck_options.gitmodules_found)) diff --git a/transport.c b/transport.c index 6cf3da19eb..50f5830eb6 100644 --- a/transport.c +++ b/transport.c @@ -427,7 +427,8 @@ static int fetch_refs_via_pack(struct transport *transport, cleanup: close(data->fd[0]); - close(data->fd[1]); + if (data->fd[1] >= 0) + close(data->fd[1]); if (finish_connect(data->conn)) ret = -1; data->conn = NULL; @@ -869,7 +870,8 @@ static int disconnect_git(struct transport *transport) if (data->got_remote_heads && !transport->stateless_rpc) packet_flush(data->fd[1]); close(data->fd[0]); - close(data->fd[1]); + if (data->fd[1] >= 0) + close(data->fd[1]); finish_connect(data->conn); } From 617480d75bdca266d4549e4047452c633ddb7a52 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 19 May 2021 15:31:28 +0000 Subject: [PATCH 082/397] refs: make explicit that ref_iterator_peel returns boolean Use -1 as error return value throughout. This removes spurious differences in the GIT_TRACE_REFS output, depending on the ref storage backend active. Before, the cached ref_iterator (but only that iterator!) would return peel_object() output directly. No callers relied on the peel_status values beyond success/failure. All calls to these functions go through peel_iterated_oid(), which returns peel_object() as a fallback, but also squashing the error values. The iteration interface already passes REF_ISSYMREF and REF_ISBROKEN through the flags argument, so the additional error values in enum peel_status provide no value. The ref iteration interface provides a separate peel() function because certain formats (eg. packed-refs and reftable) can store the peeled object next to the tag SHA1. Passing the peeled SHA1 as an optional argument to each_ref_fn maps more naturally to the implementation of ref databases. Changing the code in this way is left for a future refactoring. Signed-off-by: Han-Wen Nienhuys Signed-off-by: Junio C Hamano --- refs.c | 2 +- refs/packed-backend.c | 2 +- refs/ref-cache.c | 2 +- refs/refs-internal.h | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/refs.c b/refs.c index 261fd82beb..87cc64d562 100644 --- a/refs.c +++ b/refs.c @@ -2010,7 +2010,7 @@ int peel_iterated_oid(const struct object_id *base, struct object_id *peeled) oideq(current_ref_iter->oid, base))) return ref_iterator_peel(current_ref_iter, peeled); - return peel_object(base, peeled); + return peel_object(base, peeled) ? -1 : 0; } int refs_create_symref(struct ref_store *refs, diff --git a/refs/packed-backend.c b/refs/packed-backend.c index dfecdbc1db..66cb90c79e 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -889,7 +889,7 @@ static int packed_ref_iterator_peel(struct ref_iterator *ref_iterator, } else if ((iter->base.flags & (REF_ISBROKEN | REF_ISSYMREF))) { return -1; } else { - return !!peel_object(&iter->oid, peeled); + return peel_object(&iter->oid, peeled) ? -1 : 0; } } diff --git a/refs/ref-cache.c b/refs/ref-cache.c index 46f1e54284..49d732f6db 100644 --- a/refs/ref-cache.c +++ b/refs/ref-cache.c @@ -491,7 +491,7 @@ static int cache_ref_iterator_advance(struct ref_iterator *ref_iterator) static int cache_ref_iterator_peel(struct ref_iterator *ref_iterator, struct object_id *peeled) { - return peel_object(ref_iterator->oid, peeled); + return peel_object(ref_iterator->oid, peeled) ? -1 : 0; } static int cache_ref_iterator_abort(struct ref_iterator *ref_iterator) diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 467f4b3c93..3155708345 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -453,6 +453,9 @@ void base_ref_iterator_free(struct ref_iterator *iter); */ typedef int ref_iterator_advance_fn(struct ref_iterator *ref_iterator); +/* + * Peels the current ref, returning 0 for success or -1 for failure. + */ typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator, struct object_id *peeled); From 88dd4282d949cdafff516650c1be8aaf4d67983f Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 20 May 2021 08:43:22 +0900 Subject: [PATCH 083/397] A handful more topics before -rc1 Signed-off-by: Junio C Hamano --- Documentation/RelNotes/2.32.0.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Documentation/RelNotes/2.32.0.txt b/Documentation/RelNotes/2.32.0.txt index b47d569dc9..22f8859b40 100644 --- a/Documentation/RelNotes/2.32.0.txt +++ b/Documentation/RelNotes/2.32.0.txt @@ -371,6 +371,25 @@ Fixes since v2.31 empty directories under $GIT_DIR/refs/ for (merge 5f03e5126d wc/packed-ref-removal-cleanup later to maint). + * "git clean" and "git ls-files -i" had confusion around working on + or showing ignored paths inside an ignored directory, which has + been corrected. + (merge b548f0f156 en/dir-traversal later to maint). + + * The handling of "%(push)" formatting element of "for-each-ref" and + friends was broken when the same codepath started handling + "%(push:)", which has been corrected. + (merge 1e1c4c5eac zh/ref-filter-push-remote-fix later to maint). + + * The bash prompt script (in contrib/) did not work under "set -u". + (merge 5c0cbdb107 en/prompt-under-set-u later to maint). + + * The "chainlint" feature in the test framework is a handy way to + catch common mistakes in writing new tests, but tends to get + expensive. An knob to selectively disable it has been introduced + to help running tests that the developer has not modified. + (merge 2d86a96220 jk/test-chainlint-softer later to maint). + * Other code cleanup, docfix, build fix, etc. (merge f451960708 dl/cat-file-doc-cleanup later to maint). (merge 12604a8d0c sv/t9801-test-path-is-file-cleanup later to maint). From 09667e95163a9d51e569c50b1fdfc78d0a002c23 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Tue, 18 May 2021 00:18:55 -0600 Subject: [PATCH 084/397] fetch: improve grammar of "shallow roots" message Signed-off-by: Alex Henrie Signed-off-by: Junio C Hamano --- builtin/fetch.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/fetch.c b/builtin/fetch.c index dfde96a435..9191620e50 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1126,7 +1126,7 @@ static int store_updated_refs(const char *raw_url, const char *remote_name, if (rm->status == REF_STATUS_REJECT_SHALLOW) { if (want_status == FETCH_HEAD_MERGE) - warning(_("reject %s because shallow roots are not allowed to be updated"), + warning(_("rejected %s because shallow roots are not allowed to be updated"), rm->peer_ref ? rm->peer_ref->name : rm->name); continue; } From 8013d7d9ee7674774f6dbdbaeab11ce173bee016 Mon Sep 17 00:00:00 2001 From: Alex Henrie Date: Tue, 18 May 2021 00:19:17 -0600 Subject: [PATCH 085/397] setup: split "extensions found" messages into singular and plural It's easier to translate this way. Signed-off-by: Alex Henrie Signed-off-by: Junio C Hamano --- setup.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.c b/setup.c index 59e2facd9d..ead2f80cd8 100644 --- a/setup.c +++ b/setup.c @@ -666,7 +666,9 @@ int verify_repository_format(const struct repository_format *format, if (format->version >= 1 && format->unknown_extensions.nr) { int i; - strbuf_addstr(err, _("unknown repository extensions found:")); + strbuf_addstr(err, Q_("unknown repository extension found:", + "unknown repository extensions found:", + format->unknown_extensions.nr)); for (i = 0; i < format->unknown_extensions.nr; i++) strbuf_addf(err, "\n\t%s", @@ -678,7 +680,9 @@ int verify_repository_format(const struct repository_format *format, int i; strbuf_addstr(err, - _("repo version is 0, but v1-only extensions found:")); + Q_("repo version is 0, but v1-only extension found:", + "repo version is 0, but v1-only extensions found:", + format->v1_only_extensions.nr)); for (i = 0; i < format->v1_only_extensions.nr; i++) strbuf_addf(err, "\n\t%s", From a84216c68435f545de38d69c771a6f2ade480394 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Wed, 19 May 2021 05:17:09 -0400 Subject: [PATCH 086/397] doc: explain the use of color.pager The current documentation for color.pager is technically correct, but slightly misleading and doesn't really clarify the purpose of the variable. As explained in the original thread which added it: https://lore.kernel.org/git/E1G6zPH-00062L-Je@moooo.ath.cx/ the point is to deal with pagers that don't understand colors. And hence it being set to "true" is necessary for colorizing output to the pager, but not sufficient by itself (you must also have enabled one of the other color options, though note that these are set to "auto" by default these days). Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Documentation/config/color.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/config/color.txt b/Documentation/config/color.txt index d5daacb13a..e05d520a86 100644 --- a/Documentation/config/color.txt +++ b/Documentation/config/color.txt @@ -127,8 +127,9 @@ color.interactive.:: interactive commands. color.pager:: - A boolean to enable/disable colored output when the pager is in - use (default is true). + A boolean to specify whether `auto` color modes should colorize + output going to the pager. Defaults to true; set this to false + if your pager does not understand ANSI color codes. color.push:: A boolean to enable/disable color in push errors. May be set to From bb80333c0853b04ff789de8037bef5a4ade1da8c Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:30 +0000 Subject: [PATCH 087/397] Documentation/technical: describe remembering renames optimization Remembering renames on the upstream side of history in an early merge of a rebase or cherry-pick for re-use in a latter merge of the same operation makes pretty good intuitive sense. However, trying to show that it doesn't cause some subtle behavioral difference or some funny edge or corner case is much more involved. And, in fact, it does introduce a subtle behavioral change. Document all the assumptions, special cases, and logic involved in such an optimization, and describe why this optimization is safe under the current optimizations/features/etc. -- even when the subtle behavioral change is triggered. Part of the point of adding this document that goes over the optimization in such laborious detail, is that it is possible that significant future changes (optimizations or feature changes) could interact with this optimization in interesting ways; this document is here to help folks making big changes sanity check that the assumptions and arguments underlying this optimization are still valid. (As a side note, creating this document forced me to review things in sufficient detail that I found I was not properly caching directory-rename-induced renames, resulting in the code not being aware of those renames and causing unnecessary diffcore_rename_extended() calls in subsequent merges.) A subsequent commit will add several testcases based on this document meant to stress-test the implementation and also document the case with the subtle behavioral change. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- .../technical/remembering-renames.txt | 669 ++++++++++++++++++ 1 file changed, 669 insertions(+) create mode 100644 Documentation/technical/remembering-renames.txt diff --git a/Documentation/technical/remembering-renames.txt b/Documentation/technical/remembering-renames.txt new file mode 100644 index 0000000000..251ce477e0 --- /dev/null +++ b/Documentation/technical/remembering-renames.txt @@ -0,0 +1,669 @@ +Rebases and cherry-picks involve a sequence of merges whose results are +recorded as new single-parent commits. The first parent side of those +merges represent the "upstream" side, and often include a far larger set of +changes than the second parent side. Traditionally, the renames on the +first-parent side of that sequence of merges were repeatedly re-detected +for every merge. This file explains why it is safe and effective during +rebases and cherry-picks to remember renames on the upstream side of +history as an optimization, assuming all merges are automatic and clean +(i.e. no conflicts and not interrupted for user input or editing). + +Outline: + + 0. Assumptions + + 1. How rebasing and cherry-picking work + + 2. Why the renames on MERGE_SIDE1 in any given pick are *always* a + superset of the renames on MERGE_SIDE1 for the next pick. + + 3. Why any rename on MERGE_SIDE1 in any given pick is _almost_ always also + a rename on MERGE_SIDE1 for the next pick + + 4. A detailed description of the the counter-examples to #3. + + 5. Why the special cases in #4 are still fully reasonable to use to pair + up files for three-way content merging in the merge machinery, and why + they do not affect the correctness of the merge. + + 6. Interaction with skipping of "irrelevant" renames + + 7. Additional items that need to be cached + + 8. How directory rename detection interacts with the above and why this + optimization is still safe even if merge.directoryRenames is set to + "true". + + +=== 0. Assumptions === + +There are two assumptions that will hold throughout this document: + + * The upstream side where commits are transplanted to is treated as the + first parent side when rebase/cherry-pick call the merge machinery + + * All merges are fully automatic + +and a third that will hold in sections 2-5 for simplicity, that I'll later +address in section 8: + + * No directory renames occur + + +Let me explain more about each assumption and why I include it: + + +The first assumption is merely for the purposes of making this document +clearer; the optimization implementation does not actually depend upon it. +However, the assumption does hold in all cases because it reflects the way +that both rebase and cherry-pick were implemented; and the implementation +of cherry-pick and rebase are not readily changeable for backwards +compatibility reasons (see for example the discussion of the --ours and +--theirs flag in the documentation of `git checkout`, particularly the +comments about how they behave with rebase). The optimization avoids +checking first-parent-ness, though. It checks the conditions that make the +optimization valid instead, so it would still continue working if someone +changed the parent ordering that cherry-pick and rebase use. But making +this assumption does make this document much clearer and prevents me from +having to repeat every example twice. + +If the second assumption is violated, then the optimization simply is +turned off and thus isn't relevant to consider. The second assumption can +also be stated as "there is no interruption for a user to resolve conflicts +or to just further edit or tweak files". While real rebases and +cherry-picks are often interrupted (either because it's an interactive +rebase where the user requested to stop and edit, or because there were +conflicts that the user needs to resolve), the cache of renames is not +stored on disk, and thus is thrown away as soon as the rebase or cherry +pick stops for the user to resolve the operation. + +The third assumption makes sections 2-5 simpler, and allows people to +understand the basics of why this optimization is safe and effective, and +then I can go back and address the specifics in section 8. It is probably +also worth noting that if directory renames do occur, then the default of +merge.directoryRenames being set to "conflict" means that the operation +will stop for users to resolve the conflicts and the cache will be thrown +away, and thus that there won't be an optimization to apply. So, the only +reason we need to address directory renames specifically, is that some +users will have set merge.directoryRenames to "true" to allow the merges to +continue to proceed automatically. The optimization is still safe with +this config setting, but we have to discuss a few more cases to show why; +this discussion is deferred until section 8. + + +=== 1. How rebasing and cherry-picking work === + +Consider the following setup (from the git-rebase manpage): + + A---B---C topic + / + D---E---F---G main + +After rebasing or cherry-picking topic onto main, this will appear as: + + A'--B'--C' topic + / + D---E---F---G main + +The way the commits A', B', and C' are created is through a series of +merges, where rebase or cherry-pick sequentially uses each of the three +A-B-C commits in a special merge operation. Let's label the three commits +in the merge operation as MERGE_BASE, MERGE_SIDE1, and MERGE_SIDE2. For +this picture, the three commits for each of the three merges would be: + +To create A': + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + +To create B': + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + +To create C': + MERGE_BASE: B + MERGE_SIDE1: B' + MERGE_SIDE2: C + +Sometimes, folks are surprised that these three-way merges are done. It +can be useful in understanding these three-way merges to view them in a +slightly different light. For example, in creating C', you can view it as +either: + + * Apply the changes between B & C to B' + * Apply the changes between B & B' to C + +Conceptually the two statements above are the same as a three-way merge of +B, B', and C, at least the parts before you decide to record a commit. + + +=== 2. Why the renames on MERGE_SIDE1 in any given pick are always a === +=== superset of the renames on MERGE_SIDE1 for the next pick. === + +The merge machinery uses the filenames it is fed from MERGE_BASE, +MERGE_SIDE1, and MERGE_SIDE2. It will only move content to a different +filename under one of three conditions: + + * To make both pieces of a conflict available to a user during conflict + resolution (examples: directory/file conflict, add/add type conflict + such as symlink vs. regular file) + + * When MERGE_SIDE1 renames the file. + + * When MERGE_SIDE2 renames the file. + +First, let's remember what commits are involved in the first and second +picks of the cherry-pick or rebase sequence: + +To create A': + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + +To create B': + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + +So, in particular, we need to show that the renames between E and G are a +superset of those between A and A'. + +A' is created by the first merge. A' will only have renames for one of the +three reasons listed above. The first case, a conflict, results in a +situation where the cache is dropped and thus this optimization doesn't +take effect, so we need not consider that case. The third case, a rename +on MERGE_SIDE2 (i.e. from G to A), will show up in A' but it also shows up +in A -- therefore when diffing A and A' that path does not show up as a +rename. The only remaining way for renames to show up in A' is for the +rename to come from MERGE_SIDE1. Therefore, all renames between A and A' +are a subset of those between E and G. Equivalently, all renames between E +and G are a superset of those between A and A'. + + +=== 3. Why any rename on MERGE_SIDE1 in any given pick is _almost_ === +=== always also a rename on MERGE_SIDE1 for the next pick. === + +Let's again look at the first two picks: + +To create A': + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + +To create B': + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + +Now let's look at any given rename from MERGE_SIDE1 of the first pick, i.e. +any given rename from E to G. Let's use the filenames 'oldfile' and +'newfile' for demonstration purposes. That first pick will function as +follows; when the rename is detected, the merge machinery will do a +three-way content merge of the following: + E:oldfile + G:newfile + A:oldfile +and produce a new result: + A':newfile + +Note above that I've assumed that E->A did not rename oldfile. If that +side did rename, then we most likely have a rename/rename(1to2) conflict +that will cause the rebase or cherry-pick operation to halt and drop the +in-memory cache of renames and thus doesn't need to be considered further. +In the special case that E->A does rename the file but also renames it to +newfile, then there is no conflict from the renaming and the merge can +succeed. In this special case, the rename is not valid to cache because +the second merge will find A:newfile in the MERGE_BASE. So a +rename/rename(1to1) needs to be specially handled by pruning renames from +the cache and decrementing the dir_rename_counts in the current and leading +directories associated with those renames. Or, since these are really +rare, one could just take the easy way out and disable the remembering +renames optimization when a rename/rename(1to1) happens. + +The previous paragraph handled the cases for E->A renaming oldfile, let's +continue assuming that oldfile is not renamed in A. + +As per the diagram for creating B', MERGE_SIDE1 involves the changes from A +to A'. So, we are curious whether A:oldfile and A':newfile will be viewed +as renames. Note that: + + * There will be no A':oldfile (because there could not have been a + G:oldfile as we do not do break detection in the merge machinery and + G:newfile was detected as a rename, and by the construction of the + rename above that merged cleanly, the merge machinery will ensure there + is no 'oldfile' in the result). + + * There will be no A:newfile (if there had been, we would have had a + rename/add conflict). + + * Clearly A:oldfile and A':newfile are "related" (A':newfile came from a + clean three-way content merge involving A:oldfile). + +We can also expound on the third point above, by noting that three-way +content merges can also be viewed as applying the differences between the +base and one side to the other side. Thus we can view A':newfile as +having been created by taking the changes between E:oldfile and G:newfile +(which were detected as being related, i.e. <50% changed) to A:oldfile. + +Thus A:oldfile and A':newfile are just as related as E:oldfile and +G:newfile are -- they have exactly identical differences. Since the latter +were detected as renames, A:oldfile and A':newfile should also be +detectable as renames almost always. + + +=== 4. A detailed description of the counter-examples to #3. === + +We already noted in section 3 that rename/rename(1to1) (i.e. both sides +renaming a file the same way) was one counter-example. The more +interesting bit, though, is why did we need to use the "almost" qualifier +when stating that A:oldfile and A':newfile are "almost" always detectable +as renames? + +Let's repeat an earlier point that section 3 made: + + A':newfile was created by applying the changes between E:oldfile and + G:newfile to A:oldfile. The changes between E:oldfile and G:newfile were + <50% of the size of E:oldfile. + +If those changes that were <50% of the size of E:oldfile are also <50% of +the size of A:oldfile, then A:oldfile and A':newfile will be detectable as +renames. However, if there is a dramatic size reduction between E:oldfile +and A:oldfile (but the changes between E:oldfile, G:newfile, and A:oldfile +still somehow merge cleanly), then traditional rename detection would not +detect A:oldfile and A':newfile as renames. + +Here's an example where that can happen: + * E:oldfile had 20 lines + * G:newfile added 10 new lines at the beginning of the file + * A:oldfile kept the first 3 lines of the file, and deleted all the rest +then + => A':newfile would have 13 lines, 3 of which matches those in A:oldfile. +E:oldfile -> G:newfile would be detected as a rename, but A:oldfile and +A':newfile would not be. + + +=== 5. Why the special cases in #4 are still fully reasonable to use to === +=== pair up files for three-way content merging in the merge machinery, === +=== and why they do not affect the correctness of the merge. === + +In the rename/rename(1to1) case, A:newfile and A':newfile are not renames +since they use the *same* filename. However, files with the same filename +are obviously fine to pair up for three-way content merging (the merge +machinery has never employed break detection). The interesting +counter-example case is thus not the rename/rename(1to1) case, but the case +where A did not rename oldfile. That was the case that we spent most of +the time discussing in sections 3 and 4. The remainder of this section +will be devoted to that case as well. + +So, even if A:oldfile and A':newfile aren't detectable as renames, why is +it still reasonable to pair them up for three-way content merging in the +merge machinery? There are multiple reasons: + + * As noted in sections 3 and 4, the diff between A:oldfile and A':newfile + is *exactly* the same as the diff between E:oldfile and G:newfile. The + latter pair were detected as renames, so it seems unlikely to surprise + users for us to treat A:oldfile and A':newfile as renames. + + * In fact, "oldfile" and "newfile" were at one point detected as renames + due to how they were constructed in the E..G chain. And we used that + information once already in this rebase/cherry-pick. I think users + would be unlikely to be surprised at us continuing to treat the files + as renames and would quickly understand why we had done so. + + * Marking or declaring files as renames is *not* the end goal for merges. + Merges use renames to determine which files make sense to be paired up + for three-way content merges. + + * A:oldfile and A':newfile were _already_ paired up in a three-way + content merge; that is how A':newfile was created. In fact, that + three-way content merge was clean. So using them again in a later + three-way content merge seems very reasonable. + +However, the above is focusing on the common scenarios. Let's try to look +at all possible unusual scenarios and compare without the optimization to +with the optimization. Consider the following theoretical cases; we will +then dive into each to determine which of them are possible, +and if so, what they mean: + + 1. Without the optimization, the second merge results in a conflict. + With the optimization, the second merge also results in a conflict. + Questions: Are the conflicts confusingly different? Better in one case? + + 2. Without the optimization, the second merge results in NO conflict. + With the optimization, the second merge also results in NO conflict. + Questions: Are the merges the same? + + 3. Without the optimization, the second merge results in a conflict. + With the optimization, the second merge results in NO conflict. + Questions: Possible? Bug, bugfix, or something else? + + 4. Without the optimization, the second merge results in NO conflict. + With the optimization, the second merge results in a conflict. + Questions: Possible? Bug, bugfix, or something else? + +I'll consider all four cases, but out of order. + +The fourth case is impossible. For the code without the remembering +renames optimization to not get a conflict, B:oldfile would need to exactly +match A:oldfile -- if it doesn't, there would be a modify/delete conflict. +If A:oldfile matches B:oldfile exactly, then a three-way content merge +between A:oldfile, A':newfile, and B:oldfile would have no conflict and +just give us the version of newfile from A' as the result. + +From the same logic as the above paragraph, the second case would indeed +result in identical merges. When A:oldfile exactly matches B:oldfile, an +undetected rename would say, "Oh, I see one side didn't modify 'oldfile' +and the other side deleted it. I'll delete it. And I see you have this +brand new file named 'newfile' in A', so I'll keep it." That gives the +same results as three-way content merging A:oldfile, A':newfile, and +B:oldfile -- a removal of oldfile with the version of newfile from A' +showing up in the result. + +The third case is interesting. It means that A:oldfile and A':newfile were +not just similar enough, but that the changes between them did not conflict +with the changes between A:oldfile and B:oldfile. This would validate our +hunch that the files were similar enough to be used in a three-way content +merge, and thus seems entirely correct for us to have used them that way. +(Sidenote: One particular example here may be enlightening. Let's say that +B was an immediate revert of A. B clearly would have been a clean revert +of A, since A was B's immediate parent. One would assume that if you can +pick a commit, you should also be able to cherry-pick its immediate revert. +However, this is one of those funny corner cases; without this +optimization, we just successfully picked a commit cleanly, but we are +unable to cherry-pick its immediate revert due to the size differences +between E:oldfile and A:oldfile.) + +That leaves only the first case to consider -- when we get conflicts both +with or without the optimization. Without the optimization, we'll have a +modify/delete conflict, where both A':newfile and B:oldfile are left in the +tree for the user to deal with and no hints about the potential similarity +between the two. With the optimization, we'll have a three-way content +merged A:oldfile, A':newfile, and B:oldfile with conflict markers +suggesting we thought the files were related but giving the user the chance +to resolve. As noted above, I don't think users will find us treating +'oldfile' and 'newfile' as related as a surprise since they were between E +and G. In any event, though, this case shouldn't be concerning since we +hit a conflict in both cases, told the user what we know, and asked them to +resolve it. + +So, in summary, case 4 is impossible, case 2 yields the same behavior, and +cases 1 and 3 seem to provide as good or better behavior with the +optimization than without. + + +=== 6. Interaction with skipping of "irrelevant" renames === + +Previous optimizations involved skipping rename detection for paths +considered to be "irrelevant". See for example the following commits: + + * 32a56dfb99 ("merge-ort: precompute subset of sources for which we + need rename detection", 2021-03-11) + * 2fd9eda462 ("merge-ort: precompute whether directory rename + detection is needed", 2021-03-11) + * 9bd342137e ("diffcore-rename: determine which relevant_sources are + no longer relevant", 2021-03-13) + +Relevance is always determined by what the _other_ side of history has +done, in terms of modifing a file that our side renamed, or adding a +file to a directory which our side renamed. This means that a path +that is "irrelevant" when picking the first commit of a series in a +rebase or cherry-pick, may suddenly become "relevant" when picking the +next commit. + +The upshot of this is that we can only cache rename detection results +for relevant paths, and need to re-check relevance in subsequent +commits. If those subsequent commits have additional paths that are +relevant for rename detection, then we will need to redo rename +detection -- though we can limit it to the paths for which we have not +already detected renames. + + +=== 7. Additional items that need to be cached === + +It turns out we have to cache more than just renames; we also cache: + + A) non-renames (i.e. unpaired deletes) + B) counts of renames within directories + C) sources that were marked as RELEVANT_LOCATION, but which were + downgraded to RELEVANT_NO_MORE + D) the toplevel trees involved in the merge + +These are all stored in struct rename_info, and respectively appear in + * cached_pairs (along side actual renames, just with a value of NULL) + * dir_rename_counts + * cached_irrelevant + * merge_trees + +The reason for (A) comes from the irrelevant renames skipping +optimization discussed in section 6. The fact that irrelevant renames +are skipped means we only get a subset of the potential renames +detected and subsequent commits may need to run rename detection on +the upstream side on a subset of the remaining renames (to get the +renames that are relevant for that later commit). Since unpaired +deletes are involved in rename detection too, we don't want to +repeatedly check that those paths remain unpaired on the upstream side +with every commit we are transplanting. + +The reason for (B) is that diffcore_rename_extended() is what +generates the counts of renames by directory which is needed in +directory rename detection, and if we don't run +diffcore_rename_extended() again then we need to have the output from +it, including dir_rename_counts, from the previous run. + +The reason for (C) is that merge-ort's tree traversal will again think +those paths are relevant (marking them as RELEVANT_LOCATION), but the +fact that they were downgraded to RELEVANT_NO_MORE means that +dir_rename_counts already has the information we need for directory +rename detection. (A path which becomes RELEVANT_CONTENT in a +subsequent commit will be removed from cached_irrelevant.) + +The reason for (D) is that is how we determine whether the remember +renames optimization can be used. In particular, remembering that our +sequence of merges looks like: + + Merge 1: + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + => Creates A' + + Merge 2: + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + => Creates B' + +It is the fact that the trees A and A' appear both in Merge 1 and in +Merge 2, with A as a parent of A' that allows this optimization. So +we store the trees to compare with what we are asked to merge next +time. + + +=== 8. How directory rename detection interacts with the above and === +=== why this optimization is still safe even if === +=== merge.directoryRenames is set to "true". === + +As noted in the assumptions section: + + """ + ...if directory renames do occur, then the default of + merge.directoryRenames being set to "conflict" means that the operation + will stop for users to resolve the conflicts and the cache will be + thrown away, and thus that there won't be an optimization to apply. + So, the only reason we need to address directory renames specifically, + is that some users will have set merge.directoryRenames to "true" to + allow the merges to continue to proceed automatically. + """ + +Let's remember that we need to look at how any given pick affects the next +one. So let's again use the first two picks from the diagram in section +one: + + First pick does this three-way merge: + MERGE_BASE: E + MERGE_SIDE1: G + MERGE_SIDE2: A + => creates A' + + Second pick does this three-way merge: + MERGE_BASE: A + MERGE_SIDE1: A' + MERGE_SIDE2: B + => creates B' + +Now, directory rename detection exists so that if one side of history +renames a directory, and the other side adds a new file to the old +directory, then the merge (with merge.directoryRenames=true) can move the +file into the new directory. There are two qualitatively different ways to +add a new file to an old directory: create a new file, or rename a file +into that directory. Also, directory renames can be done on either side of +history, so there are four cases to consider: + + * MERGE_SIDE1 renames old dir, MERGE_SIDE2 adds new file to old dir + * MERGE_SIDE1 renames old dir, MERGE_SIDE2 renames file into old dir + * MERGE_SIDE1 adds new file to old dir, MERGE_SIDE2 renames old dir + * MERGE_SIDE1 renames file into old dir, MERGE_SIDE2 renames old dir + +One last note before we consider these four cases: There are some +important properties about how we implement this optimization with +respect to directory rename detection that we need to bear in mind +while considering all of these cases: + + * rename caching occurs *after* applying directory renames + + * a rename created by directory rename detection is recorded for the side + of history that did the directory rename. + + * dir_rename_counts, the nested map of + {oldname => {newname => count}}, + is cached between runs as well. This basically means that directory + rename detection is also cached, though only on the side of history + that we cache renames for (MERGE_SIDE1 as far as this document is + concerned; see the assumptions section). Two interesting sub-notes + about these counts: + + * If we need to perform rename-detection again on the given side (e.g. + some paths are relevant for rename detection that weren't before), + then we clear dir_rename_counts and recompute it, making use of + cached_pairs. The reason it is important to do this is optimizations + around RELEVANT_LOCATION exist to prevent us from computing + unnecessary renames for directory rename detection and from computing + dir_rename_counts for irrelevant directories; but those same renames + or directories may become necessary for subsequent merges. The + easiest way to "fix up" dir_rename_counts in such cases is to just + recompute it. + + * If we prune rename/rename(1to1) entries from the cache, then we also + need to update dir_rename_counts to decrement the counts for the + involved directory and any relevant parent directories (to undo what + update_dir_rename_counts() in diffcore-rename.c incremented when the + rename was initially found). If we instead just disable the + remembering renames optimization when the exceedingly rare + rename/rename(1to1) cases occur, then dir_rename_counts will get + re-computed the next time rename detection occurs, as noted above. + + * the side with multiple commits to pick, is the side of history that we + do NOT cache renames for. Thus, there are no additional commits to + change the number of renames in a directory, except for those done by + directory rename detection (which always pad the majority). + + * the "renames" we cache are modified slightly by any directory rename, + as noted below. + +Now, with those notes out of the way, let's go through the four cases +in order: + +Case 1: MERGE_SIDE1 renames old dir, MERGE_SIDE2 adds new file to old dir + + This case looks like this: + + MERGE_BASE: E, Has olddir/ + MERGE_SIDE1: G, Renames olddir/ -> newdir/ + MERGE_SIDE2: A, Adds olddir/newfile + => creates A', With newdir/newfile + + MERGE_BASE: A, Has olddir/newfile + MERGE_SIDE1: A', Has newdir/newfile + MERGE_SIDE2: B, Modifies olddir/newfile + => expected B', with threeway-merged newdir/newfile from above + + In this case, with the optimization, note that after the first commit: + * MERGE_SIDE1 remembers olddir/ -> newdir/ + * MERGE_SIDE1 has cached olddir/newfile -> newdir/newfile + Given the cached rename noted above, the second merge can proceed as + expected without needing to perform rename detection from A -> A'. + +Case 2: MERGE_SIDE1 renames old dir, MERGE_SIDE2 renames file into old dir + + This case looks like this: + MERGE_BASE: E oldfile, olddir/ + MERGE_SIDE1: G oldfile, olddir/ -> newdir/ + MERGE_SIDE2: A oldfile -> olddir/newfile + => creates A', With newdir/newfile representing original oldfile + + MERGE_BASE: A olddir/newfile + MERGE_SIDE1: A' newdir/newfile + MERGE_SIDE2: B modify olddir/newfile + => expected B', with threeway-merged newdir/newfile from above + + In this case, with the optimization, note that after the first commit: + * MERGE_SIDE1 remembers olddir/ -> newdir/ + * MERGE_SIDE1 has cached olddir/newfile -> newdir/newfile + (NOT oldfile -> newdir/newfile; compare to case with + (p->status == 'R' && new_path) in possibly_cache_new_pair()) + + Given the cached rename noted above, the second merge can proceed as + expected without needing to perform rename detection from A -> A'. + +Case 3: MERGE_SIDE1 adds new file to old dir, MERGE_SIDE2 renames old dir + + This case looks like this: + + MERGE_BASE: E, Has olddir/ + MERGE_SIDE1: G, Adds olddir/newfile + MERGE_SIDE2: A, Renames olddir/ -> newdir/ + => creates A', With newdir/newfile + + MERGE_BASE: A, Has newdir/, but no notion of newdir/newfile + MERGE_SIDE1: A', Has newdir/newfile + MERGE_SIDE2: B, Has newdir/, but no notion of newdir/newfile + => expected B', with newdir/newfile from A' + + In this case, with the optimization, note that after the first commit there + were no renames on MERGE_SIDE1, and any renames on MERGE_SIDE2 are tossed. + But the second merge didn't need any renames so this is fine. + +Case 4: MERGE_SIDE1 renames file into old dir, MERGE_SIDE2 renames old dir + + This case looks like this: + + MERGE_BASE: E, Has olddir/ + MERGE_SIDE1: G, Renames oldfile -> olddir/newfile + MERGE_SIDE2: A, Renames olddir/ -> newdir/ + => creates A', With newdir/newfile representing original oldfile + + MERGE_BASE: A, Has oldfile + MERGE_SIDE1: A', Has newdir/newfile + MERGE_SIDE2: B, Modifies oldfile + => expected B', with threeway-merged newdir/newfile from above + + In this case, with the optimization, note that after the first commit: + * MERGE_SIDE1 remembers oldfile -> newdir/newfile + (NOT oldfile -> olddir/newfile; compare to case of second + block under p->status == 'R' in possibly_cache_new_pair()) + * MERGE_SIDE2 renames are tossed because only MERGE_SIDE1 is remembered + + Given the cached rename noted above, the second merge can proceed as + expected without needing to perform rename detection from A -> A'. + +Finally, I'll just note here that interactions with the +skip-irrelevant-renames optimization means we sometimes don't detect +renames for any files within a directory that was renamed, in which +case we will not have been able to detect any rename for the directory +itself. In such a case, we do not know whether the directory was +renamed; we want to be careful to avoid cacheing some kind of "this +directory was not renamed" statement. If we did, then a subsequent +commit being rebased could add a file to the old directory, and the +user would expect it to end up in the correct directory -- something +our erroneous "this directory was not renamed" cache would preclude. From caba91c373127a316b5442a6aac7080fee9bd624 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:31 +0000 Subject: [PATCH 088/397] fast-rebase: change assert() to BUG() assert() can succinctly document expectations for the code, and do so in a way that may be useful to future folks trying to refactor the code and change basic assumptions; it allows them to more quickly find some places where their violations of previous assumptions trips things up. Unfortunately, assert() can surround a function call with important side-effects, which is a huge mistake since some users will compile with assertions disabled. I've had to debug such mistakes before in other codebases, so I should know better. Luckily, this was only in test code, but it's still very embarrassing. Change an assert() to an if (...) BUG (...). Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- t/helper/test-fast-rebase.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/helper/test-fast-rebase.c b/t/helper/test-fast-rebase.c index 373212256a..39fb7f41e8 100644 --- a/t/helper/test-fast-rebase.c +++ b/t/helper/test-fast-rebase.c @@ -124,7 +124,8 @@ int cmd__fast_rebase(int argc, const char **argv) assert(oideq(&onto->object.oid, &head)); hold_locked_index(&lock, LOCK_DIE_ON_ERROR); - assert(repo_read_index(the_repository) >= 0); + if (repo_read_index(the_repository) < 0) + BUG("Could not read index"); repo_init_revisions(the_repository, &revs, NULL); revs.verbose_header = 1; From f9500261e0aea2bebb527281462d650be1db38a4 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:32 +0000 Subject: [PATCH 089/397] fast-rebase: write conflict state to working tree, index, and HEAD Previously, when fast-rebase hit a conflict, it simply aborted and left HEAD, the index, and the working tree where they were before the operation started. While fast-rebase does not support restarting from a conflicted state, write the conflicted state out anyway as it gives us a way to see what the conflicts are and write tests that check for them. This will be important in the upcoming commits, because sequencer.c is only superficially integrated with merge-ort.c; in particular, it calls merge_switch_to_result() after EACH merge instead of only calling it at the end of all the sequence of merges (or when a conflict is hit). This not only causes needless updates to the working copy and index, but also causes all intermediate data to be freed and tossed, preventing caching information from one merge to the next. However, integrating sequencer.c more deeply with merge-ort.c is a big task, and making this small extension to fast-rebase.c provides us with a simple way to test the edge and corner cases that we want to make sure continue working. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- t/helper/test-fast-rebase.c | 51 +++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/t/helper/test-fast-rebase.c b/t/helper/test-fast-rebase.c index 39fb7f41e8..fc2d460904 100644 --- a/t/helper/test-fast-rebase.c +++ b/t/helper/test-fast-rebase.c @@ -91,7 +91,6 @@ int cmd__fast_rebase(int argc, const char **argv) struct commit *last_commit = NULL, *last_picked_commit = NULL; struct object_id head; struct lock_file lock = LOCK_INIT; - int clean = 1; struct strvec rev_walk_args = STRVEC_INIT; struct rev_info revs; struct commit *commit; @@ -176,11 +175,10 @@ int cmd__fast_rebase(int argc, const char **argv) free((char*)merge_opt.ancestor); merge_opt.ancestor = NULL; if (!result.clean) - die("Aborting: Hit a conflict and restarting is not implemented."); + break; last_picked_commit = commit; last_commit = create_commit(result.tree, commit, last_commit); } - fprintf(stderr, "\nDone.\n"); /* TODO: There should be some kind of rev_info_free(&revs) call... */ memset(&revs, 0, sizeof(revs)); @@ -189,24 +187,39 @@ int cmd__fast_rebase(int argc, const char **argv) if (result.clean < 0) exit(128); - strbuf_addf(&reflog_msg, "finish rebase %s onto %s", - oid_to_hex(&last_picked_commit->object.oid), - oid_to_hex(&last_commit->object.oid)); - if (update_ref(reflog_msg.buf, branch_name.buf, - &last_commit->object.oid, - &last_picked_commit->object.oid, - REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) { - error(_("could not update %s"), argv[4]); - die("Failed to update %s", argv[4]); - } - if (create_symref("HEAD", branch_name.buf, reflog_msg.buf) < 0) - die(_("unable to update HEAD")); - strbuf_release(&reflog_msg); - strbuf_release(&branch_name); + if (result.clean) { + fprintf(stderr, "\nDone.\n"); + strbuf_addf(&reflog_msg, "finish rebase %s onto %s", + oid_to_hex(&last_picked_commit->object.oid), + oid_to_hex(&last_commit->object.oid)); + if (update_ref(reflog_msg.buf, branch_name.buf, + &last_commit->object.oid, + &last_picked_commit->object.oid, + REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) { + error(_("could not update %s"), argv[4]); + die("Failed to update %s", argv[4]); + } + if (create_symref("HEAD", branch_name.buf, reflog_msg.buf) < 0) + die(_("unable to update HEAD")); + strbuf_release(&reflog_msg); + strbuf_release(&branch_name); - prime_cache_tree(the_repository, the_repository->index, result.tree); + prime_cache_tree(the_repository, the_repository->index, + result.tree); + } else { + fprintf(stderr, "\nAborting: Hit a conflict.\n"); + strbuf_addf(&reflog_msg, "rebase progress up to %s", + oid_to_hex(&last_picked_commit->object.oid)); + if (update_ref(reflog_msg.buf, "HEAD", + &last_commit->object.oid, + &head, + REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) { + error(_("could not update %s"), argv[4]); + die("Failed to update %s", argv[4]); + } + } if (write_locked_index(&the_index, &lock, COMMIT_LOCK | SKIP_IF_UNCHANGED)) die(_("unable to write %s"), get_index_file()); - return (clean == 0); + return (result.clean == 0); } From a22099f552d5e67dc71c2bd92c4b711387e9695f Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:33 +0000 Subject: [PATCH 090/397] t6429: testcases for remembering renames We will soon be adding an optimization that caches (in memory only, never written to disk) upstream renames during a sequence of merges such as occurs during a cherry-pick or rebase operation. Add several tests meant to stress such an implementation to ensure it does the right thing, and include a test whose outcome we will later change due to this optimization as well. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- .../technical/remembering-renames.txt | 14 +- t/t6429-merge-sequence-rename-caching.sh | 692 ++++++++++++++++++ 2 files changed, 700 insertions(+), 6 deletions(-) create mode 100755 t/t6429-merge-sequence-rename-caching.sh diff --git a/Documentation/technical/remembering-renames.txt b/Documentation/technical/remembering-renames.txt index 251ce477e0..2fd5cc88e0 100644 --- a/Documentation/technical/remembering-renames.txt +++ b/Documentation/technical/remembering-renames.txt @@ -214,12 +214,14 @@ in-memory cache of renames and thus doesn't need to be considered further. In the special case that E->A does rename the file but also renames it to newfile, then there is no conflict from the renaming and the merge can succeed. In this special case, the rename is not valid to cache because -the second merge will find A:newfile in the MERGE_BASE. So a -rename/rename(1to1) needs to be specially handled by pruning renames from -the cache and decrementing the dir_rename_counts in the current and leading -directories associated with those renames. Or, since these are really -rare, one could just take the easy way out and disable the remembering -renames optimization when a rename/rename(1to1) happens. +the second merge will find A:newfile in the MERGE_BASE (see also the new +testcases in t6429 with "rename same file identically" in their +description). So a rename/rename(1to1) needs to be specially handled by +pruning renames from the cache and decrementing the dir_rename_counts in +the current and leading directories associated with those renames. Or, +since these are really rare, one could just take the easy way out and +disable the remembering renames optimization when a rename/rename(1to1) +happens. The previous paragraph handled the cases for E->A renaming oldfile, let's continue assuming that oldfile is not renamed in A. diff --git a/t/t6429-merge-sequence-rename-caching.sh b/t/t6429-merge-sequence-rename-caching.sh new file mode 100755 index 0000000000..f47d8924ee --- /dev/null +++ b/t/t6429-merge-sequence-rename-caching.sh @@ -0,0 +1,692 @@ +#!/bin/sh + +test_description="remember regular & dir renames in sequence of merges" + +. ./test-lib.sh + +# +# NOTE 1: this testfile tends to not only rename files, but modify on both +# sides; without modifying on both sides, optimizations can kick in +# which make rename detection irrelevant or trivial. We want to make +# sure that we are triggering rename caching rather than rename +# bypassing. +# +# NOTE 2: this testfile uses 'test-tool fast-rebase' instead of either +# cherry-pick or rebase. sequencer.c is only superficially +# integrated with merge-ort; it calls merge_switch_to_result() +# after EACH merge, which updates the index and working copy AND +# throws away the cached results (because merge_switch_to_result() +# is only supposed to be called at the end of the sequence). +# Integrating them more deeply is a big task, so for now the tests +# use 'test-tool fast-rebase'. +# + + +# +# In the following simple testcase: +# Base: numbers_1, values_1 +# Upstream: numbers_2, values_2 +# Topic_1: sequence_3 +# Topic_2: scruples_3 +# or, in english, rename numbers -> sequence in the first commit, and rename +# values -> scruples in the second commit. +# +# This shouldn't be a challenge, it's just verifying that cached renames isn't +# preventing us from finding new renames. +# +test_expect_success 'caching renames does not preclude finding new ones' ' + test_create_repo caching-renames-and-new-renames && + ( + cd caching-renames-and-new-renames && + + test_seq 2 10 >numbers && + test_seq 2 10 >values && + git add numbers values && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 10 >numbers && + test_seq 1 10 >values && + git add numbers values && + git commit -m "Tweaked both files" && + + git switch topic && + + test_seq 2 12 >numbers && + git add numbers && + git mv numbers sequence && + git commit -m A && + + test_seq 2 12 >values && + git add values && + git mv values scruples && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic + + git ls-files >tracked-files && + test_line_count = 2 tracked-files && + test_seq 1 12 >expect && + test_cmp expect sequence && + test_cmp expect scruples + ) +' + +# +# In the following testcase: +# Base: numbers_1 +# Upstream: rename numbers_1 -> sequence_2 +# Topic_1: numbers_3 +# Topic_2: numbers_1 +# or, in english, the first commit on the topic branch modifies numbers by +# shrinking it (dramatically) and the second commit on topic reverts its +# parent. +# +# Can git apply both patches? +# +# Traditional cherry-pick/rebase will fail to apply the second commit, the +# one that reverted its parent, because despite detecting the rename from +# 'numbers' to 'sequence' for the first commit, it fails to detect that +# rename when picking the second commit. That's "reasonable" given the +# dramatic change in size of the file, but remembering the rename and +# reusing it is reasonable too. +# +# Rename detection (diffcore_rename_extended()) will run twice here; it is +# not needed on the topic side of history for either of the two commits +# being merged, but it is needed on the upstream side of history for each +# commit being picked. +test_expect_success 'cherry-pick both a commit and its immediate revert' ' + test_create_repo pick-commit-and-its-immediate-revert && + ( + cd pick-commit-and-its-immediate-revert && + + test_seq 11 30 >numbers && + git add numbers && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 30 >numbers && + git add numbers && + git mv numbers sequence && + git commit -m "Renamed (and modified) numbers -> sequence" && + + git switch topic && + + test_seq 11 13 >numbers && + git add numbers && + git commit -m A && + + git revert HEAD && + + # + # Actual testing + # + + git switch upstream && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test_might_fail test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 2 calls + ) +' + +# +# In the following testcase: +# Base: sequence_1 +# Upstream: rename sequence_1 -> values_2 +# Topic_1: rename sequence_1 -> values_3 +# Topic_2: add unrelated sequence_4 +# or, in english, both sides rename sequence -> values, and then the second +# commit on the topic branch adds an unrelated file called sequence. +# +# This testcase presents no problems for git traditionally, but having both +# sides do the same rename in effect "uses it up" and if it remains cached, +# could cause a spurious rename/add conflict. +# +test_expect_success 'rename same file identically, then reintroduce it' ' + test_create_repo rename-rename-1to1-then-add-old-filename && + ( + cd rename-rename-1to1-then-add-old-filename && + + test_seq 3 8 >sequence && + git add sequence && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >sequence && + git add sequence && + git mv sequence values && + git commit -m "Renamed (and modified) sequence -> values" && + + git switch topic && + + test_seq 3 10 >sequence && + git add sequence && + git mv sequence values && + git commit -m A && + + test_write_lines A B C D E F G H I J >sequence && + git add sequence && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic && + + git ls-files >tracked && + test_line_count = 2 tracked && + test_path_is_file values && + test_path_is_file sequence && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 2 calls + ) +' + +# +# In the following testcase: +# Base: olddir/{valuesZ_1, valuesY_1, valuesX_1} +# Upstream: rename olddir/valuesZ_1 -> dirA/valuesZ_2 +# rename olddir/valuesY_1 -> dirA/valuesY_2 +# rename olddir/valuesX_1 -> dirB/valuesX_2 +# Topic_1: rename olddir/valuesZ_1 -> dirA/valuesZ_3 +# rename olddir/valuesY_1 -> dirA/valuesY_3 +# Topic_2: add olddir/newfile +# Expected Pick1: dirA/{valuesZ, valuesY}, dirB/valuesX +# Expected Pick2: dirA/{valuesZ, valuesY}, dirB/{valuesX, newfile} +# +# This testcase presents no problems for git traditionally, but having both +# sides do the same renames in effect "use it up" but if the renames remain +# cached, the directory rename could put newfile in the wrong directory. +# +test_expect_success 'rename same file identically, then add file to old dir' ' + test_create_repo rename-rename-1to1-then-add-file-to-old-dir && + ( + cd rename-rename-1to1-then-add-file-to-old-dir && + + mkdir olddir/ && + test_seq 3 8 >olddir/valuesZ && + test_seq 3 8 >olddir/valuesY && + test_seq 3 8 >olddir/valuesX && + git add olddir && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >olddir/valuesZ && + test_seq 1 8 >olddir/valuesY && + test_seq 1 8 >olddir/valuesX && + git add olddir && + mkdir dirA && + git mv olddir/valuesZ olddir/valuesY dirA && + git mv olddir/ dirB/ && + git commit -m "Renamed (and modified) values*" && + + git switch topic && + + test_seq 3 10 >olddir/valuesZ && + test_seq 3 10 >olddir/valuesY && + git add olddir && + mkdir dirA && + git mv olddir/valuesZ olddir/valuesY dirA && + git commit -m A && + + >olddir/newfile && + git add olddir/newfile && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream~1..topic && + + git ls-files >tracked && + test_line_count = 4 tracked && + test_path_is_file dirA/valuesZ && + test_path_is_file dirA/valuesY && + test_path_is_file dirB/valuesX && + test_path_is_file dirB/newfile && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 3 calls + ) +' + +# +# In the following testcase, upstream renames a directory, and the topic branch +# first adds a file to the directory, then later renames the directory +# differently: +# Base: olddir/a +# olddir/b +# Upstream: rename olddir/ -> newdir/ +# Topic_1: add olddir/newfile +# Topic_2: rename olddir/ -> otherdir/ +# +# Here we are just concerned that cached renames might prevent us from seeing +# the rename conflict, and we want to ensure that we do get a conflict. +# +# While at it, also test that we do rename detection three times. We have to +# detect renames on the upstream side of history once for each merge, plus +# Topic_2 has renames. +# +test_expect_success 'cached dir rename does not prevent noticing later conflict' ' + test_create_repo dir-rename-cache-not-occluding-later-conflict && + ( + cd dir-rename-cache-not-occluding-later-conflict && + + mkdir olddir && + test_seq 3 10 >olddir/a && + test_seq 3 10 >olddir/b && + git add olddir && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 3 10 >olddir/a && + test_seq 3 10 >olddir/b && + git add olddir && + git mv olddir newdir && + git commit -m "Dir renamed" && + + git switch topic && + + >olddir/newfile && + git add olddir/newfile && + git commit -m A && + + test_seq 1 8 >olddir/a && + test_seq 1 8 >olddir/b && + git add olddir && + git mv olddir otherdir && + git commit -m B && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test_must_fail test-tool fast-rebase --onto HEAD upstream~1 topic >output && + #git cherry-pick upstream..topic && + + grep CONFLICT..rename/rename output && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 3 calls + ) +' + +# Helper for the next two tests +test_setup_upstream_rename () { + test_create_repo $1 && + ( + cd $1 && + + test_seq 3 8 >somefile && + test_seq 3 8 >relevant-rename && + git add somefile relevant-rename && + mkdir olddir && + test_write_lines a b c d e f g >olddir/a && + test_write_lines z y x w v u t >olddir/b && + git add olddir && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >somefile && + test_seq 1 8 >relevant-rename && + git add somefile relevant-rename && + git mv relevant-rename renamed && + echo h >>olddir/a && + echo s >>olddir/b && + git add olddir && + git mv olddir newdir && + git commit -m "Dir renamed" + ) +} + +# +# In the following testcase, upstream renames a file in the toplevel directory +# as well as its only directory: +# Base: relevant-rename_1 +# somefile +# olddir/a +# olddir/b +# Upstream: rename relevant-rename_1 -> renamed_2 +# rename olddir/ -> newdir/ +# Topic_1: relevant-rename_3 +# Topic_2: olddir/newfile_1 +# Topic_3: olddir/newfile_2 +# +# In this testcase, since the first commit being picked only modifies a +# file in the toplevel directory, the directory rename is irrelevant for +# that first merge. However, we need to notice the directory rename for +# the merge that picks the second commit, and we don't want the third +# commit to mess up its location either. We want to make sure that +# olddir/newfile doesn't exist in the result and that newdir/newfile does. +# +# We also expect rename detection to occur three times. Although it is +# typically needed two times per commit, there are no deleted files on the +# topic side of history, so we only need to detect renames on the upstream +# side for each of the 3 commits we need to pick. +# +test_expect_success 'dir rename unneeded, then add new file to old dir' ' + test_setup_upstream_rename dir-rename-unneeded-until-new-file && + ( + cd dir-rename-unneeded-until-new-file && + + git switch topic && + + test_seq 3 10 >relevant-rename && + git add relevant-rename && + git commit -m A && + + echo foo >olddir/newfile && + git add olddir/newfile && + git commit -m B && + + echo bar >>olddir/newfile && + git add olddir/newfile && + git commit -m C && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 3 calls && + + git ls-files >tracked && + test_line_count = 5 tracked && + test_path_is_missing olddir/newfile && + test_path_is_file newdir/newfile + ) +' + +# +# The following testcase is *very* similar to the last one, but instead of +# adding a new olddir/newfile, it renames somefile -> olddir/newfile: +# Base: relevant-rename_1 +# somefile_1 +# olddir/a +# olddir/b +# Upstream: rename relevant-rename_1 -> renamed_2 +# rename olddir/ -> newdir/ +# Topic_1: relevant-rename_3 +# Topic_2: rename somefile -> olddir/newfile_2 +# Topic_3: modify olddir/newfile_3 +# +# In this testcase, since the first commit being picked only modifies a +# file in the toplevel directory, the directory rename is irrelevant for +# that first merge. However, we need to notice the directory rename for +# the merge that picks the second commit, and we don't want the third +# commit to mess up its location either. We want to make sure that +# neither somefile or olddir/newfile exists in the result and that +# newdir/newfile does. +# +# This testcase needs one more call to rename detection than the last +# testcase, because of the somefile -> olddir/newfile rename in Topic_2. +test_expect_success 'dir rename unneeded, then rename existing file into old dir' ' + test_setup_upstream_rename dir-rename-unneeded-until-file-moved-inside && + ( + cd dir-rename-unneeded-until-file-moved-inside && + + git switch topic && + + test_seq 3 10 >relevant-rename && + git add relevant-rename && + git commit -m A && + + test_seq 1 10 >somefile && + git add somefile && + git mv somefile olddir/newfile && + git commit -m B && + + test_seq 1 12 >olddir/newfile && + git add olddir/newfile && + git commit -m C && + + # + # Actual testing + # + + git switch upstream && + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 4 calls && + + test_path_is_missing somefile && + test_path_is_missing olddir/newfile && + test_path_is_file newdir/newfile && + git ls-files >tracked && + test_line_count = 4 tracked + ) +' + +# Helper for the next two tests +test_setup_topic_rename () { + test_create_repo $1 && + ( + cd $1 && + + test_seq 3 8 >somefile && + mkdir olddir && + test_seq 3 8 >olddir/a && + echo b >olddir/b && + git add olddir somefile && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch topic && + test_seq 1 8 >somefile && + test_seq 1 8 >olddir/a && + git add somefile olddir/a && + git mv olddir newdir && + git commit -m "Dir renamed" && + + test_seq 1 10 >somefile && + git add somefile && + mkdir olddir && + >olddir/unrelated-file && + git add olddir && + git commit -m "Unrelated file in recreated old dir" + ) +} + +# +# In the following testcase, the first commit on the topic branch renames +# a directory, while the second recreates the old directory and places a +# file into it: +# Base: somefile +# olddir/a +# olddir/b +# Upstream: olddir/newfile +# Topic_1: somefile_2 +# rename olddir/ -> newdir/ +# Topic_2: olddir/unrelated-file +# +# Note that the first pick should merge: +# Base: somefile +# olddir/{a,b} +# Upstream: olddir/newfile +# Topic_1: rename olddir/ -> newdir/ +# For which the expected result (assuming merge.directoryRenames=true) is +# clearly: +# Result: somefile +# newdir/{a, b, newfile} +# +# While the second pick does the following three-way merge: +# Base (Topic_1): somefile +# newdir/{a,b} +# Upstream (Result from 1): same files as base, but adds newdir/newfile +# Topic_2: same files as base, but adds olddir/unrelated-file +# +# The second merge is pretty trivial; upstream adds newdir/newfile, and +# topic_2 adds olddir/unrelated-file. We're just testing that we don't +# accidentally cache directory renames somehow and rename +# olddir/unrelated-file to newdir/unrelated-file. +# +# This testcase should only need one call to diffcore_rename_extended(). +test_expect_success 'caching renames only on upstream side, part 1' ' + test_setup_topic_rename cache-renames-only-upstream-add-file && + ( + cd cache-renames-only-upstream-add-file && + + git switch upstream && + + >olddir/newfile && + git add olddir/newfile && + git commit -m "Add newfile" && + + # + # Actual testing + # + + git switch upstream && + + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 1 calls && + + git ls-files >tracked && + test_line_count = 5 tracked && + test_path_is_missing newdir/unrelated-file && + test_path_is_file olddir/unrelated-file && + test_path_is_file newdir/newfile && + test_path_is_file newdir/b && + test_path_is_file newdir/a && + test_path_is_file somefile + ) +' + +# +# The following testcase is *very* similar to the last one, but instead of +# adding a new olddir/newfile, it renames somefile -> olddir/newfile: +# Base: somefile +# olddir/a +# olddir/b +# Upstream: somefile_1 -> olddir/newfile +# Topic_1: rename olddir/ -> newdir/ +# somefile_2 +# Topic_2: olddir/unrelated-file +# somefile_3 +# +# Much like the previous test, this case is actually trivial and we are just +# making sure there isn't some spurious directory rename caching going on +# for the wrong side of history. +# +# +# This testcase should only need three calls to diffcore_rename_extended(), +# because there are no renames on the topic side of history for picking +# Topic_2. +# +test_expect_success 'caching renames only on upstream side, part 2' ' + test_setup_topic_rename cache-renames-only-upstream-rename-file && + ( + cd cache-renames-only-upstream-rename-file && + + git switch upstream && + + git mv somefile olddir/newfile && + git commit -m "Add newfile" && + + # + # Actual testing + # + + git switch upstream && + + git config merge.directoryRenames true && + + GIT_TRACE2_PERF="$(pwd)/trace.output" && + export GIT_TRACE2_PERF && + + test-tool fast-rebase --onto HEAD upstream~1 topic && + #git cherry-pick upstream..topic && + + grep region_enter.*diffcore_rename trace.output >calls && + test_line_count = 3 calls && + + git ls-files >tracked && + test_line_count = 4 tracked && + test_path_is_missing newdir/unrelated-file && + test_path_is_file olddir/unrelated-file && + test_path_is_file newdir/newfile && + test_path_is_file newdir/b && + test_path_is_file newdir/a + ) +' + +test_done From d29bd6d73da5c40935713731919b1950692aff15 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:34 +0000 Subject: [PATCH 091/397] merge-ort: add data structures for in-memory caching of rename detection When there are many renames between the old base of a series of commits and the new base for a series of commits, the sequence of merges employed to transplant those commits (from a cherry-pick or rebase operation) will repeatedly detect the exact same renames. This is wasted effort. Add data structures which will be used to cache rename detection results, along with the initialization and deallocation of these data structures. Future commits will populate these caches, detect the appropriate circumstances when they can be used, and employ them to avoid re-detecting the same renames repeatedly. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index 8258d3fd62..6a06c6e5a3 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -139,6 +139,48 @@ struct rename_info { int callback_data_nr, callback_data_alloc; char *callback_data_traverse_path; + /* + * cached_pairs: Caching of renames and deletions. + * + * These are mappings recording renames and deletions of individual + * files (not directories). They are thus a map from an old + * filename to either NULL (for deletions) or a new filename (for + * renames). + */ + struct strmap cached_pairs[3]; + + /* + * cached_target_names: just the destinations from cached_pairs + * + * We sometimes want a fast lookup to determine if a given filename + * is one of the destinations in cached_pairs. cached_target_names + * is thus duplicative information, but it provides a fast lookup. + */ + struct strset cached_target_names[3]; + + /* + * cached_irrelevant: Caching of rename_sources that aren't relevant. + * + * If we try to detect a rename for a source path and succeed, it's + * part of a rename. If we try to detect a rename for a source path + * and fail, then it's a delete. If we do not try to detect a rename + * for a path, then we don't know if it's a rename or a delete. If + * merge-ort doesn't think the path is relevant, then we just won't + * cache anything for that path. But there's a slight problem in + * that merge-ort can think a path is RELEVANT_LOCATION, but due to + * commit 9bd342137e ("diffcore-rename: determine which + * relevant_sources are no longer relevant", 2021-03-13), + * diffcore-rename can downgrade the path to RELEVANT_NO_MORE. To + * avoid excessive calls to diffcore_rename_extended() we still need + * to cache such paths, though we cannot record them as either + * renames or deletes. So we cache them here as a "turned out to be + * irrelevant *for this commit*" as they are often also irrelevant + * for subsequent commits, though we will have to do some extra + * checking to see whether such paths become relevant for rename + * detection when cherry-picking/rebasing subsequent commits. + */ + struct strset cached_irrelevant[3]; + /* * needed_limit: value needed for inexact rename detection to run * @@ -381,6 +423,8 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, reinitialize ? strmap_partial_clear : strmap_clear; void (*strintmap_func)(struct strintmap *) = reinitialize ? strintmap_partial_clear : strintmap_clear; + void (*strset_func)(struct strset *) = + reinitialize ? strset_partial_clear : strset_clear; /* * We marked opti->paths with strdup_strings = 0, so that we @@ -424,6 +468,9 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, strmap_func(&renames->dir_renames[i], 0); strintmap_func(&renames->relevant_sources[i]); + strset_func(&renames->cached_target_names[i]); + strmap_func(&renames->cached_pairs[i], 1); + strset_func(&renames->cached_irrelevant[i]); } if (!reinitialize) { @@ -3675,6 +3722,12 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) NULL, 0); strintmap_init_with_options(&renames->relevant_sources[i], 0, NULL, 0); + strmap_init_with_options(&renames->cached_pairs[i], + NULL, 1); + strset_init_with_options(&renames->cached_irrelevant[i], + NULL, 1); + strset_init_with_options(&renames->cached_target_names[i], + NULL, 0); } /* From 2734f2e3243ce19af1d9c9c92dffae13af7b0fe5 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:35 +0000 Subject: [PATCH 092/397] merge-ort: populate caches of rename detection results Fill in cache_pairs, cached_target_names, and cached_irrelevant based on rename detection results. Future commits will make use of these values. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/merge-ort.c b/merge-ort.c index 6a06c6e5a3..d515f5cc3f 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2344,6 +2344,67 @@ static void resolve_diffpair_statuses(struct diff_queue_struct *q) } } +static void cache_new_pair(struct rename_info *renames, + int side, + char *old_path, + char *new_path, + int free_old_value) +{ + char *old_value; + new_path = xstrdup(new_path); + old_value = strmap_put(&renames->cached_pairs[side], + old_path, new_path); + strset_add(&renames->cached_target_names[side], new_path); + if (free_old_value) + free(old_value); + else + assert(!old_value); +} + +static void possibly_cache_new_pair(struct rename_info *renames, + struct diff_filepair *p, + unsigned side, + char *new_path) +{ + int dir_renamed_side = 0; + + if (new_path) { + /* + * Directory renames happen on the other side of history from + * the side that adds new files to the old directory. + */ + dir_renamed_side = 3 - side; + } else { + int val = strintmap_get(&renames->relevant_sources[side], + p->one->path); + if (val == RELEVANT_NO_MORE) { + assert(p->status == 'D'); + strset_add(&renames->cached_irrelevant[side], + p->one->path); + } + if (val <= 0) + return; + } + + if (p->status == 'D') { + /* + * If we already had this delete, we'll just set it's value + * to NULL again, so no harm. + */ + strmap_put(&renames->cached_pairs[side], p->one->path, NULL); + } else if (p->status == 'R') { + if (!new_path) + new_path = p->two->path; + else + cache_new_pair(renames, dir_renamed_side, + p->two->path, new_path, 0); + cache_new_pair(renames, side, p->one->path, new_path, 1); + } else if (p->status == 'A' && new_path) { + cache_new_pair(renames, dir_renamed_side, + p->two->path, new_path, 0); + } +} + static int compare_pairs(const void *a_, const void *b_) { const struct diff_filepair *a = *((const struct diff_filepair **)a_); @@ -2426,6 +2487,7 @@ static int collect_renames(struct merge_options *opt, char *new_path; /* non-NULL only with directory renames */ if (p->status != 'A' && p->status != 'R') { + possibly_cache_new_pair(renames, p, side_index, NULL); diff_free_filepair(p); continue; } @@ -2437,6 +2499,7 @@ static int collect_renames(struct merge_options *opt, &collisions, &clean); + possibly_cache_new_pair(renames, p, side_index, new_path); if (p->status != 'R' && !new_path) { diff_free_filepair(p); continue; @@ -3720,8 +3783,16 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) NULL, 1); strmap_init_with_options(&renames->dir_renames[i], NULL, 0); + /* + * relevant_sources uses -1 for the default, because we need + * to be able to distinguish not-in-strintmap from valid + * relevant_source values from enum file_rename_relevance. + * In particular, possibly_cache_new_pair() expects a negative + * value for not-found entries. + */ strintmap_init_with_options(&renames->relevant_sources[i], - 0, NULL, 0); + -1 /* explicitly invalid */, + NULL, 0); strmap_init_with_options(&renames->cached_pairs[i], NULL, 1); strset_init_with_options(&renames->cached_irrelevant[i], From 64aceb6d738394130a6e215dc6de51d8452313e0 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:36 +0000 Subject: [PATCH 093/397] merge-ort: add code to check for whether cached renames can be reused We need to know when renames detected in a previous merge operation can be reused in a later merge operation. Consider the following setup (from the git-rebase manpage): A---B---C topic / D---E---F---G master After rebasing, this will appear as: A'--B'--C' topic / D---E---F---G master Further, let's say that 'oldfile' was renamed to 'newfile' between E and G. The rebase or cherry-pick of A onto G will involve a three-way merge between E (as the merge base) and G and A. After detecting the rename between E:oldfile and G:newfile, there will be a three-way content merge of the following: E:oldfile G:newfile A:oldfile and produce a new result: A':newfile Now, when we want to pick B onto A', we will need to do a three-way merge between A (as the merge-base) and A' and B. This will involve a three-way content merge of A:oldfile A':newfile B:oldfile but only if we can detect that A:oldfile is similar enough to A':newfile to be used together in a three-way content merge, i.e. only if we can detect that A:oldfile and A':newfile are a rename. But we already know that A:oldfile and A':newfile are similar enough to be used in a three-way content merge, because that is precisely where A':newfile came from in the previous merge. Note that A & A' both appear in both merges. That gives us the condition under which we can reuse renames. There are a couple important points about this optimization: - If the rebase or cherry-pick halts for user conflicts, these caches are NOT saved anywhere. Thus, resuming a halted rebase or cherry-pick will result in no reused renames for the next commit. This is intentional, as user resolution can change files significantly and in ways that violate the similarity assumptions here. - Technically, in a *very* narrow case this might give slightly different results for rename detection. Using the example above, if: * E:oldfile had 20 lines * G:newfile added 10 new lines at the beginning of the file * A:oldfile deleted all but the first three lines of the file then => A':newfile would have 13 lines, 3 of which matches those in A:oldfile. Consider the two cases: * Without this optimization: - the next step of the rebase operation (moving B to B') would not detect the rename betwen A:oldfile and A':newfile - we'd thus get a modify/delete conflict with the rebase operation halting for the user to resolve, and have both A':newfile and B:oldfile sitting in the working tree. * With this optimization: - the rename between A:oldfile and A':newfile would be detected via the cache of renames - a three-way merge between A:oldfile, A':newfile, and B:oldfile would commence and be written to A':newfile Now, is the difference in behavior a bug...or a bugfix? I can't tell. Given that A:oldfile and A':newfile are not very similar, when we three-way merge with B:oldfile it seems likely we'll hit a conflict for the user to resolve. And it shouldn't be too hard for users to see why we did that three-way merge; oldfile and newfile *were* renames somewhere in the sequence. So, most of these corner cases will still behave similarly -- namely, a conflict given to the user to resolve. Also, consider the interesting case when commit B is a clean revert of commit A. Without this optimization, a rebase could not both apply a weird patch like A and then immediately revert it; users would be forced to resolve merge conflicts. With this optimization, it would successfully apply the clean revert. So, there is certainly at least one case that behaves better. Even if it's considered a "difference in behavior", I think both behaviors are reasonable, and the time savings provided by this optimization justify using the slightly altered rename heuristics. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index d515f5cc3f..1176bae25f 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -139,6 +139,30 @@ struct rename_info { int callback_data_nr, callback_data_alloc; char *callback_data_traverse_path; + /* + * merge_trees: trees passed to the merge algorithm for the merge + * + * merge_trees records the trees passed to the merge algorithm. But, + * this data also is stored in merge_result->priv. If a sequence of + * merges are being done (such as when cherry-picking or rebasing), + * the next merge can look at this and re-use information from + * previous merges under certain circumstances. + * + * See also all the cached_* variables. + */ + struct tree *merge_trees[3]; + + /* + * cached_pairs_valid_side: which side's cached info can be reused + * + * See the description for merge_trees. For repeated merges, at most + * only one side's cached information can be used. Valid values: + * MERGE_SIDE2: cached data from side2 can be reused + * MERGE_SIDE1: cached data from side1 can be reused + * 0: no cached data can be reused + */ + int cached_pairs_valid_side; + /* * cached_pairs: Caching of renames and deletions. * @@ -472,6 +496,8 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, strmap_func(&renames->cached_pairs[i], 1); strset_func(&renames->cached_irrelevant[i]); } + renames->cached_pairs_valid_side = 0; + renames->dir_rename_mask = 0; if (!reinitialize) { struct hashmap_iter iter; @@ -494,8 +520,6 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, strmap_clear(&opti->output, 0); } - renames->dir_rename_mask = 0; - /* Clean out callback_data as well. */ FREE_AND_NULL(renames->callback_data); renames->callback_data_nr = renames->callback_data_alloc = 0; @@ -3824,6 +3848,35 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) trace2_region_leave("merge", "allocate/init", opt->repo); } +static void merge_check_renames_reusable(struct merge_options *opt, + struct merge_result *result, + struct tree *merge_base, + struct tree *side1, + struct tree *side2) +{ + struct rename_info *renames; + struct tree **merge_trees; + struct merge_options_internal *opti = result->priv; + + if (!opti) + return; + + renames = &opti->renames; + merge_trees = renames->merge_trees; + /* merge_trees[0..2] will only be NULL if opti is */ + assert(merge_trees[0] && merge_trees[1] && merge_trees[2]); + + /* Check if we meet a condition for re-using cached_pairs */ + if (oideq(&merge_base->object.oid, &merge_trees[2]->object.oid) && + oideq(&side1->object.oid, &result->tree->object.oid)) + renames->cached_pairs_valid_side = MERGE_SIDE1; + else if (oideq(&merge_base->object.oid, &merge_trees[1]->object.oid) && + oideq(&side2->object.oid, &result->tree->object.oid)) + renames->cached_pairs_valid_side = MERGE_SIDE2; + else + renames->cached_pairs_valid_side = 0; /* neither side valid */ +} + /*** Function Grouping: merge_incore_*() and their internal variants ***/ /* @@ -3971,7 +4024,16 @@ void merge_incore_nonrecursive(struct merge_options *opt, trace2_region_enter("merge", "merge_start", opt->repo); assert(opt->ancestor != NULL); + merge_check_renames_reusable(opt, result, merge_base, side1, side2); merge_start(opt, result); + /* + * Record the trees used in this merge, so if there's a next merge in + * a cherry-pick or rebase sequence it might be able to take advantage + * of the cached_pairs in that next merge. + */ + opt->priv->renames.merge_trees[0] = merge_base; + opt->priv->renames.merge_trees[1] = side1; + opt->priv->renames.merge_trees[2] = side2; trace2_region_leave("merge", "merge_start", opt->repo); merge_ort_nonrecursive_internal(opt, merge_base, side1, side2, result); From 19ceb486f8dd25fb5782724c454edb2f06f1ed71 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:37 +0000 Subject: [PATCH 094/397] merge-ort: avoid accidental API mis-use Previously, callers of the merge-ort API could have passed an uninitialized value for struct merge_result *result. However, we want to check result to see if it has cached renames from a previous merge that we can reuse; such values would be found behind result->priv. However, if result->priv is uninitialized, attempting to access behind it will give a segfault. So, we need result->priv to be NULL (which will be the case if the caller does a memset(&result, 0)), or be written by a previous call to the merge-ort machinery. Documenting this requirement may help, but despite being the person who introduced this requirement, I still missed it once and it did not fail in a very clear way and led to a long debugging session. Add a _properly_initialized field to merge_result; that value will be 0 if the caller zero'ed the merge_result, it will be set to a very specific value by a previous run by the merge-ort machinery, and if it's uninitialized it will most likely either be 0 or some value that does not match the specific one we'd expect allowing us to throw a much more meaningful error. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 7 +++++++ merge-ort.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index 1176bae25f..8b2c93fdcf 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -52,6 +52,8 @@ enum merge_side { MERGE_SIDE2 = 2 }; +static unsigned RESULT_INITIALIZED = 0x1abe11ed; /* unlikely accidental value */ + struct traversal_callback_data { unsigned long mask; unsigned long dirmask; @@ -3768,6 +3770,10 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) assert(opt->obuf.len == 0); assert(opt->priv == NULL); + if (result->_properly_initialized != 0 && + result->_properly_initialized != RESULT_INITIALIZED) + BUG("struct merge_result passed to merge_incore_*recursive() must be zeroed or filled with values from a previous run"); + assert(!!result->priv == !!result->_properly_initialized); if (result->priv) { opt->priv = result->priv; result->priv = NULL; @@ -3927,6 +3933,7 @@ static void merge_ort_nonrecursive_internal(struct merge_options *opt, result->clean &= strmap_empty(&opt->priv->conflicted); if (!opt->priv->call_depth) { result->priv = opt->priv; + result->_properly_initialized = RESULT_INITIALIZED; opt->priv = NULL; } } diff --git a/merge-ort.h b/merge-ort.h index d53a0a339f..c011864ffe 100644 --- a/merge-ort.h +++ b/merge-ort.h @@ -29,6 +29,8 @@ struct merge_result { * !clean) and to print "CONFLICT" messages. Not for external use. */ void *priv; + /* Also private */ + unsigned _properly_initialized; }; /* From d509802993e8423d459a05fcd6151ca1782caa07 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:38 +0000 Subject: [PATCH 095/397] merge-ort: preserve cached renames for the appropriate side Previous commits created an in-memory cache of the results of rename detection, and added logic to detect when that cache could appropriately be used in a subsequent merge operation -- but we were still unconditionally clearing the cache with each new merge operation anyway. If it is valid to reuse the cache from one of the two sides of history, preserve that side. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index 8b2c93fdcf..cc1adc7456 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -486,17 +486,18 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, /* Free memory used by various renames maps */ for (i = MERGE_SIDE1; i <= MERGE_SIDE2; ++i) { strintmap_func(&renames->dirs_removed[i]); - - partial_clear_dir_rename_count(&renames->dir_rename_count[i]); - if (!reinitialize) - strmap_clear(&renames->dir_rename_count[i], 1); - strmap_func(&renames->dir_renames[i], 0); - strintmap_func(&renames->relevant_sources[i]); - strset_func(&renames->cached_target_names[i]); - strmap_func(&renames->cached_pairs[i], 1); - strset_func(&renames->cached_irrelevant[i]); + if (!reinitialize) + assert(renames->cached_pairs_valid_side == 0); + if (i != renames->cached_pairs_valid_side) { + strset_func(&renames->cached_target_names[i]); + strmap_func(&renames->cached_pairs[i], 1); + strset_func(&renames->cached_irrelevant[i]); + partial_clear_dir_rename_count(&renames->dir_rename_count[i]); + if (!reinitialize) + strmap_clear(&renames->dir_rename_count[i], 1); + } } renames->cached_pairs_valid_side = 0; renames->dir_rename_mask = 0; @@ -2456,6 +2457,7 @@ static void detect_regular_renames(struct merge_options *opt, return; } + partial_clear_dir_rename_count(&renames->dir_rename_count[side_index]); repo_diff_setup(opt->repo, &diff_opts); diff_opts.flags.recursive = 1; diff_opts.flags.rename_empty = 0; From 86b41b389546e68164ed58f6e60297a391cdca83 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:39 +0000 Subject: [PATCH 096/397] merge-ort: add helper functions for using cached renames If we have a usable rename cache, then we can remove from relevant_sources all the paths that were cached; diffcore_rename_extended() can then consider an even smaller set of relevant_sources in its rename detection. However, when diffcore_rename_extended() is done, we will need to take the renames it detected and then add back in all the ones we had cached from before. Add helper functions for doing these two operations; the next commit will make use of them. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index cc1adc7456..8e38dac341 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2371,6 +2371,53 @@ static void resolve_diffpair_statuses(struct diff_queue_struct *q) } } +MAYBE_UNUSED +static void prune_cached_from_relevant(struct rename_info *renames, + unsigned side) +{ + /* Reason for this function described in add_pair() */ + struct hashmap_iter iter; + struct strmap_entry *entry; + + /* Remove from relevant_sources all entries in cached_pairs[side] */ + strmap_for_each_entry(&renames->cached_pairs[side], &iter, entry) { + strintmap_remove(&renames->relevant_sources[side], + entry->key); + } + /* Remove from relevant_sources all entries in cached_irrelevant[side] */ + strset_for_each_entry(&renames->cached_irrelevant[side], &iter, entry) { + strintmap_remove(&renames->relevant_sources[side], + entry->key); + } +} + +MAYBE_UNUSED +static void use_cached_pairs(struct merge_options *opt, + struct strmap *cached_pairs, + struct diff_queue_struct *pairs) +{ + struct hashmap_iter iter; + struct strmap_entry *entry; + + /* + * Add to side_pairs all entries from renames->cached_pairs[side_index]. + * (Info in cached_irrelevant[side_index] is not relevant here.) + */ + strmap_for_each_entry(cached_pairs, &iter, entry) { + struct diff_filespec *one, *two; + const char *old_name = entry->key; + const char *new_name = entry->value; + if (!new_name) + new_name = old_name; + + /* We don't care about oid/mode, only filenames and status */ + one = alloc_filespec(old_name); + two = alloc_filespec(new_name); + diff_queue(pairs, one, two); + pairs->queue[pairs->nr-1]->status = entry->value ? 'R' : 'D'; + } +} + static void cache_new_pair(struct rename_info *renames, int side, char *old_path, From cbdca289fbb011e7397fecfebeeac3f887ef22d1 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:40 +0000 Subject: [PATCH 097/397] merge-ort: handle interactions of caching and rename/rename(1to1) cases As documented in Documentation/technical/remembering-renames.txt, and as tested for in the two testcases in t6429 with "rename same file identically" in their description, there is one case where we need to have renames in one commit NOT be cached for the next commit in our rebase sequence -- namely, rename/rename(1to1) cases. Rather than specifically trying to uncache those and fix up dir_rename_counts() to match (which would also be valid but more work), we simply disable the optimization when this really rare type of rename occurs. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/merge-ort.c b/merge-ort.c index 8e38dac341..de96fe4f63 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2111,6 +2111,9 @@ static int process_renames(struct merge_options *opt, VERIFY_CI(side2); if (!strcmp(pathnames[1], pathnames[2])) { + struct rename_info *ri = &opt->priv->renames; + int j; + /* Both sides renamed the same way */ assert(side1 == side2); memcpy(&side1->stages[0], &base->stages[0], @@ -2120,6 +2123,16 @@ static int process_renames(struct merge_options *opt, base->merged.is_null = 1; base->merged.clean = 1; + /* + * Disable remembering renames optimization; + * rename/rename(1to1) is incredibly rare, and + * just disabling the optimization is easier + * than purging cached_pairs, + * cached_target_names, and dir_rename_counts. + */ + for (j = 0; j < 3; j++) + ri->merge_trees[j] = NULL; + /* We handled both renames, i.e. i+1 handled */ i++; /* Move to next rename */ @@ -3918,7 +3931,22 @@ static void merge_check_renames_reusable(struct merge_options *opt, renames = &opti->renames; merge_trees = renames->merge_trees; - /* merge_trees[0..2] will only be NULL if opti is */ + + /* + * Handle case where previous merge operation did not want cache to + * take effect, e.g. because rename/rename(1to1) makes it invalid. + */ + if (!merge_trees[0]) { + assert(!merge_trees[0] && !merge_trees[1] && !merge_trees[2]); + renames->cached_pairs_valid_side = 0; /* neither side valid */ + return; + } + + /* + * Handle other cases; note that merge_trees[0..2] will only + * be NULL if opti is, or if all three were manually set to + * NULL by e.g. rename/rename(1to1) handling. + */ assert(merge_trees[0] && merge_trees[1] && merge_trees[2]); /* Check if we meet a condition for re-using cached_pairs */ From 25e65b6dd52c987056f1cac00fe6073fbf8ea237 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 20 May 2021 06:09:41 +0000 Subject: [PATCH 098/397] merge-ort, diffcore-rename: employ cached renames when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When there are many renames between the old base of a series of commits and the new base, the way sequencer.c, merge-recursive.c, and diffcore-rename.c have traditionally split the work resulted in redetecting the same renames with each and every commit being transplanted. To address this, the last several commits have been creating a cache of rename detection results, determining when it was safe to use such a cache in subsequent merge operations, adding helper functions, and so on. See the previous half dozen commit messages for additional discussion of this optimization, particularly the message a few commits ago entitled "add code to check for whether cached renames can be reused". This commit finally ties all of that work together, modifying the merge algorithm to make use of these cached renames. For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 5.665 s ± 0.129 s 5.622 s ± 0.059 s mega-renames: 11.435 s ± 0.158 s 10.127 s ± 0.073 s just-one-mega: 494.2 ms ± 6.1 ms 500.3 ms ± 3.8 ms That's a fairly small improvement, but mostly because the previous optimizations were so effective for these particular testcases; this optimization only kicks in when the others don't. If we undid the basename-guided rename detection and skip-irrelevant-renames optimizations, then we'd see that this series by itself improved performance as follows: Before Basename Series After Just This Series no-renames: 13.815 s ± 0.062 s 5.697 s ± 0.080 s mega-renames: 1799.937 s ± 0.493 s 205.709 s ± 0.457 s Since this optimization kicks in to help accelerate cases where the previous optimizations do not apply, this last comparison shows that this cached-renames optimization has the potential to help signficantly in cases that don't meet the requirements for the other optimizations to be effective. The changes made in this optimization also lay some important groundwork for a future optimization around having collect_merge_info() avoid recursing into subtrees in more cases. However, for this optimization to be effective, merge_switch_to_result() should only be called when the rebase or cherry-pick operation has either completed or hit a case where the user needs to resolve a conflict or edit the result. If it is called after every commit, as sequencer.c does, then the working tree and index are needlessly updated with every commit and the cached metadata is tossed, defeating this optimization. Refactoring sequencer.c to only call merge_switch_to_result() at the end of the operation is a bigger undertaking, and the practical benefits of this optimization will not be realized until that work is performed. Since `test-tool fast-rebase` only updates at the end of the operation, it was used to obtain the timings above. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- diffcore-rename.c | 22 +++++++++-- diffcore.h | 3 +- merge-ort.c | 47 ++++++++++++++++++++--- t/t6429-merge-sequence-rename-caching.sh | 48 ++++++++++++++---------- 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index 7cc2459261..dfbe65c917 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -568,7 +568,8 @@ static void update_dir_rename_counts(struct dir_rename_info *info, static void initialize_dir_rename_info(struct dir_rename_info *info, struct strintmap *relevant_sources, struct strintmap *dirs_removed, - struct strmap *dir_rename_count) + struct strmap *dir_rename_count, + struct strmap *cached_pairs) { struct hashmap_iter iter; struct strmap_entry *entry; @@ -633,6 +634,17 @@ static void initialize_dir_rename_info(struct dir_rename_info *info, rename_dst[i].p->two->path); } + /* Add cached_pairs to counts */ + strmap_for_each_entry(cached_pairs, &iter, entry) { + const char *old_name = entry->key; + const char *new_name = entry->value; + if (!new_name) + /* known delete; ignore it */ + continue; + + update_dir_rename_counts(info, dirs_removed, old_name, new_name); + } + /* * Now we collapse * dir_rename_count: old_directory -> {new_directory -> count} @@ -1247,7 +1259,8 @@ static void handle_early_known_dir_renames(struct dir_rename_info *info, void diffcore_rename_extended(struct diff_options *options, struct strintmap *relevant_sources, struct strintmap *dirs_removed, - struct strmap *dir_rename_count) + struct strmap *dir_rename_count, + struct strmap *cached_pairs) { int detect_rename = options->detect_rename; int minimum_score = options->rename_score; @@ -1363,7 +1376,8 @@ void diffcore_rename_extended(struct diff_options *options, /* Preparation for basename-driven matching. */ trace2_region_enter("diff", "dir rename setup", options->repo); initialize_dir_rename_info(&info, relevant_sources, - dirs_removed, dir_rename_count); + dirs_removed, dir_rename_count, + cached_pairs); trace2_region_leave("diff", "dir rename setup", options->repo); /* Utilize file basenames to quickly find renames. */ @@ -1561,5 +1575,5 @@ void diffcore_rename_extended(struct diff_options *options, void diffcore_rename(struct diff_options *options) { - diffcore_rename_extended(options, NULL, NULL, NULL); + diffcore_rename_extended(options, NULL, NULL, NULL, NULL); } diff --git a/diffcore.h b/diffcore.h index cf8d4cb861..de01e64bec 100644 --- a/diffcore.h +++ b/diffcore.h @@ -181,7 +181,8 @@ void diffcore_rename(struct diff_options *); void diffcore_rename_extended(struct diff_options *options, struct strintmap *relevant_sources, struct strintmap *dirs_removed, - struct strmap *dir_rename_count); + struct strmap *dir_rename_count, + struct strmap *cached_pairs); void diffcore_merge_broken(void); void diffcore_pickaxe(struct diff_options *); void diffcore_order(const char *orderfile); diff --git a/merge-ort.c b/merge-ort.c index de96fe4f63..ed21dc8724 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -763,15 +763,48 @@ static void add_pair(struct merge_options *opt, struct rename_info *renames = &opt->priv->renames; int names_idx = is_add ? side : 0; - if (!is_add) { + if (is_add) { + if (strset_contains(&renames->cached_target_names[side], + pathname)) + return; + } else { unsigned content_relevant = (match_mask == 0); unsigned location_relevant = (dir_rename_mask == 0x07); + /* + * If pathname is found in cached_irrelevant[side] due to + * previous pick but for this commit content is relevant, + * then we need to remove it from cached_irrelevant. + */ + if (content_relevant) + /* strset_remove is no-op if strset doesn't have key */ + strset_remove(&renames->cached_irrelevant[side], + pathname); + + /* + * We do not need to re-detect renames for paths that we already + * know the pairing, i.e. for cached_pairs (or + * cached_irrelevant). However, handle_deferred_entries() needs + * to loop over the union of keys from relevant_sources[side] and + * cached_pairs[side], so for simplicity we set relevant_sources + * for all the cached_pairs too and then strip them back out in + * prune_cached_from_relevant() at the beginning of + * detect_regular_renames(). + */ if (content_relevant || location_relevant) { /* content_relevant trumps location_relevant */ strintmap_set(&renames->relevant_sources[side], pathname, content_relevant ? RELEVANT_CONTENT : RELEVANT_LOCATION); } + + /* + * Avoid creating pair if we've already cached rename results. + * Note that we do this after setting relevant_sources[side] + * as noted in the comment above. + */ + if (strmap_contains(&renames->cached_pairs[side], pathname) || + strset_contains(&renames->cached_irrelevant[side], pathname)) + return; } one = alloc_filespec(pathname); @@ -2360,7 +2393,9 @@ static inline int possible_side_renames(struct rename_info *renames, static inline int possible_renames(struct rename_info *renames) { return possible_side_renames(renames, 1) || - possible_side_renames(renames, 2); + possible_side_renames(renames, 2) || + !strmap_empty(&renames->cached_pairs[1]) || + !strmap_empty(&renames->cached_pairs[2]); } static void resolve_diffpair_statuses(struct diff_queue_struct *q) @@ -2384,7 +2419,6 @@ static void resolve_diffpair_statuses(struct diff_queue_struct *q) } } -MAYBE_UNUSED static void prune_cached_from_relevant(struct rename_info *renames, unsigned side) { @@ -2404,7 +2438,6 @@ static void prune_cached_from_relevant(struct rename_info *renames, } } -MAYBE_UNUSED static void use_cached_pairs(struct merge_options *opt, struct strmap *cached_pairs, struct diff_queue_struct *pairs) @@ -2507,6 +2540,7 @@ static void detect_regular_renames(struct merge_options *opt, struct diff_options diff_opts; struct rename_info *renames = &opt->priv->renames; + prune_cached_from_relevant(renames, side_index); if (!possible_side_renames(renames, side_index)) { /* * No rename detection needed for this side, but we still need @@ -2535,7 +2569,8 @@ static void detect_regular_renames(struct merge_options *opt, diffcore_rename_extended(&diff_opts, &renames->relevant_sources[side_index], &renames->dirs_removed[side_index], - &renames->dir_rename_count[side_index]); + &renames->dir_rename_count[side_index], + &renames->cached_pairs[side_index]); trace2_region_leave("diff", "diffcore_rename", opt->repo); resolve_diffpair_statuses(&diff_queued_diff); @@ -2643,6 +2678,8 @@ static int detect_and_process_renames(struct merge_options *opt, trace2_region_enter("merge", "regular renames", opt->repo); detect_regular_renames(opt, MERGE_SIDE1); detect_regular_renames(opt, MERGE_SIDE2); + use_cached_pairs(opt, &renames->cached_pairs[1], &renames->pairs[1]); + use_cached_pairs(opt, &renames->cached_pairs[2], &renames->pairs[2]); trace2_region_leave("merge", "regular renames", opt->repo); trace2_region_enter("merge", "directory renames", opt->repo); diff --git a/t/t6429-merge-sequence-rename-caching.sh b/t/t6429-merge-sequence-rename-caching.sh index f47d8924ee..035edc40b1 100755 --- a/t/t6429-merge-sequence-rename-caching.sh +++ b/t/t6429-merge-sequence-rename-caching.sh @@ -101,10 +101,10 @@ test_expect_success 'caching renames does not preclude finding new ones' ' # dramatic change in size of the file, but remembering the rename and # reusing it is reasonable too. # -# Rename detection (diffcore_rename_extended()) will run twice here; it is -# not needed on the topic side of history for either of the two commits -# being merged, but it is needed on the upstream side of history for each -# commit being picked. +# We do test here that we expect rename detection to only be run once total +# (the topic side of history doesn't need renames, and with caching we +# should be able to only run rename detection on the upstream side one +# time.) test_expect_success 'cherry-pick both a commit and its immediate revert' ' test_create_repo pick-commit-and-its-immediate-revert && ( @@ -140,11 +140,11 @@ test_expect_success 'cherry-pick both a commit and its immediate revert' ' GIT_TRACE2_PERF="$(pwd)/trace.output" && export GIT_TRACE2_PERF && - test_might_fail test-tool fast-rebase --onto HEAD upstream~1 topic && + test-tool fast-rebase --onto HEAD upstream~1 topic && #git cherry-pick upstream~1..topic && grep region_enter.*diffcore_rename trace.output >calls && - test_line_count = 2 calls + test_line_count = 1 calls ) ' @@ -304,9 +304,11 @@ test_expect_success 'rename same file identically, then add file to old dir' ' # Here we are just concerned that cached renames might prevent us from seeing # the rename conflict, and we want to ensure that we do get a conflict. # -# While at it, also test that we do rename detection three times. We have to -# detect renames on the upstream side of history once for each merge, plus -# Topic_2 has renames. +# While at it, though, we do test that we only try to detect renames 2 +# times and not three. (The first merge needs to detect renames on the +# upstream side. Traditionally, the second merge would need to detect +# renames on both sides of history, but our caching of upstream renames +# should avoid the need to re-detect upstream renames.) # test_expect_success 'cached dir rename does not prevent noticing later conflict' ' test_create_repo dir-rename-cache-not-occluding-later-conflict && @@ -357,7 +359,7 @@ test_expect_success 'cached dir rename does not prevent noticing later conflict' grep CONFLICT..rename/rename output && grep region_enter.*diffcore_rename trace.output >calls && - test_line_count = 3 calls + test_line_count = 2 calls ) ' @@ -412,10 +414,17 @@ test_setup_upstream_rename () { # commit to mess up its location either. We want to make sure that # olddir/newfile doesn't exist in the result and that newdir/newfile does. # -# We also expect rename detection to occur three times. Although it is -# typically needed two times per commit, there are no deleted files on the -# topic side of history, so we only need to detect renames on the upstream -# side for each of the 3 commits we need to pick. +# We also test that we only do rename detection twice. We never need +# rename detection on the topic side of history, but we do need it twice on +# the upstream side of history. For the first topic commit, we only need +# the +# relevant-rename -> renamed +# rename, because olddir is unmodified by Topic_1. For Topic_2, however, +# the new file being added to olddir means files that were previously +# irrelevant for rename detection are now relevant, forcing us to repeat +# rename detection for the paths we don't already have cached. Topic_3 also +# tweaks olddir/newfile, but the renames in olddir/ will have been cached +# from the second rename detection run. # test_expect_success 'dir rename unneeded, then add new file to old dir' ' test_setup_upstream_rename dir-rename-unneeded-until-new-file && @@ -450,7 +459,7 @@ test_expect_success 'dir rename unneeded, then add new file to old dir' ' #git cherry-pick upstream..topic && grep region_enter.*diffcore_rename trace.output >calls && - test_line_count = 3 calls && + test_line_count = 2 calls && git ls-files >tracked && test_line_count = 5 tracked && @@ -516,7 +525,7 @@ test_expect_success 'dir rename unneeded, then rename existing file into old dir #git cherry-pick upstream..topic && grep region_enter.*diffcore_rename trace.output >calls && - test_line_count = 4 calls && + test_line_count = 3 calls && test_path_is_missing somefile && test_path_is_missing olddir/newfile && @@ -648,9 +657,8 @@ test_expect_success 'caching renames only on upstream side, part 1' ' # for the wrong side of history. # # -# This testcase should only need three calls to diffcore_rename_extended(), -# because there are no renames on the topic side of history for picking -# Topic_2. +# This testcase should only need two calls to diffcore_rename_extended(), +# both for the first merge, one for each side of history. # test_expect_success 'caching renames only on upstream side, part 2' ' test_setup_topic_rename cache-renames-only-upstream-rename-file && @@ -677,7 +685,7 @@ test_expect_success 'caching renames only on upstream side, part 2' ' #git cherry-pick upstream..topic && grep region_enter.*diffcore_rename trace.output >calls && - test_line_count = 3 calls && + test_line_count = 2 calls && git ls-files >tracked && test_line_count = 4 tracked && From 4e0a64a7135b204f140058ed9f53299819db447e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Thu, 20 May 2021 13:05:45 +0200 Subject: [PATCH 099/397] trace2: refactor to avoid gcc warning under -O3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactor tr2_dst_try_uds_connect() to avoid a gcc warning[1] that appears under -O3 (but not -O2). This makes the build pass under DEVELOPER=1 without needing a DEVOPTS=no-error. This can be reproduced with GCC Debian 8.3.0-6, but not e.g. with clang 7.0.1-8+deb10u2. We've had this warning since ee4512ed481 (trace2: create new combined trace facility, 2019-02-22). As noted in [2] this warning happens because the compiler doesn't assume that errno must be non-zero after a failed syscall. Let's work around by using the well-established "saved_errno" pattern, along with returning -1 ourselves instead of "errno". The caller can thus rely on our "errno" on failure. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61846 for a related bug report against GCC. 1. trace2/tr2_dst.c: In function ‘tr2_dst_get_trace_fd.part.5’: trace2/tr2_dst.c:296:10: warning: ‘fd’ may be used uninitialized in this function [-Wmaybe-uninitialized] dst->fd = fd; ~~~~~~~~^~~~ trace2/tr2_dst.c:229:6: note: ‘fd’ was declared here int fd; ^~ 2. https://lore.kernel.org/git/20200404142131.GA679473@coredump.intra.peff.net/ Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- trace2/tr2_dst.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index 5dda0ca1cd..0031476350 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -115,15 +115,16 @@ static int tr2_dst_try_uds_connect(const char *path, int sock_type, int *out_fd) fd = socket(AF_UNIX, sock_type, 0); if (fd == -1) - return errno; + return -1; sa.sun_family = AF_UNIX; strlcpy(sa.sun_path, path, sizeof(sa.sun_path)); if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) == -1) { - int e = errno; + int saved_errno = errno; close(fd); - return e; + errno = saved_errno; + return -1; } *out_fd = fd; @@ -138,7 +139,6 @@ static int tr2_dst_try_unix_domain_socket(struct tr2_dst *dst, { unsigned int uds_try = 0; int fd; - int e; const char *path = NULL; /* @@ -182,15 +182,13 @@ static int tr2_dst_try_unix_domain_socket(struct tr2_dst *dst, } if (uds_try & TR2_DST_UDS_TRY_STREAM) { - e = tr2_dst_try_uds_connect(path, SOCK_STREAM, &fd); - if (!e) + if (!tr2_dst_try_uds_connect(path, SOCK_STREAM, &fd)) goto connected; - if (e != EPROTOTYPE) + if (errno != EPROTOTYPE) goto error; } if (uds_try & TR2_DST_UDS_TRY_DGRAM) { - e = tr2_dst_try_uds_connect(path, SOCK_DGRAM, &fd); - if (!e) + if (!tr2_dst_try_uds_connect(path, SOCK_DGRAM, &fd)) goto connected; } @@ -198,7 +196,7 @@ error: if (tr2_dst_want_warning()) warning("trace2: could not connect to socket '%s' for '%s' tracing: %s", path, tr2_sysenv_display_name(dst->sysenv_var), - strerror(e)); + strerror(errno)); tr2_dst_trace_disable(dst); return 0; From 225f7fa84790a50dc95ffc21ef8f601f0f5e9da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Avila?= Date: Thu, 20 May 2021 09:42:14 +0200 Subject: [PATCH 100/397] help: fix small typo in error message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Classic string concatenation while forgetting a space character. Signed-off-by: Jean-Noël Avila Signed-off-by: Junio C Hamano --- list-objects-filter-options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 96a605c8ad..fd8d59f653 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -102,7 +102,7 @@ static int gently_parse_list_objects_filter( } else if (skip_prefix(arg, "object:type=", &v0)) { int type = type_from_string_gently(v0, strlen(v0), 1); if (type < 0) { - strbuf_addf(errbuf, _("'%s' for 'object:type=' is" + strbuf_addf(errbuf, _("'%s' for 'object:type=' is " "not a valid object type"), v0); return 1; } From 6aac70a870fc40482eca943ff0b64003497d69c1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 20 May 2021 18:28:10 +0000 Subject: [PATCH 101/397] simple-ipc: correct ifdefs when NO_PTHREADS is defined Simple IPC always requires threads (in addition to various platform-specific IPC support). Fix the ifdefs in the Makefile to define SUPPORTS_SIMPLE_IPC when appropriate. Previously, the Unix version of the code would only verify that Unix domain sockets were available. This problem was reported here: https://lore.kernel.org/git/YKN5lXs4AoK%2FJFTO@coredump.intra.peff.net/T/#m08be8f1942ea8a2c36cfee0e51cdf06489fdeafc Reported-by: Randall S. Becker Helped-by: Jeff King Signed-off-by: Jeff Hostetler Signed-off-by: Junio C Hamano --- Makefile | 26 ++++++++++++++++++++++---- compat/simple-ipc/ipc-shared.c | 10 +++++++--- compat/simple-ipc/ipc-unix-socket.c | 8 ++++++-- compat/simple-ipc/ipc-win32.c | 8 ++++++-- contrib/buildsystems/CMakeLists.txt | 10 +++++++++- simple-ipc.h | 4 ---- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index dfc37b0480..8ca59f4943 100644 --- a/Makefile +++ b/Makefile @@ -1679,13 +1679,31 @@ ifdef NO_UNIX_SOCKETS else LIB_OBJS += unix-socket.o LIB_OBJS += unix-stream-server.o +endif + +# Simple IPC requires threads and platform-specific IPC support. +# Only platforms that have both should include these source files +# in the build. +# +# On Windows-based systems, Simple IPC requires threads and Windows +# Named Pipes. These are always available, so Simple IPC support +# is optional. +# +# On Unix-based systems, Simple IPC requires pthreads and Unix +# domain sockets. So support is only enabled when both are present. +# +ifdef USE_WIN32_IPC + BASIC_CFLAGS += -DSUPPORTS_SIMPLE_IPC + LIB_OBJS += compat/simple-ipc/ipc-shared.o + LIB_OBJS += compat/simple-ipc/ipc-win32.o +else +ifndef NO_PTHREADS +ifndef NO_UNIX_SOCKETS + BASIC_CFLAGS += -DSUPPORTS_SIMPLE_IPC LIB_OBJS += compat/simple-ipc/ipc-shared.o LIB_OBJS += compat/simple-ipc/ipc-unix-socket.o endif - -ifdef USE_WIN32_IPC - LIB_OBJS += compat/simple-ipc/ipc-shared.o - LIB_OBJS += compat/simple-ipc/ipc-win32.o +endif endif ifdef NO_ICONV diff --git a/compat/simple-ipc/ipc-shared.c b/compat/simple-ipc/ipc-shared.c index 1edec81595..1b9d359ab6 100644 --- a/compat/simple-ipc/ipc-shared.c +++ b/compat/simple-ipc/ipc-shared.c @@ -4,7 +4,13 @@ #include "pkt-line.h" #include "thread-utils.h" -#ifdef SUPPORTS_SIMPLE_IPC +#ifndef SUPPORTS_SIMPLE_IPC +/* + * This source file should only be compiled when Simple IPC is supported. + * See the top-level Makefile. + */ +#error SUPPORTS_SIMPLE_IPC not defined +#endif int ipc_server_run(const char *path, const struct ipc_server_opts *opts, ipc_server_application_cb *application_cb, @@ -24,5 +30,3 @@ int ipc_server_run(const char *path, const struct ipc_server_opts *opts, return ret; } - -#endif /* SUPPORTS_SIMPLE_IPC */ diff --git a/compat/simple-ipc/ipc-unix-socket.c b/compat/simple-ipc/ipc-unix-socket.c index 38689b278d..1927e6ef4b 100644 --- a/compat/simple-ipc/ipc-unix-socket.c +++ b/compat/simple-ipc/ipc-unix-socket.c @@ -6,8 +6,12 @@ #include "unix-socket.h" #include "unix-stream-server.h" -#ifdef NO_UNIX_SOCKETS -#error compat/simple-ipc/ipc-unix-socket.c requires Unix sockets +#ifndef SUPPORTS_SIMPLE_IPC +/* + * This source file should only be compiled when Simple IPC is supported. + * See the top-level Makefile. + */ +#error SUPPORTS_SIMPLE_IPC not defined #endif enum ipc_active_state ipc_get_active_state(const char *path) diff --git a/compat/simple-ipc/ipc-win32.c b/compat/simple-ipc/ipc-win32.c index 8f89c02037..8dc7bda087 100644 --- a/compat/simple-ipc/ipc-win32.c +++ b/compat/simple-ipc/ipc-win32.c @@ -4,8 +4,12 @@ #include "pkt-line.h" #include "thread-utils.h" -#ifndef GIT_WINDOWS_NATIVE -#error This file can only be compiled on Windows +#ifndef SUPPORTS_SIMPLE_IPC +/* + * This source file should only be compiled when Simple IPC is supported. + * See the top-level Makefile. + */ +#error SUPPORTS_SIMPLE_IPC not defined #endif static int initialize_pipe_name(const char *path, wchar_t *wpath, size_t alloc) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 629a181745..67d97d2443 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -248,8 +248,15 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") list(APPEND compat_SOURCES compat/simple-ipc/ipc-shared.c compat/simple-ipc/ipc-win32.c) + add_compile_definitions(SUPPORTS_SIMPLE_IPC) + set(SUPPORTS_SIMPLE_IPC 1) else() - list(APPEND compat_SOURCES compat/simple-ipc/ipc-shared.c compat/simple-ipc/ipc-unix-socket.c) + # Simple IPC requires both Unix sockets and pthreads on Unix-based systems. + if(NOT NO_UNIX_SOCKETS AND NOT NO_PTHREADS) + list(APPEND compat_SOURCES compat/simple-ipc/ipc-shared.c compat/simple-ipc/ipc-unix-socket.c) + add_compile_definitions(SUPPORTS_SIMPLE_IPC) + set(SUPPORTS_SIMPLE_IPC 1) + endif() endif() set(EXE_EXTENSION ${CMAKE_EXECUTABLE_SUFFIX}) @@ -966,6 +973,7 @@ file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "X='${EXE_EXTENSION}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_GETTEXT='${NO_GETTEXT}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "RUNTIME_PREFIX='${RUNTIME_PREFIX}'\n") file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_PYTHON='${NO_PYTHON}'\n") +file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "SUPPORTS_SIMPLE_IPC='${SUPPORTS_SIMPLE_IPC}'\n") if(WIN32) file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/x64-windows/bin\"\n") endif() diff --git a/simple-ipc.h b/simple-ipc.h index dc3606e30b..2c48a5ee00 100644 --- a/simple-ipc.h +++ b/simple-ipc.h @@ -5,10 +5,6 @@ * See Documentation/technical/api-simple-ipc.txt */ -#if defined(GIT_WINDOWS_NATIVE) || !defined(NO_UNIX_SOCKETS) -#define SUPPORTS_SIMPLE_IPC -#endif - #ifdef SUPPORTS_SIMPLE_IPC #include "pkt-line.h" From 7a55fa0e0bb88e2bf721bc65cc55cd6b435a8373 Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:46:54 +0300 Subject: [PATCH 102/397] t4013: test that "-m" alone has no effect in "git log" This is to notice current behavior that we are going to change when we start to imply "-p" by "-m". Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- t/t4013-diff-various.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index 87def81699..e9f67cd243 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -452,6 +452,14 @@ diff-tree --stat --compact-summary initial mode diff-tree -R --stat --compact-summary initial mode EOF +test_expect_success 'log -m matches pure log' ' + git log master >result && + process_diffs result >expected && + git log -m >result && + process_diffs result >actual && + test_cmp expected actual +' + test_expect_success 'log --diff-merges=on matches --diff-merges=separate' ' git log -p --diff-merges=separate master >result && process_diffs result >expected && From 48229c193d2e6e728d3243bacea2f1e1490ced8a Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:46:55 +0300 Subject: [PATCH 103/397] t4013: test "git log -m --raw" This is to ensure we won't break different diff formats when we start to imply "-p" by "-m". Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- t/t4013-diff-various.sh | 1 + t/t4013/diff.log_-m_--raw_master | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 t/t4013/diff.log_-m_--raw_master diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index e9f67cd243..1809355f9b 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -337,6 +337,7 @@ log -m -p --first-parent master log -m -p master log --cc -m -p master log -c -m -p master +log -m --raw master log -SF master log -S F master log -SF -p master diff --git a/t/t4013/diff.log_-m_--raw_master b/t/t4013/diff.log_-m_--raw_master new file mode 100644 index 0000000000..cd2ecc4628 --- /dev/null +++ b/t/t4013/diff.log_-m_--raw_master @@ -0,0 +1,61 @@ +$ git log -m --raw master +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +:100644 100644 cead32e... 992913c... M dir/sub +:100644 100644 b414108... 10a8a9f... M file0 + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + +:100644 100644 7289e35... 992913c... M dir/sub +:100644 100644 f4615da... 10a8a9f... M file0 +:000000 100644 0000000... b1e6722... A file1 +:100644 000000 01e79c3... 0000000... D file2 +:100644 000000 7289e35... 0000000... D file3 + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a +Author: A U Thor +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + +:100644 100644 35d242b... 7289e35... M dir/sub +:100644 100644 01e79c3... f4615da... M file0 +:000000 100644 0000000... 7289e35... A file3 + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + +:100644 100644 8422d40... cead32e... M dir/sub +:000000 100644 0000000... b1e6722... A file1 + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + +:100644 100644 35d242b... 8422d40... M dir/sub +:100644 100644 01e79c3... b414108... M file0 +:100644 000000 01e79c3... 0000000... D file2 + +commit 444ac553ac7612cc88969031b02b3767fb8a353a +Author: A U Thor +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ From faf16d4e97ed45e8e570a8d1d568449e948284f0 Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:46:56 +0300 Subject: [PATCH 104/397] t4013: test "git log -m --stat" This is to ensure we won't break different diff formats when we start to imply "-p" by "-m". Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- t/t4013-diff-various.sh | 1 + t/t4013/diff.log_-m_--stat_master | 66 +++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 t/t4013/diff.log_-m_--stat_master diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index 1809355f9b..e53ca7aa50 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -338,6 +338,7 @@ log -m -p master log --cc -m -p master log -c -m -p master log -m --raw master +log -m --stat master log -SF master log -S F master log -SF -p master diff --git a/t/t4013/diff.log_-m_--stat_master b/t/t4013/diff.log_-m_--stat_master new file mode 100644 index 0000000000..c7db084fd9 --- /dev/null +++ b/t/t4013/diff.log_-m_--stat_master @@ -0,0 +1,66 @@ +$ git log -m --stat master +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + + dir/sub | 2 ++ + file0 | 3 +++ + 2 files changed, 5 insertions(+) + +commit 59d314ad6f356dd08601a4cd5e530381da3e3c64 (from c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a) +Merge: 9a6d494 c7a2ab9 +Author: A U Thor +Date: Mon Jun 26 00:04:00 2006 +0000 + + Merge branch 'side' + + dir/sub | 4 ++++ + file0 | 3 +++ + file1 | 3 +++ + file2 | 3 --- + file3 | 4 ---- + 5 files changed, 10 insertions(+), 7 deletions(-) + +commit c7a2ab9e8eac7b117442a607d5a9b3950ae34d5a +Author: A U Thor +Date: Mon Jun 26 00:03:00 2006 +0000 + + Side + + dir/sub | 2 ++ + file0 | 3 +++ + file3 | 4 ++++ + 3 files changed, 9 insertions(+) + +commit 9a6d4949b6b76956d9d5e26f2791ec2ceff5fdc0 +Author: A U Thor +Date: Mon Jun 26 00:02:00 2006 +0000 + + Third + + dir/sub | 2 ++ + file1 | 3 +++ + 2 files changed, 5 insertions(+) + +commit 1bde4ae5f36c8d9abe3a0fce0c6aab3c4a12fe44 +Author: A U Thor +Date: Mon Jun 26 00:01:00 2006 +0000 + + Second + + This is the second commit. + + dir/sub | 2 ++ + file0 | 3 +++ + file2 | 3 --- + 3 files changed, 5 insertions(+), 3 deletions(-) + +commit 444ac553ac7612cc88969031b02b3767fb8a353a +Author: A U Thor +Date: Mon Jun 26 00:00:00 2006 +0000 + + Initial +$ From 3ae7fe2b0f8528305bc871738dd12cde23024556 Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:46:57 +0300 Subject: [PATCH 105/397] t4013: test "git diff-tree -m" We want to ensure we don't affect plumbing commands with our changes of "-m" semantics, so add corresponding test. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- t/t4013-diff-various.sh | 1 + t/t4013/diff.diff-tree_-m_master | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 t/t4013/diff.diff-tree_-m_master diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index e53ca7aa50..bdc23b1180 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -293,6 +293,7 @@ diff-tree --stat initial mode diff-tree --summary initial mode diff-tree master +diff-tree -m master diff-tree -p master diff-tree -p -m master diff-tree -c master diff --git a/t/t4013/diff.diff-tree_-m_master b/t/t4013/diff.diff-tree_-m_master new file mode 100644 index 0000000000..6d0a2207fb --- /dev/null +++ b/t/t4013/diff.diff-tree_-m_master @@ -0,0 +1,11 @@ +$ git diff-tree -m master +59d314ad6f356dd08601a4cd5e530381da3e3c64 +:040000 040000 65f5c9dd60ce3b2b3324b618ac7accf8d912c113 0564e026437809817a64fff393079714b6dd4628 M dir +:100644 100644 b414108e81e5091fe0974a1858b4d0d22b107f70 10a8a9f3657f91a156b9f0184ed79a20adef9f7f M file0 +59d314ad6f356dd08601a4cd5e530381da3e3c64 +:040000 040000 f977ed46ae6873c1c30ab878e15a4accedc3618b 0564e026437809817a64fff393079714b6dd4628 M dir +:100644 100644 f4615da674c09df322d6ba8d6b21ecfb1b1ba510 10a8a9f3657f91a156b9f0184ed79a20adef9f7f M file0 +:000000 100644 0000000000000000000000000000000000000000 b1e67221afe8461efd244b487afca22d46b95eb8 A file1 +:100644 000000 01e79c32a8c99c557f0757da7cb6d65b3414466d 0000000000000000000000000000000000000000 D file2 +:100644 000000 7289e35bff32727c08dda207511bec138fdb9ea5 0000000000000000000000000000000000000000 D file3 +$ From e0b16421b17fffa2544eebfcb570fdb766fd0bc4 Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:46:58 +0300 Subject: [PATCH 106/397] t4013: test "git diff-index -m" -m in "git diff-index" means "match missing", that differs from its meaning in "git diff". Let's check it in diff-index. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- t/t4013-diff-various.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index bdc23b1180..e561a8e485 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -494,6 +494,19 @@ test_expect_success 'git config log.diffMerges first-parent vs -m' ' test_cmp expected actual ' +# -m in "git diff-index" means "match missing", that differs +# from its meaning in "git diff". Let's check it in diff-index. +# The line in the output for removed file should disappear when +# we provide -m in diff-index. +test_expect_success 'git diff-index -m' ' + rm -f file1 && + git diff-index HEAD >without-m && + lines_count=$(wc -l with-m && + git restore file1 && + test_line_count = $((lines_count - 1)) with-m +' + test_expect_success 'log -S requires an argument' ' test_must_fail git log -S ' From 19b2517f95a0a908a8ada7417cf0717299e7e1aa Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:46:59 +0300 Subject: [PATCH 107/397] diff-merges: move specific diff-index "-m" handling to diff-index Move specific handling of "-m" for diff-index to diff-index.c, so diff-merges is left to handle only diff for merges options. Being a better design by itself, this is especially essential in preparation for letting -m imply -p, as "diff-index -m" obviously should not imply -p, as it's entirely unrelated. To handle this, in addition to moving specific diff-index "-m" code out of diff-merges, we introduce new diff_merges_suppress_options_parsing() and call it before generic options processing in cmd_diff_index(). This new diff_merges_suppress_options_parsing() could then be reused and called before invocations of setup_revisions() for other commands that don't need --diff-merges options, but that's outside of the scope of these patch series. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- builtin/diff-index.c | 9 +++++++++ diff-merges.c | 25 +++++++++++++------------ diff-merges.h | 2 ++ 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/builtin/diff-index.c b/builtin/diff-index.c index 176fe7ff2b..cf09559e42 100644 --- a/builtin/diff-index.c +++ b/builtin/diff-index.c @@ -2,6 +2,7 @@ #include "cache.h" #include "config.h" #include "diff.h" +#include "diff-merges.h" #include "commit.h" #include "revision.h" #include "builtin.h" @@ -27,6 +28,12 @@ int cmd_diff_index(int argc, const char **argv, const char *prefix) rev.abbrev = 0; prefix = precompose_argv_prefix(argc, argv, prefix); + /* + * We need no diff for merges options, and we need to avoid conflict + * with our own meaning of "-m". + */ + diff_merges_suppress_options_parsing(); + argc = setup_revisions(argc, argv, &rev, NULL); for (i = 1; i < argc; i++) { const char *arg = argv[i]; @@ -35,6 +42,8 @@ int cmd_diff_index(int argc, const char **argv, const char *prefix) option |= DIFF_INDEX_CACHED; else if (!strcmp(arg, "--merge-base")) option |= DIFF_INDEX_MERGE_BASE; + else if (!strcmp(arg, "-m")) + rev.match_missing = 1; else usage(diff_cache_usage); } diff --git a/diff-merges.c b/diff-merges.c index f3a9daed7e..9ca00cdd0c 100644 --- a/diff-merges.c +++ b/diff-merges.c @@ -6,6 +6,7 @@ typedef void (*diff_merges_setup_func_t)(struct rev_info *); static void set_separate(struct rev_info *revs); static diff_merges_setup_func_t set_to_default = set_separate; +static int suppress_parsing; static void suppress(struct rev_info *revs) { @@ -30,17 +31,6 @@ static void set_first_parent(struct rev_info *revs) revs->first_parent_merges = 1; } -static void set_m(struct rev_info *revs) -{ - /* - * To "diff-index", "-m" means "match missing", and to the "log" - * family of commands, it means "show default diff for merges". Set - * both fields appropriately. - */ - set_to_default(revs); - revs->match_missing = 1; -} - static void set_combined(struct rev_info *revs) { suppress(revs); @@ -101,14 +91,22 @@ int diff_merges_config(const char *value) return 0; } +void diff_merges_suppress_options_parsing(void) +{ + suppress_parsing = 1; +} + int diff_merges_parse_opts(struct rev_info *revs, const char **argv) { int argcount = 1; const char *optarg; const char *arg = argv[0]; + if (suppress_parsing) + return 0; + if (!strcmp(arg, "-m")) { - set_m(revs); + set_to_default(revs); } else if (!strcmp(arg, "-c")) { set_combined(revs); revs->combined_imply_patch = 1; @@ -155,6 +153,9 @@ void diff_merges_set_dense_combined_if_unset(struct rev_info *revs) void diff_merges_setup_revs(struct rev_info *revs) { + if (suppress_parsing) + return; + if (revs->combine_merges == 0) revs->dense_combined_merges = 0; if (revs->separate_merges == 0) diff --git a/diff-merges.h b/diff-merges.h index 09d9a6c9a4..b5d57f6563 100644 --- a/diff-merges.h +++ b/diff-merges.h @@ -11,6 +11,8 @@ struct rev_info; int diff_merges_config(const char *value); +void diff_merges_suppress_options_parsing(void); + int diff_merges_parse_opts(struct rev_info *revs, const char **argv); void diff_merges_suppress(struct rev_info *revs); From 23f6d40dd3e7aad72c7706bb75f77a6c73b69b56 Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:47:00 +0300 Subject: [PATCH 108/397] git-svn: stop passing "-m" to "git rev-list" rev-list doesn't utilize -m. It happens to eat it silently, so this bug went unnoticed. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- perl/Git/SVN.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/Git/SVN.pm b/perl/Git/SVN.pm index f6f1dc03c6..35ff5a6896 100644 --- a/perl/Git/SVN.pm +++ b/perl/Git/SVN.pm @@ -1636,7 +1636,7 @@ sub has_no_changes { my $commit = shift; my @revs = split / /, command_oneline( - qw(rev-list --parents -1 -m), $commit); + qw(rev-list --parents -1), $commit); # Commits with no parents, e.g. the start of a partial branch, # have changes by definition. From 1e20a407fe28b4bdba3b973b26b0b60ff6b6c97d Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:47:01 +0300 Subject: [PATCH 109/397] stash list: stop passing "-m" to "git log" Passing "-m" in "git log --first-parent -m" is not needed as --first-parent implies --diff-merges=first-parent anyway. OTOH, it will stop being harmless once we let "-m" imply "-p". While we are at it, fix corresponding test description in t3903-stash to match what it actually tests. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- builtin/stash.c | 2 +- t/t3903-stash.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/stash.c b/builtin/stash.c index ba774cce67..7c92bb51cb 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -759,7 +759,7 @@ static int list_stash(int argc, const char **argv, const char *prefix) cp.git_cmd = 1; strvec_pushl(&cp.args, "log", "--format=%gd: %gs", "-g", - "--first-parent", "-m", NULL); + "--first-parent", NULL); strvec_pushv(&cp.args, argv); strvec_push(&cp.args, ref_stash); strvec_push(&cp.args, "--"); diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index 5f282ecf61..873aa56e35 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -859,7 +859,7 @@ test_expect_success 'setup stash with index and worktree changes' ' git stash ' -test_expect_success 'stash list implies --first-parent -m' ' +test_expect_success 'stash list -p shows simple diff' ' cat >expect <<-EOF && stash@{0} From fd16a39944ce2f2967ed015379f5bd05bac639bb Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:47:02 +0300 Subject: [PATCH 110/397] diff-merges: rename "combined_imply_patch" to "merges_imply_patch" This is refactoring change in preparation for the next commit that will let -m imply -p. The old name doesn't match the intention to let not only -c/-cc imply -p, but also -m, that is not a "combined" format, so we rename the flag accordingly. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- diff-merges.c | 10 +++++----- revision.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/diff-merges.c b/diff-merges.c index 9ca00cdd0c..d897fd8a29 100644 --- a/diff-merges.c +++ b/diff-merges.c @@ -15,7 +15,7 @@ static void suppress(struct rev_info *revs) revs->combine_merges = 0; revs->dense_combined_merges = 0; revs->combined_all_paths = 0; - revs->combined_imply_patch = 0; + revs->merges_imply_patch = 0; revs->merges_need_diff = 0; } @@ -109,10 +109,10 @@ int diff_merges_parse_opts(struct rev_info *revs, const char **argv) set_to_default(revs); } else if (!strcmp(arg, "-c")) { set_combined(revs); - revs->combined_imply_patch = 1; + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "--cc")) { set_dense_combined(revs); - revs->combined_imply_patch = 1; + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "--no-diff-merges")) { suppress(revs); } else if (!strcmp(arg, "--combined-all-paths")) { @@ -162,9 +162,9 @@ void diff_merges_setup_revs(struct rev_info *revs) revs->first_parent_merges = 0; if (revs->combined_all_paths && !revs->combine_merges) die("--combined-all-paths makes no sense without -c or --cc"); - if (revs->combined_imply_patch) + if (revs->merges_imply_patch) revs->diff = 1; - if (revs->combined_imply_patch || revs->merges_need_diff) { + if (revs->merges_imply_patch || revs->merges_need_diff) { if (!revs->diffopt.output_format) revs->diffopt.output_format = DIFF_FORMAT_PATCH; } diff --git a/revision.h b/revision.h index e6be3c845e..5ea881d189 100644 --- a/revision.h +++ b/revision.h @@ -195,10 +195,10 @@ struct rev_info { /* Diff-merge flags */ explicit_diff_merges: 1, merges_need_diff: 1, + merges_imply_patch:1, separate_merges: 1, combine_merges:1, combined_all_paths:1, - combined_imply_patch:1, dense_combined_merges:1, first_parent_merges:1; From f5bfcc823ba242a46e20fb6f71c9fbf7ebb222fe Mon Sep 17 00:00:00 2001 From: Sergey Organov Date: Fri, 21 May 2021 00:47:03 +0300 Subject: [PATCH 111/397] diff-merges: let "-m" imply "-p" Fix long standing inconsistency between -c/--cc that do imply -p on one side, and -m that did not imply -p on the other side. Change corresponding test accordingly, as "log -m" output should now match one from "log -m -p", rather than from just "log". Change documentation accordingly. NOTES: After this patch git log -m produces diffs without need to provide -p as well, that improves both consistency and usability. It gets even more useful if one sets "log.diffMerges" configuration variable to "first-parent" to force -m produce usual diff with respect to first parent only. This patch, however, does not change behavior when specific diff format is explicitly provided on the command-line, so that commands like git log -m --raw git log -m --stat are not affected, nor does it change commands where specific diff format is active by default, such as: git diff-tree -m It's also worth to be noticed that exact historical semantics of -m is still provided by --diff-merges=separate. Signed-off-by: Sergey Organov Signed-off-by: Junio C Hamano --- Documentation/diff-options.txt | 8 ++++---- diff-merges.c | 1 + t/t4013-diff-various.sh | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Documentation/diff-options.txt b/Documentation/diff-options.txt index 6d968b9012..2825783049 100644 --- a/Documentation/diff-options.txt +++ b/Documentation/diff-options.txt @@ -49,10 +49,9 @@ ifdef::git-log[] --diff-merges=m::: -m::: This option makes diff output for merge commits to be shown in - the default format. `-m` will produce the output only if `-p` - is given as well. The default format could be changed using + the default format. The default format could be changed using `log.diffMerges` configuration parameter, which default value - is `separate`. + is `separate`. `-m` implies `-p`. + --diff-merges=first-parent::: --diff-merges=1::: @@ -62,7 +61,8 @@ ifdef::git-log[] --diff-merges=separate::: This makes merge commits show the full diff with respect to each of the parents. Separate log entry and diff is generated - for each parent. + for each parent. This is the format that `-m` produced + historically. + --diff-merges=combined::: --diff-merges=c::: diff --git a/diff-merges.c b/diff-merges.c index d897fd8a29..0dfcaa1b11 100644 --- a/diff-merges.c +++ b/diff-merges.c @@ -107,6 +107,7 @@ int diff_merges_parse_opts(struct rev_info *revs, const char **argv) if (!strcmp(arg, "-m")) { set_to_default(revs); + revs->merges_imply_patch = 1; } else if (!strcmp(arg, "-c")) { set_combined(revs); revs->merges_imply_patch = 1; diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index e561a8e485..7fadc985cc 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -455,8 +455,8 @@ diff-tree --stat --compact-summary initial mode diff-tree -R --stat --compact-summary initial mode EOF -test_expect_success 'log -m matches pure log' ' - git log master >result && +test_expect_success 'log -m matches log -m -p' ' + git log -m -p master >result && process_diffs result >expected && git log -m >result && process_diffs result >actual && From af5cd44b6f8f1934bc8b91f646eb2e73dcab57f3 Mon Sep 17 00:00:00 2001 From: Denton Liu Date: Fri, 21 May 2021 03:37:47 -0700 Subject: [PATCH 112/397] stash show: use stash.showIncludeUntracked even when diff options given If options pertaining to how the diff is displayed is provided to `git stash show`, the command will ignore the stash.showIncludeUntracked configuration variable, defaulting to not showing any untracked files. This is unintuitive behaviour since the format of the diff output and whether or not to display untracked files are orthogonal. Use stash.showIncludeUntracked even when diff options are given. Of course, this is still overridable via the command-line options. Update the documentation to explicitly say which configuration variables will be overridden when a diff options are given. Signed-off-by: Denton Liu Signed-off-by: Junio C Hamano --- Documentation/config/stash.txt | 6 +++--- Documentation/git-stash.txt | 6 ++++-- builtin/stash.c | 5 +---- t/t3905-stash-include-untracked.sh | 2 ++ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Documentation/config/stash.txt b/Documentation/config/stash.txt index 413f907cba..9ed775281f 100644 --- a/Documentation/config/stash.txt +++ b/Documentation/config/stash.txt @@ -6,9 +6,9 @@ stash.useBuiltin:: remaining users that setting this now does nothing. stash.showIncludeUntracked:: - If this is set to true, the `git stash show` command without an - option will show the untracked files of a stash entry. Defaults to - false. See description of 'show' command in linkgit:git-stash[1]. + If this is set to true, the `git stash show` command will show + the untracked files of a stash entry. Defaults to false. See + description of 'show' command in linkgit:git-stash[1]. stash.showPatch:: If this is set to true, the `git stash show` command without an diff --git a/Documentation/git-stash.txt b/Documentation/git-stash.txt index a8c8c32f1e..be6084ccef 100644 --- a/Documentation/git-stash.txt +++ b/Documentation/git-stash.txt @@ -91,8 +91,10 @@ show [-u|--include-untracked|--only-untracked] [] []:: By default, the command shows the diffstat, but it will accept any format known to 'git diff' (e.g., `git stash show -p stash@{1}` to view the second most recent entry in patch form). - You can use stash.showIncludeUntracked, stash.showStat, and - stash.showPatch config variables to change the default behavior. + If no `` is provided, the default behavior will be given + by the `stash.showStat`, and `stash.showPatch` config variables. You + can also use `stash.showIncludeUntracked` to set whether + `--include-untracked` is enabled by default. pop [--index] [-q|--quiet] []:: diff --git a/builtin/stash.c b/builtin/stash.c index 82e4829d44..864b6c1416 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -831,7 +831,7 @@ static int show_stash(int argc, const char **argv, const char *prefix) UNTRACKED_NONE, UNTRACKED_INCLUDE, UNTRACKED_ONLY - } show_untracked = UNTRACKED_NONE; + } show_untracked = show_include_untracked ? UNTRACKED_INCLUDE : UNTRACKED_NONE; struct option options[] = { OPT_SET_INT('u', "include-untracked", &show_untracked, N_("include untracked files in the stash"), @@ -874,9 +874,6 @@ static int show_stash(int argc, const char **argv, const char *prefix) if (show_patch) rev.diffopt.output_format |= DIFF_FORMAT_PATCH; - if (show_include_untracked) - show_untracked = UNTRACKED_INCLUDE; - if (!show_stat && !show_patch) { free_stash_info(&info); return 0; diff --git a/t/t3905-stash-include-untracked.sh b/t/t3905-stash-include-untracked.sh index 1c9765928d..f7fafcd447 100755 --- a/t/t3905-stash-include-untracked.sh +++ b/t/t3905-stash-include-untracked.sh @@ -333,6 +333,8 @@ test_expect_success 'stash show --include-untracked shows untracked files' ' git stash show -p --include-untracked >actual && test_cmp expect actual && git stash show --include-untracked -p >actual && + test_cmp expect actual && + git -c stash.showIncludeUntracked=true stash show -p >actual && test_cmp expect actual ' From 5317dfeaed30fbe647899ece261e3dd40e5ced1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Sat, 22 May 2021 14:10:08 +0700 Subject: [PATCH 113/397] t: use configured TAR instead of tar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Despite that tar is available everywhere, it's not required by POSIX. In our build system, users are allowed to specify which tar to be used in Makefile knobs. Furthermore, GNU tar (gtar) is prefered when autotools is being used. In our testsuite, 7 out of 9 tar-required-tests use "$TAR", the other two use "tar". Let's change the remaining two tests to "$TAR". Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/t3513-revert-submodule.sh | 4 ++-- t/t6041-bisect-submodule.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/t/t3513-revert-submodule.sh b/t/t3513-revert-submodule.sh index a759f12cbb..6e21278cfe 100755 --- a/t/t3513-revert-submodule.sh +++ b/t/t3513-revert-submodule.sh @@ -14,7 +14,7 @@ test_description='revert can handle submodules' git_revert () { git status -su >expect && ls -1pR * >>expect && - tar cf "$TRASH_DIRECTORY/tmp.tar" * && + "$TAR" cf "$TRASH_DIRECTORY/tmp.tar" * && may_only_be_test_must_fail "$2" && $2 git checkout "$1" && if test -n "$2" @@ -23,7 +23,7 @@ git_revert () { fi && git revert HEAD && rm -rf * && - tar xf "$TRASH_DIRECTORY/tmp.tar" && + "$TAR" xf "$TRASH_DIRECTORY/tmp.tar" && git status -su >actual && ls -1pR * >>actual && test_cmp expect actual && diff --git a/t/t6041-bisect-submodule.sh b/t/t6041-bisect-submodule.sh index df1eff0fb8..82013fc903 100755 --- a/t/t6041-bisect-submodule.sh +++ b/t/t6041-bisect-submodule.sh @@ -8,7 +8,7 @@ test_description='bisect can handle submodules' git_bisect () { git status -su >expect && ls -1pR * >>expect && - tar cf "$TRASH_DIRECTORY/tmp.tar" * && + "$TAR" cf "$TRASH_DIRECTORY/tmp.tar" * && GOOD=$(git rev-parse --verify HEAD) && may_only_be_test_must_fail "$2" && $2 git checkout "$1" && @@ -25,7 +25,7 @@ git_bisect () { git bisect start && git bisect good $GOOD && rm -rf * && - tar xf "$TRASH_DIRECTORY/tmp.tar" && + "$TAR" xf "$TRASH_DIRECTORY/tmp.tar" && git status -su >actual && ls -1pR * >>actual && test_cmp expect actual && From de88ac70f3a801262eb3aa087e5d9a712be0a54a Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sat, 22 May 2021 18:27:45 +0900 Subject: [PATCH 114/397] Git 2.32-rc1 Signed-off-by: Junio C Hamano --- Documentation/RelNotes/2.32.0.txt | 5 +++++ GIT-VERSION-GEN | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Documentation/RelNotes/2.32.0.txt b/Documentation/RelNotes/2.32.0.txt index ed590a7b56..87d56fa1aa 100644 --- a/Documentation/RelNotes/2.32.0.txt +++ b/Documentation/RelNotes/2.32.0.txt @@ -385,6 +385,11 @@ Fixes since v2.31 to help running tests that the developer has not modified. (merge 2d86a96220 jk/test-chainlint-softer later to maint). + * The "rev-parse" command did not diagnose the lack of argument to + "--path-format" option, which was introduced in v2.31 era, which + has been corrected. + (merge 99fc555188 wm/rev-parse-path-format-wo-arg later to maint). + * Other code cleanup, docfix, build fix, etc. (merge f451960708 dl/cat-file-doc-cleanup later to maint). (merge 12604a8d0c sv/t9801-test-path-is-file-cleanup later to maint). diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index dd1d8099c7..8862d57489 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=v2.32.0-rc0 +DEF_VER=v2.32.0-rc1 LF=' ' From 12d078ed2b64df220f2fe8f6a050f271133ccda9 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 21 May 2021 17:29:37 -0500 Subject: [PATCH 115/397] doc: refactor common asciidoc dependencies Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 2aae4c9cbb..46d9b98dac 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -139,6 +139,7 @@ ASCIIDOC_CONF = -f asciidoc.conf ASCIIDOC_COMMON = $(ASCIIDOC) $(ASCIIDOC_EXTRA) $(ASCIIDOC_CONF) \ -amanversion=$(GIT_VERSION) \ -amanmanual='Git Manual' -amansource='Git' +ASCIIDOC_DEPS = asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS TXT_TO_HTML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_HTML) TXT_TO_XML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_DOCBOOK) MANPAGE_XSL = manpage-normal.xsl @@ -354,12 +355,12 @@ clean: $(RM) manpage-base-url.xsl $(RM) GIT-ASCIIDOCFLAGS -$(MAN_HTML): %.html : %.txt asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS +$(MAN_HTML): %.html : %.txt $(ASCIIDOC_DEPS) $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ $(TXT_TO_HTML) -d manpage -o $@+ $< && \ mv $@+ $@ -$(OBSOLETE_HTML): %.html : %.txto asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS +$(OBSOLETE_HTML): %.html : %.txto $(ASCIIDOC_DEPS) $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ $(TXT_TO_HTML) -o $@+ $< && \ mv $@+ $@ @@ -371,7 +372,7 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(QUIET_XMLTO)$(RM) $@ && \ $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< -%.xml : %.txt asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS +%.xml : %.txt $(ASCIIDOC_DEPS) $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ $(TXT_TO_XML) -d manpage -o $@+ $< && \ mv $@+ $@ From 56da21392b10ebe161dbf426f170e74f800178cf Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 21 May 2021 17:29:38 -0500 Subject: [PATCH 116/397] doc: improve asciidoc dependencies asciidoc needs asciidoc.conf, asciidoctor asciidoctor-extensions.rb. Neither needs the other. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 46d9b98dac..0f59cc0853 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -139,7 +139,7 @@ ASCIIDOC_CONF = -f asciidoc.conf ASCIIDOC_COMMON = $(ASCIIDOC) $(ASCIIDOC_EXTRA) $(ASCIIDOC_CONF) \ -amanversion=$(GIT_VERSION) \ -amanmanual='Git Manual' -amansource='Git' -ASCIIDOC_DEPS = asciidoc.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS +ASCIIDOC_DEPS = asciidoc.conf GIT-ASCIIDOCFLAGS TXT_TO_HTML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_HTML) TXT_TO_XML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_DOCBOOK) MANPAGE_XSL = manpage-normal.xsl @@ -194,6 +194,7 @@ ASCIIDOC_DOCBOOK = docbook5 ASCIIDOC_EXTRA += -acompat-mode -atabsize=8 ASCIIDOC_EXTRA += -I. -rasciidoctor-extensions ASCIIDOC_EXTRA += -alitdd='&\#x2d;&\#x2d;' +ASCIIDOC_DEPS = asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS DBLATEX_COMMON = XMLTO_EXTRA += --skip-validation XMLTO_EXTRA += -x manpage.xsl From 471e7b2cf6ab5ef057feed03b35dde9a26357611 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 21 May 2021 17:29:39 -0500 Subject: [PATCH 117/397] doc: remove unnecessary rm instances Commits 50cff52f1a (When generating manpages, delete outdated targets first., 2007-08-02) and f9286765b2 (Documentation/Makefile: remove cmd-list.made before redirecting to it., 2007-08-06) created these rm instances for a very rare corner-case: building as root by mistake. It's odd to have workarounds here, but nowhere else in the Makefile-- which already fails in this stuation, starting from Documentation/technical/. We gain nothing but complexity, so let's remove them. Comments-by: Jeff King Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/Makefile | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 0f59cc0853..73b88c3aad 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -296,8 +296,7 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(DOC_DEP_TXT) build-docdep.perl - $(QUIET_GEN)$(RM) $@+ $@ && \ - $(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ + $(QUIET_GEN)$(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ mv $@+ $@ ifneq ($(MAKECMDGOALS),clean) @@ -318,8 +317,7 @@ cmds_txt = cmds-ancillaryinterrogators.txt \ $(cmds_txt): cmd-list.made cmd-list.made: cmd-list.perl ../command-list.txt $(MAN1_TXT) - $(QUIET_GEN)$(RM) $@ && \ - $(PERL_PATH) ./cmd-list.perl ../command-list.txt $(cmds_txt) $(QUIET_STDERR) && \ + $(QUIET_GEN)$(PERL_PATH) ./cmd-list.perl ../command-list.txt $(cmds_txt) $(QUIET_STDERR) && \ date >$@ mergetools_txt = mergetools-diff.txt mergetools-merge.txt @@ -327,7 +325,7 @@ mergetools_txt = mergetools-diff.txt mergetools-merge.txt $(mergetools_txt): mergetools-list.made mergetools-list.made: ../git-mergetool--lib.sh $(wildcard ../mergetools/*) - $(QUIET_GEN)$(RM) $@ && \ + $(QUIET_GEN) \ $(SHELL_PATH) -c 'MERGE_TOOLS_DIR=../mergetools && \ . ../git-mergetool--lib.sh && \ show_tool_names can_diff "* " || :' >mergetools-diff.txt && \ @@ -357,30 +355,25 @@ clean: $(RM) GIT-ASCIIDOCFLAGS $(MAN_HTML): %.html : %.txt $(ASCIIDOC_DEPS) - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -d manpage -o $@+ $< && \ + $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -d manpage -o $@+ $< && \ mv $@+ $@ $(OBSOLETE_HTML): %.html : %.txto $(ASCIIDOC_DEPS) - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_HTML) -o $@+ $< && \ + $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -o $@+ $< && \ mv $@+ $@ manpage-base-url.xsl: manpage-base-url.xsl.in $(QUIET_GEN)sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ %.1 %.5 %.7 : %.xml manpage-base-url.xsl $(wildcard manpage*.xsl) - $(QUIET_XMLTO)$(RM) $@ && \ - $(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< + $(QUIET_XMLTO)$(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt $(ASCIIDOC_DEPS) - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d manpage -o $@+ $< && \ + $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d manpage -o $@+ $< && \ mv $@+ $@ user-manual.xml: user-manual.txt user-manual.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ - $(TXT_TO_XML) -d book -o $@+ $< && \ + $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d book -o $@+ $< && \ mv $@+ $@ technical/api-index.txt: technical/api-index-skel.txt \ @@ -402,27 +395,24 @@ XSLTOPTS += --stringparam html.stylesheet docbook-xsl.css XSLTOPTS += --param generate.consistent.ids 1 user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)$(RM) $@+ $@ && \ - xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ + $(QUIET_XSLTPROC)xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ mv $@+ $@ git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ + $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ rm $@++ && \ mv $@+ $@ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(RM) $@+ $@ && \ - $(DBLATEX) -o $@+ $(DBLATEX_COMMON) $< && \ + $(QUIET_DBLATEX)$(DBLATEX) -o $@+ $(DBLATEX_COMMON) $< && \ mv $@+ $@ gitman.texi: $(MAN_XML) cat-texi.perl texi.xsl - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ + $(QUIET_DB2TEXI) \ ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml)+ texi.xsl $(xml) && \ $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml)+ && \ rm $(xml)+ &&) true) > $@++ && \ @@ -434,13 +424,11 @@ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(RM) $@+ $@ && \ - $(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ + $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ mv $@+ $@ howto-index.txt: howto-index.sh $(HOWTO_TXT) - $(QUIET_GEN)$(RM) $@+ $@ && \ - '$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(HOWTO_TXT)) >$@+ && \ + $(QUIET_GEN)'$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(HOWTO_TXT)) >$@+ && \ mv $@+ $@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt @@ -450,7 +438,7 @@ WEBDOC_DEST = /pub/software/scm/git/docs howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(HOWTO_TXT)): %.html : %.txt GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(RM) $@+ $@ && \ + $(QUIET_ASCIIDOC) \ sed -e '1,/^$$/d' $< | \ $(TXT_TO_HTML) - >$@+ && \ mv $@+ $@ From db10fc6c09f1f74c4d0a9294ecbb68d390f54f15 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 21 May 2021 17:29:40 -0500 Subject: [PATCH 118/397] doc: simplify Makefile using .DELETE_ON_ERROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently GNU make already removes files when catching an interruption signal, however, in order to deal with other kinds of errors a workaround is in place to store target output to a temporary file, and only move it to its right place on success. By enabling the built-in .DELETE_ON_ERROR we let make do this task, so we don't have to. This way the rules can be simplified a lot. Suggested-by: Ævar Arnfjörð Bjarmason Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/Makefile | 47 +++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 73b88c3aad..eaff97dcb8 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -296,8 +296,7 @@ docdep_prereqs = \ cmd-list.made $(cmds_txt) doc.dep : $(docdep_prereqs) $(DOC_DEP_TXT) build-docdep.perl - $(QUIET_GEN)$(PERL_PATH) ./build-docdep.perl >$@+ $(QUIET_STDERR) && \ - mv $@+ $@ + $(QUIET_GEN)$(PERL_PATH) ./build-docdep.perl >$@ $(QUIET_STDERR) ifneq ($(MAKECMDGOALS),clean) -include doc.dep @@ -355,12 +354,10 @@ clean: $(RM) GIT-ASCIIDOCFLAGS $(MAN_HTML): %.html : %.txt $(ASCIIDOC_DEPS) - $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -d manpage -o $@ $< $(OBSOLETE_HTML): %.html : %.txto $(ASCIIDOC_DEPS) - $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(TXT_TO_HTML) -o $@ $< manpage-base-url.xsl: manpage-base-url.xsl.in $(QUIET_GEN)sed "s|@@MAN_BASE_URL@@|$(MAN_BASE_URL)|" $< > $@ @@ -369,12 +366,10 @@ manpage-base-url.xsl: manpage-base-url.xsl.in $(QUIET_XMLTO)$(XMLTO) -m $(MANPAGE_XSL) $(XMLTO_EXTRA) man $< %.xml : %.txt $(ASCIIDOC_DEPS) - $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d manpage -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d manpage -o $@ $< user-manual.xml: user-manual.txt user-manual.conf asciidoctor-extensions.rb GIT-ASCIIDOCFLAGS - $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d book -o $@+ $< && \ - mv $@+ $@ + $(QUIET_ASCIIDOC)$(TXT_TO_XML) -d book -o $@ $< technical/api-index.txt: technical/api-index-skel.txt \ technical/api-index.sh $(patsubst %,%.txt,$(API_DOCS)) @@ -395,41 +390,35 @@ XSLTOPTS += --stringparam html.stylesheet docbook-xsl.css XSLTOPTS += --param generate.consistent.ids 1 user-manual.html: user-manual.xml $(XSLT) - $(QUIET_XSLTPROC)xsltproc $(XSLTOPTS) -o $@+ $(XSLT) $< && \ - mv $@+ $@ + $(QUIET_XSLTPROC)xsltproc $(XSLTOPTS) -o $@ $(XSLT) $< git.info: user-manual.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split -o $@ user-manual.texi user-manual.texi: user-manual.xml - $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@++ && \ - $(PERL_PATH) fix-texi.perl <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@+ && \ + $(PERL_PATH) fix-texi.perl <$@+ >$@ && \ + rm $@+ user-manual.pdf: user-manual.xml - $(QUIET_DBLATEX)$(DBLATEX) -o $@+ $(DBLATEX_COMMON) $< && \ - mv $@+ $@ + $(QUIET_DBLATEX)$(DBLATEX) -o $@ $(DBLATEX_COMMON) $< gitman.texi: $(MAN_XML) cat-texi.perl texi.xsl $(QUIET_DB2TEXI) \ ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml)+ texi.xsl $(xml) && \ $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml)+ && \ - rm $(xml)+ &&) true) > $@++ && \ - $(PERL_PATH) cat-texi.perl $@ <$@++ >$@+ && \ - rm $@++ && \ - mv $@+ $@ + rm $(xml)+ &&) true) > $@+ && \ + $(PERL_PATH) cat-texi.perl $@ <$@+ >$@ && \ + rm $@+ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi $(patsubst %.txt,%.texi,$(MAN_TXT)): %.texi : %.xml - $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@+ && \ - mv $@+ $@ + $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) --to-stdout $*.xml >$@ howto-index.txt: howto-index.sh $(HOWTO_TXT) - $(QUIET_GEN)'$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(HOWTO_TXT)) >$@+ && \ - mv $@+ $@ + $(QUIET_GEN)'$(SHELL_PATH_SQ)' ./howto-index.sh $(sort $(HOWTO_TXT)) >$@ $(patsubst %,%.html,$(ARTICLES)) : %.html : %.txt $(QUIET_ASCIIDOC)$(TXT_TO_HTML) $*.txt @@ -440,8 +429,7 @@ howto/%.html: ASCIIDOC_EXTRA += -a git-relative-html-prefix=../ $(patsubst %.txt,%.html,$(HOWTO_TXT)): %.html : %.txt GIT-ASCIIDOCFLAGS $(QUIET_ASCIIDOC) \ sed -e '1,/^$$/d' $< | \ - $(TXT_TO_HTML) - >$@+ && \ - mv $@+ $@ + $(TXT_TO_HTML) - >$@ install-webdoc : html '$(SHELL_PATH_SQ)' ./install-webdoc.sh $(WEBDOC_DEST) @@ -482,4 +470,7 @@ doc-l10n install-l10n:: $(MAKE) -C po $@ endif +# Delete the target file on error +.DELETE_ON_ERROR: + .PHONY: FORCE From 7ba30167291eb89f2e587b7cabfa4e7555de4ed5 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Fri, 21 May 2021 17:29:41 -0500 Subject: [PATCH 119/397] doc: avoid using rm directly That's what we have $(RM) for. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index eaff97dcb8..f5605b7767 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -398,7 +398,7 @@ git.info: user-manual.texi user-manual.texi: user-manual.xml $(QUIET_DB2TEXI)$(DOCBOOK2X_TEXI) user-manual.xml --encoding=UTF-8 --to-stdout >$@+ && \ $(PERL_PATH) fix-texi.perl <$@+ >$@ && \ - rm $@+ + $(RM) $@+ user-manual.pdf: user-manual.xml $(QUIET_DBLATEX)$(DBLATEX) -o $@ $(DBLATEX_COMMON) $< @@ -407,9 +407,9 @@ gitman.texi: $(MAN_XML) cat-texi.perl texi.xsl $(QUIET_DB2TEXI) \ ($(foreach xml,$(sort $(MAN_XML)),xsltproc -o $(xml)+ texi.xsl $(xml) && \ $(DOCBOOK2X_TEXI) --encoding=UTF-8 --to-stdout $(xml)+ && \ - rm $(xml)+ &&) true) > $@+ && \ + $(RM) $(xml)+ &&) true) > $@+ && \ $(PERL_PATH) cat-texi.perl $@ <$@+ >$@ && \ - rm $@+ + $(RM) $@+ gitman.info: gitman.texi $(QUIET_MAKEINFO)$(MAKEINFO) --no-split --no-validate $*.texi From 3127ff90ea359c5be0d61c2885b3f0f74d602faf Mon Sep 17 00:00:00 2001 From: Teng Long Date: Thu, 13 May 2021 15:15:47 +0800 Subject: [PATCH 120/397] packfile-uri.txt: fix blobPackfileUri description Fix the 'uploadpack.blobPackfileUri' description in packfile-uri.txt and the correct format also can be seen in t5702. Signed-off-by: Teng Long Reviewed-by: Jonathan Tan Signed-off-by: Junio C Hamano --- Documentation/technical/packfile-uri.txt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Documentation/technical/packfile-uri.txt b/Documentation/technical/packfile-uri.txt index f7eabc6c76..1eb525fe76 100644 --- a/Documentation/technical/packfile-uri.txt +++ b/Documentation/technical/packfile-uri.txt @@ -35,13 +35,14 @@ include some sort of non-trivial implementation in the Minimum Viable Product, at least so that we can test the client. This is the implementation: a feature, marked experimental, that allows the -server to be configured by one or more `uploadpack.blobPackfileUri= -` entries. Whenever the list of objects to be sent is assembled, all such -blobs are excluded, replaced with URIs. As noted in "Future work" below, the -server can evolve in the future to support excluding other objects (or other -implementations of servers could be made that support excluding other objects) -without needing a protocol change, so clients should not expect that packfiles -downloaded in this way only contain single blobs. +server to be configured by one or more `uploadpack.blobPackfileUri= + ` entries. Whenever the list of objects to be +sent is assembled, all such blobs are excluded, replaced with URIs. As noted +in "Future work" below, the server can evolve in the future to support +excluding other objects (or other implementations of servers could be made +that support excluding other objects) without needing a protocol change, so +clients should not expect that packfiles downloaded in this way only contain +single blobs. Client design ------------- From 53753a37d091183b4dead73fe664f1dfd58e45d2 Mon Sep 17 00:00:00 2001 From: Johannes Sixt Date: Mon, 24 May 2021 21:38:09 +0200 Subject: [PATCH 121/397] t9001-send-email.sh: fix expected absolute paths on Windows Git for Windows is a native Windows program that works with native absolute paths in the drive letter style C:\dir. The auxiliary infrastructure is based on MSYS2, which uses POSIX style /C/dir. When we test for output of absolute paths produced by git.exe, we usally have to expect C:\dir style paths. To produce such expected paths, we have to use $(pwd) in the test scripts; the alternative, $PWD, produces a POSIX style path. ($PWD is a shell variable, and the shell is bash, an MSYS2 program, and operates in the POSIX realm.) There are two recently added tests that were written to expect C:\dir paths. The output that is tested is produced by `git send-email`, but behind the scenes, this is a Perl script, which also works in the POSIX realm and produces /C/dir style output. In the first test case that is changed here, replace $(pwd) by $PWD so that the expected path is constructed using /C/dir style. The second test case sets core.hooksPath to an absolute path. Since the test script talks to native git.exe, it is supposed to place a C:/dir style path into the configuration; therefore, keep $(pwd). When this configuration value is consumed by the Perl script, it is transformed to /C/dir style by the MSYS2 layer and echoed back in this form in the error message. Hence, do use $PWD for the expected value. Signed-off-by: Johannes Sixt Signed-off-by: Junio C Hamano --- t/t9001-send-email.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 65b3035371..68bebc505b 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -539,15 +539,14 @@ test_expect_success $PREREQ "--validate respects relative core.hooksPath path" ' test_path_is_file my-hooks.ran && cat >expect <<-EOF && fatal: longline.patch: rejected by sendemail-validate hook - fatal: command '"'"'$(pwd)/my-hooks/sendemail-validate'"'"' died with exit code 1 + fatal: command '"'"'$PWD/my-hooks/sendemail-validate'"'"' died with exit code 1 warning: no patches were sent EOF test_cmp expect actual ' test_expect_success $PREREQ "--validate respects absolute core.hooksPath path" ' - hooks_path="$(pwd)/my-hooks" && - test_config core.hooksPath "$hooks_path" && + test_config core.hooksPath "$(pwd)/my-hooks" && test_when_finished "rm my-hooks.ran" && test_must_fail git send-email \ --from="Example " \ @@ -558,7 +557,7 @@ test_expect_success $PREREQ "--validate respects absolute core.hooksPath path" ' test_path_is_file my-hooks.ran && cat >expect <<-EOF && fatal: longline.patch: rejected by sendemail-validate hook - fatal: command '"'"'$hooks_path/sendemail-validate'"'"' died with exit code 1 + fatal: command '"'"'$PWD/my-hooks/sendemail-validate'"'"' died with exit code 1 warning: no patches were sent EOF test_cmp expect actual From 5b719b7552b9eca486a3a6507e9397572d16481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Tue, 25 May 2021 01:14:24 +0200 Subject: [PATCH 122/397] send-email: fix missing error message regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a regression with the "the editor exited uncleanly, aborting everything" error message going missing after my d21616c0394 (git-send-email: refactor duplicate $? checks into a function, 2021-04-06). I introduced a $msg variable, but did not actually use it. This caused us to miss the optional error message supplied by the "do_edit" codepath. Fix that, and add tests to check that this works. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 12 +++++++++++- t/t9001-send-email.sh | 23 +++++++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 175da07d94..170f42fe21 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -219,8 +219,18 @@ sub system_or_msg { my $exit_code = $? >> 8; return unless $signalled or $exit_code; + my @sprintf_args = ($args->[0], $exit_code); + if (defined $msg) { + # Quiet the 'redundant' warning category, except we + # need to support down to Perl 5.8, so we can't do a + # "no warnings 'redundant'", since that category was + # introduced in perl 5.22, and asking for it will die + # on older perls. + no warnings; + return sprintf($msg, @sprintf_args); + } return sprintf(__("fatal: command '%s' died with exit code %d"), - $args->[0], $exit_code); + @sprintf_args); } sub system_or_die { diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 65b3035371..2acf389837 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -644,14 +644,33 @@ test_expect_success $PREREQ 'In-Reply-To with --chain-reply-to' ' test_cmp expect actual ' +test_set_editor "$(pwd)/fake-editor" + +test_expect_success $PREREQ 'setup erroring fake editor' ' + write_script fake-editor <<-\EOF + echo >&2 "I am about to error" + exit 1 + EOF +' + +test_expect_success $PREREQ 'fake editor dies with error' ' + clean_fake_sendmail && + test_must_fail git send-email \ + --compose --subject foo \ + --from="Example " \ + --to=nobody@example.com \ + --smtp-server="$(pwd)/fake.sendmail" \ + $patches 2>err && + grep "I am about to error" err && + grep "the editor exited uncleanly, aborting everything" err +' + test_expect_success $PREREQ 'setup fake editor' ' write_script fake-editor <<-\EOF echo fake edit >>"$1" EOF ' -test_set_editor "$(pwd)/fake-editor" - test_expect_success $PREREQ '--compose works' ' clean_fake_sendmail && git send-email \ From a185dd58ecc17f2ea16985d59c9bb7b09bec7775 Mon Sep 17 00:00:00 2001 From: Matheus Tavares Date: Tue, 25 May 2021 00:41:01 -0300 Subject: [PATCH 123/397] init: fix bug regarding ~/ expansion in init.templateDir We used to read the init.templateDir setting at builtin/init-db.c using a git_config() callback that, in turn, called git_config_pathname(). To simplify the config reading logic at this file and plug a memory leak, this was replaced by a direct call to git_config_get_value() at e4de4502e6 ("init: remove git_init_db_config() while fixing leaks", 2021-03-14). However, this function doesn't provide path expanding semantics, like git_config_pathname() does, so paths with '~/' and '~user/' are treated literally. This makes 'git init' fail to handle init.templateDir paths using these constructs: $ git config init.templateDir '~/templates_dir' $ git init 'warning: templates not found in ~/templates_dir' Replace the git_config_get_value() call by git_config_get_pathname(), which does the '~/' and '~user/' expansions. Also add a regression test. Note that unlike git_config_get_value(), the config cache does not own the memory for the path returned by git_config_get_pathname(), so we must free() it. Reported on IRC by rkta. Signed-off-by: Matheus Tavares Signed-off-by: Junio C Hamano --- builtin/init-db.c | 3 ++- t/t0001-init.sh | 28 ++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/builtin/init-db.c b/builtin/init-db.c index efc66523e2..7d606de7ba 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -211,8 +211,9 @@ static int create_default_files(const char *template_path, * values (since we've just potentially changed what's available on * disk). */ - git_config_get_value("init.templatedir", &init_template_dir); + git_config_get_pathname("init.templatedir", &init_template_dir); copy_templates(template_path, init_template_dir); + free((char *)init_template_dir); git_config_clear(); reset_shared_repository(); git_config(git_default_config, NULL); diff --git a/t/t0001-init.sh b/t/t0001-init.sh index 0803994874..acd662e403 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -186,21 +186,33 @@ test_expect_success 'init with --template (blank)' ' test_path_is_missing template-blank/.git/info/exclude ' +init_no_templatedir_env () { + ( + sane_unset GIT_TEMPLATE_DIR && + NO_SET_GIT_TEMPLATE_DIR=t && + export NO_SET_GIT_TEMPLATE_DIR && + git init "$1" + ) +} + test_expect_success 'init with init.templatedir set' ' mkdir templatedir-source && echo Content >templatedir-source/file && test_config_global init.templatedir "${HOME}/templatedir-source" && - ( - mkdir templatedir-set && - cd templatedir-set && - sane_unset GIT_TEMPLATE_DIR && - NO_SET_GIT_TEMPLATE_DIR=t && - export NO_SET_GIT_TEMPLATE_DIR && - git init - ) && + + init_no_templatedir_env templatedir-set && test_cmp templatedir-source/file templatedir-set/.git/file ' +test_expect_success 'init with init.templatedir using ~ expansion' ' + mkdir -p templatedir-source && + echo Content >templatedir-source/file && + test_config_global init.templatedir "~/templatedir-source" && + + init_no_templatedir_env templatedir-expansion && + test_cmp templatedir-source/file templatedir-expansion/.git/file +' + test_expect_success 'init --bare/--shared overrides system/global config' ' test_config_global core.bare false && test_config_global core.sharedRepository 0640 && From e2b05746e15bad24803300122f2d95118617ca60 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 24 May 2021 19:55:07 +0000 Subject: [PATCH 124/397] t1092: use GIT_PROGRESS_DELAY for consistent results The t1092-sparse-checkout-compatibility.sh tests compare the stdout and stderr for several Git commands across both full checkouts, sparse checkouts with a full index, and sparse checkouts with a sparse index. Since these are direct comparisons, sometimes a progress indicator can flush at unpredictable points, especially on slower machines. This causes the tests to be flaky. One standard way to avoid this is to add GIT_PROGRESS_DELAY=0 to the Git commands that are run, as this will force every progress indicator created with start_progress_delay() to be created immediately. However, there are some progress indicators that are created in the case of a full index that are not created with a sparse index. Moreover, their values may be different as those indexes have a different number of entries. Instead, use GIT_PROGRESS_DELAY=-1 (which will turn into UINT_MAX) to ensure that any reasonable machine running these tests would never display delayed progress indicators. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- t/t1092-sparse-checkout-compatibility.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 12e6c45302..58ac9377e4 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -106,18 +106,18 @@ init_repos () { run_on_sparse () { ( cd sparse-checkout && - "$@" >../sparse-checkout-out 2>../sparse-checkout-err + GIT_PROGRESS_DELAY=-1 "$@" >../sparse-checkout-out 2>../sparse-checkout-err ) && ( cd sparse-index && - "$@" >../sparse-index-out 2>../sparse-index-err + GIT_PROGRESS_DELAY=-1 "$@" >../sparse-index-out 2>../sparse-index-err ) } run_on_all () { ( cd full-checkout && - "$@" >../full-checkout-out 2>../full-checkout-err + GIT_PROGRESS_DELAY=-1 "$@" >../full-checkout-out 2>../full-checkout-err ) && run_on_sparse "$@" } From a96355d84c9fe43db4e62597921667efc32b53f6 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 26 May 2021 05:52:34 +0900 Subject: [PATCH 125/397] t1092: revert the "-1" hack for emulating "no progress meter" This looked like a good idea, but it seems to break tests on 32-bit builds rather badly. Revert to just use "100 thousands must be big enough" for now. Signed-off-by: Junio C Hamano --- t/t1092-sparse-checkout-compatibility.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index 58ac9377e4..e9a815ca7a 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -106,18 +106,18 @@ init_repos () { run_on_sparse () { ( cd sparse-checkout && - GIT_PROGRESS_DELAY=-1 "$@" >../sparse-checkout-out 2>../sparse-checkout-err + GIT_PROGRESS_DELAY=100000 "$@" >../sparse-checkout-out 2>../sparse-checkout-err ) && ( cd sparse-index && - GIT_PROGRESS_DELAY=-1 "$@" >../sparse-index-out 2>../sparse-index-err + GIT_PROGRESS_DELAY=100000 "$@" >../sparse-index-out 2>../sparse-index-err ) } run_on_all () { ( cd full-checkout && - GIT_PROGRESS_DELAY=-1 "$@" >../full-checkout-out 2>../full-checkout-err + GIT_PROGRESS_DELAY=100000 "$@" >../full-checkout-out 2>../full-checkout-err ) && run_on_sparse "$@" } From 2815326f095a1126fe9b099c311fdde8d7b7b591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 26 May 2021 13:21:06 +0200 Subject: [PATCH 126/397] send-email: don't needlessly abs_path() the core.hooksPath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In c8243933c74 (git-send-email: Respect core.hooksPath setting, 2021-03-23) we started supporting core.hooksPath in "send-email". It's been reported that on Windows[1] doing this by calling abs_path() results in different canonicalizations of the absolute path. This wasn't an issue in c8243933c74 itself, but was revealed by my ea7811b37e0 (git-send-email: improve --validate error output, 2021-04-06) when we started emitting the path to the hook, which was previously only internal to git-send-email.perl. The just-landed 53753a37d09 (t9001-send-email.sh: fix expected absolute paths on Windows, 2021-05-24) narrowly fixed this issue, but I believe we can do better here. We should not be relying on whatever changes Perl's abs_path() makes to the path "rev-parse --git-path hooks" hands to us. Let's instead trust it, and hand it to Perl's system() in git-send-email.perl. It will handle either a relative or absolute path. So let's revert most of 53753a37d09 and just have "hooks_path" return what we get from "rev-parse" directly without modification. This has the added benefit of making the error message friendlier in the common case, we'll no longer print an absolute path for repository-local hook errors. 1. http://lore.kernel.org/git/bb30fe2b-cd75-4782-24a6-08bb002a0367@kdbg.org Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- perl/Git.pm | 3 +-- t/t9001-send-email.sh | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/perl/Git.pm b/perl/Git.pm index 73ebbf80cc..df6280ebab 100644 --- a/perl/Git.pm +++ b/perl/Git.pm @@ -629,8 +629,7 @@ sub hooks_path { my ($self) = @_; my $dir = $self->command_oneline('rev-parse', '--git-path', 'hooks'); - my $abs = abs_path($dir); - return $abs; + return $dir; } =item wc_path () diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 68bebc505b..9c518462c3 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -539,14 +539,15 @@ test_expect_success $PREREQ "--validate respects relative core.hooksPath path" ' test_path_is_file my-hooks.ran && cat >expect <<-EOF && fatal: longline.patch: rejected by sendemail-validate hook - fatal: command '"'"'$PWD/my-hooks/sendemail-validate'"'"' died with exit code 1 + fatal: command '"'"'my-hooks/sendemail-validate'"'"' died with exit code 1 warning: no patches were sent EOF test_cmp expect actual ' test_expect_success $PREREQ "--validate respects absolute core.hooksPath path" ' - test_config core.hooksPath "$(pwd)/my-hooks" && + hooks_path="$(pwd)/my-hooks" && + test_config core.hooksPath "$hooks_path" && test_when_finished "rm my-hooks.ran" && test_must_fail git send-email \ --from="Example " \ @@ -557,7 +558,7 @@ test_expect_success $PREREQ "--validate respects absolute core.hooksPath path" ' test_path_is_file my-hooks.ran && cat >expect <<-EOF && fatal: longline.patch: rejected by sendemail-validate hook - fatal: command '"'"'$PWD/my-hooks/sendemail-validate'"'"' died with exit code 1 + fatal: command '"'"'$hooks_path/sendemail-validate'"'"' died with exit code 1 warning: no patches were sent EOF test_cmp expect actual From 7cbc0455cc07702c5eeff1062c7e2a820758714f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Wed, 26 May 2021 13:21:07 +0200 Subject: [PATCH 127/397] send-email: move "hooks_path" invocation to git-send-email.perl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the newly added "hooks_path" API in Git.pm to its only user in git-send-email.perl. This was added in c8243933c74 (git-send-email: Respect core.hooksPath setting, 2021-03-23), meaning that it hasn't yet made it into a non-rc release of git. The consensus with Git.pm is that we need to be considerate of out-of-tree users who treat it as a public documented interface. We should therefore be less willing to add new functionality to it, least we be stuck supporting it after our own uses for it disappear. In this case the git-send-email.perl hook invocation will probably be replaced by a future "git hook run" command, and in the commit preceding this one the "hooks_path" become nothing but a trivial wrapper for "rev-parse --git-path hooks" anyway (with no Cwd::abs_path() call), so let's just inline this command in git-send-email.perl itself. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 3 ++- perl/Git.pm | 12 ------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 175da07d94..a454020f01 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -1949,7 +1949,8 @@ sub validate_patch { my ($fn, $xfer_encoding) = @_; if ($repo) { - my $validate_hook = catfile($repo->hooks_path(), + my $hooks_path = $repo->command_oneline('rev-parse', '--git-path', 'hooks'); + my $validate_hook = catfile($hooks_path, 'sendemail-validate'); my $hook_error; if (-x $validate_hook) { diff --git a/perl/Git.pm b/perl/Git.pm index df6280ebab..02eacef0c2 100644 --- a/perl/Git.pm +++ b/perl/Git.pm @@ -619,18 +619,6 @@ Return path to the git repository. Must be called on a repository instance. sub repo_path { $_[0]->{opts}->{Repository} } -=item hooks_path () - -Return path to the hooks directory. Must be called on a repository instance. - -=cut - -sub hooks_path { - my ($self) = @_; - - my $dir = $self->command_oneline('rev-parse', '--git-path', 'hooks'); - return $dir; -} =item wc_path () From ea08db7473318798527361d4b85a67d9a1325eb4 Mon Sep 17 00:00:00 2001 From: Matheus Tavares Date: Wed, 26 May 2021 20:58:56 -0300 Subject: [PATCH 128/397] t2080: fix cp invocation to copy symlinks instead of following them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t2080 makes a few copies of a test repository and later performs a branch switch on each one of the copies to verify that parallel checkout and sequential checkout produce the same results. However, the repository is copied with `cp -R` which, on some systems, defaults to following symlinks on the directory hierarchy and copying their target files instead of copying the symlinks themselves. AIX is one example of system where this happens. Because the symlinks are not preserved, the copied repositories have paths that do not match what is in the index, causing git to abort the checkout operation that we want to test. This makes the test fail on these systems. Fix this by copying the repository with the POSIX flag '-P', which forces cp to copy the symlinks instead of following them. Note that we already use this flag for other cp invocations in our test suite (see t7001). With this change, t2080 now passes on AIX. Reported-by: Ævar Arnfjörð Bjarmason Signed-off-by: Matheus Tavares Signed-off-by: Junio C Hamano --- t/t2080-parallel-checkout-basics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t2080-parallel-checkout-basics.sh b/t/t2080-parallel-checkout-basics.sh index 7087818550..3e0f8c675f 100755 --- a/t/t2080-parallel-checkout-basics.sh +++ b/t/t2080-parallel-checkout-basics.sh @@ -114,7 +114,7 @@ do test_expect_success "$mode checkout" ' repo=various_$mode && - cp -R various $repo && + cp -R -P various $repo && # The just copied files have more recent timestamps than their # associated index entries. So refresh the cached timestamps From 7d089fb9b73a0d7f9bf43a81f284fd4f499d4886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Thu, 27 May 2021 02:52:51 +0200 Subject: [PATCH 129/397] pack-objects: move static inline from a header to the sole consumer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the code that is only used in builtin/pack-objects.c out of pack-objects.h. This fixes an issue where Solaris's SunCC hasn't been able to compile git since 483fa7f42d9 (t/helper/test-bitmap.c: initial commit, 2021-03-31). The real origin of that issue is that in 898eba5e630 (pack-objects: refer to delta objects by index instead of pointer, 2018-04-14) utility functions only needed by builtin/pack-objects.c were added to pack-objects.h. Since then the header has been used in a few other places, but 483fa7f42d9 was the first time it was used by test helper. Since Solaris is stricter about linking and the oe_get_size_slow() function lives in builtin/pack-objects.c the build started failing with: Undefined first referenced symbol in file oe_get_size_slow t/helper/test-bitmap.o ld: fatal: symbol referencing errors. No output written to t/helper/test-tool On other platforms this is presumably OK because the compiler and/or linker detects that the "static inline" functions that reference oe_get_size_slow() aren't used. Let's solve this by moving the relevant code from pack-objects.h to builtin/pack-objects.c. This is almost entirely a code-only move, but because of the early macro definitions in that file referencing some of these inline functions we need to move the definition of "static struct packing_data to_pack" earlier, and declare these inline functions above the macros. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 174 +++++++++++++++++++++++++++++++++++++++-- pack-objects.h | 159 ------------------------------------- 2 files changed, 167 insertions(+), 166 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index a1e33d7507..c5c02199e5 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -37,6 +37,134 @@ #include "shallow.h" #include "promisor-remote.h" +/* + * Objects we are going to pack are collected in the `to_pack` structure. + * It contains an array (dynamically expanded) of the object data, and a map + * that can resolve SHA1s to their position in the array. + */ +static struct packing_data to_pack; + +static inline struct object_entry *oe_delta( + const struct packing_data *pack, + const struct object_entry *e) +{ + if (!e->delta_idx) + return NULL; + if (e->ext_base) + return &pack->ext_bases[e->delta_idx - 1]; + else + return &pack->objects[e->delta_idx - 1]; +} + +static inline unsigned long oe_delta_size(struct packing_data *pack, + const struct object_entry *e) +{ + if (e->delta_size_valid) + return e->delta_size_; + + /* + * pack->delta_size[] can't be NULL because oe_set_delta_size() + * must have been called when a new delta is saved with + * oe_set_delta(). + * If oe_delta() returns NULL (i.e. default state, which means + * delta_size_valid is also false), then the caller must never + * call oe_delta_size(). + */ + return pack->delta_size[e - pack->objects]; +} + +unsigned long oe_get_size_slow(struct packing_data *pack, + const struct object_entry *e); + +static inline unsigned long oe_size(struct packing_data *pack, + const struct object_entry *e) +{ + if (e->size_valid) + return e->size_; + + return oe_get_size_slow(pack, e); +} + +static inline void oe_set_delta(struct packing_data *pack, + struct object_entry *e, + struct object_entry *delta) +{ + if (delta) + e->delta_idx = (delta - pack->objects) + 1; + else + e->delta_idx = 0; +} + +static inline struct object_entry *oe_delta_sibling( + const struct packing_data *pack, + const struct object_entry *e) +{ + if (e->delta_sibling_idx) + return &pack->objects[e->delta_sibling_idx - 1]; + return NULL; +} + +static inline struct object_entry *oe_delta_child( + const struct packing_data *pack, + const struct object_entry *e) +{ + if (e->delta_child_idx) + return &pack->objects[e->delta_child_idx - 1]; + return NULL; +} + +static inline void oe_set_delta_child(struct packing_data *pack, + struct object_entry *e, + struct object_entry *delta) +{ + if (delta) + e->delta_child_idx = (delta - pack->objects) + 1; + else + e->delta_child_idx = 0; +} + +static inline void oe_set_delta_sibling(struct packing_data *pack, + struct object_entry *e, + struct object_entry *delta) +{ + if (delta) + e->delta_sibling_idx = (delta - pack->objects) + 1; + else + e->delta_sibling_idx = 0; +} + +static inline void oe_set_size(struct packing_data *pack, + struct object_entry *e, + unsigned long size) +{ + if (size < pack->oe_size_limit) { + e->size_ = size; + e->size_valid = 1; + } else { + e->size_valid = 0; + if (oe_get_size_slow(pack, e) != size) + BUG("'size' is supposed to be the object size!"); + } +} + +static inline void oe_set_delta_size(struct packing_data *pack, + struct object_entry *e, + unsigned long size) +{ + if (size < pack->oe_delta_size_limit) { + e->delta_size_ = size; + e->delta_size_valid = 1; + } else { + packing_data_lock(pack); + if (!pack->delta_size) + ALLOC_ARRAY(pack->delta_size, pack->nr_alloc); + packing_data_unlock(pack); + + pack->delta_size[e - pack->objects] = size; + e->delta_size_valid = 0; + } +} + #define IN_PACK(obj) oe_in_pack(&to_pack, obj) #define SIZE(obj) oe_size(&to_pack, obj) #define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) @@ -56,13 +184,6 @@ static const char *pack_usage[] = { NULL }; -/* - * Objects we are going to pack are collected in the `to_pack` structure. - * It contains an array (dynamically expanded) of the object data, and a map - * that can resolve SHA1s to their position in the array. - */ -static struct packing_data to_pack; - static struct pack_idx_entry **written_list; static uint32_t nr_result, nr_written, nr_seen; static struct bitmap_index *bitmap_git; @@ -301,6 +422,17 @@ static void copy_pack_data(struct hashfile *f, } } +static inline int oe_size_greater_than(struct packing_data *pack, + const struct object_entry *lhs, + unsigned long rhs) +{ + if (lhs->size_valid) + return lhs->size_ > rhs; + if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */ + return 1; + return oe_get_size_slow(pack, lhs) > rhs; +} + /* Return 0 if we will bust the pack-size limit */ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_entry *entry, unsigned long limit, int usable_delta) @@ -642,6 +774,14 @@ static int mark_tagged(const char *path, const struct object_id *oid, int flag, return 0; } +static inline unsigned char oe_layer(struct packing_data *pack, + struct object_entry *e) +{ + if (!pack->layer) + return 0; + return pack->layer[e - pack->objects]; +} + static inline void add_to_write_order(struct object_entry **wo, unsigned int *endp, struct object_entry *e) @@ -2231,6 +2371,26 @@ static pthread_mutex_t progress_mutex; * progress_mutex for protection. */ +static inline int oe_size_less_than(struct packing_data *pack, + const struct object_entry *lhs, + unsigned long rhs) +{ + if (lhs->size_valid) + return lhs->size_ < rhs; + if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */ + return 0; + return oe_get_size_slow(pack, lhs) < rhs; +} + +static inline void oe_set_tree_depth(struct packing_data *pack, + struct object_entry *e, + unsigned int tree_depth) +{ + if (!pack->tree_depth) + CALLOC_ARRAY(pack->tree_depth, pack->nr_alloc); + pack->tree_depth[e - pack->objects] = tree_depth; +} + /* * Return the size of the object without doing any delta * reconstruction (so non-deltas are true object sizes, but deltas diff --git a/pack-objects.h b/pack-objects.h index 9d88e3e518..dca2351ef9 100644 --- a/pack-objects.h +++ b/pack-objects.h @@ -268,152 +268,10 @@ static inline void oe_set_in_pack(struct packing_data *pack, pack->in_pack[e - pack->objects] = p; } -static inline struct object_entry *oe_delta( - const struct packing_data *pack, - const struct object_entry *e) -{ - if (!e->delta_idx) - return NULL; - if (e->ext_base) - return &pack->ext_bases[e->delta_idx - 1]; - else - return &pack->objects[e->delta_idx - 1]; -} - -static inline void oe_set_delta(struct packing_data *pack, - struct object_entry *e, - struct object_entry *delta) -{ - if (delta) - e->delta_idx = (delta - pack->objects) + 1; - else - e->delta_idx = 0; -} - void oe_set_delta_ext(struct packing_data *pack, struct object_entry *e, const struct object_id *oid); -static inline struct object_entry *oe_delta_child( - const struct packing_data *pack, - const struct object_entry *e) -{ - if (e->delta_child_idx) - return &pack->objects[e->delta_child_idx - 1]; - return NULL; -} - -static inline void oe_set_delta_child(struct packing_data *pack, - struct object_entry *e, - struct object_entry *delta) -{ - if (delta) - e->delta_child_idx = (delta - pack->objects) + 1; - else - e->delta_child_idx = 0; -} - -static inline struct object_entry *oe_delta_sibling( - const struct packing_data *pack, - const struct object_entry *e) -{ - if (e->delta_sibling_idx) - return &pack->objects[e->delta_sibling_idx - 1]; - return NULL; -} - -static inline void oe_set_delta_sibling(struct packing_data *pack, - struct object_entry *e, - struct object_entry *delta) -{ - if (delta) - e->delta_sibling_idx = (delta - pack->objects) + 1; - else - e->delta_sibling_idx = 0; -} - -unsigned long oe_get_size_slow(struct packing_data *pack, - const struct object_entry *e); -static inline unsigned long oe_size(struct packing_data *pack, - const struct object_entry *e) -{ - if (e->size_valid) - return e->size_; - - return oe_get_size_slow(pack, e); -} - -static inline int oe_size_less_than(struct packing_data *pack, - const struct object_entry *lhs, - unsigned long rhs) -{ - if (lhs->size_valid) - return lhs->size_ < rhs; - if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */ - return 0; - return oe_get_size_slow(pack, lhs) < rhs; -} - -static inline int oe_size_greater_than(struct packing_data *pack, - const struct object_entry *lhs, - unsigned long rhs) -{ - if (lhs->size_valid) - return lhs->size_ > rhs; - if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */ - return 1; - return oe_get_size_slow(pack, lhs) > rhs; -} - -static inline void oe_set_size(struct packing_data *pack, - struct object_entry *e, - unsigned long size) -{ - if (size < pack->oe_size_limit) { - e->size_ = size; - e->size_valid = 1; - } else { - e->size_valid = 0; - if (oe_get_size_slow(pack, e) != size) - BUG("'size' is supposed to be the object size!"); - } -} - -static inline unsigned long oe_delta_size(struct packing_data *pack, - const struct object_entry *e) -{ - if (e->delta_size_valid) - return e->delta_size_; - - /* - * pack->delta_size[] can't be NULL because oe_set_delta_size() - * must have been called when a new delta is saved with - * oe_set_delta(). - * If oe_delta() returns NULL (i.e. default state, which means - * delta_size_valid is also false), then the caller must never - * call oe_delta_size(). - */ - return pack->delta_size[e - pack->objects]; -} - -static inline void oe_set_delta_size(struct packing_data *pack, - struct object_entry *e, - unsigned long size) -{ - if (size < pack->oe_delta_size_limit) { - e->delta_size_ = size; - e->delta_size_valid = 1; - } else { - packing_data_lock(pack); - if (!pack->delta_size) - ALLOC_ARRAY(pack->delta_size, pack->nr_alloc); - packing_data_unlock(pack); - - pack->delta_size[e - pack->objects] = size; - e->delta_size_valid = 0; - } -} - static inline unsigned int oe_tree_depth(struct packing_data *pack, struct object_entry *e) { @@ -422,23 +280,6 @@ static inline unsigned int oe_tree_depth(struct packing_data *pack, return pack->tree_depth[e - pack->objects]; } -static inline void oe_set_tree_depth(struct packing_data *pack, - struct object_entry *e, - unsigned int tree_depth) -{ - if (!pack->tree_depth) - CALLOC_ARRAY(pack->tree_depth, pack->nr_alloc); - pack->tree_depth[e - pack->objects] = tree_depth; -} - -static inline unsigned char oe_layer(struct packing_data *pack, - struct object_entry *e) -{ - if (!pack->layer) - return 0; - return pack->layer[e - pack->objects]; -} - static inline void oe_set_layer(struct packing_data *pack, struct object_entry *e, unsigned char layer) From 1df046bcff6085f3800088c51e344594f822df57 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 27 May 2021 13:59:39 +0900 Subject: [PATCH 130/397] Revert "dir: introduce readdir_skip_dot_and_dotdot() helper" This reverts commit b548f0f1568f6b01e55ca69c24d3cb19489f92aa, to be replaced with a reworked version. --- builtin/clean.c | 4 +++- builtin/worktree.c | 4 +++- diff-no-index.c | 5 +++-- dir.c | 26 +++++++++----------------- dir.h | 2 -- entry.c | 5 ++++- notes-merge.c | 5 ++++- object-file.c | 4 +++- packfile.c | 5 ++++- rerere.c | 4 +++- worktree.c | 12 +++++++++--- 11 files changed, 45 insertions(+), 31 deletions(-) diff --git a/builtin/clean.c b/builtin/clean.c index a1a5747615..995053b791 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -189,8 +189,10 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, strbuf_complete(path, '/'); len = path->len; - while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((e = readdir(dir)) != NULL) { struct stat st; + if (is_dot_or_dotdot(e->d_name)) + continue; strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); diff --git a/builtin/worktree.c b/builtin/worktree.c index e081ca9bef..1cd5c2016e 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -118,8 +118,10 @@ static void prune_worktrees(void) struct dirent *d; if (!dir) return; - while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((d = readdir(dir)) != NULL) { char *path; + if (is_dot_or_dotdot(d->d_name)) + continue; strbuf_reset(&reason); if (should_prune_worktree(d->d_name, &reason, &path, expire)) prune_worktree(d->d_name, reason.buf); diff --git a/diff-no-index.c b/diff-no-index.c index e5cc878371..7814eabfe0 100644 --- a/diff-no-index.c +++ b/diff-no-index.c @@ -26,8 +26,9 @@ static int read_directory_contents(const char *path, struct string_list *list) if (!(dir = opendir(path))) return error("Could not open directory %s", path); - while ((e = readdir_skip_dot_and_dotdot(dir))) - string_list_insert(list, e->d_name); + while ((e = readdir(dir))) + if (!is_dot_or_dotdot(e->d_name)) + string_list_insert(list, e->d_name); closedir(dir); return 0; diff --git a/dir.c b/dir.c index e47b4c507f..ff004b298b 100644 --- a/dir.c +++ b/dir.c @@ -59,18 +59,6 @@ void dir_init(struct dir_struct *dir) memset(dir, 0, sizeof(*dir)); } -struct dirent * -readdir_skip_dot_and_dotdot(DIR *dirp) -{ - struct dirent *e; - - while ((e = readdir(dirp)) != NULL) { - if (!is_dot_or_dotdot(e->d_name)) - break; - } - return e; -} - int count_slashes(const char *s) { int cnt = 0; @@ -2344,7 +2332,7 @@ static int read_cached_dir(struct cached_dir *cdir) struct dirent *de; if (cdir->fdir) { - de = readdir_skip_dot_and_dotdot(cdir->fdir); + de = readdir(cdir->fdir); if (!de) { cdir->d_name = NULL; cdir->d_type = DT_UNKNOWN; @@ -2943,9 +2931,11 @@ int is_empty_dir(const char *path) if (!dir) return 0; - e = readdir_skip_dot_and_dotdot(dir); - if (e) - ret = 0; + while ((e = readdir(dir)) != NULL) + if (!is_dot_or_dotdot(e->d_name)) { + ret = 0; + break; + } closedir(dir); return ret; @@ -2985,8 +2975,10 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) strbuf_complete(path, '/'); len = path->len; - while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((e = readdir(dir)) != NULL) { struct stat st; + if (is_dot_or_dotdot(e->d_name)) + continue; strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); diff --git a/dir.h b/dir.h index 6b3fac0829..70c750e305 100644 --- a/dir.h +++ b/dir.h @@ -342,8 +342,6 @@ struct dir_struct { unsigned visited_directories; }; -struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp); - /*Count the number of slashes for string s*/ int count_slashes(const char *s); diff --git a/entry.c b/entry.c index e3d3add300..7b9f43716f 100644 --- a/entry.c +++ b/entry.c @@ -56,9 +56,12 @@ static void remove_subtree(struct strbuf *path) if (!dir) die_errno("cannot opendir '%s'", path->buf); - while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((de = readdir(dir)) != NULL) { struct stat st; + if (is_dot_or_dotdot(de->d_name)) + continue; + strbuf_addch(path, '/'); strbuf_addstr(path, de->d_name); if (lstat(path->buf, &st)) diff --git a/notes-merge.c b/notes-merge.c index e9d6f86d34..d2771fa3d4 100644 --- a/notes-merge.c +++ b/notes-merge.c @@ -695,10 +695,13 @@ int notes_merge_commit(struct notes_merge_options *o, strbuf_addch(&path, '/'); baselen = path.len; - while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((e = readdir(dir)) != NULL) { struct stat st; struct object_id obj_oid, blob_oid; + if (is_dot_or_dotdot(e->d_name)) + continue; + if (get_oid_hex(e->d_name, &obj_oid)) { if (o->verbosity >= 3) printf("Skipping non-SHA1 entry '%s%s'\n", diff --git a/object-file.c b/object-file.c index 77bdcfd21b..624af408cd 100644 --- a/object-file.c +++ b/object-file.c @@ -2304,8 +2304,10 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr, strbuf_addch(path, '/'); baselen = path->len; - while ((de = readdir_skip_dot_and_dotdot(dir))) { + while ((de = readdir(dir))) { size_t namelen; + if (is_dot_or_dotdot(de->d_name)) + continue; namelen = strlen(de->d_name); strbuf_setlen(path, baselen); diff --git a/packfile.c b/packfile.c index 463d61c877..ea29f4ba77 100644 --- a/packfile.c +++ b/packfile.c @@ -813,7 +813,10 @@ void for_each_file_in_pack_dir(const char *objdir, } strbuf_addch(&path, '/'); dirnamelen = path.len; - while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((de = readdir(dir)) != NULL) { + if (is_dot_or_dotdot(de->d_name)) + continue; + strbuf_setlen(&path, dirnamelen); strbuf_addstr(&path, de->d_name); diff --git a/rerere.c b/rerere.c index d83d58df4f..dee60dc6df 100644 --- a/rerere.c +++ b/rerere.c @@ -1190,11 +1190,13 @@ void rerere_gc(struct repository *r, struct string_list *rr) if (!dir) die_errno(_("unable to open rr-cache directory")); /* Collect stale conflict IDs ... */ - while ((e = readdir_skip_dot_and_dotdot(dir))) { + while ((e = readdir(dir))) { struct rerere_dir *rr_dir; struct rerere_id id; int now_empty; + if (is_dot_or_dotdot(e->d_name)) + continue; if (!is_rr_cache_dirname(e->d_name)) continue; /* or should we remove e->d_name? */ diff --git a/worktree.c b/worktree.c index 237517baee..f35ac40a84 100644 --- a/worktree.c +++ b/worktree.c @@ -128,8 +128,10 @@ struct worktree **get_worktrees(void) dir = opendir(path.buf); strbuf_release(&path); if (dir) { - while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { + while ((d = readdir(dir)) != NULL) { struct worktree *linked = NULL; + if (is_dot_or_dotdot(d->d_name)) + continue; if ((linked = get_linked_worktree(d->d_name))) { ALLOC_GROW(list, counter + 1, alloc); @@ -484,9 +486,13 @@ int submodule_uses_worktrees(const char *path) if (!dir) return 0; - d = readdir_skip_dot_and_dotdot(dir); - if (d != NULL) + while ((d = readdir(dir)) != NULL) { + if (is_dot_or_dotdot(d->d_name)) + continue; + ret = 1; + break; + } closedir(dir); return ret; } From 2c9f1bfdb47acde21052bb33ff083347cbcb574d Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 27 May 2021 14:00:00 +0900 Subject: [PATCH 131/397] Revert "dir: update stale description of treat_directory()" This reverts commit 4e689d81718eb6e939cace317ea3e33cb994dcbb, to be replaced with a reworked version. --- dir.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dir.c b/dir.c index ff004b298b..ed68b7e641 100644 --- a/dir.c +++ b/dir.c @@ -1740,13 +1740,13 @@ static enum exist_status directory_exists_in_index(struct index_state *istate, * Case 3: if we didn't have it in the index previously, we * have a few sub-cases: * - * (a) if DIR_SHOW_OTHER_DIRECTORIES flag is set, we show it as - * just a directory, unless DIR_HIDE_EMPTY_DIRECTORIES is + * (a) if "show_other_directories" is true, we show it as + * just a directory, unless "hide_empty_directories" is * also true, in which case we need to check if it contains any * untracked and / or ignored files. - * (b) if it looks like a git directory and we don't have the - * DIR_NO_GITLINKS flag, then we treat it as a gitlink, and - * show it as a directory. + * (b) if it looks like a git directory, and we don't have + * 'no_gitlinks' set we treat it as a gitlink, and show it + * as a directory. * (c) otherwise, we recurse into it. */ static enum path_treatment treat_directory(struct dir_struct *dir, @@ -1834,6 +1834,7 @@ static enum path_treatment treat_directory(struct dir_struct *dir, return path_recurse; } + /* This is the "show_other_directories" case */ assert(dir->flags & DIR_SHOW_OTHER_DIRECTORIES); /* @@ -1848,7 +1849,7 @@ static enum path_treatment treat_directory(struct dir_struct *dir, /* Special cases for where this directory is excluded/ignored */ if (excluded) { /* - * If DIR_SHOW_OTHER_DIRECTORIES is set and we're not + * In the show_other_directories case, if we're not * hiding empty directories, there is no need to * recurse into an ignored directory. */ From eef814828f175290ca1fcd17ad74faff95346c85 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 27 May 2021 04:53:55 +0000 Subject: [PATCH 132/397] dir: update stale description of treat_directory() The documentation comment for treat_directory() was originally written in 095952 (Teach directory traversal about subprojects, 2007-04-11) which was before the 'struct dir_struct' split its bitfield of named options into a 'flags' enum in 7c4c97c0 (Turn the flags in struct dir_struct into a single variable, 2009-02-16). When those flags changed, the comment became stale, since members like 'show_other_directories' transitioned into flags like DIR_SHOW_OTHER_DIRECTORIES. Update the comments for treat_directory() to use these flag names rather than the old member names. Signed-off-by: Derrick Stolee Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- dir.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/dir.c b/dir.c index ed68b7e641..ff004b298b 100644 --- a/dir.c +++ b/dir.c @@ -1740,13 +1740,13 @@ static enum exist_status directory_exists_in_index(struct index_state *istate, * Case 3: if we didn't have it in the index previously, we * have a few sub-cases: * - * (a) if "show_other_directories" is true, we show it as - * just a directory, unless "hide_empty_directories" is + * (a) if DIR_SHOW_OTHER_DIRECTORIES flag is set, we show it as + * just a directory, unless DIR_HIDE_EMPTY_DIRECTORIES is * also true, in which case we need to check if it contains any * untracked and / or ignored files. - * (b) if it looks like a git directory, and we don't have - * 'no_gitlinks' set we treat it as a gitlink, and show it - * as a directory. + * (b) if it looks like a git directory and we don't have the + * DIR_NO_GITLINKS flag, then we treat it as a gitlink, and + * show it as a directory. * (c) otherwise, we recurse into it. */ static enum path_treatment treat_directory(struct dir_struct *dir, @@ -1834,7 +1834,6 @@ static enum path_treatment treat_directory(struct dir_struct *dir, return path_recurse; } - /* This is the "show_other_directories" case */ assert(dir->flags & DIR_SHOW_OTHER_DIRECTORIES); /* @@ -1849,7 +1848,7 @@ static enum path_treatment treat_directory(struct dir_struct *dir, /* Special cases for where this directory is excluded/ignored */ if (excluded) { /* - * In the show_other_directories case, if we're not + * If DIR_SHOW_OTHER_DIRECTORIES is set and we're not * hiding empty directories, there is no need to * recurse into an ignored directory. */ From 906fc557b70b2b2995785c9b37e212d2f86b469e Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Thu, 27 May 2021 04:53:56 +0000 Subject: [PATCH 133/397] dir: introduce readdir_skip_dot_and_dotdot() helper Many places in the code were doing while ((d = readdir(dir)) != NULL) { if (is_dot_or_dotdot(d->d_name)) continue; ...process d... } Introduce a readdir_skip_dot_and_dotdot() helper to make that a one-liner: while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { ...process d... } This helper particularly simplifies checks for empty directories. Also use this helper in read_cached_dir() so that our statistics are consistent across platforms. (In other words, read_cached_dir() should have been using is_dot_or_dotdot() and skipping such entries, but did not and left it to treat_path() to detect and mark such entries as path_none.) Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- builtin/clean.c | 4 +--- builtin/worktree.c | 4 +--- diff-no-index.c | 5 ++--- dir.c | 25 ++++++++++++++++--------- dir.h | 2 ++ entry.c | 5 +---- notes-merge.c | 5 +---- object-file.c | 4 +--- packfile.c | 5 +---- rerere.c | 4 +--- worktree.c | 12 +++--------- 11 files changed, 30 insertions(+), 45 deletions(-) diff --git a/builtin/clean.c b/builtin/clean.c index 995053b791..a1a5747615 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -189,10 +189,8 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, strbuf_complete(path, '/'); len = path->len; - while ((e = readdir(dir)) != NULL) { + while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; - if (is_dot_or_dotdot(e->d_name)) - continue; strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); diff --git a/builtin/worktree.c b/builtin/worktree.c index 1cd5c2016e..e081ca9bef 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -118,10 +118,8 @@ static void prune_worktrees(void) struct dirent *d; if (!dir) return; - while ((d = readdir(dir)) != NULL) { + while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { char *path; - if (is_dot_or_dotdot(d->d_name)) - continue; strbuf_reset(&reason); if (should_prune_worktree(d->d_name, &reason, &path, expire)) prune_worktree(d->d_name, reason.buf); diff --git a/diff-no-index.c b/diff-no-index.c index 7814eabfe0..e5cc878371 100644 --- a/diff-no-index.c +++ b/diff-no-index.c @@ -26,9 +26,8 @@ static int read_directory_contents(const char *path, struct string_list *list) if (!(dir = opendir(path))) return error("Could not open directory %s", path); - while ((e = readdir(dir))) - if (!is_dot_or_dotdot(e->d_name)) - string_list_insert(list, e->d_name); + while ((e = readdir_skip_dot_and_dotdot(dir))) + string_list_insert(list, e->d_name); closedir(dir); return 0; diff --git a/dir.c b/dir.c index ff004b298b..b909cf9b03 100644 --- a/dir.c +++ b/dir.c @@ -59,6 +59,17 @@ void dir_init(struct dir_struct *dir) memset(dir, 0, sizeof(*dir)); } +struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp) +{ + struct dirent *e; + + while ((e = readdir(dirp)) != NULL) { + if (!is_dot_or_dotdot(e->d_name)) + break; + } + return e; +} + int count_slashes(const char *s) { int cnt = 0; @@ -2332,7 +2343,7 @@ static int read_cached_dir(struct cached_dir *cdir) struct dirent *de; if (cdir->fdir) { - de = readdir(cdir->fdir); + de = readdir_skip_dot_and_dotdot(cdir->fdir); if (!de) { cdir->d_name = NULL; cdir->d_type = DT_UNKNOWN; @@ -2931,11 +2942,9 @@ int is_empty_dir(const char *path) if (!dir) return 0; - while ((e = readdir(dir)) != NULL) - if (!is_dot_or_dotdot(e->d_name)) { - ret = 0; - break; - } + e = readdir_skip_dot_and_dotdot(dir); + if (e) + ret = 0; closedir(dir); return ret; @@ -2975,10 +2984,8 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) strbuf_complete(path, '/'); len = path->len; - while ((e = readdir(dir)) != NULL) { + while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; - if (is_dot_or_dotdot(e->d_name)) - continue; strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); diff --git a/dir.h b/dir.h index 70c750e305..6b3fac0829 100644 --- a/dir.h +++ b/dir.h @@ -342,6 +342,8 @@ struct dir_struct { unsigned visited_directories; }; +struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp); + /*Count the number of slashes for string s*/ int count_slashes(const char *s); diff --git a/entry.c b/entry.c index 7b9f43716f..e3d3add300 100644 --- a/entry.c +++ b/entry.c @@ -56,12 +56,9 @@ static void remove_subtree(struct strbuf *path) if (!dir) die_errno("cannot opendir '%s'", path->buf); - while ((de = readdir(dir)) != NULL) { + while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; - if (is_dot_or_dotdot(de->d_name)) - continue; - strbuf_addch(path, '/'); strbuf_addstr(path, de->d_name); if (lstat(path->buf, &st)) diff --git a/notes-merge.c b/notes-merge.c index d2771fa3d4..e9d6f86d34 100644 --- a/notes-merge.c +++ b/notes-merge.c @@ -695,13 +695,10 @@ int notes_merge_commit(struct notes_merge_options *o, strbuf_addch(&path, '/'); baselen = path.len; - while ((e = readdir(dir)) != NULL) { + while ((e = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct stat st; struct object_id obj_oid, blob_oid; - if (is_dot_or_dotdot(e->d_name)) - continue; - if (get_oid_hex(e->d_name, &obj_oid)) { if (o->verbosity >= 3) printf("Skipping non-SHA1 entry '%s%s'\n", diff --git a/object-file.c b/object-file.c index 624af408cd..77bdcfd21b 100644 --- a/object-file.c +++ b/object-file.c @@ -2304,10 +2304,8 @@ int for_each_file_in_obj_subdir(unsigned int subdir_nr, strbuf_addch(path, '/'); baselen = path->len; - while ((de = readdir(dir))) { + while ((de = readdir_skip_dot_and_dotdot(dir))) { size_t namelen; - if (is_dot_or_dotdot(de->d_name)) - continue; namelen = strlen(de->d_name); strbuf_setlen(path, baselen); diff --git a/packfile.c b/packfile.c index ea29f4ba77..463d61c877 100644 --- a/packfile.c +++ b/packfile.c @@ -813,10 +813,7 @@ void for_each_file_in_pack_dir(const char *objdir, } strbuf_addch(&path, '/'); dirnamelen = path.len; - while ((de = readdir(dir)) != NULL) { - if (is_dot_or_dotdot(de->d_name)) - continue; - + while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) { strbuf_setlen(&path, dirnamelen); strbuf_addstr(&path, de->d_name); diff --git a/rerere.c b/rerere.c index dee60dc6df..d83d58df4f 100644 --- a/rerere.c +++ b/rerere.c @@ -1190,13 +1190,11 @@ void rerere_gc(struct repository *r, struct string_list *rr) if (!dir) die_errno(_("unable to open rr-cache directory")); /* Collect stale conflict IDs ... */ - while ((e = readdir(dir))) { + while ((e = readdir_skip_dot_and_dotdot(dir))) { struct rerere_dir *rr_dir; struct rerere_id id; int now_empty; - if (is_dot_or_dotdot(e->d_name)) - continue; if (!is_rr_cache_dirname(e->d_name)) continue; /* or should we remove e->d_name? */ diff --git a/worktree.c b/worktree.c index f35ac40a84..237517baee 100644 --- a/worktree.c +++ b/worktree.c @@ -128,10 +128,8 @@ struct worktree **get_worktrees(void) dir = opendir(path.buf); strbuf_release(&path); if (dir) { - while ((d = readdir(dir)) != NULL) { + while ((d = readdir_skip_dot_and_dotdot(dir)) != NULL) { struct worktree *linked = NULL; - if (is_dot_or_dotdot(d->d_name)) - continue; if ((linked = get_linked_worktree(d->d_name))) { ALLOC_GROW(list, counter + 1, alloc); @@ -486,13 +484,9 @@ int submodule_uses_worktrees(const char *path) if (!dir) return 0; - while ((d = readdir(dir)) != NULL) { - if (is_dot_or_dotdot(d->d_name)) - continue; - + d = readdir_skip_dot_and_dotdot(dir); + if (d != NULL) ret = 1; - break; - } closedir(dir); return ret; } From 4e42405f00ecbbee412846f48cb0253efeebe726 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 28 May 2021 13:05:29 +0900 Subject: [PATCH 134/397] Git 2.32-rc2 Signed-off-by: Junio C Hamano --- GIT-VERSION-GEN | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 8862d57489..6fcb8fdf3b 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=v2.32.0-rc1 +DEF_VER=v2.32.0-rc2 LF=' ' From ecc4ee9cac60600f38542cebd940b4bd8b6497c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:40 +0200 Subject: [PATCH 135/397] send-email tests: support GIT_TEST_PERL_FATAL_WARNINGS=true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for the "GIT_TEST_PERL_FATAL_WARNINGS=true" test mode to "send-email". This was added to e.g. git-svn in 5338ed2b26 (perl: check for perl warnings while running tests, 2020-10-21), but not "send-email". Let's rectify that. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-send-email.perl b/git-send-email.perl index 25be2ebd2a..1630e87cd4 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -18,7 +18,7 @@ use 5.008; use strict; -use warnings; +use warnings $ENV{GIT_PERL_FATAL_WARNINGS} ? qw(FATAL all) : (); use POSIX qw/strftime/; use Term::ReadLine; use Getopt::Long; From 879be4319f70a2d6c65c3444a269bff671d182cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:41 +0200 Subject: [PATCH 136/397] send-email tests: test for boolean variables without a value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Git.pm code does its own Perl-ifying of boolean variables, let's ensure that empty values = true for boolean variables, as in the C code. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t9001-send-email.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh index 3b7540050c..612de095fd 100755 --- a/t/t9001-send-email.sh +++ b/t/t9001-send-email.sh @@ -1368,6 +1368,16 @@ test_expect_success $PREREQ 'sendemail.identity: bool variable fallback' ' ! grep "X-Mailer" stdout ' +test_expect_success $PREREQ 'sendemail.identity: bool variable without a value' ' + git -c sendemail.xmailer \ + send-email \ + --dry-run \ + --from="nobody@example.com" \ + $patches >stdout && + grep "To: default@example.com" stdout && + grep "X-Mailer" stdout +' + test_expect_success $PREREQ '--no-to overrides sendemail.to' ' git send-email \ --dry-run \ @@ -2092,6 +2102,18 @@ test_expect_success $PREREQ '--[no-]xmailer with sendemail.xmailer=true' ' do_xmailer_test 1 "--xmailer" ' +test_expect_success $PREREQ '--[no-]xmailer with sendemail.xmailer' ' + test_when_finished "test_unconfig sendemail.xmailer" && + cat >>.git/config <<-\EOF && + [sendemail] + xmailer + EOF + test_config sendemail.xmailer true && + do_xmailer_test 1 "" && + do_xmailer_test 0 "--no-xmailer" && + do_xmailer_test 1 "--xmailer" +' + test_expect_success $PREREQ '--[no-]xmailer with sendemail.xmailer=false' ' test_config sendemail.xmailer false && do_xmailer_test 0 "" && @@ -2099,6 +2121,13 @@ test_expect_success $PREREQ '--[no-]xmailer with sendemail.xmailer=false' ' do_xmailer_test 1 "--xmailer" ' +test_expect_success $PREREQ '--[no-]xmailer with sendemail.xmailer=' ' + test_config sendemail.xmailer "" && + do_xmailer_test 0 "" && + do_xmailer_test 0 "--no-xmailer" && + do_xmailer_test 1 "--xmailer" +' + test_expect_success $PREREQ 'setup expected-list' ' git send-email \ --dry-run \ From 671818ab0b8206481d38054477059fee46db0ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:42 +0200 Subject: [PATCH 137/397] send-email: remove non-working support for "sendemail.smtpssl" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the already dead code to support "sendemail.smtpssl" by finally removing the dead code supporting the configuration option. In f6bebd121ac (git-send-email: add support for TLS via Net::SMTP::SSL, 2008-06-25) the --smtp-ssl command-line option was documented as deprecated, later in 65180c66186 (List send-email config options in config.txt., 2009-07-22) the "sendemail.smtpssl" configuration option was also documented as such. Then in in 3ff15040e22 (send-email: fix regression in sendemail.identity parsing, 2019-05-17) I unintentionally removed support for it by introducing a bug in read_config(). As can be seen from the diff context we've already returned unless $enc i defined, so it's not possible for us to reach the "elsif" branch here. This code was therefore already dead since Git v2.23.0. So let's just remove it. We were already 11 years into a stated deprecation period of this variable when 3ff15040e22 landed, now it's around 13. Since it hasn't worked anyway for around 2 years it looks like we can safely remove it. The --smtp-ssl option is still deprecated, if someone cares they can follow-up and remove that too, but unlike the config option that one could still be in use in the wild. I'm just removing this code that's provably unused already. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- Documentation/config/sendemail.txt | 3 --- git-send-email.perl | 6 +----- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Documentation/config/sendemail.txt b/Documentation/config/sendemail.txt index cbc5af42fd..50baa5d6bf 100644 --- a/Documentation/config/sendemail.txt +++ b/Documentation/config/sendemail.txt @@ -8,9 +8,6 @@ sendemail.smtpEncryption:: See linkgit:git-send-email[1] for description. Note that this setting is not subject to the 'identity' mechanism. -sendemail.smtpssl (deprecated):: - Deprecated alias for 'sendemail.smtpEncryption = ssl'. - sendemail.smtpsslcertpath:: Path to ca-certificates (either a directory or a single file). Set it to an empty string to disable certificate verification. diff --git a/git-send-email.perl b/git-send-email.perl index 1630e87cd4..9a1a4898e3 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -393,11 +393,7 @@ sub read_config { my $enc = Git::config(@repo, $setting); return unless defined $enc; return if $configured->{$setting}++; - if (defined $enc) { - $smtp_encryption = $enc; - } elsif (Git::config_bool(@repo, "$prefix.smtpssl")) { - $smtp_encryption = 'ssl'; - } + $smtp_encryption = $enc; } } From 119974e9e7f3569400aa1950529f69431534a617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:43 +0200 Subject: [PATCH 138/397] send-email: refactor sendemail.smtpencryption config parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the removal of the support for sendemail.smtpssl in the preceding commit the parsing of sendemail.smtpencryption is no longer special, and can by moved to %config_settings. This gets us rid of an unconditional call to Git::config(), which as we'll see in subsequent commits matters for startup performance. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 9a1a4898e3..e2fe112aa5 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -287,6 +287,7 @@ my %config_bool_settings = ( ); my %config_settings = ( + "smtpencryption" => \$smtp_encryption, "smtpserver" => \$smtp_server, "smtpserverport" => \$smtp_server_port, "smtpserveroption" => \@smtp_server_options, @@ -387,14 +388,6 @@ sub read_config { $$target = $v; } } - - if (!defined $smtp_encryption) { - my $setting = "$prefix.smtpencryption"; - my $enc = Git::config(@repo, $setting); - return unless defined $enc; - return if $configured->{$setting}++; - $smtp_encryption = $enc; - } } # sendemail.identity yields to --identity. We must parse this From 2b110e9ba255120a2d78a5d2cccc842519e341f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:44 +0200 Subject: [PATCH 139/397] send-email: copy "config_regxp" into git-send-email.perl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The config_regexp() function was added in dd84e528a3 (git-send-email: die if sendmail.* config is set, 2020-07-23) for use in git-send-email, and it's the only in-tree user of it. However, the consensus is that Git.pm is a public interface, so even though it's a recently added function we can't change it. So let's copy over a minimal version of it to git-send-email.perl itself. In a subsequent commit it'll be changed further for our own use. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/git-send-email.perl b/git-send-email.perl index e2fe112aa5..73e3d3fd26 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -390,6 +390,24 @@ sub read_config { } } +sub config_regexp { + my ($regex) = @_; + my @ret; + eval { + @ret = Git::command( + 'config', + '--name-only', + '--get-regexp', + $regex, + ); + 1; + } or do { + # If we have no keys we're OK, otherwise re-throw + die $@ if $@->value != 1; + }; + return @ret; +} + # sendemail.identity yields to --identity. We must parse this # special-case first before the rest of the config is read. $identity = Git::config(@repo, "sendemail.identity"); @@ -488,7 +506,7 @@ unless ($rc) { usage(); } -if ($forbid_sendmail_variables && (scalar Git::config_regexp("^sendmail[.]")) != 0) { +if ($forbid_sendmail_variables && (scalar config_regexp("^sendmail[.]")) != 0) { die __("fatal: found configuration options for 'sendmail'\n" . "git-send-email is configured with the sendemail.* options - note the 'e'.\n" . "Set sendemail.forbidSendmailVariables to false to disable this check.\n"); From 9264d29bf6da9e1789d71807dd72bbd4fb447887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:45 +0200 Subject: [PATCH 140/397] send-email: lazily load config for a big speedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce the time it takes git-send-email to get to even the most trivial of tasks (such as serving up its "-h" output) by first listing config keys that exist, and only then only call e.g. "git config --bool" on them if they do. Over a lot of runs this speeds the time to "-h" up for me from ~250ms to ~150ms, and the runtime of t9001-send-email.sh goes from ~25s to ~20s. This introduces a race condition where we'll do the "wrong" thing if a config key were to be inserted between us discovering the list and calling read_config(), i.e. we won't know about the racily added key. In theory this is a change in behavior, in practice it doesn't matter. The config_regexp() function being changed here was added in dd84e528a34 (git-send-email: die if sendmail.* config is set, 2020-07-23) for use by git-send-email. So we can change its odd return value in the case where no values are found by "git config". The difference in the *.pm code would matter if it was invoked in scalar context, but now it no longer is. Arguably this caching belongs in Git.pm itself, but in lieu of modifying it for all its callers let's only do this for "git send-email". The other big potential win would be "git svn", but unlike "git send-email" it doesn't check tens of config variables one at a time at startup (in my brief testing it doesn't check any). Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 73e3d3fd26..de62cbf250 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -347,11 +347,13 @@ $SIG{INT} = \&signal_handler; # Read our sendemail.* config sub read_config { - my ($configured, $prefix) = @_; + my ($known_keys, $configured, $prefix) = @_; foreach my $setting (keys %config_bool_settings) { my $target = $config_bool_settings{$setting}; - my $v = Git::config_bool(@repo, "$prefix.$setting"); + my $key = "$prefix.$setting"; + next unless exists $known_keys->{$key}; + my $v = Git::config_bool(@repo, $key); next unless defined $v; next if $configured->{$setting}++; $$target = $v; @@ -359,8 +361,10 @@ sub read_config { foreach my $setting (keys %config_path_settings) { my $target = $config_path_settings{$setting}; + my $key = "$prefix.$setting"; + next unless exists $known_keys->{$key}; if (ref($target) eq "ARRAY") { - my @values = Git::config_path(@repo, "$prefix.$setting"); + my @values = Git::config_path(@repo, $key); next unless @values; next if $configured->{$setting}++; @$target = @values; @@ -375,14 +379,16 @@ sub read_config { foreach my $setting (keys %config_settings) { my $target = $config_settings{$setting}; + my $key = "$prefix.$setting"; + next unless exists $known_keys->{$key}; if (ref($target) eq "ARRAY") { - my @values = Git::config(@repo, "$prefix.$setting"); + my @values = Git::config(@repo, $key); next unless @values; next if $configured->{$setting}++; @$target = @values; } else { - my $v = Git::config(@repo, "$prefix.$setting"); + my $v = Git::config(@repo, $key); next unless defined $v; next if $configured->{$setting}++; $$target = $v; @@ -408,9 +414,20 @@ sub config_regexp { return @ret; } +# Save ourselves a lot of work of shelling out to 'git config' (it +# parses 'bool' etc.) by only doing so for config keys that exist. +my %known_config_keys; +{ + my @known_config_keys = config_regexp("^sende?mail[.]"); + @known_config_keys{@known_config_keys} = (); +} + # sendemail.identity yields to --identity. We must parse this # special-case first before the rest of the config is read. -$identity = Git::config(@repo, "sendemail.identity"); +{ + my $key = "sendemail.identity"; + $identity = Git::config(@repo, $key) if exists $known_config_keys{$key}; +} my $rc = GetOptions( "identity=s" => \$identity, "no-identity" => \$no_identity, @@ -421,8 +438,8 @@ undef $identity if $no_identity; # Now we know enough to read the config { my %configured; - read_config(\%configured, "sendemail.$identity") if defined $identity; - read_config(\%configured, "sendemail"); + read_config(\%known_config_keys, \%configured, "sendemail.$identity") if defined $identity; + read_config(\%known_config_keys, \%configured, "sendemail"); } # Begin by accumulating all the variables (defined above), that we will end up @@ -506,7 +523,7 @@ unless ($rc) { usage(); } -if ($forbid_sendmail_variables && (scalar config_regexp("^sendmail[.]")) != 0) { +if ($forbid_sendmail_variables && grep { /^sendmail/s } keys %known_config_keys) { die __("fatal: found configuration options for 'sendmail'\n" . "git-send-email is configured with the sendemail.* options - note the 'e'.\n" . "Set sendemail.forbidSendmailVariables to false to disable this check.\n"); From fef381e6dcbc328c4ccabaf4e8d4cdf19d1aba00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:46 +0200 Subject: [PATCH 141/397] send-email: lazily shell out to "git var" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimize git-send-email by only shelling out to "git var" if we need to. This is easily done by re-inventing our own small version of perl's Memoize module. I suppose I could just use Memoize itself, but in a subsequent patch I'll be micro-optimizing send-email's use of dependencies. Using Memoize is a measly extra 5-10 milliseconds, but as we'll see that'll end up mattering for us in the end. This brings the runtime of a plain "send-email" from around ~160-170ms to ~140m-150ms. The runtime of the tests is around the same, or around ~20s. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index de62cbf250..38408ec75a 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -588,8 +588,18 @@ if (0) { } my ($repoauthor, $repocommitter); -($repoauthor) = Git::ident_person(@repo, 'author'); -($repocommitter) = Git::ident_person(@repo, 'committer'); +{ + my %cache; + my ($author, $committer); + my $common = sub { + my ($what) = @_; + return $cache{$what} if exists $cache{$what}; + ($cache{$what}) = Git::ident_person(@repo, $what); + return $cache{$what}; + }; + $repoauthor = sub { $common->('author') }; + $repocommitter = sub { $common->('committer') }; +} sub parse_address_line { return map { $_->format } Mail::Address->parse($_[0]); @@ -777,7 +787,7 @@ if ($compose) { or die sprintf(__("Failed to open for writing %s: %s"), $compose_filename, $!); - my $tpl_sender = $sender || $repoauthor || $repocommitter || ''; + my $tpl_sender = $sender || $repoauthor->() || $repocommitter->() || ''; my $tpl_subject = $initial_subject || ''; my $tpl_in_reply_to = $initial_in_reply_to || ''; my $tpl_reply_to = $reply_to || ''; @@ -983,7 +993,7 @@ if (defined $sender) { $sender =~ s/^\s+|\s+$//g; ($sender) = expand_aliases($sender); } else { - $sender = $repoauthor || $repocommitter || ''; + $sender = $repoauthor->() || $repocommitter->() || ''; } # $sender could be an already sanitized address @@ -1132,7 +1142,7 @@ sub make_message_id { $uniq = "$message_id_stamp-$message_id_serial"; my $du_part; - for ($sender, $repocommitter, $repoauthor) { + for ($sender, $repocommitter->(), $repoauthor->()) { $du_part = extract_valid_address(sanitize_address($_)); last if (defined $du_part and $du_part ne ''); } From 4adbf387bfd4b03b838282757e0bdf67e8f9fc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:47 +0200 Subject: [PATCH 142/397] send-email: use function syntax instead of barewords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change calls like "__ 'foo'" to "__('foo')" so the Perl compiler doesn't have to guess that "__" is a function. This makes the code more readable. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 38408ec75a..44dc3f6eb1 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -706,7 +706,7 @@ sub is_format_patch_arg { if (defined($format_patch)) { return $format_patch; } - die sprintf(__ < Date: Fri, 28 May 2021 11:23:48 +0200 Subject: [PATCH 143/397] send-email: get rid of indirect object syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change indirect object syntax such as "new X ARGS" to "X->new(ARGS)". This allows perl to see what "new" is at compile-time without having loaded Term::ReadLine. This doesn't matter now, but will in a subsequent commit when we start lazily loading it. Let's do the same for the adjacent "FakeTerm" package for consistency, even though we're not going to conditionally load it. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 44dc3f6eb1..cc1027d877 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -194,11 +194,11 @@ my $repo = eval { Git->repository() }; my @repo = $repo ? ($repo) : (); my $term = eval { $ENV{"GIT_SEND_EMAIL_NOTTY"} - ? new Term::ReadLine 'git-send-email', \*STDIN, \*STDOUT - : new Term::ReadLine 'git-send-email'; + ? Term::ReadLine->new('git-send-email', \*STDIN, \*STDOUT) + : Term::ReadLine->new('git-send-email'); }; if ($@) { - $term = new FakeTerm "$@: going non-interactive"; + $term = FakeTerm->new("$@: going non-interactive"); } # Behavior modification variables From f4dc9432fd287bde9100488943baf3c6a04d90d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:49 +0200 Subject: [PATCH 144/397] send-email: lazily load modules for a big speedup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimize the time git-send-email takes to do even the simplest of things (such as serving up "-h") from around ~150ms to ~80ms-~90ms by lazily loading the modules it requires. Before this change Devel::TraceUse would report 99/97 used modules under NO_GETTEXT=[|Y], respectively. Now it's 52/37. It now takes ~15s to run t9001-send-email.sh, down from ~20s. Changing File::Spec::Functions::{catdir,catfile} to invoking class methods on File::Spec itself is idiomatic. See [1] for a more elaborate explanation, the resulting code behaves the same way, just without the now-pointless function wrapper. 1. http://lore.kernel.org/git/8735u8mmj9.fsf@evledraar.gmail.com Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 71 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index cc1027d877..a8949c9d31 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -19,20 +19,10 @@ use 5.008; use strict; use warnings $ENV{GIT_PERL_FATAL_WARNINGS} ? qw(FATAL all) : (); -use POSIX qw/strftime/; -use Term::ReadLine; use Getopt::Long; -use Text::ParseWords; -use Term::ANSIColor; -use File::Temp qw/ tempdir tempfile /; -use File::Spec::Functions qw(catdir catfile); use Git::LoadCPAN::Error qw(:try); -use Cwd qw(abs_path cwd); use Git; use Git::I18N; -use Net::Domain (); -use Net::SMTP (); -use Git::LoadCPAN::Mail::Address; Getopt::Long::Configure qw/ pass_through /; @@ -166,7 +156,6 @@ sub format_2822_time { ); } -my $have_email_valid = eval { require Email::Valid; 1 }; my $smtp; my $auth; my $num_sent = 0; @@ -192,14 +181,6 @@ my (@config_bcc, @getopt_bcc); my $repo = eval { Git->repository() }; my @repo = $repo ? ($repo) : (); -my $term = eval { - $ENV{"GIT_SEND_EMAIL_NOTTY"} - ? Term::ReadLine->new('git-send-email', \*STDIN, \*STDOUT) - : Term::ReadLine->new('git-send-email'); -}; -if ($@) { - $term = FakeTerm->new("$@: going non-interactive"); -} # Behavior modification variables my ($quiet, $dry_run) = (0, 0); @@ -319,9 +300,9 @@ my %config_path_settings = ( # Handle Uncouth Termination sub signal_handler { - # Make text normal - print color("reset"), "\n"; + require Term::ANSIColor; + print Term::ANSIColor::color("reset"), "\n"; # SMTP password masked system "stty echo"; @@ -602,11 +583,13 @@ my ($repoauthor, $repocommitter); } sub parse_address_line { + require Git::LoadCPAN::Mail::Address; return map { $_->format } Mail::Address->parse($_[0]); } sub split_addrs { - return quotewords('\s*,\s*', 1, @_); + require Text::ParseWords; + return Text::ParseWords::quotewords('\s*,\s*', 1, @_); } my %aliases; @@ -655,10 +638,11 @@ my %parse_alias = ( s/\\"/"/g foreach @addr; $aliases{$alias} = \@addr }}}, - mailrc => sub { my $fh = shift; while (<$fh>) { + mailrc => sub { my $fh = shift; while (<$fh>) { if (/^alias\s+(\S+)\s+(.*?)\s*$/) { + require Text::ParseWords; # spaces delimit multiple addresses - $aliases{$1} = [ quotewords('\s+', 0, $2) ]; + $aliases{$1} = [ Text::ParseWords::quotewords('\s+', 0, $2) ]; }}}, pine => sub { my $fh = shift; my $f='\t[^\t]*'; for (my $x = ''; defined($x); $x = $_) { @@ -730,7 +714,8 @@ while (defined(my $f = shift @ARGV)) { opendir my $dh, $f or die sprintf(__("Failed to opendir %s: %s"), $f, $!); - push @files, grep { -f $_ } map { catfile($f, $_) } + require File::Spec; + push @files, grep { -f $_ } map { File::Spec->catfile($f, $_) } sort readdir $dh; closedir $dh; } elsif ((-f $f or -p $f) and !is_format_patch_arg($f)) { @@ -743,7 +728,8 @@ while (defined(my $f = shift @ARGV)) { if (@rev_list_opts) { die __("Cannot run git format-patch from outside a repository\n") unless $repo; - push @files, $repo->command('format-patch', '-o', tempdir(CLEANUP => 1), @rev_list_opts); + require File::Temp; + push @files, $repo->command('format-patch', '-o', File::Temp::tempdir(CLEANUP => 1), @rev_list_opts); } @files = handle_backup_files(@files); @@ -780,9 +766,10 @@ sub get_patch_subject { if ($compose) { # Note that this does not need to be secure, but we will make a small # effort to have it be unique + require File::Temp; $compose_filename = ($repo ? - tempfile(".gitsendemail.msg.XXXXXX", DIR => $repo->repo_path()) : - tempfile(".gitsendemail.msg.XXXXXX", DIR => "."))[1]; + File::Temp::tempfile(".gitsendemail.msg.XXXXXX", DIR => $repo->repo_path()) : + File::Temp::tempfile(".gitsendemail.msg.XXXXXX", DIR => "."))[1]; open my $c, ">", $compose_filename or die sprintf(__("Failed to open for writing %s: %s"), $compose_filename, $!); @@ -889,6 +876,19 @@ EOT3 do_edit(@files); } +sub term { + my $term = eval { + require Term::ReadLine; + $ENV{"GIT_SEND_EMAIL_NOTTY"} + ? Term::ReadLine->new('git-send-email', \*STDIN, \*STDOUT) + : Term::ReadLine->new('git-send-email'); + }; + if ($@) { + $term = FakeTerm->new("$@: going non-interactive"); + } + return $term; +} + sub ask { my ($prompt, %arg) = @_; my $valid_re = $arg{valid_re}; @@ -896,6 +896,7 @@ sub ask { my $confirm_only = $arg{confirm_only}; my $resp; my $i = 0; + my $term = term(); return defined $default ? $default : undef unless defined $term->IN and defined fileno($term->IN) and defined $term->OUT and defined fileno($term->OUT); @@ -1076,6 +1077,7 @@ sub extract_valid_address { return $address if ($address =~ /^($local_part_regexp)$/); $address =~ s/^\s*<(.*)>\s*$/$1/; + my $have_email_valid = eval { require Email::Valid; 1 }; if ($have_email_valid) { return scalar Email::Valid->address($address); } @@ -1135,7 +1137,8 @@ my ($message_id_stamp, $message_id_serial); sub make_message_id { my $uniq; if (!defined $message_id_stamp) { - $message_id_stamp = strftime("%Y%m%d%H%M%S.$$", gmtime(time)); + require POSIX; + $message_id_stamp = POSIX::strftime("%Y%m%d%H%M%S.$$", gmtime(time)); $message_id_serial = 0; } $message_id_serial++; @@ -1305,6 +1308,7 @@ sub valid_fqdn { sub maildomain_net { my $maildomain; + require Net::Domain; my $domain = Net::Domain::domainname(); $maildomain = $domain if valid_fqdn($domain); @@ -1315,6 +1319,7 @@ sub maildomain_mta { my $maildomain; for my $host (qw(mailhost localhost)) { + require Net::SMTP; my $smtp = Net::SMTP->new($host); if (defined $smtp) { my $domain = $smtp->domain; @@ -1994,13 +1999,15 @@ sub validate_patch { if ($repo) { my $hooks_path = $repo->command_oneline('rev-parse', '--git-path', 'hooks'); - my $validate_hook = catfile($hooks_path, + require File::Spec; + my $validate_hook = File::Spec->catfile($hooks_path, 'sendemail-validate'); my $hook_error; if (-x $validate_hook) { - my $target = abs_path($fn); + require Cwd; + my $target = Cwd::abs_path($fn); # The hook needs a correct cwd and GIT_DIR. - my $cwd_save = cwd(); + my $cwd_save = Cwd::cwd(); chdir($repo->wc_path() or $repo->repo_path()) or die("chdir: $!"); local $ENV{"GIT_DIR"} = $repo->repo_path(); From 5a544a4e11e2a08a813215c1b9cc80cc1555c7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:50 +0200 Subject: [PATCH 145/397] perl: lazily load some common Git.pm setup code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of unconditionally requiring modules such as File::Spec, let's only load them when needed. This speeds up code that only needs a subset of the features Git.pm provides. This brings a plain invocation of "git send-email" down from 52/37 loaded modules under NO_GETTEXT=[|Y] to 39/18, and it now takes ~60-~70ms instead of ~80-~90ms. The runtime of t9001-send-email.sh test is down to ~13s from ~15s. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- perl/Git.pm | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/perl/Git.pm b/perl/Git.pm index 02eacef0c2..5562c0cede 100644 --- a/perl/Git.pm +++ b/perl/Git.pm @@ -11,9 +11,6 @@ use 5.008; use strict; use warnings $ENV{GIT_PERL_FATAL_WARNINGS} ? qw(FATAL all) : (); -use File::Temp (); -use File::Spec (); - BEGIN { our ($VERSION, @ISA, @EXPORT, @EXPORT_OK); @@ -103,12 +100,9 @@ increase notwithstanding). =cut -use Carp qw(carp croak); # but croak is bad - throw instead +sub carp { require Carp; goto &Carp::carp } +sub croak { require Carp; goto &Carp::croak } use Git::LoadCPAN::Error qw(:try); -use Cwd qw(abs_path cwd); -use IPC::Open2 qw(open2); -use Fcntl qw(SEEK_SET SEEK_CUR); -use Time::Local qw(timegm); } @@ -191,13 +185,15 @@ sub repository { $dir = undef; }; + require Cwd; if ($dir) { + require File::Spec; File::Spec->file_name_is_absolute($dir) or $dir = $opts{Directory} . '/' . $dir; - $opts{Repository} = abs_path($dir); + $opts{Repository} = Cwd::abs_path($dir); # If --git-dir went ok, this shouldn't die either. my $prefix = $search->command_oneline('rev-parse', '--show-prefix'); - $dir = abs_path($opts{Directory}) . '/'; + $dir = Cwd::abs_path($opts{Directory}) . '/'; if ($prefix) { if (substr($dir, -length($prefix)) ne $prefix) { throw Error::Simple("rev-parse confused me - $dir does not have trailing $prefix"); @@ -223,7 +219,7 @@ sub repository { throw Error::Simple("fatal: Not a git repository: $dir"); } - $opts{Repository} = abs_path($dir); + $opts{Repository} = Cwd::abs_path($dir); } delete $opts{Directory}; @@ -408,10 +404,12 @@ sub command_bidi_pipe { my $cwd_save = undef; if ($self) { shift; - $cwd_save = cwd(); + require Cwd; + $cwd_save = Cwd::cwd(); _setup_git_cmd_env($self); } - $pid = open2($in, $out, 'git', @_); + require IPC::Open2; + $pid = IPC::Open2::open2($in, $out, 'git', @_); chdir($cwd_save) if $cwd_save; return ($pid, $in, $out, join(' ', @_)); } @@ -538,7 +536,8 @@ sub get_tz_offset { my $t = shift || time; my @t = localtime($t); $t[5] += 1900; - my $gm = timegm(@t); + require Time::Local; + my $gm = Time::Local::timegm(@t); my $sign = qw( + + - )[ $gm <=> $t ]; return sprintf("%s%02d%02d", $sign, (gmtime(abs($t - $gm)))[2,1]); } @@ -1340,6 +1339,7 @@ sub _temp_cache { my $n = $name; $n =~ s/\W/_/g; # no strange chars + require File::Temp; ($$temp_fd, $fname) = File::Temp::tempfile( "Git_${n}_XXXXXX", UNLINK => 1, DIR => $tmpdir, ) or throw Error::Simple("couldn't open new temp file"); @@ -1362,9 +1362,9 @@ sub temp_reset { truncate $temp_fd, 0 or throw Error::Simple("couldn't truncate file"); - sysseek($temp_fd, 0, SEEK_SET) and seek($temp_fd, 0, SEEK_SET) + sysseek($temp_fd, 0, Fcntl::SEEK_SET()) and seek($temp_fd, 0, Fcntl::SEEK_SET()) or throw Error::Simple("couldn't seek to beginning of file"); - sysseek($temp_fd, 0, SEEK_CUR) == 0 and tell($temp_fd) == 0 + sysseek($temp_fd, 0, Fcntl::SEEK_CUR()) == 0 and tell($temp_fd) == 0 or throw Error::Simple("expected file position to be reset"); } From c95e3a3f0b8107b5dc7eac9dfdb9e5238280c9fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:51 +0200 Subject: [PATCH 146/397] send-email: move trivial config handling to Perl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimize the startup time of git-send-email by using an amended config_regexp() function to retrieve the list of config keys and values we're interested in. For boolean keys we can handle the [true|false] case ourselves, and the "--get" case didn't need any parsing. Let's leave "--path" and other "--bool" cases to "git config". I'm not bothering with the "undef" or "" case (true and false, respectively), let's just punt on those and others and have "git config --type=bool" handle it. The "grep { defined } @values" here covers a rather subtle case. For list values such as sendemail.to it is possible as with any other config key to provide a plain "-c sendemail.to", i.e. to set the key as a boolean true. In that case the Git::config() API will return an empty string, but this new parser will correctly return "undef". However, that means we can end up with "undef" in the middle of a list. E.g. for sendemail.smtpserveroption in conjuction with sendemail.smtpserver as a path this would have produce a warning. For most of the other keys we'd behave the same despite the subtle change in the value, e.g. sendemail.to would behave the same because Mail::Address->parse() happens to return an empty list if fed "undef". For the boolean values we were already prepared to handle these variables being initialized as undef anyway. This brings the runtime of "git send-email" from ~60-~70ms to a very steady ~40ms on my test box. We now run just one "git config" invocation on startup instead of 8, the exact number will differ based on the local sendemail.* config. I happen to have 8 of those set. This brings the runtime of t9001-send-email.sh from ~13s down to ~12s for me. The change there is less impressive as many of those tests set various config values, and we're also getting to the point of diminishing returns for optimizing "git send-email" itself. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index a8949c9d31..5791138683 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -334,7 +334,11 @@ sub read_config { my $target = $config_bool_settings{$setting}; my $key = "$prefix.$setting"; next unless exists $known_keys->{$key}; - my $v = Git::config_bool(@repo, $key); + my $v = (@{$known_keys->{$key}} == 1 && + (defined $known_keys->{$key}->[0] && + $known_keys->{$key}->[0] =~ /^(?:true|false)$/s)) + ? $known_keys->{$key}->[0] eq 'true' + : Git::config_bool(@repo, $key); next unless defined $v; next if $configured->{$setting}++; $$target = $v; @@ -363,13 +367,13 @@ sub read_config { my $key = "$prefix.$setting"; next unless exists $known_keys->{$key}; if (ref($target) eq "ARRAY") { - my @values = Git::config(@repo, $key); - next unless @values; + my @values = @{$known_keys->{$key}}; + @values = grep { defined } @values; next if $configured->{$setting}++; @$target = @values; } else { - my $v = Git::config(@repo, $key); + my $v = $known_keys->{$key}->[0]; next unless defined $v; next if $configured->{$setting}++; $$target = $v; @@ -381,12 +385,19 @@ sub config_regexp { my ($regex) = @_; my @ret; eval { - @ret = Git::command( + my $ret = Git::command( 'config', - '--name-only', + '--null', '--get-regexp', $regex, ); + @ret = map { + # We must always return ($k, $v) here, since + # empty config values will be just "key\0", + # not "key\nvalue\0". + my ($k, $v) = split /\n/, $_, 2; + ($k, $v); + } split /\0/, $ret; 1; } or do { # If we have no keys we're OK, otherwise re-throw @@ -399,8 +410,10 @@ sub config_regexp { # parses 'bool' etc.) by only doing so for config keys that exist. my %known_config_keys; { - my @known_config_keys = config_regexp("^sende?mail[.]"); - @known_config_keys{@known_config_keys} = (); + my @kv = config_regexp("^sende?mail[.]"); + while (my ($k, $v) = splice @kv, 0, 2) { + push @{$known_config_keys{$k}} => $v; + } } # sendemail.identity yields to --identity. We must parse this From 17530b2ed2eac62706b8bbbcf93f62866f651ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Fri, 28 May 2021 11:23:52 +0200 Subject: [PATCH 147/397] perl: nano-optimize by replacing Cwd::cwd() with Cwd::getcwd() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It has been pointed out[1] that cwd() invokes "pwd(1)" while getcwd() is a Perl-native XS function. For what we're using these for we can use getcwd(). The performance difference is miniscule, we're saving on the order of a millisecond or so, see [2] below for the benchmark. I don't think this matters in practice for optimizing git-send-email or perl execution (unlike the patches leading up to this one). But let's do it regardless of that, if only so we don't have to think about this as a low-hanging fruit anymore. 1. https://lore.kernel.org/git/20210512180517.GA11354@dcvr/ 2. $ perl -MBenchmark=:all -MCwd -wE 'cmpthese(10000, { getcwd => sub { getcwd }, cwd => sub { cwd }, pwd => sub { system "pwd >/dev/null" }})' (warning: too few iterations for a reliable count) Rate pwd cwd getcwd pwd 982/s -- -48% -100% cwd 1890/s 92% -- -100% getcwd 10000000000000000000/s 1018000000000000000% 529000000000000064% - Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- git-send-email.perl | 2 +- perl/Git.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/git-send-email.perl b/git-send-email.perl index 5791138683..0efe85c0b0 100755 --- a/git-send-email.perl +++ b/git-send-email.perl @@ -2020,7 +2020,7 @@ sub validate_patch { require Cwd; my $target = Cwd::abs_path($fn); # The hook needs a correct cwd and GIT_DIR. - my $cwd_save = Cwd::cwd(); + my $cwd_save = Cwd::getcwd(); chdir($repo->wc_path() or $repo->repo_path()) or die("chdir: $!"); local $ENV{"GIT_DIR"} = $repo->repo_path(); diff --git a/perl/Git.pm b/perl/Git.pm index 5562c0cede..090a7df63f 100644 --- a/perl/Git.pm +++ b/perl/Git.pm @@ -405,7 +405,7 @@ sub command_bidi_pipe { if ($self) { shift; require Cwd; - $cwd_save = Cwd::cwd(); + $cwd_save = Cwd::getcwd(); _setup_git_cmd_env($self); } require IPC::Open2; From d1e7c2cac9f44f411700397a91b2fb1ec640cdd4 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Sun, 30 May 2021 13:31:35 +0200 Subject: [PATCH 148/397] completion: add --anchored to diff's options This flag was introduced in 2477ab2e (diff: support anchoring line(s), 2017-11-27) but back then, the bash completion script did not learn about the new flag. Add it. Signed-off-by: Thomas Braun Signed-off-by: Junio C Hamano --- contrib/completion/git-completion.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 3c5739b905..4dd268470a 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1729,6 +1729,7 @@ __git_diff_common_options="--stat --numstat --shortstat --summary --indent-heuristic --no-indent-heuristic --textconv --no-textconv --patch --no-patch + --anchored= " __git_diff_difftool_options="--cached --staged --pickaxe-all --pickaxe-regex From 28abf260a52b2fb79342d5010e921602e078149f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Tue, 1 Jun 2021 02:05:59 +0200 Subject: [PATCH 149/397] builtin/fsck.c: don't conflate "int" and "enum" in callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix a warning on AIX's xlc compiler that's been emitted since my a1aad71601a (fsck.h: use "enum object_type" instead of "int", 2021-03-28): "builtin/fsck.c", line 805.32: 1506-068 (W) Operation between types "int(*)(struct object*,enum object_type,void*,struct fsck_options*)" and "int(*)(struct object*,int,void*,struct fsck_options*)" is not allowed. I.e. it complains about us assigning a function with a prototype "int" where we're expecting "enum object_type". Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- builtin/fsck.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/fsck.c b/builtin/fsck.c index 70ff95837a..d606c4444d 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -109,7 +109,8 @@ static int fsck_error_func(struct fsck_options *o, static struct object_array pending; -static int mark_object(struct object *obj, int type, void *data, struct fsck_options *options) +static int mark_object(struct object *obj, enum object_type type, + void *data, struct fsck_options *options) { struct object *parent = data; From 230356ba704fc36e848e94b13c5b92acfa4c6fc6 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:16 +0000 Subject: [PATCH 150/397] t4202: split testcase for invalid HEAD symref and HEAD hash Reftable will prohibit invalid hashes at the storage level, but git-symbolic-ref can still create branches ending in ".lock". Signed-off-by: Han-Wen Nienhuys Signed-off-by: Junio C Hamano --- t/t4202-log.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/t/t4202-log.sh b/t/t4202-log.sh index 350cfa3593..327991fcea 100755 --- a/t/t4202-log.sh +++ b/t/t4202-log.sh @@ -1834,14 +1834,18 @@ test_expect_success 'log --graph --no-walk is forbidden' ' test_must_fail git log --graph --no-walk ' -test_expect_success 'log diagnoses bogus HEAD' ' +test_expect_success 'log diagnoses bogus HEAD hash' ' git init empty && + test_when_finished "rm -rf empty" && test_must_fail git -C empty log 2>stderr && test_i18ngrep does.not.have.any.commits stderr && echo 1234abcd >empty/.git/refs/heads/main && test_must_fail git -C empty log 2>stderr && - test_i18ngrep broken stderr && - echo "ref: refs/heads/invalid.lock" >empty/.git/HEAD && + test_i18ngrep broken stderr' + +test_expect_success 'log diagnoses bogus HEAD symref' ' + git init empty && + git --git-dir empty/.git symbolic-ref HEAD refs/heads/invalid.lock && test_must_fail git -C empty log 2>stderr && test_i18ngrep broken stderr && test_must_fail git -C empty log --default totally-bogus 2>stderr && From 0221eb86782bdd2cbbe9d6223d7ac9bcb61961b1 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:17 +0000 Subject: [PATCH 151/397] t/helper/ref-store: initialize oid in resolve-ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will print $ZERO_OID when asking for a non-existent ref from the test-helper. Since resolve-ref provides direct access to refs_resolve_ref_unsafe(), it provides a reliable mechanism for accessing REFNAME, while avoiding the implicit resolution to refs/heads/REFNAME. Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Han-Wen Nienhuys Signed-off-by: Junio C Hamano --- t/helper/test-ref-store.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index bba5f841c6..b314b81a45 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -118,7 +118,7 @@ static int cmd_for_each_ref(struct ref_store *refs, const char **argv) static int cmd_resolve_ref(struct ref_store *refs, const char **argv) { - struct object_id oid; + struct object_id oid = *null_oid(); const char *refname = notnull(*argv++, "refname"); int resolve_flags = arg_flags(*argv++, "resolve-flags"); int flags; From 62038c81f35455745066e6f746d5f0b0a1e1f917 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:18 +0000 Subject: [PATCH 152/397] t9300: check ref existence using test-helper rather than a file system check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t9300-fast-import.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index 5c47ac4465..1aea943bef 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -392,7 +392,7 @@ test_expect_success 'B: accept branch name "TEMP_TAG"' ' git gc git prune" && git fast-import Date: Mon, 31 May 2021 16:56:19 +0000 Subject: [PATCH 153/397] t5601: read HEAD using rev-parse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t5601-clone.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 329ae599fd..7223372c76 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -305,7 +305,8 @@ test_expect_success 'clone from original with relative alternate' ' test_expect_success 'clone checking out a tag' ' git clone --branch=some-tag src dst.tag && GIT_DIR=src/.git git rev-parse some-tag >expected && - test_cmp expected dst.tag/.git/HEAD && + GIT_DIR=dst.tag/.git git rev-parse HEAD >actual && + test_cmp expected actual && GIT_DIR=dst.tag/.git git config remote.origin.fetch >fetch.actual && echo "+refs/heads/*:refs/remotes/origin/*" >fetch.expected && test_cmp fetch.expected fetch.actual From 9910cbb6f988341001ded0c879fb54865a19d592 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:20 +0000 Subject: [PATCH 154/397] t1401: use tar to snapshot and restore repo state This is agnostic to the ref storage format Signed-off-by: Han-Wen Nienhuys Signed-off-by: Junio C Hamano --- t/t1401-symbolic-ref.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/t/t1401-symbolic-ref.sh b/t/t1401-symbolic-ref.sh index a4ebb0b65f..7a9629fb9f 100755 --- a/t/t1401-symbolic-ref.sh +++ b/t/t1401-symbolic-ref.sh @@ -7,9 +7,16 @@ test_description='basic symbolic-ref tests' # the git repo, meaning that further tests will operate on # the surrounding git repo instead of the trash directory. reset_to_sane() { - echo ref: refs/heads/foo >.git/HEAD + rm -rf .git && + "$TAR" xf .git.tar } +test_expect_success 'setup' ' + git symbolic-ref HEAD refs/heads/foo && + test_commit file && + "$TAR" cf .git.tar .git/ +' + test_expect_success 'symbolic-ref writes HEAD' ' git symbolic-ref HEAD refs/heads/foo && echo ref: refs/heads/foo >expect && From b1259ecff9a51de22ac8a70fc26c20dde8a18f80 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:21 +0000 Subject: [PATCH 155/397] t1401-symbolic-ref: avoid direct filesystem access MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use symbolic-ref and rev-parse to inspect refs. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1401-symbolic-ref.sh | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/t/t1401-symbolic-ref.sh b/t/t1401-symbolic-ref.sh index 7a9629fb9f..132a1b885a 100755 --- a/t/t1401-symbolic-ref.sh +++ b/t/t1401-symbolic-ref.sh @@ -17,14 +17,9 @@ test_expect_success 'setup' ' "$TAR" cf .git.tar .git/ ' -test_expect_success 'symbolic-ref writes HEAD' ' - git symbolic-ref HEAD refs/heads/foo && - echo ref: refs/heads/foo >expect && - test_cmp expect .git/HEAD -' - -test_expect_success 'symbolic-ref reads HEAD' ' - echo refs/heads/foo >expect && +test_expect_success 'symbolic-ref read/write roundtrip' ' + git symbolic-ref HEAD refs/heads/read-write-roundtrip && + echo refs/heads/read-write-roundtrip >expect && git symbolic-ref HEAD >actual && test_cmp expect actual ' @@ -32,12 +27,13 @@ test_expect_success 'symbolic-ref reads HEAD' ' test_expect_success 'symbolic-ref refuses non-ref for HEAD' ' test_must_fail git symbolic-ref HEAD foo ' + reset_to_sane test_expect_success 'symbolic-ref refuses bare sha1' ' - echo content >file && git add file && git commit -m one && test_must_fail git symbolic-ref HEAD $(git rev-parse HEAD) ' + reset_to_sane test_expect_success 'HEAD cannot be removed' ' @@ -49,16 +45,16 @@ reset_to_sane test_expect_success 'symbolic-ref can be deleted' ' git symbolic-ref NOTHEAD refs/heads/foo && git symbolic-ref -d NOTHEAD && - test_path_is_file .git/refs/heads/foo && - test_path_is_missing .git/NOTHEAD + git rev-parse refs/heads/foo && + test_must_fail git symbolic-ref NOTHEAD ' reset_to_sane test_expect_success 'symbolic-ref can delete dangling symref' ' git symbolic-ref NOTHEAD refs/heads/missing && git symbolic-ref -d NOTHEAD && - test_path_is_missing .git/refs/heads/missing && - test_path_is_missing .git/NOTHEAD + test_must_fail git rev-parse refs/heads/missing && + test_must_fail git symbolic-ref NOTHEAD ' reset_to_sane From 1142746cbbf016fe2568076d6f30b2b73b29e731 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:22 +0000 Subject: [PATCH 156/397] t1413: use tar to save and restore entire .git directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes the test independent of the particulars of the storage formats. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1413-reflog-detach.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/t/t1413-reflog-detach.sh b/t/t1413-reflog-detach.sh index bde05208ae..934688a1ee 100755 --- a/t/t1413-reflog-detach.sh +++ b/t/t1413-reflog-detach.sh @@ -7,8 +7,7 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh reset_state () { - git checkout main && - cp saved_reflog .git/logs/HEAD + rm -rf .git && "$TAR" xf .git-saved.tar } test_expect_success setup ' @@ -17,7 +16,7 @@ test_expect_success setup ' git branch side && test_tick && git commit --allow-empty -m second && - cat .git/logs/HEAD >saved_reflog + "$TAR" cf .git-saved.tar .git ' test_expect_success baseline ' From 0218ad5d5c41d7b4adc2ff817391d9e488d328d3 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:23 +0000 Subject: [PATCH 157/397] t1301: fix typo in error message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1301-shared-repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t1301-shared-repo.sh b/t/t1301-shared-repo.sh index ac947bff9f..84bf1970d8 100755 --- a/t/t1301-shared-repo.sh +++ b/t/t1301-shared-repo.sh @@ -124,7 +124,7 @@ test_expect_success POSIXPERM 'git reflog expire honors core.sharedRepository' ' : happy ;; *) - echo Ooops, .git/logs/refs/heads/main is not 0662 [$actual] + echo Ooops, .git/logs/refs/heads/main is not 066x [$actual] false ;; esac From 9c8e7e968c0a22774754f8c8577872dfbb308456 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:24 +0000 Subject: [PATCH 158/397] t5000: reformat indentation to the latest fashion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t5000-tar-tree.sh | 113 ++++++++++++++++++++++++-------------------- 1 file changed, 62 insertions(+), 51 deletions(-) diff --git a/t/t5000-tar-tree.sh b/t/t5000-tar-tree.sh index 7204799a0b..8c5867b6c8 100755 --- a/t/t5000-tar-tree.sh +++ b/t/t5000-tar-tree.sh @@ -111,25 +111,34 @@ test_expect_success 'setup' ' EOF ' -test_expect_success \ - 'populate workdir' \ - 'mkdir a && - echo simple textfile >a/a && - ten=0123456789 && hundred=$ten$ten$ten$ten$ten$ten$ten$ten$ten$ten && - echo long filename >a/four$hundred && - mkdir a/bin && - test-tool genrandom "frotz" 500000 >a/bin/sh && - printf "A\$Format:%s\$O" "$SUBSTFORMAT" >a/substfile1 && - printf "A not substituted O" >a/substfile2 && - if test_have_prereq SYMLINKS; then - ln -s a a/l1 - else - printf %s a > a/l1 - fi && - (p=long_path_to_a_file && cd a && - for depth in 1 2 3 4 5; do mkdir $p && cd $p; done && - echo text >file_with_long_path) && - (cd a && find .) | sort >a.lst' +test_expect_success 'populate workdir' ' + mkdir a && + echo simple textfile >a/a && + ten=0123456789 && + hundred="$ten$ten$ten$ten$ten$ten$ten$ten$ten$ten" && + echo long filename >"a/four$hundred" && + mkdir a/bin && + test-tool genrandom "frotz" 500000 >a/bin/sh && + printf "A\$Format:%s\$O" "$SUBSTFORMAT" >a/substfile1 && + printf "A not substituted O" >a/substfile2 && + if test_have_prereq SYMLINKS + then + ln -s a a/l1 + else + printf %s a >a/l1 + fi && + ( + p=long_path_to_a_file && + cd a && + for depth in 1 2 3 4 5 + do + mkdir $p && + cd $p + done && + echo text >file_with_long_path + ) && + (cd a && find .) | sort >a.lst +' test_expect_success \ 'add ignored file' \ @@ -147,18 +156,18 @@ test_expect_success 'setup export-subst' ' >a/substfile1 ' -test_expect_success \ - 'create bare clone' \ - 'git clone --bare . bare.git && - cp .git/info/attributes bare.git/info/attributes' +test_expect_success 'create bare clone' ' + git clone --bare . bare.git && + cp .git/info/attributes bare.git/info/attributes +' -test_expect_success \ - 'remove ignored file' \ - 'rm a/ignored' +test_expect_success 'remove ignored file' ' + rm a/ignored +' -test_expect_success \ - 'git archive' \ - 'git archive HEAD >b.tar' +test_expect_success 'git archive' ' + git archive HEAD >b.tar +' check_tar b @@ -194,26 +203,28 @@ check_added with_untracked2 untracked one/untracked check_added with_untracked2 untracked two/untracked test_expect_success 'git archive on large files' ' - test_config core.bigfilethreshold 1 && - git archive HEAD >b3.tar && - test_cmp_bin b.tar b3.tar + test_config core.bigfilethreshold 1 && + git archive HEAD >b3.tar && + test_cmp_bin b.tar b3.tar ' -test_expect_success \ - 'git archive in a bare repo' \ - '(cd bare.git && git archive HEAD) >b3.tar' +test_expect_success 'git archive in a bare repo' ' + git --git-dir bare.git archive HEAD >b3.tar +' -test_expect_success \ - 'git archive vs. the same in a bare repo' \ - 'test_cmp_bin b.tar b3.tar' +test_expect_success 'git archive vs. the same in a bare repo' ' + test_cmp_bin b.tar b3.tar +' -test_expect_success 'git archive with --output' \ - 'git archive --output=b4.tar HEAD && - test_cmp_bin b.tar b4.tar' +test_expect_success 'git archive with --output' ' + git archive --output=b4.tar HEAD && + test_cmp_bin b.tar b4.tar +' -test_expect_success 'git archive --remote' \ - 'git archive --remote=. HEAD >b5.tar && - test_cmp_bin b.tar b5.tar' +test_expect_success 'git archive --remote' ' + git archive --remote=. HEAD >b5.tar && + test_cmp_bin b.tar b5.tar +' test_expect_success 'git archive --remote with configured remote' ' git config remote.foo.url . && @@ -224,13 +235,13 @@ test_expect_success 'git archive --remote with configured remote' ' test_cmp_bin b.tar b5-nick.tar ' -test_expect_success \ - 'validate file modification time' \ - 'mkdir extract && - "$TAR" xf b.tar -C extract a/a && - test-tool chmtime --get extract/a/a >b.mtime && - echo "1117231200" >expected.mtime && - test_cmp expected.mtime b.mtime' +test_expect_success 'validate file modification time' ' + mkdir extract && + "$TAR" xf b.tar -C extract a/a && + test-tool chmtime --get extract/a/a >b.mtime && + echo "1117231200" >expected.mtime && + test_cmp expected.mtime b.mtime +' test_expect_success \ 'git get-tar-commit-id' \ From f1ed224753728b6defc6bc6e9058717a0645482d Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:25 +0000 Subject: [PATCH 159/397] t5000: inspect HEAD using git-rev-parse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t5000-tar-tree.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/t/t5000-tar-tree.sh b/t/t5000-tar-tree.sh index 8c5867b6c8..2c88d1c159 100755 --- a/t/t5000-tar-tree.sh +++ b/t/t5000-tar-tree.sh @@ -243,10 +243,11 @@ test_expect_success 'validate file modification time' ' test_cmp expected.mtime b.mtime ' -test_expect_success \ - 'git get-tar-commit-id' \ - 'git get-tar-commit-id b.commitid && - test_cmp .git/$(git symbolic-ref HEAD) b.commitid' +test_expect_success 'git get-tar-commit-id' ' + git get-tar-commit-id actual && + git rev-parse HEAD >expect && + test_cmp expect actual +' test_expect_success 'git archive with --output, override inferred format' ' git archive --format=tar --output=d4.zip HEAD && From fdc8acc7066650b967d8957bc24d58eb54b537f8 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:26 +0000 Subject: [PATCH 160/397] t7003: use rev-parse rather than FS inspection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t7003-filter-branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7003-filter-branch.sh b/t/t7003-filter-branch.sh index 1349e5b232..cf30055c88 100755 --- a/t/t7003-filter-branch.sh +++ b/t/t7003-filter-branch.sh @@ -395,7 +395,7 @@ test_expect_success '--prune-empty is able to prune root commit' ' test_expect_success '--prune-empty is able to prune entire branch' ' git branch prune-entire B && git filter-branch -f --prune-empty --index-filter "git update-index --remove A.t B.t" prune-entire && - test_path_is_missing .git/refs/heads/prune-entire && + test_must_fail git rev-parse refs/heads/prune-entire && test_must_fail git reflog exists refs/heads/prune-entire ' From 1fa9cf6ea14eba36d72eb48e82069012c6c8af90 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:27 +0000 Subject: [PATCH 161/397] t5304: restyle: trim empty lines, drop ':' before > MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t5304-prune.sh | 74 +++++++++++++----------------------------------- 1 file changed, 20 insertions(+), 54 deletions(-) diff --git a/t/t5304-prune.sh b/t/t5304-prune.sh index b447ce56a9..7f47f13c78 100755 --- a/t/t5304-prune.sh +++ b/t/t5304-prune.sh @@ -22,30 +22,25 @@ add_blob() { } test_expect_success setup ' - - : > file && + >file && git add file && test_tick && git commit -m initial && git gc - ' test_expect_success 'prune stale packs' ' - orig_pack=$(echo .git/objects/pack/*.pack) && - : > .git/objects/tmp_1.pack && - : > .git/objects/tmp_2.pack && + >.git/objects/tmp_1.pack && + >.git/objects/tmp_2.pack && test-tool chmtime =-86501 .git/objects/tmp_1.pack && git prune --expire 1.day && test_path_is_file $orig_pack && test_path_is_file .git/objects/tmp_2.pack && test_path_is_missing .git/objects/tmp_1.pack - ' test_expect_success 'prune --expire' ' - add_blob && git prune --expire=1.hour.ago && verbose test $((1 + $before)) = $(git count-objects | sed "s/ .*//") && @@ -54,11 +49,9 @@ test_expect_success 'prune --expire' ' git prune --expire 1.day && verbose test $before = $(git count-objects | sed "s/ .*//") && test_path_is_missing $BLOB_FILE - ' test_expect_success 'gc: implicit prune --expire' ' - add_blob && test-tool chmtime =-$((2*$week-30)) $BLOB_FILE && git gc && @@ -68,33 +61,25 @@ test_expect_success 'gc: implicit prune --expire' ' git gc && verbose test $before = $(git count-objects | sed "s/ .*//") && test_path_is_missing $BLOB_FILE - ' test_expect_success 'gc: refuse to start with invalid gc.pruneExpire' ' - git config gc.pruneExpire invalid && test_must_fail git gc - ' test_expect_success 'gc: start with ok gc.pruneExpire' ' - git config gc.pruneExpire 2.days.ago && git gc - ' test_expect_success 'prune: prune nonsense parameters' ' - test_must_fail git prune garbage && test_must_fail git prune --- && test_must_fail git prune --no-such-option - ' test_expect_success 'prune: prune unreachable heads' ' - git config core.logAllRefUpdates false && mv .git/logs .git/logs.old && : > file2 && @@ -104,11 +89,9 @@ test_expect_success 'prune: prune unreachable heads' ' git reset HEAD^ && git prune && test_must_fail git reset $tmp_head -- - ' test_expect_success 'prune: do not prune detached HEAD with no reflog' ' - git checkout --detach --quiet && git commit --allow-empty -m "detached commit" && # verify that there is no reflogs @@ -116,75 +99,61 @@ test_expect_success 'prune: do not prune detached HEAD with no reflog' ' test_path_is_missing .git/logs && git prune -n >prune_actual && test_must_be_empty prune_actual - ' test_expect_success 'prune: prune former HEAD after checking out branch' ' - head_oid=$(git rev-parse HEAD) && git checkout --quiet main && git prune -v >prune_actual && grep "$head_oid" prune_actual - ' test_expect_success 'prune: do not prune heads listed as an argument' ' - - : > file2 && + >file2 && git add file2 && git commit -m temporary && tmp_head=$(git rev-list -1 HEAD) && git reset HEAD^ && git prune -- $tmp_head && git reset $tmp_head -- - ' test_expect_success 'gc --no-prune' ' - add_blob && test-tool chmtime =-$((5001*$day)) $BLOB_FILE && git config gc.pruneExpire 2.days.ago && git gc --no-prune && verbose test 1 = $(git count-objects | sed "s/ .*//") && test_path_is_file $BLOB_FILE - ' test_expect_success 'gc respects gc.pruneExpire' ' - git config gc.pruneExpire 5002.days.ago && git gc && test_path_is_file $BLOB_FILE && git config gc.pruneExpire 5000.days.ago && git gc && test_path_is_missing $BLOB_FILE - ' test_expect_success 'gc --prune=' ' - add_blob && test-tool chmtime =-$((5001*$day)) $BLOB_FILE && git gc --prune=5002.days.ago && test_path_is_file $BLOB_FILE && git gc --prune=5000.days.ago && test_path_is_missing $BLOB_FILE - ' test_expect_success 'gc --prune=never' ' - add_blob && git gc --prune=never && test_path_is_file $BLOB_FILE && git gc --prune=now && test_path_is_missing $BLOB_FILE - ' test_expect_success 'gc respects gc.pruneExpire=never' ' - git config gc.pruneExpire never && add_blob && git gc && @@ -192,17 +161,14 @@ test_expect_success 'gc respects gc.pruneExpire=never' ' git config gc.pruneExpire now && git gc && test_path_is_missing $BLOB_FILE - ' test_expect_success 'prune --expire=never' ' - add_blob && git prune --expire=never && test_path_is_file $BLOB_FILE && git prune && test_path_is_missing $BLOB_FILE - ' test_expect_success 'gc: prune old objects after local clone' ' @@ -222,16 +188,16 @@ test_expect_success 'gc: prune old objects after local clone' ' test_expect_success 'garbage report in count-objects -v' ' test_when_finished "rm -f .git/objects/pack/fake*" && test_when_finished "rm -f .git/objects/pack/foo*" && - : >.git/objects/pack/foo && - : >.git/objects/pack/foo.bar && - : >.git/objects/pack/foo.keep && - : >.git/objects/pack/foo.pack && - : >.git/objects/pack/fake.bar && - : >.git/objects/pack/fake.keep && - : >.git/objects/pack/fake.pack && - : >.git/objects/pack/fake.idx && - : >.git/objects/pack/fake2.keep && - : >.git/objects/pack/fake3.idx && + >.git/objects/pack/foo && + >.git/objects/pack/foo.bar && + >.git/objects/pack/foo.keep && + >.git/objects/pack/foo.pack && + >.git/objects/pack/fake.bar && + >.git/objects/pack/fake.keep && + >.git/objects/pack/fake.pack && + >.git/objects/pack/fake.idx && + >.git/objects/pack/fake2.keep && + >.git/objects/pack/fake3.idx && git count-objects -v 2>stderr && grep "index file .git/objects/pack/fake.idx is too small" stderr && grep "^warning:" stderr | sort >actual && @@ -250,12 +216,12 @@ EOF test_expect_success 'clean pack garbage with gc' ' test_when_finished "rm -f .git/objects/pack/fake*" && test_when_finished "rm -f .git/objects/pack/foo*" && - : >.git/objects/pack/foo.keep && - : >.git/objects/pack/foo.pack && - : >.git/objects/pack/fake.idx && - : >.git/objects/pack/fake2.keep && - : >.git/objects/pack/fake2.idx && - : >.git/objects/pack/fake3.keep && + >.git/objects/pack/foo.keep && + >.git/objects/pack/foo.pack && + >.git/objects/pack/fake.idx && + >.git/objects/pack/fake2.keep && + >.git/objects/pack/fake2.idx && + >.git/objects/pack/fake3.keep && git gc && git count-objects -v 2>stderr && grep "^warning:" stderr | sort >actual && From d491f5ea07e5bb85be75060d3939894892f9478a Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:28 +0000 Subject: [PATCH 162/397] t5304: use "reflog expire --all" to clear the reflog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test checks that unreachable objects are really removed. For the test to work, it has to ensure that no reflog retain any reachable objects. Previously, it did this by manipulating the file system to remove reflog in the first test, and relying on git not updating the reflog if the relevant logfile doesn't exist in follow-up tests. Now, explicitly clear the reflog using 'reflog expire'. This reduces the dependency between test functions. It also is more amenable to use with reftable, which has no concept of (non)-existence of a reflog Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t5304-prune.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/t/t5304-prune.sh b/t/t5304-prune.sh index 7f47f13c78..7b850ae261 100755 --- a/t/t5304-prune.sh +++ b/t/t5304-prune.sh @@ -81,12 +81,12 @@ test_expect_success 'prune: prune nonsense parameters' ' test_expect_success 'prune: prune unreachable heads' ' git config core.logAllRefUpdates false && - mv .git/logs .git/logs.old && - : > file2 && + >file2 && git add file2 && git commit -m temporary && tmp_head=$(git rev-list -1 HEAD) && git reset HEAD^ && + git reflog expire --all && git prune && test_must_fail git reset $tmp_head -- ' @@ -94,9 +94,7 @@ test_expect_success 'prune: prune unreachable heads' ' test_expect_success 'prune: do not prune detached HEAD with no reflog' ' git checkout --detach --quiet && git commit --allow-empty -m "detached commit" && - # verify that there is no reflogs - # (should be removed and disabled by previous test) - test_path_is_missing .git/logs && + git reflog expire --all && git prune -n >prune_actual && test_must_be_empty prune_actual ' @@ -104,6 +102,7 @@ test_expect_success 'prune: do not prune detached HEAD with no reflog' ' test_expect_success 'prune: prune former HEAD after checking out branch' ' head_oid=$(git rev-parse HEAD) && git checkout --quiet main && + git reflog expire --all && git prune -v >prune_actual && grep "$head_oid" prune_actual ' From c305e667e0cad4acb40a4d408e44b823fa47c111 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:29 +0000 Subject: [PATCH 163/397] test-lib: provide test prereq REFFILES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REFFILES can be used to mark tests that are specific to the packed/loose ref storage format and its limitations. Marking such tests is a preparation for introducing the reftable storage backend. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/README | 6 ++++++ t/test-lib.sh | 2 ++ 2 files changed, 8 insertions(+) diff --git a/t/README b/t/README index fd9375b146..723bd3387f 100644 --- a/t/README +++ b/t/README @@ -1114,6 +1114,12 @@ use these, and "test_set_prereq" for how to define your own. Git wasn't compiled with NO_PTHREADS=YesPlease. + - REFFILES + + Test is specific to packed/loose ref storage, and should be + disabled for other ref storage backends + + Tips for Writing Tests ---------------------- diff --git a/t/test-lib.sh b/t/test-lib.sh index d3f6af6a65..ea7397c633 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1481,6 +1481,8 @@ parisc* | hppa*) ;; esac +test_set_prereq REFFILES + ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1 test -z "$NO_PERL" && test_set_prereq PERL test -z "$NO_PTHREADS" && test_set_prereq PTHREADS From 759d02d1aeb2e9d51afe9f948fe0dc15093722a0 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:30 +0000 Subject: [PATCH 164/397] t1407: require REFFILES for for_each_reflog test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add extensive comment why this test needs a REFFILES annotation. I tried forcing universal reflog creation with core.logAllRefUpdates=true, but that apparently also doesn't cause reflogs to be created for pseudorefs Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1407-worktree-ref-store.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/t/t1407-worktree-ref-store.sh b/t/t1407-worktree-ref-store.sh index d3fe777511..ad8006c813 100755 --- a/t/t1407-worktree-ref-store.sh +++ b/t/t1407-worktree-ref-store.sh @@ -52,7 +52,14 @@ test_expect_success 'create_symref(FOO, refs/heads/main)' ' test_cmp expected actual ' -test_expect_success 'for_each_reflog()' ' +# Some refs (refs/bisect/*, pseudorefs) are kept per worktree, so they should +# only appear in the for-each-reflog output if it is called from the correct +# worktree, which is exercised in this test. This test is poorly written (and +# therefore marked REFFILES) for mulitple reasons: 1) it creates invalidly +# formatted log entres. 2) it uses direct FS access for creating the reflogs. 3) +# PSEUDO-WT and refs/bisect/random do not create reflogs by default, so it is +# not testing a realistic scenario. +test_expect_success REFFILES 'for_each_reflog()' ' echo $ZERO_OID > .git/logs/PSEUDO-MAIN && mkdir -p .git/logs/refs/bisect && echo $ZERO_OID > .git/logs/refs/bisect/random && From 41e2e177c7b8d3e8861c30342c5e57edd5425997 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:31 +0000 Subject: [PATCH 165/397] t1414: mark corruption test with REFFILES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test checks what happens if reflog and ref database disagree on the state of the latest commit. This seems to require accessing reflog storage directly. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1414-reflog-walk.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/t/t1414-reflog-walk.sh b/t/t1414-reflog-walk.sh index 80d94704d0..ea64cecf47 100755 --- a/t/t1414-reflog-walk.sh +++ b/t/t1414-reflog-walk.sh @@ -119,7 +119,9 @@ test_expect_success 'min/max age uses entry date to limit' ' test_cmp expect actual ' -test_expect_success 'walk prefers reflog to ref tip' ' +# Create a situation where the reflog and ref database disagree about the latest +# state of HEAD. +test_expect_success REFFILES 'walk prefers reflog to ref tip' ' head=$(git rev-parse HEAD) && one=$(git rev-parse one) && ident="$GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE" && From a5709636d9958b78ed6cd708ac1c5f80f4b9dc91 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:32 +0000 Subject: [PATCH 166/397] t2017: mark --orphan/logAllRefUpdates=false test as REFFILES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In reftable, there is no notion of a per-ref 'existence' of a reflog. Each reflog entry has its own key, so it is not possible to distinguish between {reflog doesn't exist,reflog exists but is empty}. This makes the logic in log_ref_setup() (file refs/files-backend.c), which depends on the existence of the reflog file infeasible. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t2017-checkout-orphan.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t2017-checkout-orphan.sh b/t/t2017-checkout-orphan.sh index c7adbdd39a..88d6992a5e 100755 --- a/t/t2017-checkout-orphan.sh +++ b/t/t2017-checkout-orphan.sh @@ -76,7 +76,7 @@ test_expect_success '--orphan makes reflog by default' ' git rev-parse --verify delta@{0} ' -test_expect_success '--orphan does not make reflog when core.logAllRefUpdates = false' ' +test_expect_success REFFILES '--orphan does not make reflog when core.logAllRefUpdates = false' ' git checkout main && git config core.logAllRefUpdates false && git checkout --orphan epsilon && From fe8fc09f34c353b744d793d45c2ee62fb1a5f699 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:33 +0000 Subject: [PATCH 167/397] t1404: mark tests that muck with .git directly as REFFILES. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The packed/loose ref storage is an overlay combination of packed-refs (refs and tags in a single file) and one-file-per-ref. This creates all kinds of edge cases related to directory/file conflicts, (non-)empty directories, and the locking scheme, none of which applies to reftable. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1404-update-ref-errors.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/t/t1404-update-ref-errors.sh b/t/t1404-update-ref-errors.sh index 8b51c4efc1..b729c1f480 100755 --- a/t/t1404-update-ref-errors.sh +++ b/t/t1404-update-ref-errors.sh @@ -189,7 +189,7 @@ test_expect_success 'one new ref is a simple prefix of another' ' ' -test_expect_success 'empty directory should not fool rev-parse' ' +test_expect_success REFFILES 'empty directory should not fool rev-parse' ' prefix=refs/e-rev-parse && git update-ref $prefix/foo $C && git pack-refs --all && @@ -199,7 +199,7 @@ test_expect_success 'empty directory should not fool rev-parse' ' test_cmp expected actual ' -test_expect_success 'empty directory should not fool for-each-ref' ' +test_expect_success REFFILES 'empty directory should not fool for-each-ref' ' prefix=refs/e-for-each-ref && git update-ref $prefix/foo $C && git for-each-ref $prefix >expected && @@ -209,14 +209,14 @@ test_expect_success 'empty directory should not fool for-each-ref' ' test_cmp expected actual ' -test_expect_success 'empty directory should not fool create' ' +test_expect_success REFFILES 'empty directory should not fool create' ' prefix=refs/e-create && mkdir -p .git/$prefix/foo/bar/baz && printf "create %s $C\n" $prefix/foo | git update-ref --stdin ' -test_expect_success 'empty directory should not fool verify' ' +test_expect_success REFFILES 'empty directory should not fool verify' ' prefix=refs/e-verify && git update-ref $prefix/foo $C && git pack-refs --all && @@ -225,7 +225,7 @@ test_expect_success 'empty directory should not fool verify' ' git update-ref --stdin ' -test_expect_success 'empty directory should not fool 1-arg update' ' +test_expect_success REFFILES 'empty directory should not fool 1-arg update' ' prefix=refs/e-update-1 && git update-ref $prefix/foo $C && git pack-refs --all && @@ -234,7 +234,7 @@ test_expect_success 'empty directory should not fool 1-arg update' ' git update-ref --stdin ' -test_expect_success 'empty directory should not fool 2-arg update' ' +test_expect_success REFFILES 'empty directory should not fool 2-arg update' ' prefix=refs/e-update-2 && git update-ref $prefix/foo $C && git pack-refs --all && @@ -243,7 +243,7 @@ test_expect_success 'empty directory should not fool 2-arg update' ' git update-ref --stdin ' -test_expect_success 'empty directory should not fool 0-arg delete' ' +test_expect_success REFFILES 'empty directory should not fool 0-arg delete' ' prefix=refs/e-delete-0 && git update-ref $prefix/foo $C && git pack-refs --all && @@ -252,7 +252,7 @@ test_expect_success 'empty directory should not fool 0-arg delete' ' git update-ref --stdin ' -test_expect_success 'empty directory should not fool 1-arg delete' ' +test_expect_success REFFILES 'empty directory should not fool 1-arg delete' ' prefix=refs/e-delete-1 && git update-ref $prefix/foo $C && git pack-refs --all && @@ -466,7 +466,7 @@ test_expect_success 'incorrect old value blocks indirect no-deref delete' ' test_cmp expected output.err ' -test_expect_success 'non-empty directory blocks create' ' +test_expect_success REFFILES 'non-empty directory blocks create' ' prefix=refs/ne-create && mkdir -p .git/$prefix/foo/bar && : >.git/$prefix/foo/bar/baz.lock && @@ -485,7 +485,7 @@ test_expect_success 'non-empty directory blocks create' ' test_cmp expected output.err ' -test_expect_success 'broken reference blocks create' ' +test_expect_success REFFILES 'broken reference blocks create' ' prefix=refs/broken-create && mkdir -p .git/$prefix && echo "gobbledigook" >.git/$prefix/foo && @@ -504,7 +504,7 @@ test_expect_success 'broken reference blocks create' ' test_cmp expected output.err ' -test_expect_success 'non-empty directory blocks indirect create' ' +test_expect_success REFFILES 'non-empty directory blocks indirect create' ' prefix=refs/ne-indirect-create && git symbolic-ref $prefix/symref $prefix/foo && mkdir -p .git/$prefix/foo/bar && @@ -524,7 +524,7 @@ test_expect_success 'non-empty directory blocks indirect create' ' test_cmp expected output.err ' -test_expect_success 'broken reference blocks indirect create' ' +test_expect_success REFFILES 'broken reference blocks indirect create' ' prefix=refs/broken-indirect-create && git symbolic-ref $prefix/symref $prefix/foo && echo "gobbledigook" >.git/$prefix/foo && @@ -543,7 +543,7 @@ test_expect_success 'broken reference blocks indirect create' ' test_cmp expected output.err ' -test_expect_success 'no bogus intermediate values during delete' ' +test_expect_success REFFILES 'no bogus intermediate values during delete' ' prefix=refs/slow-transaction && # Set up a reference with differing loose and packed versions: git update-ref $prefix/foo $C && @@ -600,7 +600,7 @@ test_expect_success 'no bogus intermediate values during delete' ' test_must_fail git rev-parse --verify --quiet $prefix/foo ' -test_expect_success 'delete fails cleanly if packed-refs file is locked' ' +test_expect_success REFFILES 'delete fails cleanly if packed-refs file is locked' ' prefix=refs/locked-packed-refs && # Set up a reference with differing loose and packed versions: git update-ref $prefix/foo $C && @@ -616,7 +616,7 @@ test_expect_success 'delete fails cleanly if packed-refs file is locked' ' test_cmp unchanged actual ' -test_expect_success 'delete fails cleanly if packed-refs.new write fails' ' +test_expect_success REFFILES 'delete fails cleanly if packed-refs.new write fails' ' # Setup and expectations are similar to the test above. prefix=refs/failed-packed-refs && git update-ref $prefix/foo $C && From e740873c4778434737e36eecf2a9393abe709a88 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:34 +0000 Subject: [PATCH 168/397] t7900: stop checking for loose refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Given that git-maintenance simply calls out git-pack-refs, it seems superfluous to test the functionality of pack-refs itself, as that is covered by t3210-pack-refs.sh. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t7900-maintenance.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 2412d8c5c0..70563cc269 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -350,8 +350,6 @@ test_expect_success 'pack-refs task' ' done && GIT_TRACE2_EVENT="$(pwd)/pack-refs.txt" \ git maintenance run --task=pack-refs && - ls .git/refs/heads/ >after && - test_must_be_empty after && test_subcommand git pack-refs --all --prune Date: Mon, 31 May 2021 16:56:35 +0000 Subject: [PATCH 169/397] t7003: check reflog existence only for REFFILES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t7003-filter-branch.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/t/t7003-filter-branch.sh b/t/t7003-filter-branch.sh index cf30055c88..e18a218952 100755 --- a/t/t7003-filter-branch.sh +++ b/t/t7003-filter-branch.sh @@ -396,7 +396,10 @@ test_expect_success '--prune-empty is able to prune entire branch' ' git branch prune-entire B && git filter-branch -f --prune-empty --index-filter "git update-index --remove A.t B.t" prune-entire && test_must_fail git rev-parse refs/heads/prune-entire && - test_must_fail git reflog exists refs/heads/prune-entire + if test_have_prereq REFFILES + then + test_must_fail git reflog exists refs/heads/prune-entire + fi ' test_expect_success '--remap-to-ancestor with filename filters' ' From dc474899e7ad19be0a9f76b2ee2915b8bf79b683 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:36 +0000 Subject: [PATCH 170/397] t4202: mark bogus head hash test with REFFILES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In reftable, hashes are correctly formed by design. Split off test for git-log in empty repo. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t4202-log.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/t/t4202-log.sh b/t/t4202-log.sh index 327991fcea..39e746fbcb 100755 --- a/t/t4202-log.sh +++ b/t/t4202-log.sh @@ -1834,14 +1834,20 @@ test_expect_success 'log --graph --no-walk is forbidden' ' test_must_fail git log --graph --no-walk ' -test_expect_success 'log diagnoses bogus HEAD hash' ' +test_expect_success 'log on empty repo fails' ' git init empty && test_when_finished "rm -rf empty" && test_must_fail git -C empty log 2>stderr && - test_i18ngrep does.not.have.any.commits stderr && + test_i18ngrep does.not.have.any.commits stderr +' + +test_expect_success REFFILES 'log diagnoses bogus HEAD hash' ' + git init empty && + test_when_finished "rm -rf empty" && echo 1234abcd >empty/.git/refs/heads/main && test_must_fail git -C empty log 2>stderr && - test_i18ngrep broken stderr' + test_i18ngrep broken stderr +' test_expect_success 'log diagnoses bogus HEAD symref' ' git init empty && From 1231cab341e3e14065e39a5403ad8026c9bc19a8 Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Mon, 31 May 2021 16:56:37 +0000 Subject: [PATCH 171/397] t1415: set REFFILES for test specific to storage format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Packing refs (and therefore checking that certain refs are not packed) is a property of the packed/loose ref storage. Add a comment to explain what the test checks. Signed-off-by: Han-Wen Nienhuys Reviewed-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- t/t1415-worktree-refs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/t/t1415-worktree-refs.sh b/t/t1415-worktree-refs.sh index 7ab91241ab..66f27d0fdf 100755 --- a/t/t1415-worktree-refs.sh +++ b/t/t1415-worktree-refs.sh @@ -16,7 +16,10 @@ test_expect_success 'setup' ' git -C wt2 update-ref refs/worktree/foo HEAD ' -test_expect_success 'refs/worktree must not be packed' ' +# The 'packed-refs' file is stored directly in .git/. This means it is global +# to the repository, and can only contain refs that are shared across all +# worktrees. +test_expect_success REFFILES 'refs/worktree must not be packed' ' git pack-refs --all && test_path_is_missing .git/refs/tags/wt1 && test_path_is_file .git/refs/worktree/foo && From 050f76b9af75bf69110d542e0cee569d0112f320 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:31 -0500 Subject: [PATCH 172/397] push: rename !triangular to same_remote The typical case is what git was designed for: distributed remotes. It's only the atypical case--fetching and pushing to the same remote--that we need to keep an eye on. No functional changes. Liked-by: Junio C Hamano Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 194967ed79..06406353ce 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -186,7 +186,7 @@ static const char message_detached_head_die[] = " git push %s HEAD:\n"); static void setup_push_upstream(struct remote *remote, struct branch *branch, - int triangular, int simple) + int same_remote, int simple) { if (!branch) die(_(message_detached_head_die), remote->name); @@ -201,7 +201,7 @@ static void setup_push_upstream(struct remote *remote, struct branch *branch, if (branch->merge_nr != 1) die(_("The current branch %s has multiple upstream branches, " "refusing to push."), branch->name); - if (triangular) + if (!same_remote) die(_("You are pushing to remote '%s', which is not the upstream of\n" "your current branch '%s', without telling me what to push\n" "to update which remote branch."), @@ -223,16 +223,16 @@ static void setup_push_current(struct remote *remote, struct branch *branch) refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); } -static int is_workflow_triangular(struct remote *remote) +static int is_same_remote(struct remote *remote) { struct remote *fetch_remote = remote_get(NULL); - return (fetch_remote && fetch_remote != remote); + return (!fetch_remote || fetch_remote == remote); } static void setup_default_push_refspecs(struct remote *remote) { struct branch *branch = branch_get(NULL); - int triangular = is_workflow_triangular(remote); + int same_remote = is_same_remote(remote); switch (push_default) { default: @@ -242,14 +242,14 @@ static void setup_default_push_refspecs(struct remote *remote) case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: - if (triangular) + if (!same_remote) setup_push_current(remote, branch); else - setup_push_upstream(remote, branch, triangular, 1); + setup_push_upstream(remote, branch, same_remote, 1); break; case PUSH_DEFAULT_UPSTREAM: - setup_push_upstream(remote, branch, triangular, 0); + setup_push_upstream(remote, branch, same_remote, 0); break; case PUSH_DEFAULT_CURRENT: From 3b9fd8361f1ed98f06df24d406b9a618c7f39df1 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:32 -0500 Subject: [PATCH 173/397] push: hedge code of default=simple `simple` is the most important mode so move the relevant code to its own function to make it easier to see what it's doing. Reviewed-by: Elijah Newren Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 06406353ce..48c38fe25a 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -223,6 +223,14 @@ static void setup_push_current(struct remote *remote, struct branch *branch) refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); } +static void setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) +{ + if (!same_remote) + setup_push_current(remote, branch); + else + setup_push_upstream(remote, branch, same_remote, 1); +} + static int is_same_remote(struct remote *remote) { struct remote *fetch_remote = remote_get(NULL); @@ -242,10 +250,7 @@ static void setup_default_push_refspecs(struct remote *remote) case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: - if (!same_remote) - setup_push_current(remote, branch); - else - setup_push_upstream(remote, branch, same_remote, 1); + setup_push_simple(remote, branch, same_remote); break; case PUSH_DEFAULT_UPSTREAM: From d099b9c9c723e8ca974e5ece027165e2824a1b89 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:33 -0500 Subject: [PATCH 174/397] push: copy code to setup_push_simple() In order to avoid doing unnecessary things and simplify it in further patches. In particular moving the additional name safety out of setup_push_upstream() and into setup_push_simple() and thus making both more straightforward. The code is copied exactly as-is; no functional changes. Reviewed-by: Elijah Newren Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 48c38fe25a..6a620a90e3 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -225,10 +225,38 @@ static void setup_push_current(struct remote *remote, struct branch *branch) static void setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) { - if (!same_remote) - setup_push_current(remote, branch); - else - setup_push_upstream(remote, branch, same_remote, 1); + if (!same_remote) { + if (!branch) + die(_(message_detached_head_die), remote->name); + refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); + } else { + if (!branch) + die(_(message_detached_head_die), remote->name); + if (!branch->merge_nr || !branch->merge || !branch->remote_name) + die(_("The current branch %s has no upstream branch.\n" + "To push the current branch and set the remote as upstream, use\n" + "\n" + " git push --set-upstream %s %s\n"), + branch->name, + remote->name, + branch->name); + if (branch->merge_nr != 1) + die(_("The current branch %s has multiple upstream branches, " + "refusing to push."), branch->name); + if (!same_remote) + die(_("You are pushing to remote '%s', which is not the upstream of\n" + "your current branch '%s', without telling me what to push\n" + "to update which remote branch."), + remote->name, branch->name); + + if (1) { + /* Additional safety */ + if (strcmp(branch->refname, branch->merge[0]->src)) + die_push_simple(branch, remote); + } + + refspec_appendf(&rs, "%s:%s", branch->refname, branch->merge[0]->src); + } } static int is_same_remote(struct remote *remote) From 6b010c80a2596c75d562afa5ab098ffff21bab06 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:34 -0500 Subject: [PATCH 175/397] push: reorganize setup_push_simple() Simply move the code around and remove dead code. In particular the '!same_remote' conditional is a no-op since that part of the code is the same_remote leg of the conditional beforehand. No functional changes. Suggestions-by: Elijah Newren Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 6a620a90e3..972d8e1cfd 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -225,13 +225,14 @@ static void setup_push_current(struct remote *remote, struct branch *branch) static void setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) { + const char *dst; + + if (!branch) + die(_(message_detached_head_die), remote->name); + if (!same_remote) { - if (!branch) - die(_(message_detached_head_die), remote->name); - refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); + dst = branch->refname; } else { - if (!branch) - die(_(message_detached_head_die), remote->name); if (!branch->merge_nr || !branch->merge || !branch->remote_name) die(_("The current branch %s has no upstream branch.\n" "To push the current branch and set the remote as upstream, use\n" @@ -243,20 +244,14 @@ static void setup_push_simple(struct remote *remote, struct branch *branch, int if (branch->merge_nr != 1) die(_("The current branch %s has multiple upstream branches, " "refusing to push."), branch->name); - if (!same_remote) - die(_("You are pushing to remote '%s', which is not the upstream of\n" - "your current branch '%s', without telling me what to push\n" - "to update which remote branch."), - remote->name, branch->name); - if (1) { - /* Additional safety */ - if (strcmp(branch->refname, branch->merge[0]->src)) - die_push_simple(branch, remote); - } + /* Additional safety */ + if (strcmp(branch->refname, branch->merge[0]->src)) + die_push_simple(branch, remote); - refspec_appendf(&rs, "%s:%s", branch->refname, branch->merge[0]->src); + dst = branch->merge[0]->src; } + refspec_appendf(&rs, "%s:%s", branch->refname, dst); } static int is_same_remote(struct remote *remote) From b8e8b98647f02e721fc3c1b5db6936ba7c1c1167 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:35 -0500 Subject: [PATCH 176/397] push: simplify setup_push_simple() There's a safety check to make sure branch->refname isn't different from branch->merge[0]->src, otherwise we die(). Therefore we always push to branch->refname. Suggestions-by: Elijah Newren Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 972d8e1cfd..e37c751268 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -225,14 +225,10 @@ static void setup_push_current(struct remote *remote, struct branch *branch) static void setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) { - const char *dst; - if (!branch) die(_(message_detached_head_die), remote->name); - if (!same_remote) { - dst = branch->refname; - } else { + if (same_remote) { if (!branch->merge_nr || !branch->merge || !branch->remote_name) die(_("The current branch %s has no upstream branch.\n" "To push the current branch and set the remote as upstream, use\n" @@ -248,10 +244,8 @@ static void setup_push_simple(struct remote *remote, struct branch *branch, int /* Additional safety */ if (strcmp(branch->refname, branch->merge[0]->src)) die_push_simple(branch, remote); - - dst = branch->merge[0]->src; } - refspec_appendf(&rs, "%s:%s", branch->refname, dst); + refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); } static int is_same_remote(struct remote *remote) From 7e6d72bb111b080ef37e26fbdfe080672521c570 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:36 -0500 Subject: [PATCH 177/397] push: remove unused code in setup_push_upstream() Now it's not used for the simple mode. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index e37c751268..29fea70ff1 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -186,7 +186,7 @@ static const char message_detached_head_die[] = " git push %s HEAD:\n"); static void setup_push_upstream(struct remote *remote, struct branch *branch, - int same_remote, int simple) + int same_remote) { if (!branch) die(_(message_detached_head_die), remote->name); @@ -207,12 +207,6 @@ static void setup_push_upstream(struct remote *remote, struct branch *branch, "to update which remote branch."), remote->name, branch->name); - if (simple) { - /* Additional safety */ - if (strcmp(branch->refname, branch->merge[0]->src)) - die_push_simple(branch, remote); - } - refspec_appendf(&rs, "%s:%s", branch->refname, branch->merge[0]->src); } @@ -271,7 +265,7 @@ static void setup_default_push_refspecs(struct remote *remote) break; case PUSH_DEFAULT_UPSTREAM: - setup_push_upstream(remote, branch, same_remote, 0); + setup_push_upstream(remote, branch, same_remote); break; case PUSH_DEFAULT_CURRENT: From 90cfb2666b5913e0be4ffb84630866287dde4f9a Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:32:37 -0500 Subject: [PATCH 178/397] doc: push: explain default=simple correctly Now that the code has been simplified and it's clear what it's actually doing, update the documentation to reflect that. Namely; the simple mode only barfs when working on a centralized workflow, and there's no configured upstream branch with the same name. Cc: Elijah Newren Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/config/push.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Documentation/config/push.txt b/Documentation/config/push.txt index 21b256e0a4..5dbf343bd0 100644 --- a/Documentation/config/push.txt +++ b/Documentation/config/push.txt @@ -24,15 +24,14 @@ push.default:: * `tracking` - This is a deprecated synonym for `upstream`. -* `simple` - in centralized workflow, work like `upstream` with an - added safety to refuse to push if the upstream branch's name is - different from the local one. +* `simple` - pushes the current branch with the same name on the remote. + -When pushing to a remote that is different from the remote you normally -pull from, work as `current`. This is the safest option and is suited -for beginners. +If you are working on a centralized workflow (pushing to the same repository you +pull from, which is typically `origin`), then you need to configure an upstream +branch with the same name. + -This mode has become the default in Git 2.0. +This mode is the default since Git 2.0, and is the safest option suited for +beginners. * `matching` - push all branches having the same name on both ends. This makes the repository you are pushing to remember the set of From 533e0325abcf51d6b3ff689dfacf8fa95e3b6da1 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:12 -0500 Subject: [PATCH 179/397] push: create new get_upstream_ref() helper This code is duplicated among multiple functions. No functional changes. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 29fea70ff1..e3e792c69c 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -185,29 +185,37 @@ static const char message_detached_head_die[] = "\n" " git push %s HEAD:\n"); -static void setup_push_upstream(struct remote *remote, struct branch *branch, - int same_remote) +static const char *get_upstream_ref(struct branch *branch, const char *remote_name) { - if (!branch) - die(_(message_detached_head_die), remote->name); if (!branch->merge_nr || !branch->merge || !branch->remote_name) die(_("The current branch %s has no upstream branch.\n" "To push the current branch and set the remote as upstream, use\n" "\n" " git push --set-upstream %s %s\n"), branch->name, - remote->name, + remote_name, branch->name); if (branch->merge_nr != 1) die(_("The current branch %s has multiple upstream branches, " "refusing to push."), branch->name); + + return branch->merge[0]->src; +} + +static void setup_push_upstream(struct remote *remote, struct branch *branch, + int same_remote) +{ + const char *upstream_ref; + if (!branch) + die(_(message_detached_head_die), remote->name); + upstream_ref = get_upstream_ref(branch, remote->name); if (!same_remote) die(_("You are pushing to remote '%s', which is not the upstream of\n" "your current branch '%s', without telling me what to push\n" "to update which remote branch."), remote->name, branch->name); - refspec_appendf(&rs, "%s:%s", branch->refname, branch->merge[0]->src); + refspec_appendf(&rs, "%s:%s", branch->refname, upstream_ref); } static void setup_push_current(struct remote *remote, struct branch *branch) @@ -223,20 +231,12 @@ static void setup_push_simple(struct remote *remote, struct branch *branch, int die(_(message_detached_head_die), remote->name); if (same_remote) { - if (!branch->merge_nr || !branch->merge || !branch->remote_name) - die(_("The current branch %s has no upstream branch.\n" - "To push the current branch and set the remote as upstream, use\n" - "\n" - " git push --set-upstream %s %s\n"), - branch->name, - remote->name, - branch->name); - if (branch->merge_nr != 1) - die(_("The current branch %s has multiple upstream branches, " - "refusing to push."), branch->name); + const char *upstream_ref; + + upstream_ref = get_upstream_ref(branch, remote->name); /* Additional safety */ - if (strcmp(branch->refname, branch->merge[0]->src)) + if (strcmp(branch->refname, upstream_ref)) die_push_simple(branch, remote); } refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); From 72739680fc912c6e8dedaf06af4969f8e52ffb4d Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:13 -0500 Subject: [PATCH 180/397] push: return immediately in trivial switch case There's no need to break when nothing else will be executed. Will help next patches. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index e3e792c69c..0aa1d0f07d 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -257,25 +257,25 @@ static void setup_default_push_refspecs(struct remote *remote) default: case PUSH_DEFAULT_MATCHING: refspec_append(&rs, ":"); - break; + return; case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: setup_push_simple(remote, branch, same_remote); - break; + return; case PUSH_DEFAULT_UPSTREAM: setup_push_upstream(remote, branch, same_remote); - break; + return; case PUSH_DEFAULT_CURRENT: setup_push_current(remote, branch); - break; + return; case PUSH_DEFAULT_NOTHING: die(_("You didn't specify any refspecs to push, and " "push.default is \"nothing\".")); - break; + return; } } From 04159fba42b61ffa954dfb1fa13df1862c210ad8 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:14 -0500 Subject: [PATCH 181/397] push: split switch cases We want all the cases that don't do anything with a branch first, and then the rest. That way we will be able to get the branch and die if there's a problem in the parent function, instead of inside the function of each mode. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 0aa1d0f07d..f64b7100f0 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -254,11 +254,20 @@ static void setup_default_push_refspecs(struct remote *remote) int same_remote = is_same_remote(remote); switch (push_default) { - default: case PUSH_DEFAULT_MATCHING: refspec_append(&rs, ":"); return; + case PUSH_DEFAULT_NOTHING: + die(_("You didn't specify any refspecs to push, and " + "push.default is \"nothing\".")); + return; + default: + break; + } + + switch (push_default) { + default: case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: setup_push_simple(remote, branch, same_remote); @@ -271,11 +280,6 @@ static void setup_default_push_refspecs(struct remote *remote) case PUSH_DEFAULT_CURRENT: setup_push_current(remote, branch); return; - - case PUSH_DEFAULT_NOTHING: - die(_("You didn't specify any refspecs to push, and " - "push.default is \"nothing\".")); - return; } } From cc16f95d212d8e0bba07ddfb82aaf416dc4df1c2 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:15 -0500 Subject: [PATCH 182/397] push: factor out null branch check No need to do it in every single function. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index f64b7100f0..8fcbd2878d 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -206,8 +206,6 @@ static void setup_push_upstream(struct remote *remote, struct branch *branch, int same_remote) { const char *upstream_ref; - if (!branch) - die(_(message_detached_head_die), remote->name); upstream_ref = get_upstream_ref(branch, remote->name); if (!same_remote) die(_("You are pushing to remote '%s', which is not the upstream of\n" @@ -220,16 +218,11 @@ static void setup_push_upstream(struct remote *remote, struct branch *branch, static void setup_push_current(struct remote *remote, struct branch *branch) { - if (!branch) - die(_(message_detached_head_die), remote->name); refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); } static void setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) { - if (!branch) - die(_(message_detached_head_die), remote->name); - if (same_remote) { const char *upstream_ref; @@ -266,6 +259,9 @@ static void setup_default_push_refspecs(struct remote *remote) break; } + if (!branch) + die(_(message_detached_head_die), remote->name); + switch (push_default) { default: case PUSH_DEFAULT_UNSPECIFIED: From 65c63a005434306e7c064eb463ff1f61de56d38e Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:16 -0500 Subject: [PATCH 183/397] push: only get the branch when needed Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/push.c b/builtin/push.c index 8fcbd2878d..d9f9d20f39 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -243,7 +243,7 @@ static int is_same_remote(struct remote *remote) static void setup_default_push_refspecs(struct remote *remote) { - struct branch *branch = branch_get(NULL); + struct branch *branch; int same_remote = is_same_remote(remote); switch (push_default) { @@ -259,6 +259,7 @@ static void setup_default_push_refspecs(struct remote *remote) break; } + branch = branch_get(NULL); if (!branch) die(_(message_detached_head_die), remote->name); From 00458dc5f191f52ef965ca7185b739e8c800b7cb Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:17 -0500 Subject: [PATCH 184/397] push: make setup_push_* return the dst All of the setup_push_* functions are appending a refspec. Do this only once on the parent function. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index d9f9d20f39..933b1cc6c0 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -202,8 +202,8 @@ static const char *get_upstream_ref(struct branch *branch, const char *remote_na return branch->merge[0]->src; } -static void setup_push_upstream(struct remote *remote, struct branch *branch, - int same_remote) +static const char *setup_push_upstream(struct remote *remote, struct branch *branch, + int same_remote) { const char *upstream_ref; upstream_ref = get_upstream_ref(branch, remote->name); @@ -212,16 +212,15 @@ static void setup_push_upstream(struct remote *remote, struct branch *branch, "your current branch '%s', without telling me what to push\n" "to update which remote branch."), remote->name, branch->name); - - refspec_appendf(&rs, "%s:%s", branch->refname, upstream_ref); + return upstream_ref; } -static void setup_push_current(struct remote *remote, struct branch *branch) +static const char *setup_push_current(struct remote *remote, struct branch *branch) { - refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); + return branch->refname; } -static void setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) +static const char *setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) { if (same_remote) { const char *upstream_ref; @@ -232,7 +231,7 @@ static void setup_push_simple(struct remote *remote, struct branch *branch, int if (strcmp(branch->refname, upstream_ref)) die_push_simple(branch, remote); } - refspec_appendf(&rs, "%s:%s", branch->refname, branch->refname); + return branch->refname; } static int is_same_remote(struct remote *remote) @@ -245,6 +244,7 @@ static void setup_default_push_refspecs(struct remote *remote) { struct branch *branch; int same_remote = is_same_remote(remote); + const char *dst; switch (push_default) { case PUSH_DEFAULT_MATCHING: @@ -267,17 +267,19 @@ static void setup_default_push_refspecs(struct remote *remote) default: case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: - setup_push_simple(remote, branch, same_remote); - return; + dst = setup_push_simple(remote, branch, same_remote); + break; case PUSH_DEFAULT_UPSTREAM: - setup_push_upstream(remote, branch, same_remote); - return; + dst = setup_push_upstream(remote, branch, same_remote); + break; case PUSH_DEFAULT_CURRENT: - setup_push_current(remote, branch); - return; + dst = setup_push_current(remote, branch); + break; } + + refspec_appendf(&rs, "%s:%s", branch->refname, dst); } static const char message_advice_pull_before_push[] = From d371a9ef4cfc4610989b0a3d71c79cbe19e0073a Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:18 -0500 Subject: [PATCH 185/397] push: trivial simplifications Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 933b1cc6c0..43c039a2e3 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -205,14 +205,12 @@ static const char *get_upstream_ref(struct branch *branch, const char *remote_na static const char *setup_push_upstream(struct remote *remote, struct branch *branch, int same_remote) { - const char *upstream_ref; - upstream_ref = get_upstream_ref(branch, remote->name); if (!same_remote) die(_("You are pushing to remote '%s', which is not the upstream of\n" "your current branch '%s', without telling me what to push\n" "to update which remote branch."), remote->name, branch->name); - return upstream_ref; + return get_upstream_ref(branch, remote->name); } static const char *setup_push_current(struct remote *remote, struct branch *branch) @@ -222,15 +220,9 @@ static const char *setup_push_current(struct remote *remote, struct branch *bran static const char *setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) { - if (same_remote) { - const char *upstream_ref; - - upstream_ref = get_upstream_ref(branch, remote->name); - - /* Additional safety */ - if (strcmp(branch->refname, upstream_ref)) + if (same_remote) + if (strcmp(branch->refname, get_upstream_ref(branch, remote->name))) die_push_simple(branch, remote); - } return branch->refname; } From 0add899baf5a9ee478659821327db719ce74454a Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:19 -0500 Subject: [PATCH 186/397] push: get rid of all the setup_push_* functions Their code is much simpler now and can move into the parent function. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index 43c039a2e3..da406fc890 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -202,30 +202,6 @@ static const char *get_upstream_ref(struct branch *branch, const char *remote_na return branch->merge[0]->src; } -static const char *setup_push_upstream(struct remote *remote, struct branch *branch, - int same_remote) -{ - if (!same_remote) - die(_("You are pushing to remote '%s', which is not the upstream of\n" - "your current branch '%s', without telling me what to push\n" - "to update which remote branch."), - remote->name, branch->name); - return get_upstream_ref(branch, remote->name); -} - -static const char *setup_push_current(struct remote *remote, struct branch *branch) -{ - return branch->refname; -} - -static const char *setup_push_simple(struct remote *remote, struct branch *branch, int same_remote) -{ - if (same_remote) - if (strcmp(branch->refname, get_upstream_ref(branch, remote->name))) - die_push_simple(branch, remote); - return branch->refname; -} - static int is_same_remote(struct remote *remote) { struct remote *fetch_remote = remote_get(NULL); @@ -259,15 +235,23 @@ static void setup_default_push_refspecs(struct remote *remote) default: case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: - dst = setup_push_simple(remote, branch, same_remote); + if (same_remote) + if (strcmp(branch->refname, get_upstream_ref(branch, remote->name))) + die_push_simple(branch, remote); + dst = branch->refname; break; case PUSH_DEFAULT_UPSTREAM: - dst = setup_push_upstream(remote, branch, same_remote); + if (!same_remote) + die(_("You are pushing to remote '%s', which is not the upstream of\n" + "your current branch '%s', without telling me what to push\n" + "to update which remote branch."), + remote->name, branch->name); + dst = get_upstream_ref(branch, remote->name); break; case PUSH_DEFAULT_CURRENT: - dst = setup_push_current(remote, branch); + dst = branch->refname; break; } From 1f934725f7597366c981b72be6597124e2c21a77 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:20 -0500 Subject: [PATCH 187/397] push: factor out the typical case Only override dst on the odd case. This allows a preemptive break on the `simple` case. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index da406fc890..b5e951bf59 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -231,14 +231,16 @@ static void setup_default_push_refspecs(struct remote *remote) if (!branch) die(_(message_detached_head_die), remote->name); + dst = branch->refname; + switch (push_default) { default: case PUSH_DEFAULT_UNSPECIFIED: case PUSH_DEFAULT_SIMPLE: - if (same_remote) - if (strcmp(branch->refname, get_upstream_ref(branch, remote->name))) - die_push_simple(branch, remote); - dst = branch->refname; + if (!same_remote) + break; + if (strcmp(branch->refname, get_upstream_ref(branch, remote->name))) + die_push_simple(branch, remote); break; case PUSH_DEFAULT_UPSTREAM: @@ -251,7 +253,6 @@ static void setup_default_push_refspecs(struct remote *remote) break; case PUSH_DEFAULT_CURRENT: - dst = branch->refname; break; } From 1afd78fb5c29b8ad95de072bb8a0edacc69db61a Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:21 -0500 Subject: [PATCH 188/397] push: remove redundant check If fetch_remote is NULL (i.e. the branch remote is invalid), then it can't possibly be same as remote, which can't be NULL. The check is redundant, and so is the extra variable. Also, fix the Yoda condition: we want to check if remote is the same as the branch remote, not the other way around. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index b5e951bf59..aa22d6a8e5 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -204,8 +204,7 @@ static const char *get_upstream_ref(struct branch *branch, const char *remote_na static int is_same_remote(struct remote *remote) { - struct remote *fetch_remote = remote_get(NULL); - return (!fetch_remote || fetch_remote == remote); + return remote == remote_get(NULL); } static void setup_default_push_refspecs(struct remote *remote) From c5b09cf771ea3c340335aeff501ebec342f78aa4 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:22 -0500 Subject: [PATCH 189/397] push: remove trivial function It's a single line that is used in a single place, and the variable has the same name as the function. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/builtin/push.c b/builtin/push.c index aa22d6a8e5..a873f8da92 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -202,15 +202,10 @@ static const char *get_upstream_ref(struct branch *branch, const char *remote_na return branch->merge[0]->src; } -static int is_same_remote(struct remote *remote) -{ - return remote == remote_get(NULL); -} - static void setup_default_push_refspecs(struct remote *remote) { struct branch *branch; - int same_remote = is_same_remote(remote); + int same_remote = remote == remote_get(NULL); const char *dst; switch (push_default) { From e0c91cffde8477ababb5163180f08e29da54db3e Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:23 -0500 Subject: [PATCH 190/397] push: only check same_remote when needed Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builtin/push.c b/builtin/push.c index a873f8da92..f3916c66d1 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -205,8 +205,8 @@ static const char *get_upstream_ref(struct branch *branch, const char *remote_na static void setup_default_push_refspecs(struct remote *remote) { struct branch *branch; - int same_remote = remote == remote_get(NULL); const char *dst; + int same_remote; switch (push_default) { case PUSH_DEFAULT_MATCHING: @@ -226,6 +226,7 @@ static void setup_default_push_refspecs(struct remote *remote) die(_(message_detached_head_die), remote->name); dst = branch->refname; + same_remote = remote == remote_get(NULL); switch (push_default) { default: From 7088ce71918a0f9fde2ceb421d4b332888a202c7 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 31 May 2021 14:51:24 -0500 Subject: [PATCH 191/397] push: don't get a full remote object All we need to know is that their names are the same. Additionally this might be easier to parse for some since remote_for_branch is more descriptive than remote_get(NULL). Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- builtin/push.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/push.c b/builtin/push.c index f3916c66d1..e8b10a9b7e 100644 --- a/builtin/push.c +++ b/builtin/push.c @@ -226,7 +226,7 @@ static void setup_default_push_refspecs(struct remote *remote) die(_(message_detached_head_die), remote->name); dst = branch->refname; - same_remote = remote == remote_get(NULL); + same_remote = !strcmp(remote->name, remote_for_branch(branch, NULL)); switch (push_default) { default: From 0b18023d000ed8c0851a025fcd4fd518cb2df673 Mon Sep 17 00:00:00 2001 From: David Aguilar Date: Tue, 1 Jun 2021 13:52:29 -0700 Subject: [PATCH 192/397] contrib/completion: fix zsh completion regression from 59d85a2a05 A recent change to make git-completion.bash use $__git_cmd_idx in more places broke a number of completions on zsh because it modified __git_main but did not update __git_zsh_main. Notably, completions for "add", "branch", "mv" and "push" were broken as a result of this change. In addition to the undefined variable usage, "git mv " also prints the following error: __git_count_arguments:7: bad math expression: operand expected at `"1"' _git_mv:[:7: unknown condition: -gt Remove the quotes around $__git_cmd_idx in __git_count_arguments and set __git_cmd_idx=1 early in __git_zsh_main to fix the regressions from 59d85a2a05. This was tested on zsh 5.7.1 (x86_64-apple-darwin19.0). Suggested-by: Felipe Contreras Signed-off-by: David Aguilar Acked-by: Felipe Contreras Signed-off-by: Junio C Hamano --- contrib/completion/git-completion.bash | 2 +- contrib/completion/git-completion.zsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index 3c5739b905..b50c5d0ea3 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -1306,7 +1306,7 @@ __git_count_arguments () local word i c=0 # Skip "git" (first argument) - for ((i="$__git_cmd_idx"; i < ${#words[@]}; i++)); do + for ((i=$__git_cmd_idx; i < ${#words[@]}; i++)); do word="${words[i]}" case "$word" in diff --git a/contrib/completion/git-completion.zsh b/contrib/completion/git-completion.zsh index 6c56296997..cac6f61881 100644 --- a/contrib/completion/git-completion.zsh +++ b/contrib/completion/git-completion.zsh @@ -251,7 +251,7 @@ __git_zsh_main () done ;; (arg) - local command="${words[1]}" __git_dir + local command="${words[1]}" __git_dir __git_cmd_idx=1 if (( $+opt_args[--bare] )); then __git_dir='.' From c09b6306c6ca275ed9d0348a8c8014b2ff723cfb Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Wed, 2 Jun 2021 12:51:09 +0900 Subject: [PATCH 193/397] Git 2.32-rc3 Signed-off-by: Junio C Hamano --- GIT-VERSION-GEN | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index 6fcb8fdf3b..c0942f12a3 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=v2.32.0-rc2 +DEF_VER=v2.32.0-rc3 LF=' ' From e16acc80a78ae5e931b94e861aff53a4af485f77 Mon Sep 17 00:00:00 2001 From: ZheNing Hu Date: Thu, 3 Jun 2021 16:29:25 +0000 Subject: [PATCH 194/397] cat-file: handle trivial --batch format with --batch-all-objects The --batch code to print an object assumes we found out the type of the object from calling oid_object_info_extended(). This is true for the default format, but even in a custom format, we manually modify the object_info struct to ask for the type. This assumption was broken by 845de33a5b (cat-file: avoid noop calls to sha1_object_info_extended, 2016-05-18). That commit skips the call to oid_object_info_extended() entirely when --batch-all-objects is in use, and the custom format does not include any placeholders that require calling it. Or when the custom format only include placeholders like %(objectname) or %(rest), oid_object_info_extended() will not get the type of the object. This results in an error when we try to confirm that the type didn't change: $ git cat-file --batch=batman --batch-all-objects batman fatal: object 000023961a0c02d6e21dc51ea3484ff71abf1c74 changed type!? and also has other subtle effects (e.g., we'd fail to stream a blob, since we don't realize it's a blob in the first place). We can fix this by flipping the order of the setup. The check for "do we need to get the object info" must come _after_ we've decided whether we need to look up the type. Helped-by: Jeff King Signed-off-by: ZheNing Hu Acked-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/cat-file.c | 13 +++++++------ t/t1006-cat-file.sh | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 5ebf13359e..02461bb5ea 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -512,12 +512,6 @@ static int batch_objects(struct batch_options *opt) if (opt->cmdmode) data.split_on_whitespace = 1; - if (opt->all_objects) { - struct object_info empty = OBJECT_INFO_INIT; - if (!memcmp(&data.info, &empty, sizeof(empty))) - data.skip_object_info = 1; - } - /* * If we are printing out the object, then always fill in the type, * since we will want to decide whether or not to stream. @@ -525,6 +519,13 @@ static int batch_objects(struct batch_options *opt) if (opt->print_contents) data.info.typep = &data.type; + if (opt->all_objects) { + struct object_info empty = OBJECT_INFO_INIT; + + if (!memcmp(&data.info, &empty, sizeof(empty))) + data.skip_object_info = 1; + } + if (opt->all_objects) { struct object_cb_data cb; diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 2f501d2dc9..fa0759d7a4 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -586,4 +586,26 @@ test_expect_success 'cat-file --unordered works' ' test_cmp expect actual ' +test_expect_success 'set up object list for --batch-all-objects tests' ' + git -C all-two cat-file --batch-all-objects --batch-check="%(objectname)" >objects +' + +test_expect_success 'cat-file --batch="%(objectname)" with --batch-all-objects will work' ' + git -C all-two cat-file --batch="%(objectname)" expect && + git -C all-two cat-file --batch-all-objects --batch="%(objectname)" >actual && + cmp expect actual +' + +test_expect_success 'cat-file --batch="%(rest)" with --batch-all-objects will work' ' + git -C all-two cat-file --batch="%(rest)" expect && + git -C all-two cat-file --batch-all-objects --batch="%(rest)" >actual && + cmp expect actual +' + +test_expect_success 'cat-file --batch="batman" with --batch-all-objects will work' ' + git -C all-two cat-file --batch="batman" expect && + git -C all-two cat-file --batch-all-objects --batch="batman" >actual && + cmp expect actual +' + test_done From ee02ac616435cb1da1e02c8b9c220649d3cec40a Mon Sep 17 00:00:00 2001 From: ZheNing Hu Date: Thu, 3 Jun 2021 16:29:26 +0000 Subject: [PATCH 195/397] cat-file: merge two block into one There are two "if (opt->all_objects)" blocks next to each other, merge them into one to provide better readability. Helped-by: Jeff King Signed-off-by: ZheNing Hu Acked-by: Jeff King Signed-off-by: Junio C Hamano --- builtin/cat-file.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 02461bb5ea..243fe6844b 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -520,14 +520,11 @@ static int batch_objects(struct batch_options *opt) data.info.typep = &data.type; if (opt->all_objects) { + struct object_cb_data cb; struct object_info empty = OBJECT_INFO_INIT; if (!memcmp(&data.info, &empty, sizeof(empty))) data.skip_object_info = 1; - } - - if (opt->all_objects) { - struct object_cb_data cb; if (has_promisor_remote()) warning("This repository uses promisor remotes. Some objects may not be loaded."); From ace6d8e3d66b3559addd2988d5c0eb97d255db95 Mon Sep 17 00:00:00 2001 From: Tao Klerks Date: Wed, 2 Jun 2021 11:47:26 +0000 Subject: [PATCH 196/397] Remove warning that repack only works on non-promisor packfiles The git-repack doc clearly states that it *does* operate on promisor packfiles (in a separate partition), with "-a" specified. Presumably the statements here are outdated, as they feature from the first doc in 2017 (and the repack support was added in 2018) Signed-off-by: Tao Klerks Reviewed-by: Taylor Blau Acked-by: Jonathan Tan Signed-off-by: Junio C Hamano --- Documentation/technical/partial-clone.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Documentation/technical/partial-clone.txt b/Documentation/technical/partial-clone.txt index 0780d30cac..a0dd7c66f2 100644 --- a/Documentation/technical/partial-clone.txt +++ b/Documentation/technical/partial-clone.txt @@ -242,8 +242,7 @@ remote in a specific order. repository and can satisfy all such requests. - Repack essentially treats promisor and non-promisor packfiles as 2 - distinct partitions and does not mix them. Repack currently only works - on non-promisor packfiles and loose objects. + distinct partitions and does not mix them. - Dynamic object fetching invokes fetch-pack once *for each item* because most algorithms stumble upon a missing object and need to have @@ -273,9 +272,6 @@ to use those promisor remotes in that order." The user might want to work in a triangular work flow with multiple promisor remotes that each have an incomplete view of the repository. -- Allow repack to work on promisor packfiles (while keeping them distinct - from non-promisor packfiles). - - Allow non-pathname-based filters to make use of packfile bitmaps (when present). This was just an omission during the initial implementation. From 7ba68e0cf1df53e56ec96cb20d02d971db45a4b7 Mon Sep 17 00:00:00 2001 From: Josh Steadmon Date: Thu, 3 Jun 2021 19:41:30 -0700 Subject: [PATCH 197/397] docs: fix api-trace2 doc for "too_many_files" event In 87db61a (trace2: write discard message to sentinel files, 2019-10-04), we added a new "too_many_files" event for when trace2 logging would create too many files in an output directory. Unfortunately, the api-trace2 doc described a "discard" event instead. Fix the doc to use the correct event name. Signed-off-by: Josh Steadmon Signed-off-by: Junio C Hamano --- Documentation/technical/api-trace2.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/technical/api-trace2.txt b/Documentation/technical/api-trace2.txt index a045dbe422..d04828d15a 100644 --- a/Documentation/technical/api-trace2.txt +++ b/Documentation/technical/api-trace2.txt @@ -621,14 +621,14 @@ only present on the "start" and "atexit" events. } ------------ -`"discard"`:: +`"too_many_files"`:: This event is written to the git-trace2-discard sentinel file if there are too many files in the target trace directory (see the trace2.maxFiles config option). + ------------ { - "event":"discard", + "event":"too_many_files", ... } ------------ From cccdfd22436ede80b37311e73a5c8339054a7ff1 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 4 Jun 2021 10:36:11 +0900 Subject: [PATCH 198/397] fsync(): be prepared to see EINTR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some platforms, like NonStop do not automatically restart fsync() when interrupted by a signal, even when that signal is setup with SA_RESTART. This can lead to test breakage, e.g., where "--progress" is used, thus SIGALRM is sent often, and can interrupt an fsync() syscall. Make sure we deal with such a case by retrying the syscall ourselves. Luckily, we call fsync() fron a single wrapper, fsync_or_die(), so the fix is fairly isolated. Reported-by: Randall S. Becker Helped-by: Jeff King Helped-by: Taylor Blau [jc: the above two did most of the work---I just tied the loose end] Helped-by: René Scharfe Signed-off-by: Junio C Hamano --- write-or-die.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/write-or-die.c b/write-or-die.c index eab8c8d0b9..d33e68f6ab 100644 --- a/write-or-die.c +++ b/write-or-die.c @@ -57,8 +57,9 @@ void fprintf_or_die(FILE *f, const char *fmt, ...) void fsync_or_die(int fd, const char *msg) { - if (fsync(fd) < 0) { - die_errno("fsync error on '%s'", msg); + while (fsync(fd) < 0) { + if (errno != EINTR) + die_errno("fsync error on '%s'", msg); } } From ebee5580ca7e4dba06405713de14507909615966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= Date: Sun, 6 Jun 2021 03:01:57 +0200 Subject: [PATCH 199/397] parallel-checkout: avoid dash local bug in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dash bug https://bugs.launchpad.net/ubuntu/+source/dash/+bug/139097 lets the shell erroneously perform field splitting on the expansion of a command substitution during declaration of a local variable. It causes the parallel-checkout tests to fail e.g. when running them with /bin/dash on MacOS 11.4, where they error out like this: ./t2080-parallel-checkout-basics.sh: 33: local: 0: bad variable name That's because the output of wc -l contains leading spaces and the returned number of lines is treated as another variable to declare, i.e. as in "local workers= 0". Work around it by enclosing the command substitution in quotes. Helped-by: Matheus Tavares Bernardino Helped-by: SZEDER Gábor Helped-by: Ævar Arnfjörð Bjarmason Signed-off-by: René Scharfe Signed-off-by: Junio C Hamano --- t/lib-parallel-checkout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/lib-parallel-checkout.sh b/t/lib-parallel-checkout.sh index 21f5759732..83b279a846 100644 --- a/t/lib-parallel-checkout.sh +++ b/t/lib-parallel-checkout.sh @@ -27,7 +27,7 @@ test_checkout_workers () { rm -f "$trace_file" && GIT_TRACE2="$(pwd)/$trace_file" "$@" 2>&8 && - local workers=$(grep "child_start\[..*\] git checkout--worker" "$trace_file" | wc -l) && + local workers="$(grep "child_start\[..*\] git checkout--worker" "$trace_file" | wc -l)" && test $workers -eq $expected_workers && rm "$trace_file" } 8>&2 2>&4 From ebf3c04b262aa27fbb97f8a0156c2347fecafafb Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Sun, 6 Jun 2021 15:40:01 +0900 Subject: [PATCH 200/397] Git 2.32 Signed-off-by: Junio C Hamano --- GIT-VERSION-GEN | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index c0942f12a3..9c125f298a 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=GIT-VERSION-FILE -DEF_VER=v2.32.0-rc3 +DEF_VER=v2.32.0 LF=' ' From 52ff891c034d625cd3e5f17f75ced42c8dde3438 Mon Sep 17 00:00:00 2001 From: Andrei Rybak Date: Mon, 7 Jun 2021 15:13:20 +0200 Subject: [PATCH 201/397] t: fix whitespace around && Add missing spaces before '&&' and switch tabs around '&&' to spaces. These issues were found using `git grep '[^ ]&&$'` and `git grep -P '&&\t'`. Signed-off-by: Andrei Rybak Signed-off-by: Junio C Hamano --- contrib/mw-to-git/t/t9360-mw-to-git-clone.sh | 2 +- contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh | 4 ++-- t/t1092-sparse-checkout-compatibility.sh | 2 +- t/t3920-crlf-messages.sh | 2 +- t/t4203-mailmap.sh | 2 +- t/t4205-log-pretty-formats.sh | 2 +- t/t7800-difftool.sh | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh b/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh index 4c39bda7bf..f08890d9e7 100755 --- a/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh +++ b/contrib/mw-to-git/t/t9360-mw-to-git-clone.sh @@ -86,7 +86,7 @@ test_expect_success 'Git clone works with page added' ' test_expect_success 'Git clone works with an edited page ' ' wiki_reset && wiki_editpage foo "this page will be edited" \ - false -s "first edition of page foo"&& + false -s "first edition of page foo" && wiki_editpage foo "this page has been edited and must be on the clone " true && git clone mediawiki::'"$WIKI_URL"' mw_dir_6 && test_path_is_file mw_dir_6/Foo.mw && diff --git a/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh b/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh index 6b0dbdac4d..526d92850f 100755 --- a/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh +++ b/contrib/mw-to-git/t/t9362-mw-to-git-utf8.sh @@ -287,7 +287,7 @@ test_expect_success 'git push with \' ' git add \\ko\\o.mw && git commit -m " \\ko\\o added" && git push - )&& + ) && wiki_page_exist \\ko\\o && wiki_check_content mw_dir_18/\\ko\\o.mw \\ko\\o @@ -311,7 +311,7 @@ test_expect_success 'git push with \ in format control' ' git add \\fo\\o.mw && git commit -m " \\fo\\o added" && git push - )&& + ) && wiki_page_exist \\fo\\o && wiki_check_content mw_dir_20/\\fo\\o.mw \\fo\\o diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh index e9a815ca7a..d028b73eba 100755 --- a/t/t1092-sparse-checkout-compatibility.sh +++ b/t/t1092-sparse-checkout-compatibility.sh @@ -268,7 +268,7 @@ test_expect_success 'diff with renames' ' for branch in rename-out-to-out rename-out-to-in rename-in-to-out do test_all_match git checkout rename-base && - test_all_match git checkout $branch -- .&& + test_all_match git checkout $branch -- . && test_all_match git diff --staged --no-renames && test_all_match git diff --staged --find-renames || return 1 done diff --git a/t/t3920-crlf-messages.sh b/t/t3920-crlf-messages.sh index 70ddce3a2e..a8ad5462d9 100755 --- a/t/t3920-crlf-messages.sh +++ b/t/t3920-crlf-messages.sh @@ -64,7 +64,7 @@ test_crlf_subject_body_and_contents() { while test -n "${atoms}" do set ${atoms} && atom=$1 && shift && atoms="$*" && - set ${files} && file=$1 && shift && files="$*" && + set ${files} && file=$1 && shift && files="$*" && test_expect_success "${command}: --format='%${atom}' works with messages using CRLF" " rm -f expect && for ref in ${LIB_CRLF_BRANCHES} diff --git a/t/t4203-mailmap.sh b/t/t4203-mailmap.sh index d8e7374234..0b2d21ec55 100755 --- a/t/t4203-mailmap.sh +++ b/t/t4203-mailmap.sh @@ -959,7 +959,7 @@ test_expect_success SYMLINKS 'symlinks not respected in-tree' ' test_when_finished "rm .mailmap" && ln -s map .mailmap && git log -1 --format=%aE >actual && - echo "orig@example.com" >expect&& + echo "orig@example.com" >expect && test_cmp expect actual ' diff --git a/t/t4205-log-pretty-formats.sh b/t/t4205-log-pretty-formats.sh index 8272d94ce6..5865daa8f8 100755 --- a/t/t4205-log-pretty-formats.sh +++ b/t/t4205-log-pretty-formats.sh @@ -988,7 +988,7 @@ test_expect_success '%(describe) vs git describe' ' test_expect_success '%(describe:match=...) vs git describe --match ...' ' test_when_finished "git tag -d tag-match" && - git tag -a -m tagged tag-match&& + git tag -a -m tagged tag-match && git describe --match "*-match" >expect && git log -1 --format="%(describe:match=*-match)" >actual && test_cmp expect actual diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 3e041e83ae..a173f564bc 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -770,7 +770,7 @@ test_expect_success 'difftool --rotate-to' ' echo 4 >4 && git add 1 2 4 && git commit -a -m "124" && - git difftool --no-prompt --extcmd=cat --rotate-to="2" HEAD^ >output&& + git difftool --no-prompt --extcmd=cat --rotate-to="2" HEAD^ >output && cat >expect <<-\EOF && 2 4 From d94f9b8e9049177d4148b57ecf5f44bfdcc4648d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 7 Jun 2021 16:33:05 +0200 Subject: [PATCH 202/397] protocol-caps.h: add newline at end of file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a trailing newline to the protocol-caps.h file added in the recent a2ba162cda (object-info: support for retrieving object info, 2021-04-20). Various editors add this implicitly, and some compilers warn about the lack of a \n here. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- protocol-caps.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol-caps.h b/protocol-caps.h index 6351648e37..0a9f49df11 100644 --- a/protocol-caps.h +++ b/protocol-caps.h @@ -7,4 +7,4 @@ struct packet_reader; int cap_object_info(struct repository *r, struct strvec *keys, struct packet_reader *request); -#endif /* PROTOCOL_CAPS_H */ \ No newline at end of file +#endif /* PROTOCOL_CAPS_H */ From b7b793d1e7b1ec8104eef2da450ff47c02c9c365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 7 Jun 2021 16:43:22 +0200 Subject: [PATCH 203/397] read-cache.c: don't guard calls to progress.c API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't guard the calls to the progress.c API with "if (progress)". The API itself will check this. This doesn't change any behavior, but makes this code consistent with the rest of the codebase. See ae9af12287b (status: show progress bar if refreshing the index takes too long, 2018-09-15) for the commit that added the pattern we're changing here. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- read-cache.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/read-cache.c b/read-cache.c index 5a907af2fb..8dfdeee93f 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1584,8 +1584,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, t2_sum_scan += t2_did_scan; if (new_entry == ce) continue; - if (progress) - display_progress(progress, i); + display_progress(progress, i); if (!new_entry) { const char *fmt; @@ -1620,10 +1619,8 @@ int refresh_index(struct index_state *istate, unsigned int flags, trace2_data_intmax("index", NULL, "refresh/sum_lstat", t2_sum_lstat); trace2_data_intmax("index", NULL, "refresh/sum_scan", t2_sum_scan); trace2_region_leave("index", "refresh", NULL); - if (progress) { - display_progress(progress, istate->cache_nr); - stop_progress(&progress); - } + display_progress(progress, istate->cache_nr); + stop_progress(&progress); trace_performance_leave("refresh index"); return has_errors; } From 546096a5cbb4806462d0e8e8fa4a562fc173aa8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Mon, 7 Jun 2021 18:43:49 +0200 Subject: [PATCH 204/397] xdiff: use BUG(...), not xdl_bug(...) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The xdl_bug() function was introduced in e8adf23d1e (xdl_change_compact(): introduce the concept of a change group, 2016-08-22), let's use our usual BUG() function instead. We'll now have meaningful line numbers if we encounter bugs in xdiff, and less code duplication. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- xdiff/xdiffi.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index 380eb728ed..a4542c05b6 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -796,12 +796,6 @@ static int group_slide_up(xdfile_t *xdf, struct xdlgroup *g, long flags) } } -static void xdl_bug(const char *msg) -{ - fprintf(stderr, "BUG: %s\n", msg); - exit(1); -} - /* * Move back and forward change groups for a consistent and pretty diff output. * This also helps in finding joinable change groups and reducing the diff @@ -841,7 +835,7 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) { /* Shift the group backward as much as possible: */ while (!group_slide_up(xdf, &g, flags)) if (group_previous(xdfo, &go)) - xdl_bug("group sync broken sliding up"); + BUG("group sync broken sliding up"); /* * This is this highest that this group can be shifted. @@ -857,7 +851,7 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) { if (group_slide_down(xdf, &g, flags)) break; if (group_next(xdfo, &go)) - xdl_bug("group sync broken sliding down"); + BUG("group sync broken sliding down"); if (go.end > go.start) end_matching_other = g.end; @@ -882,9 +876,9 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) { */ while (go.end == go.start) { if (group_slide_up(xdf, &g, flags)) - xdl_bug("match disappeared"); + BUG("match disappeared"); if (group_previous(xdfo, &go)) - xdl_bug("group sync broken sliding to match"); + BUG("group sync broken sliding to match"); } } else if (flags & XDF_INDENT_HEURISTIC) { /* @@ -925,9 +919,9 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) { while (g.end > best_shift) { if (group_slide_up(xdf, &g, flags)) - xdl_bug("best shift unreached"); + BUG("best shift unreached"); if (group_previous(xdfo, &go)) - xdl_bug("group sync broken sliding to blank line"); + BUG("group sync broken sliding to blank line"); } } @@ -936,11 +930,11 @@ int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) { if (group_next(xdf, &g)) break; if (group_next(xdfo, &go)) - xdl_bug("group sync broken moving to next group"); + BUG("group sync broken moving to next group"); } if (!group_next(xdfo, &go)) - xdl_bug("group sync broken at end of file"); + BUG("group sync broken at end of file"); return 0; } From 47eb4c689073eddc082007255564e1d3e142b727 Mon Sep 17 00:00:00 2001 From: Michael Schindler Date: Mon, 7 Jun 2021 20:18:33 +0000 Subject: [PATCH 205/397] mergetools/kdiff3: make kdiff3 work on Windows too The native kdiff3 mergetool is not found by git mergetool on Windows. The message "The merge tool kdiff3 is not available as 'kdiff3'" is displayed. Just like we translate the name of the binary and look for it on the search path for WinMerge, do the same for kdiff3 to find it. Signed-off-by: Michael Schindler michael@compressconsult.com Signed-off-by: Junio C Hamano --- mergetools/kdiff3 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mergetools/kdiff3 b/mergetools/kdiff3 index 0264ed5b20..520cb914a1 100644 --- a/mergetools/kdiff3 +++ b/mergetools/kdiff3 @@ -25,3 +25,12 @@ merge_cmd () { exit_code_trustable () { true } + +translate_merge_tool_path() { + if type kdiff3 >/dev/null 2>/dev/null + then + echo kdiff3 + else + mergetool_find_win32_cmd "kdiff3.exe" "Kdiff3" + fi +} From a0538e5c8baf9985a71d18a1495985441041efba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Tue, 8 Jun 2021 08:58:40 +0700 Subject: [PATCH 206/397] doc/log: correct default for --decorate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There're two different default options for log --decorate: * Should `--decorate` be given without any arguments, it's default to `short` * Should neither `--decorate` nor `--no-decorate` be given, it's default to the `log.decorate` or `auto`. We documented the former, but not the latter. Let's document them, too. Reported-by: Andy AO Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- Documentation/git-log.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/git-log.txt b/Documentation/git-log.txt index 1bbf865a1b..0498e7bacb 100644 --- a/Documentation/git-log.txt +++ b/Documentation/git-log.txt @@ -39,7 +39,9 @@ OPTIONS full ref name (including prefix) will be printed. If 'auto' is specified, then if the output is going to a terminal, the ref names are shown as if 'short' were given, otherwise no ref names are - shown. The default option is 'short'. + shown. The option `--decorate` is short-hand for `--decorate=short`. + Default to configuration value of `log.decorate` if configured, + otherwise, `auto`. --decorate-refs=:: --decorate-refs-exclude=:: From 8603c419d3b7619b9958541461eb024c767cf212 Mon Sep 17 00:00:00 2001 From: Felipe Contreras Date: Mon, 7 Jun 2021 20:58:07 -0500 Subject: [PATCH 207/397] doc: merge: mention default of defaulttoupstream Commit a01f7f2ba0 (merge: enable defaulttoupstream by default, 2014-04-20) forgot to mention the new default in the configuration documentation. Signed-off-by: Felipe Contreras Signed-off-by: Junio C Hamano --- Documentation/config/merge.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/config/merge.txt b/Documentation/config/merge.txt index cb2ed58907..6b66c83eab 100644 --- a/Documentation/config/merge.txt +++ b/Documentation/config/merge.txt @@ -14,7 +14,7 @@ merge.defaultToUpstream:: branches at the remote named by `branch..remote` are consulted, and then they are mapped via `remote..fetch` to their corresponding remote-tracking branches, and the tips of - these tracking branches are merged. + these tracking branches are merged. Defaults to true. merge.ff:: By default, Git does not create an extra merge commit when merging From 482c962de4af32471a5ab770160731a028fcd9d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Tue, 8 Jun 2021 13:56:28 +0700 Subject: [PATCH 208/397] t: use user-specified utf-8 locale for testing svn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In some test-cases, UTF-8 locale is required. To find such locale, we're using the first available UTF-8 locale that returned by "locale -a". However, the locale(1) utility is unavailable on some systems, e.g. Linux with musl libc. However, without "locale -a", we can't guess provided UTF-8 locale. Add a Makefile knob GIT_TEST_UTF8_LOCALE and activate it for linux-musl in our CI system. Rename t/lib-git-svn.sh:prepare_a_utf8_locale to prepare_utf8_locale, since we no longer prepare the variable named "a_utf8_locale", but set up a fallback value for GIT_TEST_UTF8_LOCALE instead. The fallback will be LC_ALL, LANG environment variable, or the first UTF-8 locale from output of "locale -a", in that order. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- Makefile | 7 +++++++ ci/lib.sh | 1 + t/lib-git-svn.sh | 22 ++++++++++++++++------ t/t9100-git-svn-basic.sh | 14 +++----------- t/t9115-git-svn-dcommit-funky-renames.sh | 6 +++--- t/t9129-git-svn-i18n-commitencoding.sh | 4 ++-- 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index f3dc217832..5d6a50f118 100644 --- a/Makefile +++ b/Makefile @@ -398,6 +398,10 @@ all:: # with a different indexfile format version. If it isn't set the index # file format used is index-v[23]. # +# Define GIT_TEST_UTF8_LOCALE to preferred utf-8 locale for testing. +# If it isn't set, fallback to $LC_ALL, $LANG or use the first utf-8 +# locale returned by "locale -a". +# # Define HAVE_CLOCK_GETTIME if your platform has clock_gettime. # # Define HAVE_CLOCK_MONOTONIC if your platform has CLOCK_MONOTONIC. @@ -2740,6 +2744,9 @@ ifdef GIT_TEST_CMP endif ifdef GIT_TEST_CMP_USE_COPIED_CONTEXT @echo GIT_TEST_CMP_USE_COPIED_CONTEXT=YesPlease >>$@+ +endif +ifdef GIT_TEST_UTF8_LOCALE + @echo GIT_TEST_UTF8_LOCALE=\''$(subst ','\'',$(subst ','\'',$(GIT_TEST_UTF8_LOCALE)))'\' >>$@+ endif @echo NO_GETTEXT=\''$(subst ','\'',$(subst ','\'',$(NO_GETTEXT)))'\' >>$@+ ifdef GIT_PERF_REPEAT_COUNT diff --git a/ci/lib.sh b/ci/lib.sh index d848c036c5..476c3f369f 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -229,6 +229,7 @@ linux-musl) CC=gcc MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" + MAKEFLAGS="$MAKEFLAGS GIT_TEST_UTF8_LOCALE=C.UTF-8" ;; esac diff --git a/t/lib-git-svn.sh b/t/lib-git-svn.sh index 547eb3c31a..2fde2353fd 100644 --- a/t/lib-git-svn.sh +++ b/t/lib-git-svn.sh @@ -121,12 +121,22 @@ start_svnserve () { --listen-host 127.0.0.1 & } -prepare_a_utf8_locale () { - a_utf8_locale=$(locale -a | sed -n '/\.[uU][tT][fF]-*8$/{ - p - q -}') - if test -n "$a_utf8_locale" +prepare_utf8_locale () { + if test -z "$GIT_TEST_UTF8_LOCALE" + then + case "${LC_ALL:-$LANG}" in + *.[Uu][Tt][Ff]8 | *.[Uu][Tt][Ff]-8) + GIT_TEST_UTF8_LOCALE="${LC_ALL:-$LANG}" + ;; + *) + GIT_TEST_UTF8_LOCALE=$(locale -a | sed -n '/\.[uU][tT][fF]-*8$/{ + p + q + }') + ;; + esac + fi + if test -n "$GIT_TEST_UTF8_LOCALE" then test_set_prereq UTF8 else diff --git a/t/t9100-git-svn-basic.sh b/t/t9100-git-svn-basic.sh index 1d3fdcc997..d5563ec35f 100755 --- a/t/t9100-git-svn-basic.sh +++ b/t/t9100-git-svn-basic.sh @@ -4,21 +4,13 @@ # test_description='git svn basic tests' -GIT_SVN_LC_ALL=${LC_ALL:-$LANG} GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./lib-git-svn.sh -case "$GIT_SVN_LC_ALL" in -*.UTF-8) - test_set_prereq UTF8 - ;; -*) - say "# UTF-8 locale not set, some tests skipped ($GIT_SVN_LC_ALL)" - ;; -esac +prepare_utf8_locale test_expect_success 'git svn --version works anywhere' ' nongit git svn --version @@ -187,8 +179,8 @@ test_expect_success POSIXPERM,SYMLINKS "$name" ' test ! -h "$SVN_TREE"/exec-2.sh && test_cmp help "$SVN_TREE"/exec-2.sh' -name="commit with UTF-8 message: locale: $GIT_SVN_LC_ALL" -LC_ALL="$GIT_SVN_LC_ALL" +name="commit with UTF-8 message: locale: $GIT_TEST_UTF8_LOCALE" +LC_ALL="$GIT_TEST_UTF8_LOCALE" export LC_ALL # This test relies on the previous test, hence requires POSIXPERM,SYMLINKS test_expect_success UTF8,POSIXPERM,SYMLINKS "$name" " diff --git a/t/t9115-git-svn-dcommit-funky-renames.sh b/t/t9115-git-svn-dcommit-funky-renames.sh index 9b44a44bc1..743fbe1fe4 100755 --- a/t/t9115-git-svn-dcommit-funky-renames.sh +++ b/t/t9115-git-svn-dcommit-funky-renames.sh @@ -93,9 +93,9 @@ test_expect_success 'git svn rebase works inside a fresh-cloned repository' ' # > ... All of the above characters, except for the backslash, are converted # > to special UNICODE characters in the range 0xf000 to 0xf0ff (the # > "Private use area") when creating or accessing files. -prepare_a_utf8_locale +prepare_utf8_locale test_expect_success UTF8,!MINGW,!UTF8_NFD_TO_NFC 'svn.pathnameencoding=cp932 new file on dcommit' ' - LC_ALL=$a_utf8_locale && + LC_ALL=$GIT_TEST_UTF8_LOCALE && export LC_ALL && neq=$(printf "\201\202") && git config svn.pathnameencoding cp932 && @@ -107,7 +107,7 @@ test_expect_success UTF8,!MINGW,!UTF8_NFD_TO_NFC 'svn.pathnameencoding=cp932 new # See the comment on the above test for setting of LC_ALL. test_expect_success !MINGW,!UTF8_NFD_TO_NFC 'svn.pathnameencoding=cp932 rename on dcommit' ' - LC_ALL=$a_utf8_locale && + LC_ALL=$GIT_TEST_UTF8_LOCALE && export LC_ALL && inf=$(printf "\201\207") && git config svn.pathnameencoding cp932 && diff --git a/t/t9129-git-svn-i18n-commitencoding.sh b/t/t9129-git-svn-i18n-commitencoding.sh index 2c213ae654..01e1e8a8f7 100755 --- a/t/t9129-git-svn-i18n-commitencoding.sh +++ b/t/t9129-git-svn-i18n-commitencoding.sh @@ -14,12 +14,12 @@ compare_git_head_with () { test_cmp current "$1" } -prepare_a_utf8_locale +prepare_utf8_locale compare_svn_head_with () { # extract just the log message and strip out committer info. # don't use --limit here since svn 1.1.x doesn't have it, - LC_ALL="$a_utf8_locale" svn log $(git svn info --url) | perl -w -e ' + LC_ALL="$GIT_TEST_UTF8_LOCALE" svn log $(git svn info --url) | perl -w -e ' use bytes; $/ = ("-"x72) . "\n"; my @x = ; From 6fb9195f6c6df047949efcdb8a1fb9eaed0c925e Mon Sep 17 00:00:00 2001 From: Jeff King Date: Tue, 8 Jun 2021 03:24:48 -0400 Subject: [PATCH 209/397] doc: warn people against --max-pack-size This option is almost never a good idea, as the resulting repository is larger and slower (see the new explanations in the docs). I outlined the potential problems. We could go further and make the option harder to find (or at least, make the command-line option descriptions a much more terse "you probably don't want this; see pack.packsizeLimit for details"). But this seems like a minimal change that may prevent people from thinking it's more useful than it is. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- Documentation/config/pack.txt | 23 +++++++++++++++++------ Documentation/git-pack-objects.txt | 6 +++--- Documentation/git-repack.txt | 4 +++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/Documentation/config/pack.txt b/Documentation/config/pack.txt index 3da4ea98e2..10c778a942 100644 --- a/Documentation/config/pack.txt +++ b/Documentation/config/pack.txt @@ -99,12 +99,23 @@ pack.packSizeLimit:: packing to a file when repacking, i.e. the git:// protocol is unaffected. It can be overridden by the `--max-pack-size` option of linkgit:git-repack[1]. Reaching this limit results - in the creation of multiple packfiles; which in turn prevents - bitmaps from being created. - The minimum size allowed is limited to 1 MiB. - The default is unlimited. - Common unit suffixes of 'k', 'm', or 'g' are - supported. + in the creation of multiple packfiles. ++ +Note that this option is rarely useful, and may result in a larger total +on-disk size (because Git will not store deltas between packs), as well +as worse runtime performance (object lookup within multiple packs is +slower than a single pack, and optimizations like reachability bitmaps +cannot cope with multiple packs). ++ +If you need to actively run Git using smaller packfiles (e.g., because your +filesystem does not support large files), this option may help. But if +your goal is to transmit a packfile over a medium that supports limited +sizes (e.g., removable media that cannot store the whole repository), +you are likely better off creating a single large packfile and splitting +it using a generic multi-volume archive tool (e.g., Unix `split`). ++ +The minimum size allowed is limited to 1 MiB. The default is unlimited. +Common unit suffixes of 'k', 'm', or 'g' are supported. pack.useBitmaps:: When true, git will use pack bitmaps (if available) when packing diff --git a/Documentation/git-pack-objects.txt b/Documentation/git-pack-objects.txt index f85cb7ea93..c20b978510 100644 --- a/Documentation/git-pack-objects.txt +++ b/Documentation/git-pack-objects.txt @@ -118,10 +118,10 @@ depth is 4095. into multiple independent packfiles, each not larger than the given size. The size can be suffixed with "k", "m", or "g". The minimum size allowed is limited to 1 MiB. - This option - prevents the creation of a bitmap index. The default is unlimited, unless the config variable - `pack.packSizeLimit` is set. + `pack.packSizeLimit` is set. Note that this option may result in + a larger and slower repository; see the discussion in + `pack.packSizeLimit`. --honor-pack-keep:: This flag causes an object already in a local pack that diff --git a/Documentation/git-repack.txt b/Documentation/git-repack.txt index fbd4b4ae06..7bb797cbe8 100644 --- a/Documentation/git-repack.txt +++ b/Documentation/git-repack.txt @@ -121,7 +121,9 @@ depth is 4095. If specified, multiple packfiles may be created, which also prevents the creation of a bitmap index. The default is unlimited, unless the config variable - `pack.packSizeLimit` is set. + `pack.packSizeLimit` is set. Note that this option may result in + a larger and slower repository; see the discussion in + `pack.packSizeLimit`. -b:: --write-bitmap-index:: From 338abb0f045b87df5e628543800e74dec0e72cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?= Date: Tue, 8 Jun 2021 12:48:03 +0200 Subject: [PATCH 210/397] builtins + test helpers: use return instead of exit() in cmd_* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change various cmd_* functions that claim to return an "int" to use "return" instead of exit() to indicate an exit code. These were not marked with NORETURN, and by directly exit()-ing we'll skip the cleanup git.c would otherwise do (e.g. closing fd's, erroring if we can't). See run_builtin() in git.c. In the case of shell.c and sh-i18n--envsubst.c this was the result of an incomplete migration to using a cmd_main() in 3f2e2297b9 (add an extra level of indirection to main(), 2016-07-01). This was spotted by SunCC 12.5 on Solaris 10 (gcc210 on the gccfarm). Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- builtin/difftool.c | 5 ++--- builtin/merge-ours.c | 4 ++-- builtin/mktree.c | 2 +- sh-i18n--envsubst.c | 6 +++--- shell.c | 2 +- t/helper/test-hash-speed.c | 2 +- t/helper/test-hash.c | 2 +- t/helper/test-match-trees.c | 2 +- t/helper/test-reach.c | 2 +- 9 files changed, 13 insertions(+), 14 deletions(-) diff --git a/builtin/difftool.c b/builtin/difftool.c index 6e18e623fd..6861f33db6 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -671,7 +671,7 @@ static int run_file_diff(int prompt, const char *prefix, "GIT_PAGER=", "GIT_EXTERNAL_DIFF=git-difftool--helper", NULL, NULL }; - int ret = 0, i; + int i; if (prompt > 0) env[2] = "GIT_DIFFTOOL_PROMPT=true"; @@ -682,8 +682,7 @@ static int run_file_diff(int prompt, const char *prefix, strvec_push(&args, "diff"); for (i = 0; i < argc; i++) strvec_push(&args, argv[i]); - ret = run_command_v_opt_cd_env(args.v, RUN_GIT_CMD, prefix, env); - exit(ret); + return run_command_v_opt_cd_env(args.v, RUN_GIT_CMD, prefix, env); } int cmd_difftool(int argc, const char **argv, const char *prefix) diff --git a/builtin/merge-ours.c b/builtin/merge-ours.c index 4594507420..3583cff71c 100644 --- a/builtin/merge-ours.c +++ b/builtin/merge-ours.c @@ -28,6 +28,6 @@ int cmd_merge_ours(int argc, const char **argv, const char *prefix) if (read_cache() < 0) die_errno("read_cache failed"); if (index_differs_from(the_repository, "HEAD", NULL, 0)) - exit(2); - exit(0); + return 2; + return 0; } diff --git a/builtin/mktree.c b/builtin/mktree.c index 891991b00d..ae78ca1c02 100644 --- a/builtin/mktree.c +++ b/builtin/mktree.c @@ -189,5 +189,5 @@ int cmd_mktree(int ac, const char **av, const char *prefix) used=0; /* reset tree entry buffer for re-use in batch mode */ } strbuf_release(&sb); - exit(0); + return 0; } diff --git a/sh-i18n--envsubst.c b/sh-i18n--envsubst.c index e7430b9aa8..6cd307ac2c 100644 --- a/sh-i18n--envsubst.c +++ b/sh-i18n--envsubst.c @@ -104,12 +104,12 @@ cmd_main (int argc, const char *argv[]) if (ferror (stderr) || fflush (stderr)) { fclose (stderr); - exit (EXIT_FAILURE); + return (EXIT_FAILURE); } if (fclose (stderr) && errno != EBADF) - exit (EXIT_FAILURE); + return (EXIT_FAILURE); - exit (EXIT_SUCCESS); + return (EXIT_SUCCESS); } /* Parse the string and invoke the callback each time a $VARIABLE or diff --git a/shell.c b/shell.c index cef7ffdc9e..811e13b9c9 100644 --- a/shell.c +++ b/shell.c @@ -177,7 +177,7 @@ int cmd_main(int argc, const char **argv) default: continue; } - exit(cmd->exec(cmd->name, arg)); + return cmd->exec(cmd->name, arg); } cd_to_homedir(); diff --git a/t/helper/test-hash-speed.c b/t/helper/test-hash-speed.c index 432233c7f0..f40d9ad0c2 100644 --- a/t/helper/test-hash-speed.c +++ b/t/helper/test-hash-speed.c @@ -57,5 +57,5 @@ int cmd__hash_speed(int ac, const char **av) free(p); } - exit(0); + return 0; } diff --git a/t/helper/test-hash.c b/t/helper/test-hash.c index 0a31de66f3..261c545b9d 100644 --- a/t/helper/test-hash.c +++ b/t/helper/test-hash.c @@ -54,5 +54,5 @@ int cmd_hash_impl(int ac, const char **av, int algo) fwrite(hash, 1, algop->rawsz, stdout); else puts(hash_to_hex_algop(hash, algop)); - exit(0); + return 0; } diff --git a/t/helper/test-match-trees.c b/t/helper/test-match-trees.c index b9fd427571..4079fdee06 100644 --- a/t/helper/test-match-trees.c +++ b/t/helper/test-match-trees.c @@ -23,5 +23,5 @@ int cmd__match_trees(int ac, const char **av) shift_tree(the_repository, &one->object.oid, &two->object.oid, &shifted, -1); printf("shifted: %s\n", oid_to_hex(&shifted)); - exit(0); + return 0; } diff --git a/t/helper/test-reach.c b/t/helper/test-reach.c index cda804ed79..2f65c7f6a5 100644 --- a/t/helper/test-reach.c +++ b/t/helper/test-reach.c @@ -166,5 +166,5 @@ int cmd__reach(int ac, const char **av) print_sorted_commit_ids(list); } - exit(0); + return 0; } From 91d2347033b72d53c222faeb474ae0d46c01065f Mon Sep 17 00:00:00 2001 From: Atharva Raykar Date: Wed, 9 Jun 2021 00:36:12 +0530 Subject: [PATCH 211/397] MyFirstContribution: link #git-devel to Libera Chat Many of the regulars on #git-devel are now on Libera Chat, to the extent that the community page now lists it as the IRC Channel[1]. This will help new contributors find the right place, if they choose to ask questions on `#git-devel`. Relevant discussion on the IRC transition: https://lore.kernel.org/git/CAJoAoZ=e62sceNpcR5L5zjsj177uczTnXjcAg+BbOoOkeH8vXQ@mail.gmail.com/ [1] https://git-scm.com/community Signed-off-by: Atharva Raykar Reviewed-by: Emily Shaffer Signed-off-by: Junio C Hamano --- Documentation/MyFirstContribution.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/MyFirstContribution.txt b/Documentation/MyFirstContribution.txt index af0a9da62e..015cf24631 100644 --- a/Documentation/MyFirstContribution.txt +++ b/Documentation/MyFirstContribution.txt @@ -47,7 +47,7 @@ Veteran contributors who are especially interested in helping mentor newcomers are present on the list. In order to avoid search indexers, group membership is required to view messages; anyone can join and no approval is required. -==== https://webchat.freenode.net/#git-devel[#git-devel] on Freenode +==== https://web.libera.chat/#git-devel[#git-devel] on Libera Chat This IRC channel is for conversations between Git contributors. If someone is currently online and knows the answer to your question, you can receive help @@ -827,7 +827,7 @@ either examining recent pull requests where someone has been granted `/allow` (https://github.com/gitgitgadget/git/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+%22%2Fallow%22[Search: is:pr is:open "/allow"]), in which case both the author and the person who granted the `/allow` can now `/allow` you, or by inquiring on the -https://webchat.freenode.net/#git-devel[#git-devel] IRC channel on Freenode +https://web.libera.chat/#git-devel[#git-devel] IRC channel on Libera Chat linking your pull request and asking for someone to `/allow` you. If the CI fails, you can update your changes with `git rebase -i` and push your From 4184cbd635140e83c5f0d57c377eec93a9b6eedf Mon Sep 17 00:00:00 2001 From: Andrei Rybak Date: Tue, 8 Jun 2021 22:48:41 +0200 Subject: [PATCH 212/397] mailinfo: use starts_with() when checking scissors Existing checks for scissors characters using memcmp(3) never read past the end of the line, because all substrings we are interested in are two characters long, and the outer loop guarantees we have at least one character. So at most we will look at the NUL. However, this is too subtle and may lead to bugs in code which copies this behavior without realizing substring length requirement. So use starts_with() instead, which will stop at NUL regardless of the length of the prefix. Remove extra pair of parentheses while we are here. Helped-by: Jeff King Signed-off-by: Andrei Rybak Acked-by: Jeff King Signed-off-by: Junio C Hamano --- mailinfo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailinfo.c b/mailinfo.c index 5681d9130d..6f3bb5a22e 100644 --- a/mailinfo.c +++ b/mailinfo.c @@ -705,8 +705,8 @@ static int is_scissors_line(const char *line) perforation++; continue; } - if ((!memcmp(c, ">8", 2) || !memcmp(c, "8<", 2) || - !memcmp(c, ">%", 2) || !memcmp(c, "%<", 2))) { + if (starts_with(c, ">8") || starts_with(c, "8<") || + starts_with(c, ">%") || starts_with(c, "%<")) { in_perforation = 1; perforation += 2; scissors += 2; From 5a3743da326b2e041c65ac12ba0d04a1fed45b58 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:39 +0000 Subject: [PATCH 213/397] merge-ort: replace string_list_df_name_compare with faster alternative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gathering accumulated times from trace2 output on the mega-renames testcase, I saw the following timings (where I'm only showing a few lines to highlight the portions of interest): 10.120 : label:incore_nonrecursive 4.462 : ..label:process_entries 3.143 : ....label:process_entries setup 2.988 : ......label:plist special sort 1.305 : ....label:processing 2.604 : ..label:collect_merge_info 2.018 : ..label:merge_start 1.018 : ..label:renames In the above output, note that the 4.462 seconds for process_entries was split as 3.143 seconds for "process_entries setup" and 1.305 seconds for "processing" (and a little time for other stuff removed from the highlight). Most of the "process_entries setup" time was spent on "plist special sort" which corresponds to the following code: trace2_region_enter("merge", "plist special sort", opt->repo); plist.cmp = string_list_df_name_compare; string_list_sort(&plist); trace2_region_leave("merge", "plist special sort", opt->repo); In other words, in a merge strategy that would be invoked by passing "-sort" to either rebase or merge, sorting an array takes more time than anything else. Serves me right for naming my merge strategy this way. Rewrite the comparison function in a way that does not require finding out the lengths of the strings when comparing them. While at it, tweak the code for our specific case -- no need to handle a variety of modes, for example. The combination of these changes reduced the time spent in "plist special sort" by ~25% in the mega-renames case. For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 5.622 s ± 0.059 s 5.235 s ± 0.042 s mega-renames: 10.127 s ± 0.073 s 9.419 s ± 0.107 s just-one-mega: 500.3 ms ± 3.8 ms 480.1 ms ± 3.9 ms Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- merge-ort.c | 67 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index ed21dc8724..be80143f4d 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2746,31 +2746,58 @@ simple_cleanup: /*** Function Grouping: functions related to process_entries() ***/ -static int string_list_df_name_compare(const char *one, const char *two) +static int sort_dirs_next_to_their_children(const char *one, const char *two) { - int onelen = strlen(one); - int twolen = strlen(two); + unsigned char c1, c2; + /* - * Here we only care that entries for D/F conflicts are - * adjacent, in particular with the file of the D/F conflict - * appearing before files below the corresponding directory. - * The order of the rest of the list is irrelevant for us. + * Here we only care that entries for directories appear adjacent + * to and before files underneath the directory. We can achieve + * that by pretending to add a trailing slash to every file and + * then sorting. In other words, we do not want the natural + * sorting of + * foo + * foo.txt + * foo/bar + * Instead, we want "foo" to sort as though it were "foo/", so that + * we instead get + * foo.txt + * foo + * foo/bar + * To achieve this, we basically implement our own strcmp, except that + * if we get to the end of either string instead of comparing NUL to + * another character, we compare '/' to it. * - * To achieve this, we sort with df_name_compare and provide - * the mode S_IFDIR so that D/F conflicts will sort correctly. - * We use the mode S_IFDIR for everything else for simplicity, - * since in other cases any changes in their order due to - * sorting cause no problems for us. + * If this unusual "sort as though '/' were appended" perplexes + * you, perhaps it will help to note that this is not the final + * sort. write_tree() will sort again without the trailing slash + * magic, but just on paths immediately under a given tree. + * + * The reason to not use df_name_compare directly was that it was + * just too expensive (we don't have the string lengths handy), so + * it was reimplemented. */ - int cmp = df_name_compare(one, onelen, S_IFDIR, - two, twolen, S_IFDIR); + /* - * Now that 'foo' and 'foo/bar' compare equal, we have to make sure - * that 'foo' comes before 'foo/bar'. + * NOTE: This function will never be called with two equal strings, + * because it is used to sort the keys of a strmap, and strmaps have + * unique keys by construction. That simplifies our c1==c2 handling + * below. */ - if (cmp) - return cmp; - return onelen - twolen; + + while (*one && (*one == *two)) { + one++; + two++; + } + + c1 = *one ? *one : '/'; + c2 = *two ? *two : '/'; + + if (c1 == c2) { + /* Getting here means one is a leading directory of the other */ + return (*one) ? 1 : -1; + } else + return c1 - c2; } static int read_oid_strbuf(struct merge_options *opt, @@ -3489,7 +3516,7 @@ static void process_entries(struct merge_options *opt, trace2_region_leave("merge", "plist copy", opt->repo); trace2_region_enter("merge", "plist special sort", opt->repo); - plist.cmp = string_list_df_name_compare; + plist.cmp = sort_dirs_next_to_their_children; string_list_sort(&plist); trace2_region_leave("merge", "plist special sort", opt->repo); From 61bf4490afffb946787edec63b932551b3089d27 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:40 +0000 Subject: [PATCH 214/397] diffcore-rename: avoid unnecessary strdup'ing in break_idx The keys of break_idx are strings from the diff_filepairs of diff_queued_diff. break_idx is only used in location_rename_dst(), and that usage is always before any free'ing of the pairs (and thus the strings in the pairs). As such, there is no need to strdup these keys; we can just reuse the existing strings as-is. The merge logic doesn't make use of break detection, so this does not affect the performance of any of my testcases. It was just a minor unrelated optimization noted in passing while looking at the code. Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- diffcore-rename.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index dfbe65c917..9c0bc5afb4 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -54,7 +54,7 @@ static void register_rename_src(struct diff_filepair *p) if (p->broken_pair) { if (!break_idx) { break_idx = xmalloc(sizeof(*break_idx)); - strintmap_init(break_idx, -1); + strintmap_init_with_options(break_idx, -1, NULL, 0); } strintmap_set(break_idx, p->one->path, rename_dst_nr); } From 356da0f98b2c6631b72e69d8638517dd849d28f8 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:41 +0000 Subject: [PATCH 215/397] Fix various issues found in comments A random hodge-podge of incorrect or out-of-date comments that I found: * t6423 had a comment that has referred to the wrong test for years; fix it to refer to the right one. * diffcore-rename had a FIXME comment meant to remind myself to investigate if I could make another code change. I later investigated and removed the FIXME, but while cherry-picking the patch to submit upstream I missed the later update. Remove the comment now. * merge-ort had the early part of a comment for a function; I had meant to include the more involved description when I updated the function. Update the comment now. Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- diffcore-rename.c | 2 +- merge-ort.c | 8 +++++--- t/t6423-merge-rename-directories.sh | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index 9c0bc5afb4..eb77e5c4f0 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -1544,7 +1544,7 @@ void diffcore_rename_extended(struct diff_options *options, /* all the usual ones need to be kept */ diff_q(&outq, p); else - /* no need to keep unmodified pairs; FIXME: remove earlier? */ + /* no need to keep unmodified pairs */ pair_to_free = p; if (pair_to_free) diff --git a/merge-ort.c b/merge-ort.c index be80143f4d..a3bb3e1132 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2533,7 +2533,7 @@ static int compare_pairs(const void *a_, const void *b_) return strcmp(a->one->path, b->one->path); } -/* Call diffcore_rename() to compute which files have changed on given side */ +/* Call diffcore_rename() to update deleted/added pairs into rename pairs */ static void detect_regular_renames(struct merge_options *opt, unsigned side_index) { @@ -2586,8 +2586,10 @@ static void detect_regular_renames(struct merge_options *opt, } /* - * Get information of all renames which occurred in 'side_pairs', discarding - * non-renames. + * Get information of all renames which occurred in 'side_pairs', making use + * of any implicit directory renames in side_dir_renames (also making use of + * implicit directory renames rename_exclusions as needed by + * check_for_directory_rename()). Add all (updated) renames into result. */ static int collect_renames(struct merge_options *opt, struct diff_queue_struct *result, diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh index 3037c5c9bf..85d947dd27 100755 --- a/t/t6423-merge-rename-directories.sh +++ b/t/t6423-merge-rename-directories.sh @@ -454,7 +454,7 @@ test_expect_success '1f: Split a directory into two other directories' ' # the directory renamed, but the files within it. (see 1b) # # If renames split a directory into two or more others, the directory -# with the most renames, "wins" (see 1c). However, see the testcases +# with the most renames, "wins" (see 1f). However, see the testcases # in section 2, plus testcases 3a and 4a. ########################################################################### From ef68c3d800b9c4516f1a13038e047610882cd26f Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Tue, 8 Jun 2021 16:11:42 +0000 Subject: [PATCH 216/397] merge-ort: miscellaneous touch-ups Add some notes in the code about invariants with match_mask when adding pairs. Also add a comment that seems to have been left out in my work of pushing these changes upstream. Signed-off-by: Elijah Newren Reviewed-by: Derrick Stolee Signed-off-by: Junio C Hamano --- merge-ort.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index a3bb3e1132..53571742a0 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -764,6 +764,7 @@ static void add_pair(struct merge_options *opt, int names_idx = is_add ? side : 0; if (is_add) { + assert(match_mask == 0 || match_mask == 6); if (strset_contains(&renames->cached_target_names[side], pathname)) return; @@ -771,6 +772,8 @@ static void add_pair(struct merge_options *opt, unsigned content_relevant = (match_mask == 0); unsigned location_relevant = (dir_rename_mask == 0x07); + assert(match_mask == 0 || match_mask == 3 || match_mask == 5); + /* * If pathname is found in cached_irrelevant[side] due to * previous pick but for this commit content is relevant, @@ -3485,6 +3488,8 @@ static void process_entry(struct merge_options *opt, */ if (!ci->merged.clean) strmap_put(&opt->priv->conflicted, path, ci); + + /* Record metadata for ci->merged in dir_metadata */ record_entry_for_tree(dir_metadata, path, &ci->merged); } From 211eca0895794362184da2be2a2d812d070719d3 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 10 Jun 2021 11:37:04 +0900 Subject: [PATCH 217/397] The first batch post Git 2.32 Signed-off-by: Junio C Hamano --- Documentation/RelNotes/2.33.0.txt | 32 +++++++++++++++++++++++++++++++ RelNotes | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 Documentation/RelNotes/2.33.0.txt diff --git a/Documentation/RelNotes/2.33.0.txt b/Documentation/RelNotes/2.33.0.txt new file mode 100644 index 0000000000..6795a2734f --- /dev/null +++ b/Documentation/RelNotes/2.33.0.txt @@ -0,0 +1,32 @@ +Git 2.33 Release Notes +====================== + +Updates since Git 2.32 +---------------------- + +UI, Workflows & Features + +Performance, Internal Implementation, Development Support etc. + + +Fixes since v2.31 +----------------- + + * We historically rejected a very short string as an author name + while accepting a patch e-mail, which has been loosened. + (merge 72ee47ceeb ef/mailinfo-short-name later to maint). + + * The parallel checkout codepath did not initialize object ID field + used to talk to the worker processes in a futureproof way. + + * Rewrite code that triggers undefined behaiour warning. + (merge aafa5df0df jn/size-t-casted-to-off-t-fix later to maint). + + * The description of "fast-forward" in the glossary has been updated. + (merge e22f2daed0 ry/clarify-fast-forward-in-glossary later to maint). + + * Other code cleanup, docfix, build fix, etc. + (merge bfe35a6165 ah/doc-describe later to maint). + (merge f302c1e4aa jc/clarify-revision-range later to maint). + (merge 3127ff90ea tl/fix-packfile-uri-doc later to maint). + (merge a84216c684 jk/doc-color-pager later to maint). diff --git a/RelNotes b/RelNotes index aece21e8a4..f071367dc3 120000 --- a/RelNotes +++ b/RelNotes @@ -1 +1 @@ -Documentation/RelNotes/2.32.0.txt \ No newline at end of file +Documentation/RelNotes/2.33.0.txt \ No newline at end of file From 7d879ad7e02f1f4d7f504862b280e503db403a36 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 10 Jun 2021 08:57:05 -0400 Subject: [PATCH 218/397] ll_binary_merge(): handle XDL_MERGE_FAVOR_UNION Prior to commit a944af1d86 (merge: teach -Xours/-Xtheirs to binary ll-merge driver, 2012-09-08), we always reported a conflict from ll_binary_merge() by returning "1" (in the xdl_merge and ll_merge code, this value is the number of conflict hunks). After that commit, we report zero conflicts if the "variant" flag is set, under the assumption that it is one of XDL_MERGE_FAVOR_OURS or XDL_MERGE_FAVOR_THEIRS. But this gets confused by XDL_MERGE_FAVOR_UNION. We do not know how to do a binary union merge, but erroneously report no conflicts anyway (and just blindly use the "ours" content as the result). Let's tighten our check to just the cases that a944af1d86 meant to cover. This fixes the union case (which existed already back when that commit was made), as well as future-proofing us against any other variants that get added later. Note that you can't trigger this from "git merge-file --union", as that bails on binary files before even calling into the ll-merge machinery. The test here uses the "union" merge attribute, which does erroneously report a successful merge. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- ll-merge.c | 4 +++- t/t6406-merge-attr.sh | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ll-merge.c b/ll-merge.c index 9a8a2c365c..145deb12fa 100644 --- a/ll-merge.c +++ b/ll-merge.c @@ -91,7 +91,9 @@ static int ll_binary_merge(const struct ll_merge_driver *drv_unused, * With -Xtheirs or -Xours, we have cleanly merged; * otherwise we got a conflict. */ - return (opts->variant ? 0 : 1); + return opts->variant == XDL_MERGE_FAVOR_OURS || + opts->variant == XDL_MERGE_FAVOR_THEIRS ? + 0 : 1; } static int ll_xdl_merge(const struct ll_merge_driver *drv_unused, diff --git a/t/t6406-merge-attr.sh b/t/t6406-merge-attr.sh index d5a4ac2d81..c1c458d933 100755 --- a/t/t6406-merge-attr.sh +++ b/t/t6406-merge-attr.sh @@ -207,4 +207,21 @@ test_expect_success 'custom merge does not lock index' ' git merge main ' +test_expect_success 'binary files with union attribute' ' + git checkout -b bin-main && + printf "base\0" >bin.txt && + echo "bin.txt merge=union" >.gitattributes && + git add bin.txt .gitattributes && + git commit -m base && + + printf "one\0" >bin.txt && + git commit -am one && + + git checkout -b bin-side HEAD^ && + printf "two\0" >bin.txt && + git commit -am two && + + test_must_fail git merge bin-main +' + test_done From 7f53f78b04b49c2060c23df6566aceb3ba394aea Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 10 Jun 2021 08:58:43 -0400 Subject: [PATCH 219/397] ll_union_merge(): pass name labels to ll_xdl_merge() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since cd1d61c44f (make union merge an xdl merge favor, 2010-03-01), we pass NULL to ll_xdl_merge() for the "name" labels of the ancestor, ours and theirs buffers. We usually use these for annotating conflict markers left in a file. For a union merge, these shouldn't matter; the point of it is that we'd never leave conflict markers in the first place. But there is one code path where we may dereference them: if the file contents appear to be binary, ll_binary_merge() will give up and pass them to warning() to generate a message for the user (that was true even when cd1d61c44f was written, though the warning was in ll_xdl_merge() back then). That can result in a segfault, though on many systems (including glibc), the printf routines will helpfully just say "(null)" instead. We can extend our binary-union test in t6406 to check stderr, which catches the problem on all systems. This also fixes a warning from "gcc -O3". Unlike lower optimization levels, it inlines enough to see that the NULL can make it to warning() and complains: In function ‘ll_binary_merge’, inlined from ‘ll_xdl_merge’ at ll-merge.c:115:10, inlined from ‘ll_union_merge’ at ll-merge.c:151:9: ll-merge.c:74:4: warning: ‘%s’ directive argument is null [-Wformat-overflow=] 74 | warning("Cannot merge binary files: %s (%s vs. %s)", | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 75 | path, name1, name2); | ~~~~~~~~~~~~~~~~~~~ Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- ll-merge.c | 2 +- t/t6406-merge-attr.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ll-merge.c b/ll-merge.c index 145deb12fa..0ee34d8a01 100644 --- a/ll-merge.c +++ b/ll-merge.c @@ -151,7 +151,7 @@ static int ll_union_merge(const struct ll_merge_driver *drv_unused, o = *opts; o.variant = XDL_MERGE_FAVOR_UNION; return ll_xdl_merge(drv_unused, result, path_unused, - orig, NULL, src1, NULL, src2, NULL, + orig, orig_name, src1, name1, src2, name2, &o, marker_size); } diff --git a/t/t6406-merge-attr.sh b/t/t6406-merge-attr.sh index c1c458d933..8494645837 100755 --- a/t/t6406-merge-attr.sh +++ b/t/t6406-merge-attr.sh @@ -221,7 +221,8 @@ test_expect_success 'binary files with union attribute' ' printf "two\0" >bin.txt && git commit -am two && - test_must_fail git merge bin-main + test_must_fail git merge bin-main 2>stderr && + grep -i "warning.*cannot merge.*HEAD vs. bin-main" stderr ' test_done From 382b601acde12b298bb84faa11b3f42868716a0d Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 10 Jun 2021 12:14:12 -0400 Subject: [PATCH 220/397] ll_union_merge(): rename path_unused parameter The "path" parameter to ll_union_merge() is named "path_unused", since we don't ourselves use it. But we do pass it to ll_xdl_merge(), which may look at it (it gets passed to ll_binary_merge(), which may pass it to warning()). Let's rename it to correct this inaccuracy (both of the other functions correctly do not call this "unused"). Note that we also pass drv_unused, but it truly is unused by the rest of the stack (it only exists at all to provide a generic interface that matches what ll_ext_merge() needs). Reported-by: Elijah Newren Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- ll-merge.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ll-merge.c b/ll-merge.c index 0ee34d8a01..261657578c 100644 --- a/ll-merge.c +++ b/ll-merge.c @@ -138,7 +138,7 @@ static int ll_xdl_merge(const struct ll_merge_driver *drv_unused, static int ll_union_merge(const struct ll_merge_driver *drv_unused, mmbuffer_t *result, - const char *path_unused, + const char *path, mmfile_t *orig, const char *orig_name, mmfile_t *src1, const char *name1, mmfile_t *src2, const char *name2, @@ -150,7 +150,7 @@ static int ll_union_merge(const struct ll_merge_driver *drv_unused, assert(opts); o = *opts; o.variant = XDL_MERGE_FAVOR_UNION; - return ll_xdl_merge(drv_unused, result, path_unused, + return ll_xdl_merge(drv_unused, result, path, orig, orig_name, src1, name1, src2, name2, &o, marker_size); } From 1d72b604ef638155f7af91c968ccf1c95234ecee Mon Sep 17 00:00:00 2001 From: Jeff King Date: Thu, 10 Jun 2021 09:06:43 -0400 Subject: [PATCH 221/397] add_pending_object_with_path(): work around "gcc -O3" complaint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When compiling with -O3, some gcc versions (10.2.1 here) complain about an out-of-bounds subscript: revision.c: In function ‘do_add_index_objects_to_pending’: revision.c:321:22: error: array subscript [1, 2147483647] is outside array bounds of ‘char[1]’ [-Werror=array-bounds] 321 | if (0 < len && name[len] && buf.len) | ~~~~^~~~~ The "len" parameter here comes from calling interpret_branch_name(), which intends to return the number of characters of "name" it parsed. But the compiler doesn't realize this. It knows the size of the empty string "name" passed in from do_add_index_objects_to_pending(), but it has no clue that the "len" we get back will be constrained to "0" in that case. And I don't think the warning is telling us about some subtle or clever bug. The implementation of interpret_branch_name() is in another file entirely, and the compiler can't see it (you can even verify there is no clever LTO going on by replacing it with "return 0" and still getting the warning). We can work around this by replacing our "did we hit the trailing NUL" subscript dereference with a length check. We do not even have to pay the cost for an extra strlen(), as we can pass our new length into interpret_branch_name(), which was converting our "0" into a call to strlen() anyway. Signed-off-by: Jeff King Signed-off-by: Junio C Hamano --- revision.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/revision.c b/revision.c index 8140561b6c..cddd0542a6 100644 --- a/revision.c +++ b/revision.c @@ -316,9 +316,10 @@ static void add_pending_object_with_path(struct rev_info *revs, revs->no_walk = 0; if (revs->reflog_info && obj->type == OBJ_COMMIT) { struct strbuf buf = STRBUF_INIT; - int len = interpret_branch_name(name, 0, &buf, &options); + size_t namelen = strlen(name); + int len = interpret_branch_name(name, namelen, &buf, &options); - if (0 < len && name[len] && buf.len) + if (0 < len && len < namelen && buf.len) strbuf_addstr(&buf, name + len); add_reflog_for_walk(revs->reflog_info, (struct commit *)obj, From f96178c5297adc20e495318780de0b8e804b3f0d Mon Sep 17 00:00:00 2001 From: Andrzej Hunt Date: Thu, 10 Jun 2021 16:48:30 +0000 Subject: [PATCH 222/397] bulk-checkin: make buffer reuse more obvious and safer ibuf can be reused for multiple iterations of the loop. Specifically: deflate() overwrites s.avail_in to show how much of the input buffer has not been processed yet - and sometimes leaves 'avail_in > 0', in which case ibuf will be processed again during the loop's subsequent iteration. But if we declare ibuf within the loop, then (in theory) we get a new (and uninitialised) buffer for every iteration. In practice, my compiler seems to resue the same buffer - meaning that this code does work - but it doesn't seem safe to rely on this behaviour. MSAN correctly catches this issue - as soon as we hit the 's.avail_in > 0' condition, we end up reading from what seems to be uninitialised memory. Therefore, we move ibuf out of the loop, making this reuse safe. See MSAN output from t1050-large below - the interesting part is the ibuf creation at the end, although there's a lot of indirection before we reach the read from unitialised memory: ==11294==WARNING: MemorySanitizer: use-of-uninitialized-value #0 0x7f75db58fb1c in crc32_little crc32.c:283:9 #1 0x7f75db58d5b3 in crc32_z crc32.c:220:20 #2 0x7f75db59668c in crc32 crc32.c:242:12 #3 0x8c94f8 in hashwrite csum-file.c:101:15 #4 0x825faf in stream_to_pack bulk-checkin.c:154:5 #5 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #6 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #7 0xa7cff2 in index_stream object-file.c:2234:9 #8 0xa7bff7 in index_fd object-file.c:2256:9 #9 0xa7d22d in index_path object-file.c:2274:7 #10 0xb3c8c9 in add_to_index read-cache.c:802:7 #11 0xb3e039 in add_file_to_index read-cache.c:835:9 #12 0x4a99c3 in add_files add.c:458:7 #13 0x4a7276 in cmd_add add.c:670:18 #14 0x4a1e76 in run_builtin git.c:461:11 #15 0x49e1e7 in handle_builtin git.c:714:3 #16 0x4a0c08 in run_argv git.c:781:4 #17 0x49d5a8 in cmd_main git.c:912:19 #18 0x7974da in main common-main.c:52:11 #19 0x7f75da66f349 in __libc_start_main (/lib64/libc.so.6+0x24349) #20 0x421bd9 in _start start.S:120 Uninitialized value was stored to memory at #0 0x7f75db58fa6b in crc32_little crc32.c:283:9 #1 0x7f75db58d5b3 in crc32_z crc32.c:220:20 #2 0x7f75db59668c in crc32 crc32.c:242:12 #3 0x8c94f8 in hashwrite csum-file.c:101:15 #4 0x825faf in stream_to_pack bulk-checkin.c:154:5 #5 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #6 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #7 0xa7cff2 in index_stream object-file.c:2234:9 #8 0xa7bff7 in index_fd object-file.c:2256:9 #9 0xa7d22d in index_path object-file.c:2274:7 #10 0xb3c8c9 in add_to_index read-cache.c:802:7 #11 0xb3e039 in add_file_to_index read-cache.c:835:9 #12 0x4a99c3 in add_files add.c:458:7 #13 0x4a7276 in cmd_add add.c:670:18 #14 0x4a1e76 in run_builtin git.c:461:11 #15 0x49e1e7 in handle_builtin git.c:714:3 #16 0x4a0c08 in run_argv git.c:781:4 #17 0x49d5a8 in cmd_main git.c:912:19 #18 0x7974da in main common-main.c:52:11 #19 0x7f75da66f349 in __libc_start_main (/lib64/libc.so.6+0x24349) Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db5c2011 in flush_pending deflate.c:746:5 #2 0x7f75db5cafa0 in deflate_stored deflate.c:1815:9 #3 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #4 0xd80b7f in git_deflate zlib.c:244:12 #5 0x825dff in stream_to_pack bulk-checkin.c:140:12 #6 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #7 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #8 0xa7cff2 in index_stream object-file.c:2234:9 #9 0xa7bff7 in index_fd object-file.c:2256:9 #10 0xa7d22d in index_path object-file.c:2274:7 #11 0xb3c8c9 in add_to_index read-cache.c:802:7 #12 0xb3e039 in add_file_to_index read-cache.c:835:9 #13 0x4a99c3 in add_files add.c:458:7 #14 0x4a7276 in cmd_add add.c:670:18 #15 0x4a1e76 in run_builtin git.c:461:11 #16 0x49e1e7 in handle_builtin git.c:714:3 #17 0x4a0c08 in run_argv git.c:781:4 #18 0x49d5a8 in cmd_main git.c:912:19 #19 0x7974da in main common-main.c:52:11 Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db644241 in _tr_stored_block trees.c:873:5 #2 0x7f75db5cad7c in deflate_stored deflate.c:1813:9 #3 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #4 0xd80b7f in git_deflate zlib.c:244:12 #5 0x825dff in stream_to_pack bulk-checkin.c:140:12 #6 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #7 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #8 0xa7cff2 in index_stream object-file.c:2234:9 #9 0xa7bff7 in index_fd object-file.c:2256:9 #10 0xa7d22d in index_path object-file.c:2274:7 #11 0xb3c8c9 in add_to_index read-cache.c:802:7 #12 0xb3e039 in add_file_to_index read-cache.c:835:9 #13 0x4a99c3 in add_files add.c:458:7 #14 0x4a7276 in cmd_add add.c:670:18 #15 0x4a1e76 in run_builtin git.c:461:11 #16 0x49e1e7 in handle_builtin git.c:714:3 #17 0x4a0c08 in run_argv git.c:781:4 #18 0x49d5a8 in cmd_main git.c:912:19 #19 0x7974da in main common-main.c:52:11 Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db5c8fcf in deflate_stored deflate.c:1783:9 #2 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #3 0xd80b7f in git_deflate zlib.c:244:12 #4 0x825dff in stream_to_pack bulk-checkin.c:140:12 #5 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #6 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #7 0xa7cff2 in index_stream object-file.c:2234:9 #8 0xa7bff7 in index_fd object-file.c:2256:9 #9 0xa7d22d in index_path object-file.c:2274:7 #10 0xb3c8c9 in add_to_index read-cache.c:802:7 #11 0xb3e039 in add_file_to_index read-cache.c:835:9 #12 0x4a99c3 in add_files add.c:458:7 #13 0x4a7276 in cmd_add add.c:670:18 #14 0x4a1e76 in run_builtin git.c:461:11 #15 0x49e1e7 in handle_builtin git.c:714:3 #16 0x4a0c08 in run_argv git.c:781:4 #17 0x49d5a8 in cmd_main git.c:912:19 #18 0x7974da in main common-main.c:52:11 #19 0x7f75da66f349 in __libc_start_main (/lib64/libc.so.6+0x24349) Uninitialized value was stored to memory at #0 0x447eb9 in __msan_memcpy msan_interceptors.cpp:1558:3 #1 0x7f75db5ea545 in read_buf deflate.c:1181:5 #2 0x7f75db5c97f7 in deflate_stored deflate.c:1791:9 #3 0x7f75db5bb7d2 in deflate deflate.c:1005:34 #4 0xd80b7f in git_deflate zlib.c:244:12 #5 0x825dff in stream_to_pack bulk-checkin.c:140:12 #6 0x82467b in deflate_to_pack bulk-checkin.c:225:8 #7 0x823ff1 in index_bulk_checkin bulk-checkin.c:264:15 #8 0xa7cff2 in index_stream object-file.c:2234:9 #9 0xa7bff7 in index_fd object-file.c:2256:9 #10 0xa7d22d in index_path object-file.c:2274:7 #11 0xb3c8c9 in add_to_index read-cache.c:802:7 #12 0xb3e039 in add_file_to_index read-cache.c:835:9 #13 0x4a99c3 in add_files add.c:458:7 #14 0x4a7276 in cmd_add add.c:670:18 #15 0x4a1e76 in run_builtin git.c:461:11 #16 0x49e1e7 in handle_builtin git.c:714:3 #17 0x4a0c08 in run_argv git.c:781:4 #18 0x49d5a8 in cmd_main git.c:912:19 #19 0x7974da in main common-main.c:52:11 Uninitialized value was created by an allocation of 'ibuf' in the stack frame of function 'stream_to_pack' #0 0x825710 in stream_to_pack bulk-checkin.c:101 SUMMARY: MemorySanitizer: use-of-uninitialized-value crc32.c:283:9 in crc32_little Exiting Signed-off-by: Andrzej Hunt Signed-off-by: Junio C Hamano --- bulk-checkin.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bulk-checkin.c b/bulk-checkin.c index 127312acd1..b023d9959a 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -100,6 +100,7 @@ static int stream_to_pack(struct bulk_checkin_state *state, const char *path, unsigned flags) { git_zstream s; + unsigned char ibuf[16384]; unsigned char obuf[16384]; unsigned hdrlen; int status = Z_OK; @@ -113,8 +114,6 @@ static int stream_to_pack(struct bulk_checkin_state *state, s.avail_out = sizeof(obuf) - hdrlen; while (status != Z_STREAM_END) { - unsigned char ibuf[16384]; - if (size && !s.avail_in) { ssize_t rsize = size < sizeof(ibuf) ? size : sizeof(ibuf); ssize_t read_result = read_in_full(fd, ibuf, rsize); From f74d11471fa8870d4bcbe7dbff2f3ddac6783e7f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 10 Jun 2021 08:29:24 +0000 Subject: [PATCH 223/397] multimail: stop shipping a copy The multimail project is developed independently and has its own project page. Traditionally, we shipped a copy in contrib/. However, such a copy is prone to become stale, and users are much better served to be directed to the actual project instead. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- contrib/hooks/multimail/CHANGES | 285 -- contrib/hooks/multimail/CONTRIBUTING.rst | 60 - contrib/hooks/multimail/README.Git | 12 +- .../README.migrate-from-post-receive-email | 145 - contrib/hooks/multimail/README.rst | 774 --- .../multimail/doc/customizing-emails.rst | 56 - contrib/hooks/multimail/doc/gerrit.rst | 56 - contrib/hooks/multimail/doc/gitolite.rst | 118 - .../hooks/multimail/doc/troubleshooting.rst | 78 - contrib/hooks/multimail/git_multimail.py | 4346 ----------------- .../hooks/multimail/migrate-mailhook-config | 274 -- contrib/hooks/multimail/post-receive.example | 101 - 12 files changed, 2 insertions(+), 6303 deletions(-) delete mode 100644 contrib/hooks/multimail/CHANGES delete mode 100644 contrib/hooks/multimail/CONTRIBUTING.rst delete mode 100644 contrib/hooks/multimail/README.migrate-from-post-receive-email delete mode 100644 contrib/hooks/multimail/README.rst delete mode 100644 contrib/hooks/multimail/doc/customizing-emails.rst delete mode 100644 contrib/hooks/multimail/doc/gerrit.rst delete mode 100644 contrib/hooks/multimail/doc/gitolite.rst delete mode 100644 contrib/hooks/multimail/doc/troubleshooting.rst delete mode 100755 contrib/hooks/multimail/git_multimail.py delete mode 100755 contrib/hooks/multimail/migrate-mailhook-config delete mode 100755 contrib/hooks/multimail/post-receive.example diff --git a/contrib/hooks/multimail/CHANGES b/contrib/hooks/multimail/CHANGES deleted file mode 100644 index 35791fd02c..0000000000 --- a/contrib/hooks/multimail/CHANGES +++ /dev/null @@ -1,285 +0,0 @@ -Release 1.5.0 -============= - -Backward-incompatible change ----------------------------- - -The name of classes for environment was misnamed as `*Environement`. -It is now `*Environment`. - -New features ------------- - -* A Thread-Index header is now added to each email sent (except for - combined emails where it would not make sense), so that MS Outlook - properly groups messages by threads even though they have a - different subject line. Unfortunately, even adding this header the - threading still seems to be unreliable, but it is unclear whether - this is an issue on our side or on MS Outlook's side (see discussion - here: https://github.com/git-multimail/git-multimail/pull/194). - -* A new variable multimailhook.ExcludeMergeRevisions was added to send - notification emails only for non-merge commits. - -* For gitolite environment, it is now possible to specify the mail map - in a separate file in addition to gitolite.conf, using the variable - multimailhook.MailaddressMap. - -Internal changes ----------------- - -* The testsuite now uses GIT_PRINT_SHA1_ELLIPSIS where needed for - compatibility with recent Git versions. Only tests are affected. - -* We don't try to install pyflakes in the continuous integration job - for old Python versions where it's no longer available. - -* Stop using the deprecated cgi.escape in Python 3. - -* New flake8 warnings have been fixed. - -* Python 3.6 is now tested against on Travis-CI. - -* A bunch of lgtm.com warnings have been fixed. - -Bug fixes ---------- - -* SMTPMailer logs in only once now. It used to re-login for each email - sent which triggered errors for some SMTP servers. - -* migrate-mailhook-config was broken by internal refactoring, it - should now work again. - -This version was tested with Python 2.6 to 3.7. It was tested with Git -1.7.10.406.gdc801, 2.15.1 and 2.20.1.98.gecbdaf0. - -Release 1.4.0 -============= - -New features to troubleshoot a git-multimail installation ---------------------------------------------------------- - -* One can now perform a basic check of git-multimail's setup by - running the hook with the environment variable - GIT_MULTIMAIL_CHECK_SETUP set to a non-empty string. See - doc/troubleshooting.rst for details. - -* A new log files system was added. See the multimailhook.logFile, - multimailhook.errorLogFile and multimailhook.debugLogFile variables. - -* git_multimail.py can now be made more verbose using - multimailhook.verbose. - -* A new option --check-ref-filter is now available to help debugging - the refFilter* options. - -Formatting emails ------------------ - -* Formatting of emails was made slightly more compact, to reduce the - odds of having long subject lines truncated or wrapped in short list - of commits. - -* multimailhook.emailPrefix may now use the '%(repo_shortname)s' - placeholder for the repository's short name. - -* A new option multimailhook.subjectMaxLength is available to truncate - overly long subject lines. - -Bug fixes and minor changes ---------------------------- - -* Options refFilterDoSendRegex and refFilterDontSendRegex were - essentially broken. They should work now. - -* The behavior when both refFilter{Do,Dont}SendRegex and - refFilter{Exclusion,Inclusion}Regex are set have been slightly - changed. Exclusion/Inclusion is now strictly stronger than - DoSend/DontSend. - -* The management of precedence when a setting can be computed in - multiple ways has been considerably refactored and modified. - multimailhook.from and multimailhook.reponame now have precedence - over the environment-specific settings ($GL_REPO/$GL_USER for - gitolite, --stash-user/repo for Stash, --submitter/--project for - Gerrit). - -* The coverage of the testsuite has been considerably improved. All - configuration variables now appear at least once in the testsuite. - -This version was tested with Python 2.6 to 3.5. It also mostly works -with Python 2.4, but there is one known breakage in the testsuite -related to non-ascii characters. It was tested with Git -1.7.10.406.gdc801, 1.8.5.6, 2.1.4, and 2.10.0.rc0.1.g07c9292. - -Release 1.3.1 (bugfix-only release) -=================================== - -* Generate links to commits in combined emails (it was done only for - commit emails in 1.3.0). - -* Fix broken links on PyPi. - -Release 1.3.0 -============= - -* New options multimailhook.htmlInIntro and multimailhook.htmlInFooter - now allow using HTML in the introduction and footer of emails (e.g. - for a more pleasant formatting or to insert a link to the commit on - a web interface). - -* A new option multimailhook.commitBrowseURL gives a simpler (and less - flexible) way to add a link to a web interface for commit emails - than multimailhook.htmlInIntro and multimailhook.htmlInFooter. - -* A new public function config.add_config_parameters was added to - allow custom hooks to set specific Git configuration variables - without modifying the configuration files. See an example in - post-receive.example. - -* Error handling for SMTP has been improved (we used to print Python - backtraces for legitimate errors). - -* The SMTP mailer can now check TLS certificates when the newly added - configuration variable multimailhook.smtpCACerts. - -* Python 3 portability has been improved. - -* The documentation's formatting has been improved. - -* The testsuite has been improved (we now use pyflakes to check for - errors in the code). - -This version has been tested with Python 2.4 and 2.6 to 3.5, and Git -v1.7.10-406-gdc801e7, 2.1.4 and 2.8.1.339.g3ad15fd. - -No change since 1.3 RC1. - -Release 1.2.0 -============= - -* It is now possible to exclude some refs (e.g. exclude some branches - or tags). See refFilterDoSendRegex, refFilterDontSendRegex, - refFilterInclusionRegex and refFilterExclusionRegex. - -* New commitEmailFormat option which can be set to "html" to generate - simple colorized diffs using HTML for the commit emails. - -* git-multimail can now be ran as a Gerrit ref-updated hook, or from - Atlassian BitBucket Server (formerly known as Atlassian Stash). - -* The From: field is now more customizeable. It can be set - independently for refchange emails and commit emails (see - fromCommit, fromRefChange). The special values pusher and author can - be used in these configuration variable. - -* A new command-line option, --version, was added. The version is also - available in the X-Git-Multimail-Version header of sent emails. - -* Set X-Git-NotificationType header to differentiate the various types - of notifications. Current values are: diff, ref_changed_plus_diff, - ref_changed. - -* Preliminary support for Python 3. The testsuite passes with Python 3, - but it has not received as much testing as the Python 2 version yet. - -* Several encoding-related fixes. UTF-8 characters work in more - situations (but non-ascii characters in email address are still not - supported). - -* The testsuite and its documentation has been greatly improved. - -Plus all the bugfixes from version 1.1.1. - -This version has been tested with Python 2.4 and 2.6 to 3.5, and Git -v1.7.10-406-gdc801e7, git-1.8.2.3 and 2.6.0. Git versions prior to -v1.7.10-406-gdc801e7 probably work, but cannot run the testsuite -properly. - -Release 1.1.1 (bugfix-only release) -=================================== - -* The SMTP mailer was not working with Python 2.4. - -Release 1.1.0 -============= - -* When a single commit is pushed, omit the reference changed email. - Set multimailhook.combineWhenSingleCommit to false to disable this - new feature. - -* In gitolite environments, the pusher's email address can be used as - the From address by creating a specially formatted comment block in - gitolite.conf (see multimailhook.from in README). - -* Support for SMTP authentication and SSL/TLS encryption was added, - see smtpUser, smtpPass, smtpEncryption in README. - -* A new option scanCommitForCc was added to allow git-multimail to - search the commit message for 'Cc: ...' lines, and add the - corresponding emails in Cc. - -* If $USER is not set, use the variable $USERNAME. This is needed on - Windows platform to recognize the pusher. - -* The emailPrefix variable can now be set to an empty string to remove - the prefix. - -* A short tutorial was added in doc/gitolite.rst to set up - git-multimail with gitolite. - -* The post-receive file was renamed to post-receive.example. It has - always been an example (the standard way to call git-multimail is to - call git_multimail.py), but it was unclear to many users. - -* A new refchangeShowGraph option was added to make it possible to - include both a graph and a log in the summary emails. The options - to control the graph formatting can be set via the new graphOpts - option. - -* New option --force-send was added to disable new commit detection - for update hook. One use-case is to run git_multimail.py after - running "git fetch" to send emails about commits that have just been - fetched (the detection of new commits was unreliable in this mode). - -* The testing infrastructure was considerably improved (continuous - integration with travis-ci, automatic check of PEP8 and RST syntax, - many improvements to the test scripts). - -This version has been tested with Python 2.4 to 2.7, and Git 1.7.1 to -2.4. - -Release 1.0.0 -============= - -* Fix encoding of non-ASCII email addresses in email headers. - -* Fix backwards-compatibility bugs for older Python 2.x versions. - -* Fix a backwards-compatibility bug for Git 1.7.1. - -* Add an option commitDiffOpts to customize logs for revisions. - -* Pass "-oi" to sendmail by default to prevent premature termination - on a line containing only ".". - -* Stagger email "Date:" values in an attempt to help mail clients - thread the emails in the right order. - -* If a mailing list setting is missing, just skip sending the - corresponding email (with a warning) instead of failing. - -* Add a X-Git-Host header that can be used for email filtering. - -* Allow the sender's fully-qualified domain name to be configured. - -* Minor documentation improvements. - -* Add this CHANGES file. - - -Release 0.9.0 -============= - -* Initial release. diff --git a/contrib/hooks/multimail/CONTRIBUTING.rst b/contrib/hooks/multimail/CONTRIBUTING.rst deleted file mode 100644 index de20a54287..0000000000 --- a/contrib/hooks/multimail/CONTRIBUTING.rst +++ /dev/null @@ -1,60 +0,0 @@ -Contributing -============ - -git-multimail is an open-source project, built by volunteers. We would -welcome your help! - -The current maintainers are `Matthieu Moy `__ and -`Michael Haggerty `__. - -Please note that although a copy of git-multimail is distributed in -the "contrib" section of the main Git project, development takes place -in a separate `git-multimail repository on GitHub`_. - -Whenever enough changes to git-multimail have accumulated, a new -code-drop of git-multimail will be submitted for inclusion in the Git -project. - -We use the GitHub issue tracker to keep track of bugs and feature -requests, and we use GitHub pull requests to exchange patches (though, -if you prefer, you can send patches via the Git mailing list with CC -to the maintainers). Please sign off your patches as per the `Git -project practice -`__. - -Please vote for issues you would like to be addressed in priority -(click "add your reaction" and then the "+1" thumbs-up button on the -GitHub issue). - -General discussion of git-multimail can take place on the main `Git -mailing list`_. - -Please CC emails regarding git-multimail to the maintainers so that we -don't overlook them. - -Help needed: testers/maintainer for specific environments/OS ------------------------------------------------------------- - -The current maintainer uses and tests git-multimail on Linux with the -Generic environment. More testers, or better contributors are needed -to test git-multimail on other real-life setups: - -* Mac OS X, Windows: git-multimail is currently not supported on these - platforms. But since we have no external dependencies and try to - write code as portable as possible, it is possible that - git-multimail already runs there and if not, it is likely that it - could be ported easily. - - Patches to improve support for Windows and OS X are welcome. - Ideally, there would be a sub-maintainer for each OS who would test - at least once before each release (around twice a year). - -* Gerrit, Stash, Gitolite environments: although the testsuite - contains tests for these environments, a tester/maintainer for each - environment would be welcome to test and report failure (or success) - on real-life environments periodically (here also, feedback before - each release would be highly appreciated). - - -.. _`git-multimail repository on GitHub`: https://github.com/git-multimail/git-multimail -.. _`Git mailing list`: git@vger.kernel.org diff --git a/contrib/hooks/multimail/README.Git b/contrib/hooks/multimail/README.Git index 044444245d..c427efc7bd 100644 --- a/contrib/hooks/multimail/README.Git +++ b/contrib/hooks/multimail/README.Git @@ -1,15 +1,7 @@ -This copy of git-multimail is distributed as part of the "contrib" -section of the Git project as a convenience to Git users. git-multimail is developed as an independent project at the following website: https://github.com/git-multimail/git-multimail -The version in this directory was obtained from the upstream project -on January 07 2019 and consists of the "git-multimail" subdirectory from -revision - - 04e80e6c40be465cc62b6c246f0fcb8fd2cfd454 refs/tags/1.5.0 - -Please see the README file in this directory for information about how -to report bugs or contribute to git-multimail. +Please refer to that project page for information about how to report +bugs or contribute to git-multimail. diff --git a/contrib/hooks/multimail/README.migrate-from-post-receive-email b/contrib/hooks/multimail/README.migrate-from-post-receive-email deleted file mode 100644 index 1e6a976699..0000000000 --- a/contrib/hooks/multimail/README.migrate-from-post-receive-email +++ /dev/null @@ -1,145 +0,0 @@ -git-multimail is close to, but not exactly, a plug-in replacement for -the old Git project script contrib/hooks/post-receive-email. This -document describes the differences and explains how to configure -git-multimail to get behavior closest to that of post-receive-email. - -If you are in a hurry -===================== - -A script called migrate-mailhook-config is included with -git-multimail. If you run this script within a Git repository that is -configured to use post-receive-email, it will convert the -configuration settings into the approximate equivalent settings for -git-multimail. For more information, run - - migrate-mailhook-config --help - - -Configuration differences -========================= - -* The names of the config options for git-multimail are in namespace - "multimailhook.*" instead of "hooks.*". (Editorial comment: - post-receive-email should never have used such a generic top-level - namespace.) - -* In emails about new annotated tags, post-receive-email includes a - shortlog of all changes since the previous annotated tag. To get - this behavior with git-multimail, you need to set - multimailhook.announceshortlog to true: - - git config multimailhook.announceshortlog true - -* multimailhook.commitlist -- This is a new configuration variable. - Recipients listed here will receive a separate email for each new - commit. However, if this variable is *not* set, it defaults to the - value of multimailhook.mailinglist. Therefore, if you *don't* want - the members of multimailhook.mailinglist to receive one email per - commit, then set this value to the empty string: - - git config multimailhook.commitlist '' - -* multimailhook.emailprefix -- If this value is not set, then the - subjects of generated emails are prefixed with the short name of the - repository enclosed in square brackets; e.g., "[myrepo]". - post-receive-email defaults to prefix "[SCM]" if this option is not - set. So if you were using the old default and want to retain it - (for example, to avoid having to change your email filters), set - this variable explicitly to the old value: - - git config multimailhook.emailprefix "[SCM]" - -* The "multimailhook.showrev" configuration option is not supported. - Its main use is obsoleted by the one-email-per-commit feature of - git-multimail. - - -Other differences -================= - -This section describes other differences in the behavior of -git-multimail vs. post-receive-email. For full details, please refer -to the main README file: - -* One email per commit. For each reference change, the script first - outputs one email summarizing the reference change (including - one-line summaries of the new commits), then it outputs a separate - email for each new commit that was introduced, including patches. - These one-email-per-commit emails go to the addresses listed in - multimailhook.commitlist. post-receive-email sends only one email - for each *reference* that is changed, no matter how many commits - were added to the reference. - -* Better algorithm for detecting new commits. post-receive-email - processes one reference change at a time, which causes it to fail to - describe new commits that were included in multiple branches. For - example, if a single push adds the "*" commits in the diagram below, - then post-receive-email would never include the details of the two - commits that are common to "master" and "branch" in its - notifications. - - o---o---o---*---*---* <-- master - \ - *---* <-- branch - - git-multimail analyzes all reference modifications to determine - which commits were not present before the change, therefore avoiding - that error. - -* In reference change emails, git-multimail tells which commits have - been added to the reference vs. are entirely new to the repository, - and which commits that have been omitted from the reference - vs. entirely discarded from the repository. - -* The environment in which Git is running can be configured via an - "Environment" abstraction. - -* Built-in support for Gitolite-managed repositories. - -* Instead of using full SHA1 object names in emails, git-multimail - mostly uses abbreviated SHA1s, plus one-line log message summaries - where appropriate. - -* In the schematic diagrams that explain non-fast-forward commits, - git-multimail shows the names of the branches involved. - -* The emails generated by git-multimail include the name of the Git - repository that was modified; this is convenient for recipients who - are monitoring multiple repositories. - -* git-multimail allows the email "From" addresses to be configured. - -* The recipients lists (multimailhook.mailinglist, - multimailhook.refchangelist, multimailhook.announcelist, and - multimailhook.commitlist) can be comma-separated values and/or - multivalued settings in the config file; e.g., - - [multimailhook] - mailinglist = mr.brown@example.com, mr.black@example.com - announcelist = Him - announcelist = Jim - announcelist = pop@example.com - - This might make it easier to maintain short recipients lists without - requiring full-fledged mailing list software. - -* By default, git-multimail sets email "Reply-To" headers to reply to - the pusher (for reference updates) and to the author (for commit - notifications). By default, the pusher's email address is - constructed by appending "multimailhook.emaildomain" to the pusher's - username. - -* The generated emails contain a configurable footer. By default, it - lists the name of the administrator who should be contacted to - unsubscribe from notification emails. - -* New option multimailhook.emailmaxlinelength to limit the length of - lines in the main part of the email body. The default limit is 500 - characters. - -* New option multimailhook.emailstrictutf8 to ensure that the main - part of the email body is valid UTF-8. Invalid characters are - turned into the Unicode replacement character, U+FFFD. By default - this option is turned on. - -* Written in Python. Easier to add new features. diff --git a/contrib/hooks/multimail/README.rst b/contrib/hooks/multimail/README.rst deleted file mode 100644 index 7c0fc4a6ef..0000000000 --- a/contrib/hooks/multimail/README.rst +++ /dev/null @@ -1,774 +0,0 @@ -git-multimail version 1.5.0 -=========================== - -.. image:: https://travis-ci.org/git-multimail/git-multimail.svg?branch=master - :target: https://travis-ci.org/git-multimail/git-multimail - -git-multimail is a tool for sending notification emails on pushes to a -Git repository. It includes a Python module called ``git_multimail.py``, -which can either be used as a hook script directly or can be imported -as a Python module into another script. - -git-multimail is derived from the Git project's old -contrib/hooks/post-receive-email, and is mostly compatible with that -script. See README.migrate-from-post-receive-email for details about -the differences and for how to migrate from post-receive-email to -git-multimail. - -git-multimail, like the rest of the Git project, is licensed under -GPLv2 (see the COPYING file for details). - -Please note: although, as a convenience, git-multimail may be -distributed along with the main Git project, development of -git-multimail takes place in its own, separate project. Please, read -``__ for more information. - - -By default, for each push received by the repository, git-multimail: - -1. Outputs one email summarizing each reference that was changed. - These "reference change" (called "refchange" below) emails describe - the nature of the change (e.g., was the reference created, deleted, - fast-forwarded, etc.) and include a one-line summary of each commit - that was added to the reference. - -2. Outputs one email for each new commit that was introduced by the - reference change. These "commit" emails include a list of the - files changed by the commit, followed by the diffs of files - modified by the commit. The commit emails are threaded to the - corresponding reference change email via "In-Reply-To". This style - (similar to the "git format-patch" style used on the Git mailing - list) makes it easy to scan through the emails, jump to patches - that need further attention, and write comments about specific - commits. Commits are handled in reverse topological order (i.e., - parents shown before children). For example:: - - [git] branch master updated - + [git] 01/08: doc: fix xref link from api docs to manual pages - + [git] 02/08: api-credentials.txt: show the big picture first - + [git] 03/08: api-credentials.txt: mention credential.helper explicitly - + [git] 04/08: api-credentials.txt: add "see also" section - + [git] 05/08: t3510 (cherry-pick-sequence): add missing '&&' - + [git] 06/08: Merge branch 'rr/maint-t3510-cascade-fix' - + [git] 07/08: Merge branch 'mm/api-credentials-doc' - + [git] 08/08: Git 1.7.11-rc2 - - By default, each commit appears in exactly one commit email, the - first time that it is pushed to the repository. If a commit is later - merged into another branch, then a one-line summary of the commit - is included in the reference change email (as usual), but no - additional commit email is generated. See - `multimailhook.refFilter(Inclusion|Exclusion|DoSend|DontSend)Regex` - below to configure which branches and tags are watched by the hook. - - By default, reference change emails have their "Reply-To" field set - to the person who pushed the change, and commit emails have their - "Reply-To" field set to the author of the commit. - -3. Output one "announce" mail for each new annotated tag, including - information about the tag and optionally a shortlog describing the - changes since the previous tag. Such emails might be useful if you - use annotated tags to mark releases of your project. - - -Requirements ------------- - -* Python 2.x, version 2.4 or later. No non-standard Python modules - are required. git-multimail has preliminary support for Python 3 - (but it has been better tested with Python 2). - -* The ``git`` command must be in your PATH. git-multimail is known to - work with Git versions back to 1.7.1. (Earlier versions have not - been tested; if you do so, please report your results.) - -* To send emails using the default configuration, a standard sendmail - program must be located at '/usr/sbin/sendmail' or - '/usr/lib/sendmail' and must be configured correctly to send emails. - If this is not the case, set multimailhook.sendmailCommand, or see - the multimailhook.mailer configuration variable below for how to - configure git-multimail to send emails via an SMTP server. - -* git-multimail is currently tested only on Linux. It may or may not - work on other platforms such as Windows and Mac OS. See - ``__ to improve the situation. - - -Invocation ----------- - -``git_multimail.py`` is designed to be used as a ``post-receive`` hook in a -Git repository (see githooks(5)). Link or copy it to -$GIT_DIR/hooks/post-receive within the repository for which email -notifications are desired. Usually it should be installed on the -central repository for a project, to which all commits are eventually -pushed. - -For use on pre-v1.5.1 Git servers, ``git_multimail.py`` can also work as -an ``update`` hook, taking its arguments on the command line. To use -this script in this manner, link or copy it to $GIT_DIR/hooks/update. -Please note that the script is not completely reliable in this mode -[1]_. - -Alternatively, ``git_multimail.py`` can be imported as a Python module -into your own Python post-receive script. This method is a bit more -work, but allows the behavior of the hook to be customized using -arbitrary Python code. For example, you can use a custom environment -(perhaps inheriting from GenericEnvironment or GitoliteEnvironment) to - -* change how the user who did the push is determined - -* read users' email addresses from an LDAP server or from a database - -* decide which users should be notified about which commits based on - the contents of the commits (e.g., for users who want to be notified - only about changes affecting particular files or subdirectories) - -Or you can change how emails are sent by writing your own Mailer -class. The ``post-receive`` script in this directory demonstrates how -to use ``git_multimail.py`` as a Python module. (If you make interesting -changes of this type, please consider sharing them with the -community.) - - -Troubleshooting/FAQ -------------------- - -Please read ``__ for frequently asked -questions and common issues with git-multimail. - - -Configuration -------------- - -By default, git-multimail mostly takes its configuration from the -following ``git config`` settings: - -multimailhook.environment - This describes the general environment of the repository. In most - cases, you do not need to specify a value for this variable: - `git-multimail` will autodetect which environment to use. - Currently supported values: - - generic - the username of the pusher is read from $USER or $USERNAME and - the repository name is derived from the repository's path. - - gitolite - Environment to use when ``git-multimail`` is ran as a gitolite_ - hook. - - The username of the pusher is read from $GL_USER, the repository - name is read from $GL_REPO, and the From: header value is - optionally read from gitolite.conf (see multimailhook.from). - - For more information about gitolite and git-multimail, read - ``__ - - stash - Environment to use when ``git-multimail`` is ran as an Atlassian - BitBucket Server (formerly known as Atlassian Stash) hook. - - **Warning:** this mode was provided by a third-party contributor - and never tested by the git-multimail maintainers. It is - provided as-is and may or may not work for you. - - This value is automatically assumed when the stash-specific - flags (``--stash-user`` and ``--stash-repo``) are specified on - the command line. When this environment is active, the username - and repo come from these two command line flags, which must be - specified. - - gerrit - Environment to use when ``git-multimail`` is ran as a - ``ref-updated`` Gerrit hook. - - This value is used when the gerrit-specific command line flags - (``--oldrev``, ``--newrev``, ``--refname``, ``--project``) for - gerrit's ref-updated hook are present. When this environment is - active, the username of the pusher is taken from the - ``--submitter`` argument if that command line option is passed, - otherwise 'Gerrit' is used. The repository name is taken from - the ``--project`` option on the command line, which must be passed. - - For more information about gerrit and git-multimail, read - ``__ - - If none of these environments is suitable for your setup, then you - can implement a Python class that inherits from Environment and - instantiate it via a script that looks like the example - post-receive script. - - The environment value can be specified on the command line using - the ``--environment`` option. If it is not specified on the - command line or by ``multimailhook.environment``, the value is - guessed as follows: - - * If stash-specific (respectively gerrit-specific) command flags - are present on the command-line, then ``stash`` (respectively - ``gerrit``) is used. - - * If the environment variables $GL_USER and $GL_REPO are set, then - ``gitolite`` is used. - - * If none of the above apply, then ``generic`` is used. - -multimailhook.repoName - A short name of this Git repository, to be used in various places - in the notification email text. The default is to use $GL_REPO - for gitolite repositories, or otherwise to derive this value from - the repository path name. - -multimailhook.mailingList - The list of email addresses to which notification emails should be - sent, as RFC 2822 email addresses separated by commas. This - configuration option can be multivalued. Leave it unset or set it - to the empty string to not send emails by default. The next few - settings can be used to configure specific address lists for - specific types of notification email. - -multimailhook.refchangeList - The list of email addresses to which summary emails about - reference changes should be sent, as RFC 2822 email addresses - separated by commas. This configuration option can be - multivalued. The default is the value in - multimailhook.mailingList. Set this value to "none" (or the empty - string) to prevent reference change emails from being sent even if - multimailhook.mailingList is set. - -multimailhook.announceList - The list of email addresses to which emails about new annotated - tags should be sent, as RFC 2822 email addresses separated by - commas. This configuration option can be multivalued. The - default is the value in multimailhook.refchangeList or - multimailhook.mailingList. Set this value to "none" (or the empty - string) to prevent annotated tag announcement emails from being sent - even if one of the other values is set. - -multimailhook.commitList - The list of email addresses to which emails about individual new - commits should be sent, as RFC 2822 email addresses separated by - commas. This configuration option can be multivalued. The - default is the value in multimailhook.mailingList. Set this value - to "none" (or the empty string) to prevent notification emails about - individual commits from being sent even if - multimailhook.mailingList is set. - -multimailhook.announceShortlog - If this option is set to true, then emails about changes to - annotated tags include a shortlog of changes since the previous - tag. This can be useful if the annotated tags represent releases; - then the shortlog will be a kind of rough summary of what has - happened since the last release. But if your tagging policy is - not so straightforward, then the shortlog might be confusing - rather than useful. Default is false. - -multimailhook.commitEmailFormat - The format of email messages for the individual commits, can be "text" or - "html". In the latter case, the emails will include diffs using colorized - HTML instead of plain text used by default. Note that this currently the - ref change emails are always sent in plain text. - - Note that when using "html", the formatting is done by parsing the - output of ``git log`` with ``-p``. When using - ``multimailhook.commitLogOpts`` to specify a ``--format`` for - ``git log``, one may get false positive (e.g. lines in the body of - the message starting with ``+++`` or ``---`` colored in red or - green). - - By default, all the message is HTML-escaped. See - ``multimailhook.htmlInIntro`` to change this behavior. - -multimailhook.commitBrowseURL - Used to generate a link to an online repository browser in commit - emails. This variable must be a string. Format directives like - ``%()s`` will be expanded the same way as template - strings. In particular, ``%(id)s`` will be replaced by the full - Git commit identifier (40-chars hexadecimal). - - If the string does not contain any format directive, then - ``%(id)s`` will be automatically added to the string. If you don't - want ``%(id)s`` to be automatically added, use the empty format - directive ``%()s`` anywhere in the string. - - For example, a suitable value for the git-multimail project itself - would be - ``https://github.com/git-multimail/git-multimail/commit/%(id)s``. - -multimailhook.htmlInIntro, multimailhook.htmlInFooter - When generating an HTML message, git-multimail escapes any HTML - sequence by default. This means that if a template contains HTML - like ``link``, the reader will see the HTML - source code and not a proper link. - - Set ``multimailhook.htmlInIntro`` to true to allow writing HTML - formatting in introduction templates. Similarly, set - ``multimailhook.htmlInFooter`` for HTML in the footer. - - Variables expanded in the template are still escaped. For example, - if a repository's path contains a ``<``, it will be rendered as - such in the message. - - Read ``__ for more details and - examples. - -multimailhook.refchangeShowGraph - If this option is set to true, then summary emails about reference - changes will additionally include: - - * a graph of the added commits (if any) - - * a graph of the discarded commits (if any) - - The log is generated by running ``git log --graph`` with the options - specified in graphOpts. The default is false. - -multimailhook.refchangeShowLog - If this option is set to true, then summary emails about reference - changes will include a detailed log of the added commits in - addition to the one line summary. The log is generated by running - ``git log`` with the options specified in multimailhook.logOpts. - Default is false. - -multimailhook.mailer - This option changes the way emails are sent. Accepted values are: - - * **sendmail (the default)**: use the command ``/usr/sbin/sendmail`` or - ``/usr/lib/sendmail`` (or sendmailCommand, if configured). This - mode can be further customized via the following options: - - multimailhook.sendmailCommand - The command used by mailer ``sendmail`` to send emails. Shell - quoting is allowed in the value of this setting, but remember that - Git requires double-quotes to be escaped; e.g.:: - - git config multimailhook.sendmailcommand '/usr/sbin/sendmail -oi -t -F \"Git Repo\"' - - Default is '/usr/sbin/sendmail -oi -t' or - '/usr/lib/sendmail -oi -t' (depending on which file is - present and executable). - - multimailhook.envelopeSender - If set then pass this value to sendmail via the -f option to set - the envelope sender address. - - * **smtp**: use Python's smtplib. This is useful when the sendmail - command is not available on the system. This mode can be - further customized via the following options: - - multimailhook.smtpServer - The name of the SMTP server to connect to. The value can - also include a colon and a port number; e.g., - ``mail.example.com:25``. Default is 'localhost' using port 25. - - multimailhook.smtpUser, multimailhook.smtpPass - Server username and password. Required if smtpEncryption is 'ssl'. - Note that the username and password currently need to be - set cleartext in the configuration file, which is not - recommended. If you need to use this option, be sure your - configuration file is read-only. - - multimailhook.envelopeSender - The sender address to be passed to the SMTP server. If - unset, then the value of multimailhook.from is used. - - multimailhook.smtpServerTimeout - Timeout in seconds. Default is 10. - - multimailhook.smtpEncryption - Set the security type. Allowed values: ``none``, ``ssl``, ``tls`` (starttls). - Default is ``none``. - - multimailhook.smtpCACerts - Set the path to a list of trusted CA certificate to verify the - server certificate, only supported when ``smtpEncryption`` is - ``tls``. If unset or empty, the server certificate is not - verified. If it targets a file containing a list of trusted CA - certificates (PEM format) these CAs will be used to verify the - server certificate. For debian, you can set - ``/etc/ssl/certs/ca-certificates.crt`` for using the system - trusted CAs. For self-signed server, you can add your server - certificate to the system store:: - - cd /usr/local/share/ca-certificates/ - openssl s_client -starttls smtp \ - -connect mail.example.net:587 -showcerts \ - /dev/null \ - | openssl x509 -outform PEM >mail.example.net.crt - update-ca-certificates - - and used the updated ``/etc/ssl/certs/ca-certificates.crt``. Or - directly use your ``/path/to/mail.example.net.crt``. Default is - unset. - - multimailhook.smtpServerDebugLevel - Integer number. Set to greater than 0 to activate debugging. - -multimailhook.from, multimailhook.fromCommit, multimailhook.fromRefchange - If set, use this value in the From: field of generated emails. - ``fromCommit`` is used for commit emails, ``fromRefchange`` is - used for refchange emails, and ``from`` is used as fall-back in - all cases. - - The value for these variables can be either: - - - An email address, which will be used directly. - - - The value ``pusher``, in which case the pusher's address (if - available) will be used. - - - The value ``author`` (meaningful only for ``fromCommit``), in which - case the commit author's address will be used. - - If config values are unset, the value of the From: header is - determined as follows: - - 1. (gitolite environment only) - 1.a) If ``multimailhook.MailaddressMap`` is set, and is a path - to an existing file (if relative, it is considered relative to - the place where ``gitolite.conf`` is located), then this file - should contain lines like:: - - username Firstname Lastname - - git-multimail will then look for a line where ``$GL_USER`` - matches the ``username`` part, and use the rest of the line for - the ``From:`` header. - - 1.b) Parse gitolite.conf, looking for a block of comments that - looks like this:: - - # BEGIN USER EMAILS - # username Firstname Lastname - # END USER EMAILS - - If that block exists, and there is a line between the BEGIN - USER EMAILS and END USER EMAILS lines where the first field - matches the gitolite username ($GL_USER), use the rest of the - line for the From: header. - - 2. If the user.email configuration setting is set, use its value - (and the value of user.name, if set). - - 3. Use the value of multimailhook.envelopeSender. - -multimailhook.MailaddressMap - (gitolite environment only) - File to look for a ``From:`` address based on the user doing the - push. Defaults to unset. See ``multimailhook.from`` for details. - -multimailhook.administrator - The name and/or email address of the administrator of the Git - repository; used in FOOTER_TEMPLATE. Default is - multimailhook.envelopesender if it is set; otherwise a generic - string is used. - -multimailhook.emailPrefix - All emails have this string prepended to their subjects, to aid - email filtering (though filtering based on the X-Git-* email - headers is probably more robust). Default is the short name of - the repository in square brackets; e.g., ``[myrepo]``. Set this - value to the empty string to suppress the email prefix. You may - use the placeholder ``%(repo_shortname)s`` for the short name of - the repository. - -multimailhook.emailMaxLines - The maximum number of lines that should be included in the body of - a generated email. If not specified, there is no limit. Lines - beyond the limit are suppressed and counted, and a final line is - added indicating the number of suppressed lines. - -multimailhook.emailMaxLineLength - The maximum length of a line in the email body. Lines longer than - this limit are truncated to this length with a trailing ``[...]`` - added to indicate the missing text. The default is 500, because - (a) diffs with longer lines are probably from binary files, for - which a diff is useless, and (b) even if a text file has such long - lines, the diffs are probably unreadable anyway. To disable line - truncation, set this option to 0. - -multimailhook.subjectMaxLength - The maximum length of the subject line (i.e. the ``oneline`` field - in templates, not including the prefix). Lines longer than this - limit are truncated to this length with a trailing ``[...]`` added - to indicate the missing text. This option The default is to use - ``multimailhook.emailMaxLineLength``. This option avoids sending - emails with overly long subject lines, but should not be needed if - the commit messages follow the Git convention (one short subject - line, then a blank line, then the message body). To disable line - truncation, set this option to 0. - -multimailhook.maxCommitEmails - The maximum number of commit emails to send for a given change. - When the number of patches is larger that this value, only the - summary refchange email is sent. This can avoid accidental - mailbombing, for example on an initial push. To disable commit - emails limit, set this option to 0. The default is 500. - -multimailhook.excludeMergeRevisions - When sending out revision emails, do not consider merge commits (the - functional equivalent of `rev-list --no-merges`). - The default is `false` (send merge commit emails). - -multimailhook.emailStrictUTF8 - If this boolean option is set to `true`, then the main part of the - email body is forced to be valid UTF-8. Any characters that are - not valid UTF-8 are converted to the Unicode replacement - character, U+FFFD. The default is `true`. - - This option is ineffective with Python 3, where non-UTF-8 - characters are unconditionally replaced. - -multimailhook.diffOpts - Options passed to ``git diff-tree`` when generating the summary - information for ReferenceChange emails. Default is ``--stat - --summary --find-copies-harder``. Add -p to those options to - include a unified diff of changes in addition to the usual summary - output. Shell quoting is allowed; see ``multimailhook.logOpts`` for - details. - -multimailhook.graphOpts - Options passed to ``git log --graph`` when generating graphs for the - reference change summary emails (used only if refchangeShowGraph - is true). The default is '--oneline --decorate'. - - Shell quoting is allowed; see logOpts for details. - -multimailhook.logOpts - Options passed to ``git log`` to generate additional info for - reference change emails (used only if refchangeShowLog is set). - For example, adding -p will show each commit's complete diff. The - default is empty. - - Shell quoting is allowed; for example, a log format that contains - spaces can be specified using something like:: - - git config multimailhook.logopts '--pretty=format:"%h %aN <%aE>%n%s%n%n%b%n"' - - If you want to set this by editing your configuration file - directly, remember that Git requires double-quotes to be escaped - (see git-config(1) for more information):: - - [multimailhook] - logopts = --pretty=format:\"%h %aN <%aE>%n%s%n%n%b%n\" - -multimailhook.commitLogOpts - Options passed to ``git log`` to generate additional info for - revision change emails. For example, adding --ignore-all-spaces - will suppress whitespace changes. The default options are ``-C - --stat -p --cc``. Shell quoting is allowed; see - multimailhook.logOpts for details. - -multimailhook.dateSubstitute - String to use as a substitute for ``Date:`` in the output of ``git - log`` while formatting commit messages. This is useful to avoid - emitting a line that can be interpreted by mailers as the start of - a cited message (Zimbra webmail in particular). Defaults to - ``CommitDate:``. Set to an empty string or ``none`` to deactivate - the behavior. - -multimailhook.emailDomain - Domain name appended to the username of the person doing the push - to convert it into an email address - (via ``"%s@%s" % (username, emaildomain)``). More complicated - schemes can be implemented by overriding Environment and - overriding its get_pusher_email() method. - -multimailhook.replyTo, multimailhook.replyToCommit, multimailhook.replyToRefchange - Addresses to use in the Reply-To: field for commit emails - (replyToCommit) and refchange emails (replyToRefchange). - multimailhook.replyTo is used as default when replyToCommit or - replyToRefchange is not set. The shortcuts ``pusher`` and - ``author`` are allowed with the same semantics as for - ``multimailhook.from``. In addition, the value ``none`` can be - used to omit the ``Reply-To:`` field. - - The default is ``pusher`` for refchange emails, and ``author`` for - commit emails. - -multimailhook.quiet - Do not output the list of email recipients from the hook - -multimailhook.stdout - For debugging, send emails to stdout rather than to the - mailer. Equivalent to the --stdout command line option - -multimailhook.scanCommitForCc - If this option is set to true, than recipients from lines in commit body - that starts with ``CC:`` will be added to CC list. - Default: false - -multimailhook.combineWhenSingleCommit - If this option is set to true and a single new commit is pushed to - a branch, combine the summary and commit email messages into a - single email. - Default: true - -multimailhook.refFilterInclusionRegex, multimailhook.refFilterExclusionRegex, multimailhook.refFilterDoSendRegex, multimailhook.refFilterDontSendRegex - **Warning:** these options are experimental. They should work, but - the user-interface is not stable yet (in particular, the option - names may change). If you want to participate in stabilizing the - feature, please contact the maintainers and/or send pull-requests. - If you are happy with the current shape of the feature, please - report it too. - - Regular expressions that can be used to limit refs for which email - updates will be sent. It is an error to specify both an inclusion - and an exclusion regex. If a ``refFilterInclusionRegex`` is - specified, emails will only be sent for refs which match this - regex. If a ``refFilterExclusionRegex`` regex is specified, - emails will be sent for all refs except those that match this - regex (or that match a predefined regex specific to the - environment, such as "^refs/notes" for most environments and - "^refs/notes|^refs/changes" for the gerrit environment). - - The expressions are matched against the complete refname, and is - considered to match if any substring matches. For example, to - filter-out all tags, set ``refFilterExclusionRegex`` to - ``^refs/tags/`` (note the leading ``^`` but no trailing ``$``). If - you set ``refFilterExclusionRegex`` to ``master``, then any ref - containing ``master`` will be excluded (the ``master`` branch, but - also ``refs/tags/master`` or ``refs/heads/foo-master-bar``). - - ``refFilterDoSendRegex`` and ``refFilterDontSendRegex`` are - analogous to ``refFilterInclusionRegex`` and - ``refFilterExclusionRegex`` with one difference: with - ``refFilterDoSendRegex`` and ``refFilterDontSendRegex``, commits - introduced by one excluded ref will not be considered as new when - they reach an included ref. Typically, if you add a branch ``foo`` - to ``refFilterDontSendRegex``, push commits to this branch, and - later merge branch ``foo`` into ``master``, then the notification - email for ``master`` will contain a commit email only for the - merge commit. If you include ``foo`` in - ``refFilterExclusionRegex``, then at the time of merge, you will - receive one commit email per commit in the branch. - - These variables can be multi-valued, like:: - - [multimailhook] - refFilterExclusionRegex = ^refs/tags/ - refFilterExclusionRegex = ^refs/heads/master$ - - You can also provide a whitespace-separated list like:: - - [multimailhook] - refFilterExclusionRegex = ^refs/tags/ ^refs/heads/master$ - - Both examples exclude tags and the master branch, and are - equivalent to:: - - [multimailhook] - refFilterExclusionRegex = ^refs/tags/|^refs/heads/master$ - - ``refFilterInclusionRegex`` and ``refFilterExclusionRegex`` are - strictly stronger than ``refFilterDoSendRegex`` and - ``refFilterDontSendRegex``. In other words, adding a ref to a - DoSend/DontSend regex has no effect if it is already excluded by a - Exclusion/Inclusion regex. - -multimailhook.logFile, multimailhook.errorLogFile, multimailhook.debugLogFile - - When set, these variable designate path to files where - git-multimail will log some messages. Normal messages and error - messages are sent to ``logFile``, and error messages are also sent - to ``errorLogFile``. Debug messages and all other messages are - sent to ``debugLogFile``. The recommended way is to set only one - of these variables, but it is also possible to set several of them - (part of the information is then duplicated in several log files, - for example errors are duplicated to all log files). - - Relative path are relative to the Git repository where the push is - done. - -multimailhook.verbose - - Verbosity level of git-multimail on its standard output. By - default, show only error and info messages. If set to true, show - also debug messages. - -Email filtering aids --------------------- - -All emails include extra headers to enable fine tuned filtering and -give information for debugging. All emails include the headers -``X-Git-Host``, ``X-Git-Repo``, ``X-Git-Refname``, and ``X-Git-Reftype``. -ReferenceChange emails also include headers ``X-Git-Oldrev`` and ``X-Git-Newrev``; -Revision emails also include header ``X-Git-Rev``. - - -Customizing email contents --------------------------- - -git-multimail mostly generates emails by expanding templates. The -templates can be customized. To avoid the need to edit -``git_multimail.py`` directly, the preferred way to change the templates -is to write a separate Python script that imports ``git_multimail.py`` as -a module, then replaces the templates in place. See the provided -post-receive script for an example of how this is done. - - -Customizing git-multimail for your environment ----------------------------------------------- - -git-multimail is mostly customized via an "environment" that describes -the local environment in which Git is running. Two types of -environment are built in: - -GenericEnvironment - a stand-alone Git repository. - -GitoliteEnvironment - a Git repository that is managed by gitolite_. For such - repositories, the identity of the pusher is read from - environment variable $GL_USER, the name of the repository is read - from $GL_REPO (if it is not overridden by multimailhook.reponame), - and the From: header value is optionally read from gitolite.conf - (see multimailhook.from). - -By default, git-multimail assumes GitoliteEnvironment if $GL_USER and -$GL_REPO are set, and otherwise assumes GenericEnvironment. -Alternatively, you can choose one of these two environments explicitly -by setting a ``multimailhook.environment`` config setting (which can -have the value `generic` or `gitolite`) or by passing an --environment -option to the script. - -If you need to customize the script in ways that are not supported by -the existing environments, you can define your own environment class -class using arbitrary Python code. To do so, you need to import -``git_multimail.py`` as a Python module, as demonstrated by the example -post-receive script. Then implement your environment class; it should -usually inherit from one of the existing Environment classes and -possibly one or more of the EnvironmentMixin classes. Then set the -``environment`` variable to an instance of your own environment class -and pass it to ``run_as_post_receive_hook()``. - -The standard environment classes, GenericEnvironment and -GitoliteEnvironment, are in fact themselves put together out of a -number of mixin classes, each of which handles one aspect of the -customization. For the finest control over your configuration, you -can specify exactly which mixin classes your own environment class -should inherit from, and override individual methods (or even add your -own mixin classes) to implement entirely new behaviors. If you -implement any mixins that might be useful to other people, please -consider sharing them with the community! - - -Getting involved ----------------- - -Please, read ``__ for instructions on how to -contribute to git-multimail. - - -Footnotes ---------- - -.. [1] Because of the way information is passed to update hooks, the - script's method of determining whether a commit has already - been seen does not work when it is used as an ``update`` script. - In particular, no notification email will be generated for a - new commit that is added to multiple references in the same - push. A workaround is to use --force-send to force sending the - emails. - -.. _gitolite: https://github.com/sitaramc/gitolite diff --git a/contrib/hooks/multimail/doc/customizing-emails.rst b/contrib/hooks/multimail/doc/customizing-emails.rst deleted file mode 100644 index 3f5b67f768..0000000000 --- a/contrib/hooks/multimail/doc/customizing-emails.rst +++ /dev/null @@ -1,56 +0,0 @@ -Customizing the content and formatting of emails -================================================ - -Overloading template strings ----------------------------- - -The content of emails is generated based on template strings defined -in ``git_multimail.py``. You can customize these template strings -without changing the script itself, by defining a Python wrapper -around it. The python wrapper should ``import git_multimail`` and then -override the ``git_multimail.*`` strings like this:: - - import sys # needed for sys.argv - - # Import and customize git_multimail: - import git_multimail - git_multimail.REVISION_INTRO_TEMPLATE = """...""" - git_multimail.COMBINED_INTRO_TEMPLATE = git_multimail.REVISION_INTRO_TEMPLATE - - # start git_multimail itself: - git_multimail.main(sys.argv[1:]) - -The template strings can use any value already used in the existing -templates (read the source code). - -Using HTML in template strings ------------------------------- - -If ``multimailhook.commitEmailFormat`` is set to HTML, then -git-multimail will generate HTML emails for commit notifications. The -log and diff will be formatted automatically by git-multimail. By -default, any HTML special character in the templates will be escaped. - -To use HTML formatting in the introduction of the email, set -``multimailhook.htmlInIntro`` to ``true``. Then, the template can -contain any HTML tags, that will be sent as-is in the email. For -example, to add some formatting and a link to the online commit, use -a format like:: - - git_multimail.REVISION_INTRO_TEMPLATE = """\ - This is an automated email from the git hooks/post-receive script.

- - %(pusher)s pushed a commit to %(refname_type)s %(short_refname)s - in repository %(repo_shortname)s.
- - View on GitHub. - """ - -Note that the values expanded from ``%(variable)s`` in the format -strings will still be escaped. - -For a less flexible but easier to set up way to add a link to commit -emails, see ``multimailhook.commitBrowseURL``. - -Similarly, one can set ``multimailhook.htmlInFooter`` and override any -of the ``*_FOOTER*`` template strings. diff --git a/contrib/hooks/multimail/doc/gerrit.rst b/contrib/hooks/multimail/doc/gerrit.rst deleted file mode 100644 index 8011d05dec..0000000000 --- a/contrib/hooks/multimail/doc/gerrit.rst +++ /dev/null @@ -1,56 +0,0 @@ -Setting up git-multimail on Gerrit -================================== - -Gerrit has its own email-sending system, but you may prefer using -``git-multimail`` instead. It supports Gerrit natively as a Gerrit -``ref-updated`` hook (Warning: `Gerrit hooks -`__ -are distinct from Git hooks). Setting up ``git-multimail`` on a Gerrit -installation can be done following the instructions below. - -The explanations show an easy way to set up ``git-multimail``, -but leave ``git-multimail`` installed and unconfigured for a while. If -you run Gerrit on a production server, it is advised that you -execute the step "Set up the hook" last to avoid confusing your users -in the meantime. - -Set up the hook ---------------- - -Create a directory ``$site_path/hooks/`` if it does not exist (if you -don't know what ``$site_path`` is, run ``gerrit.sh status`` and look -for a ``GERRIT_SITE`` line). Either copy ``git_multimail.py`` to -``$site_path/hooks/ref-updated`` or create a wrapper script like -this:: - - #! /bin/sh - exec /path/to/git_multimail.py "$@" - -In both cases, make sure the file is named exactly -``$site_path/hooks/ref-updated`` and is executable. - -(Alternatively, you may configure the ``[hooks]`` section of -gerrit.config) - -Configuration -------------- - -Log on the gerrit server and edit ``$site_path/git/$project/config`` -to configure ``git-multimail``. - -Troubleshooting ---------------- - -Warning: this will disable ``git-multimail`` during the debug, and -could confuse your users. Don't run on a production server. - -To debug configuration issues with ``git-multimail``, you can add the -``--stdout`` option when calling ``git_multimail.py`` like this:: - - #!/bin/sh - exec /path/to/git-multimail/git-multimail/git_multimail.py \ - --stdout "$@" >> /tmp/log.txt - -and try pushing from a test repository. You should see the source of -the email that would have been sent in the output of ``git push`` in -the file ``/tmp/log.txt``. diff --git a/contrib/hooks/multimail/doc/gitolite.rst b/contrib/hooks/multimail/doc/gitolite.rst deleted file mode 100644 index 5054833105..0000000000 --- a/contrib/hooks/multimail/doc/gitolite.rst +++ /dev/null @@ -1,118 +0,0 @@ -Setting up git-multimail on gitolite -==================================== - -``git-multimail`` supports gitolite 3 natively. -The explanations below show an easy way to set up ``git-multimail``, -but leave ``git-multimail`` installed and unconfigured for a while. If -you run gitolite on a production server, it is advised that you -execute the step "Set up the hook" last to avoid confusing your users -in the meantime. - -Set up the hook ---------------- - -Log in as your gitolite user. - -Create a file ``.gitolite/hooks/common/post-receive`` on your gitolite -account containing (adapt the path, obviously):: - - #!/bin/sh - exec /path/to/git-multimail/git-multimail/git_multimail.py "$@" - -Make sure it's executable (``chmod +x``). Record the hook in -gitolite:: - - gitolite setup - -Configuration -------------- - -First, you have to allow the admin to set Git configuration variables. - -As gitolite user, edit the line containing ``GIT_CONFIG_KEYS`` in file -``.gitolite.rc``, to make it look like:: - - GIT_CONFIG_KEYS => 'multimailhook\..*', - -You can now log out and return to your normal user. - -In the ``gitolite-admin`` clone, edit the file ``conf/gitolite.conf`` -and add:: - - repo @all - # Not strictly needed as git_multimail.py will chose gitolite if - # $GL_USER is set. - config multimailhook.environment = gitolite - config multimailhook.mailingList = # Where emails should be sent - config multimailhook.from = # From address to use - -Note that by default, gitolite forbids ``<`` and ``>`` in variable -values (for security/paranoia reasons, see -`compensating for UNSAFE_PATT -`__ -in gitolite's documentation for explanations and a way to disable -this). As a consequence, you will not be able to use ``First Last -`` as recipient email, but specifying -``First.Last@example.com`` alone works. - -Obviously, you can customize all parameters on a per-repository basis by -adding these ``config multimailhook.*`` lines in the section -corresponding to a repository or set of repositories. - -To activate ``git-multimail`` on a per-repository basis, do not set -``multimailhook.mailingList`` in the ``@all`` section and set it only -for repositories for which you want ``git-multimail``. - -Alternatively, you can set up the ``From:`` field on a per-user basis -by adding a ``BEGIN USER EMAILS``/``END USER EMAILS`` section (see -``../README``). - -Specificities of Gitolite for Configuration -------------------------------------------- - -Empty configuration variables -............................. - -With gitolite, the syntax ``config multimailhook.commitList = ""`` -unsets the variable instead of setting it to an empty string (see -`here -`__). -As a result, there is no way to set a variable to the empty string. -In all most places where an empty value is required, git-multimail -now allows to specify special ``"none"`` value (case-sensitive) to -mean the same. - -Alternatively, one can use ``" "`` (a single space) instead of ``""``. -In most cases (in particular ``multimailhook.*List`` variables), this -will be equivalent to an empty string. - -If you have a use-case where ``"none"`` is not an acceptable value and -you need ``" "`` or ``""`` instead, please report it as a bug to -git-multimail. - -Allowing Regular Expressions in Configuration -............................................. - -gitolite has a mechanism to prevent unsafe configuration variable -values, which prevent characters like ``|`` commonly used in regular -expressions. If you do not need the safety feature of gitolite and -need to use regular expressions in your configuration (e.g. for -``multimailhook.refFilter*`` variables), set -`UNSAFE_PATT -`__ to a -less restrictive value. - -Troubleshooting ---------------- - -Warning: this will disable ``git-multimail`` during the debug, and -could confuse your users. Don't run on a production server. - -To debug configuration issues with ``git-multimail``, you can add the -``--stdout`` option when calling ``git_multimail.py`` like this:: - - #!/bin/sh - exec /path/to/git-multimail/git-multimail/git_multimail.py --stdout "$@" - -and try pushing from a test repository. You should see the source of -the email that would have been sent in the output of ``git push``. diff --git a/contrib/hooks/multimail/doc/troubleshooting.rst b/contrib/hooks/multimail/doc/troubleshooting.rst deleted file mode 100644 index 651b509ee6..0000000000 --- a/contrib/hooks/multimail/doc/troubleshooting.rst +++ /dev/null @@ -1,78 +0,0 @@ -Troubleshooting issues with git-multimail: a FAQ -================================================ - -How to check that git-multimail is properly set up? ---------------------------------------------------- - -Since version 1.4.0, git-multimail allows a simple self-checking of -its configuration: run it with the environment variable -``GIT_MULTIMAIL_CHECK_SETUP`` set to a non-empty string. You should -get something like this:: - - $ GIT_MULTIMAIL_CHECK_SETUP=true /home/moy/dev/git-multimail/git-multimail/git_multimail.py - Environment values: - administrator : 'the administrator of this repository' - charset : 'utf-8' - emailprefix : '[git-multimail] ' - fqdn : 'anie' - projectdesc : 'UNNAMED PROJECT' - pusher : 'moy' - repo_path : '/home/moy/dev/git-multimail' - repo_shortname : 'git-multimail' - - Now, checking that git-multimail's standard input is properly set ... - Please type some text and then press Return - foo - You have just entered: - foo - git-multimail seems properly set up. - -If you forgot to set an important variable, you may get instead:: - - $ GIT_MULTIMAIL_CHECK_SETUP=true /home/moy/dev/git-multimail/git-multimail/git_multimail.py - No email recipients configured! - -Do not set ``$GIT_MULTIMAIL_CHECK_SETUP`` other than for testing your -configuration: it would disable the hook completely. - -Git is not using the right address in the From/To/Reply-To field ----------------------------------------------------------------- - -First, make sure that git-multimail actually uses what you think it is -using. A lot happens to your email (especially when posting to a -mailing-list) between the time `git_multimail.py` sends it and the -time it reaches your inbox. - -A simple test (to do on a test repository, do not use in production as -it would disable email sending): change your post-receive hook to call -`git_multimail.py` with the `--stdout` option, and try to push to the -repository. You should see something like:: - - Counting objects: 3, done. - Writing objects: 100% (3/3), 263 bytes | 0 bytes/s, done. - Total 3 (delta 0), reused 0 (delta 0) - remote: Sending notification emails to: foo.bar@example.com - remote: =========================================================================== - remote: Date: Mon, 25 Apr 2016 18:39:59 +0200 - remote: To: foo.bar@example.com - remote: Subject: [git] branch master updated: foo - remote: MIME-Version: 1.0 - remote: Content-Type: text/plain; charset=utf-8 - remote: Content-Transfer-Encoding: 8bit - remote: Message-ID: <20160425163959.2311.20498@anie> - remote: From: Auth Or - remote: Reply-To: Auth Or - remote: X-Git-Host: example - ... - remote: -- - remote: To stop receiving notification emails like this one, please contact - remote: the administrator of this repository. - remote: =========================================================================== - To /path/to/repo - 6278f04..e173f20 master -> master - -Note: this does not include the sender (Return-Path: header), as it is -not part of the message content but passed to the mailer. Some mailer -show the ``Sender:`` field instead of the ``From:`` field (for -example, Zimbra Webmail shows ``From: on behalf of -``). diff --git a/contrib/hooks/multimail/git_multimail.py b/contrib/hooks/multimail/git_multimail.py deleted file mode 100755 index f563be82fc..0000000000 --- a/contrib/hooks/multimail/git_multimail.py +++ /dev/null @@ -1,4346 +0,0 @@ -#! /usr/bin/env python - -__version__ = '1.5.0' - -# Copyright (c) 2015-2016 Matthieu Moy and others -# Copyright (c) 2012-2014 Michael Haggerty and others -# Derived from contrib/hooks/post-receive-email, which is -# Copyright (c) 2007 Andy Parkins -# and also includes contributions by other authors. -# -# This file is part of git-multimail. -# -# git-multimail is free software: you can redistribute it and/or -# modify it under the terms of the GNU General Public License version -# 2 as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see -# . - -"""Generate notification emails for pushes to a git repository. - -This hook sends emails describing changes introduced by pushes to a -git repository. For each reference that was changed, it emits one -ReferenceChange email summarizing how the reference was changed, -followed by one Revision email for each new commit that was introduced -by the reference change. - -Each commit is announced in exactly one Revision email. If the same -commit is merged into another branch in the same or a later push, then -the ReferenceChange email will list the commit's SHA1 and its one-line -summary, but no new Revision email will be generated. - -This script is designed to be used as a "post-receive" hook in a git -repository (see githooks(5)). It can also be used as an "update" -script, but this usage is not completely reliable and is deprecated. - -To help with debugging, this script accepts a --stdout option, which -causes the emails to be written to standard output rather than sent -using sendmail. - -See the accompanying README file for the complete documentation. - -""" - -import sys -import os -import re -import bisect -import socket -import subprocess -import shlex -import optparse -import logging -import smtplib -try: - import ssl -except ImportError: - # Python < 2.6 do not have ssl, but that's OK if we don't use it. - pass -import time - -import uuid -import base64 - -PYTHON3 = sys.version_info >= (3, 0) - -if sys.version_info <= (2, 5): - def all(iterable): - for element in iterable: - if not element: - return False - return True - - -def is_ascii(s): - return all(ord(c) < 128 and ord(c) > 0 for c in s) - - -if PYTHON3: - def is_string(s): - return isinstance(s, str) - - def str_to_bytes(s): - return s.encode(ENCODING) - - def bytes_to_str(s, errors='strict'): - return s.decode(ENCODING, errors) - - unicode = str - - def write_str(f, msg): - # Try outputting with the default encoding. If it fails, - # try UTF-8. - try: - f.buffer.write(msg.encode(sys.getdefaultencoding())) - except UnicodeEncodeError: - f.buffer.write(msg.encode(ENCODING)) - - def read_line(f): - # Try reading with the default encoding. If it fails, - # try UTF-8. - out = f.buffer.readline() - try: - return out.decode(sys.getdefaultencoding()) - except UnicodeEncodeError: - return out.decode(ENCODING) - - import html - - def html_escape(s): - return html.escape(s) - -else: - def is_string(s): - try: - return isinstance(s, basestring) - except NameError: # Silence Pyflakes warning - raise - - def str_to_bytes(s): - return s - - def bytes_to_str(s, errors='strict'): - return s - - def write_str(f, msg): - f.write(msg) - - def read_line(f): - return f.readline() - - def next(it): - return it.next() - - import cgi - - def html_escape(s): - return cgi.escape(s, True) - -try: - from email.charset import Charset - from email.utils import make_msgid - from email.utils import getaddresses - from email.utils import formataddr - from email.utils import formatdate - from email.header import Header -except ImportError: - # Prior to Python 2.5, the email module used different names: - from email.Charset import Charset - from email.Utils import make_msgid - from email.Utils import getaddresses - from email.Utils import formataddr - from email.Utils import formatdate - from email.Header import Header - - -DEBUG = False - -ZEROS = '0' * 40 -LOGBEGIN = '- Log -----------------------------------------------------------------\n' -LOGEND = '-----------------------------------------------------------------------\n' - -ADDR_HEADERS = set(['from', 'to', 'cc', 'bcc', 'reply-to', 'sender']) - -# It is assumed in many places that the encoding is uniformly UTF-8, -# so changing these constants is unsupported. But define them here -# anyway, to make it easier to find (at least most of) the places -# where the encoding is important. -(ENCODING, CHARSET) = ('UTF-8', 'utf-8') - - -REF_CREATED_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s created' - ' (now %(newrev_short)s)' - ) -REF_UPDATED_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s updated' - ' (%(oldrev_short)s -> %(newrev_short)s)' - ) -REF_DELETED_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s deleted' - ' (was %(oldrev_short)s)' - ) - -COMBINED_REFCHANGE_REVISION_SUBJECT_TEMPLATE = ( - '%(emailprefix)s%(refname_type)s %(short_refname)s updated: %(oneline)s' - ) - -REFCHANGE_HEADER_TEMPLATE = """\ -Date: %(send_date)s -To: %(recipients)s -Subject: %(subject)s -MIME-Version: 1.0 -Content-Type: text/%(contenttype)s; charset=%(charset)s -Content-Transfer-Encoding: 8bit -Message-ID: %(msgid)s -From: %(fromaddr)s -Reply-To: %(reply_to)s -Thread-Index: %(thread_index)s -X-Git-Host: %(fqdn)s -X-Git-Repo: %(repo_shortname)s -X-Git-Refname: %(refname)s -X-Git-Reftype: %(refname_type)s -X-Git-Oldrev: %(oldrev)s -X-Git-Newrev: %(newrev)s -X-Git-NotificationType: ref_changed -X-Git-Multimail-Version: %(multimail_version)s -Auto-Submitted: auto-generated -""" - -REFCHANGE_INTRO_TEMPLATE = """\ -This is an automated email from the git hooks/post-receive script. - -%(pusher)s pushed a change to %(refname_type)s %(short_refname)s -in repository %(repo_shortname)s. - -""" - - -FOOTER_TEMPLATE = """\ - --- \n\ -To stop receiving notification emails like this one, please contact -%(administrator)s. -""" - - -REWIND_ONLY_TEMPLATE = """\ -This update removed existing revisions from the reference, leaving the -reference pointing at a previous point in the repository history. - - * -- * -- N %(refname)s (%(newrev_short)s) - \\ - O -- O -- O (%(oldrev_short)s) - -Any revisions marked "omit" are not gone; other references still -refer to them. Any revisions marked "discard" are gone forever. -""" - - -NON_FF_TEMPLATE = """\ -This update added new revisions after undoing existing revisions. -That is to say, some revisions that were in the old version of the -%(refname_type)s are not in the new version. This situation occurs -when a user --force pushes a change and generates a repository -containing something like this: - - * -- * -- B -- O -- O -- O (%(oldrev_short)s) - \\ - N -- N -- N %(refname)s (%(newrev_short)s) - -You should already have received notification emails for all of the O -revisions, and so the following emails describe only the N revisions -from the common base, B. - -Any revisions marked "omit" are not gone; other references still -refer to them. Any revisions marked "discard" are gone forever. -""" - - -NO_NEW_REVISIONS_TEMPLATE = """\ -No new revisions were added by this update. -""" - - -DISCARDED_REVISIONS_TEMPLATE = """\ -This change permanently discards the following revisions: -""" - - -NO_DISCARDED_REVISIONS_TEMPLATE = """\ -The revisions that were on this %(refname_type)s are still contained in -other references; therefore, this change does not discard any commits -from the repository. -""" - - -NEW_REVISIONS_TEMPLATE = """\ -The %(tot)s revisions listed above as "new" are entirely new to this -repository and will be described in separate emails. The revisions -listed as "add" were already present in the repository and have only -been added to this reference. - -""" - - -TAG_CREATED_TEMPLATE = """\ - at %(newrev_short)-8s (%(newrev_type)s) -""" - - -TAG_UPDATED_TEMPLATE = """\ -*** WARNING: tag %(short_refname)s was modified! *** - - from %(oldrev_short)-8s (%(oldrev_type)s) - to %(newrev_short)-8s (%(newrev_type)s) -""" - - -TAG_DELETED_TEMPLATE = """\ -*** WARNING: tag %(short_refname)s was deleted! *** - -""" - - -# The template used in summary tables. It looks best if this uses the -# same alignment as TAG_CREATED_TEMPLATE and TAG_UPDATED_TEMPLATE. -BRIEF_SUMMARY_TEMPLATE = """\ -%(action)8s %(rev_short)-8s %(text)s -""" - - -NON_COMMIT_UPDATE_TEMPLATE = """\ -This is an unusual reference change because the reference did not -refer to a commit either before or after the change. We do not know -how to provide full information about this reference change. -""" - - -REVISION_HEADER_TEMPLATE = """\ -Date: %(send_date)s -To: %(recipients)s -Cc: %(cc_recipients)s -Subject: %(emailprefix)s%(num)02d/%(tot)02d: %(oneline)s -MIME-Version: 1.0 -Content-Type: text/%(contenttype)s; charset=%(charset)s -Content-Transfer-Encoding: 8bit -From: %(fromaddr)s -Reply-To: %(reply_to)s -In-Reply-To: %(reply_to_msgid)s -References: %(reply_to_msgid)s -Thread-Index: %(thread_index)s -X-Git-Host: %(fqdn)s -X-Git-Repo: %(repo_shortname)s -X-Git-Refname: %(refname)s -X-Git-Reftype: %(refname_type)s -X-Git-Rev: %(rev)s -X-Git-NotificationType: diff -X-Git-Multimail-Version: %(multimail_version)s -Auto-Submitted: auto-generated -""" - -REVISION_INTRO_TEMPLATE = """\ -This is an automated email from the git hooks/post-receive script. - -%(pusher)s pushed a commit to %(refname_type)s %(short_refname)s -in repository %(repo_shortname)s. - -""" - -LINK_TEXT_TEMPLATE = """\ -View the commit online: -%(browse_url)s - -""" - -LINK_HTML_TEMPLATE = """\ -

View the commit online.

-""" - - -REVISION_FOOTER_TEMPLATE = FOOTER_TEMPLATE - - -# Combined, meaning refchange+revision email (for single-commit additions) -COMBINED_HEADER_TEMPLATE = """\ -Date: %(send_date)s -To: %(recipients)s -Subject: %(subject)s -MIME-Version: 1.0 -Content-Type: text/%(contenttype)s; charset=%(charset)s -Content-Transfer-Encoding: 8bit -Message-ID: %(msgid)s -From: %(fromaddr)s -Reply-To: %(reply_to)s -X-Git-Host: %(fqdn)s -X-Git-Repo: %(repo_shortname)s -X-Git-Refname: %(refname)s -X-Git-Reftype: %(refname_type)s -X-Git-Oldrev: %(oldrev)s -X-Git-Newrev: %(newrev)s -X-Git-Rev: %(rev)s -X-Git-NotificationType: ref_changed_plus_diff -X-Git-Multimail-Version: %(multimail_version)s -Auto-Submitted: auto-generated -""" - -COMBINED_INTRO_TEMPLATE = """\ -This is an automated email from the git hooks/post-receive script. - -%(pusher)s pushed a commit to %(refname_type)s %(short_refname)s -in repository %(repo_shortname)s. - -""" - -COMBINED_FOOTER_TEMPLATE = FOOTER_TEMPLATE - - -class CommandError(Exception): - def __init__(self, cmd, retcode): - self.cmd = cmd - self.retcode = retcode - Exception.__init__( - self, - 'Command "%s" failed with retcode %s' % (' '.join(cmd), retcode,) - ) - - -class ConfigurationException(Exception): - pass - - -# The "git" program (this could be changed to include a full path): -GIT_EXECUTABLE = 'git' - - -# How "git" should be invoked (including global arguments), as a list -# of words. This variable is usually initialized automatically by -# read_git_output() via choose_git_command(), but if a value is set -# here then it will be used unconditionally. -GIT_CMD = None - - -def choose_git_command(): - """Decide how to invoke git, and record the choice in GIT_CMD.""" - - global GIT_CMD - - if GIT_CMD is None: - try: - # Check to see whether the "-c" option is accepted (it was - # only added in Git 1.7.2). We don't actually use the - # output of "git --version", though if we needed more - # specific version information this would be the place to - # do it. - cmd = [GIT_EXECUTABLE, '-c', 'foo.bar=baz', '--version'] - read_output(cmd) - GIT_CMD = [GIT_EXECUTABLE, '-c', 'i18n.logoutputencoding=%s' % (ENCODING,)] - except CommandError: - GIT_CMD = [GIT_EXECUTABLE] - - -def read_git_output(args, input=None, keepends=False, **kw): - """Read the output of a Git command.""" - - if GIT_CMD is None: - choose_git_command() - - return read_output(GIT_CMD + args, input=input, keepends=keepends, **kw) - - -def read_output(cmd, input=None, keepends=False, **kw): - if input: - stdin = subprocess.PIPE - input = str_to_bytes(input) - else: - stdin = None - errors = 'strict' - if 'errors' in kw: - errors = kw['errors'] - del kw['errors'] - p = subprocess.Popen( - tuple(str_to_bytes(w) for w in cmd), - stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw - ) - (out, err) = p.communicate(input) - out = bytes_to_str(out, errors=errors) - retcode = p.wait() - if retcode: - raise CommandError(cmd, retcode) - if not keepends: - out = out.rstrip('\n\r') - return out - - -def read_git_lines(args, keepends=False, **kw): - """Return the lines output by Git command. - - Return as single lines, with newlines stripped off.""" - - return read_git_output(args, keepends=True, **kw).splitlines(keepends) - - -def git_rev_list_ish(cmd, spec, args=None, **kw): - """Common functionality for invoking a 'git rev-list'-like command. - - Parameters: - * cmd is the Git command to run, e.g., 'rev-list' or 'log'. - * spec is a list of revision arguments to pass to the named - command. If None, this function returns an empty list. - * args is a list of extra arguments passed to the named command. - * All other keyword arguments (if any) are passed to the - underlying read_git_lines() function. - - Return the output of the Git command in the form of a list, one - entry per output line. - """ - if spec is None: - return [] - if args is None: - args = [] - args = [cmd, '--stdin'] + args - spec_stdin = ''.join(s + '\n' for s in spec) - return read_git_lines(args, input=spec_stdin, **kw) - - -def git_rev_list(spec, **kw): - """Run 'git rev-list' with the given list of revision arguments. - - See git_rev_list_ish() for parameter and return value - documentation. - """ - return git_rev_list_ish('rev-list', spec, **kw) - - -def git_log(spec, **kw): - """Run 'git log' with the given list of revision arguments. - - See git_rev_list_ish() for parameter and return value - documentation. - """ - return git_rev_list_ish('log', spec, **kw) - - -def header_encode(text, header_name=None): - """Encode and line-wrap the value of an email header field.""" - - # Convert to unicode, if required. - if not isinstance(text, unicode): - text = unicode(text, 'utf-8') - - if is_ascii(text): - charset = 'ascii' - else: - charset = 'utf-8' - - return Header(text, header_name=header_name, charset=Charset(charset)).encode() - - -def addr_header_encode(text, header_name=None): - """Encode and line-wrap the value of an email header field containing - email addresses.""" - - # Convert to unicode, if required. - if not isinstance(text, unicode): - text = unicode(text, 'utf-8') - - text = ', '.join( - formataddr((header_encode(name), emailaddr)) - for name, emailaddr in getaddresses([text]) - ) - - if is_ascii(text): - charset = 'ascii' - else: - charset = 'utf-8' - - return Header(text, header_name=header_name, charset=Charset(charset)).encode() - - -class Config(object): - def __init__(self, section, git_config=None): - """Represent a section of the git configuration. - - If git_config is specified, it is passed to "git config" in - the GIT_CONFIG environment variable, meaning that "git config" - will read the specified path rather than the Git default - config paths.""" - - self.section = section - if git_config: - self.env = os.environ.copy() - self.env['GIT_CONFIG'] = git_config - else: - self.env = None - - @staticmethod - def _split(s): - """Split NUL-terminated values.""" - - words = s.split('\0') - assert words[-1] == '' - return words[:-1] - - @staticmethod - def add_config_parameters(c): - """Add configuration parameters to Git. - - c is either an str or a list of str, each element being of the - form 'var=val' or 'var', with the same syntax and meaning as - the argument of 'git -c var=val'. - """ - if isinstance(c, str): - c = (c,) - parameters = os.environ.get('GIT_CONFIG_PARAMETERS', '') - if parameters: - parameters += ' ' - # git expects GIT_CONFIG_PARAMETERS to be of the form - # "'name1=value1' 'name2=value2' 'name3=value3'" - # including everything inside the double quotes (but not the double - # quotes themselves). Spacing is critical. Also, if a value contains - # a literal single quote that quote must be represented using the - # four character sequence: '\'' - parameters += ' '.join("'" + x.replace("'", "'\\''") + "'" for x in c) - os.environ['GIT_CONFIG_PARAMETERS'] = parameters - - def get(self, name, default=None): - try: - values = self._split(read_git_output( - ['config', '--get', '--null', '%s.%s' % (self.section, name)], - env=self.env, keepends=True, - )) - assert len(values) == 1 - return values[0] - except CommandError: - return default - - def get_bool(self, name, default=None): - try: - value = read_git_output( - ['config', '--get', '--bool', '%s.%s' % (self.section, name)], - env=self.env, - ) - except CommandError: - return default - return value == 'true' - - def get_all(self, name, default=None): - """Read a (possibly multivalued) setting from the configuration. - - Return the result as a list of values, or default if the name - is unset.""" - - try: - return self._split(read_git_output( - ['config', '--get-all', '--null', '%s.%s' % (self.section, name)], - env=self.env, keepends=True, - )) - except CommandError: - t, e, traceback = sys.exc_info() - if e.retcode == 1: - # "the section or key is invalid"; i.e., there is no - # value for the specified key. - return default - else: - raise - - def set(self, name, value): - read_git_output( - ['config', '%s.%s' % (self.section, name), value], - env=self.env, - ) - - def add(self, name, value): - read_git_output( - ['config', '--add', '%s.%s' % (self.section, name), value], - env=self.env, - ) - - def __contains__(self, name): - return self.get_all(name, default=None) is not None - - # We don't use this method anymore internally, but keep it here in - # case somebody is calling it from their own code: - def has_key(self, name): - return name in self - - def unset_all(self, name): - try: - read_git_output( - ['config', '--unset-all', '%s.%s' % (self.section, name)], - env=self.env, - ) - except CommandError: - t, e, traceback = sys.exc_info() - if e.retcode == 5: - # The name doesn't exist, which is what we wanted anyway... - pass - else: - raise - - def set_recipients(self, name, value): - self.unset_all(name) - for pair in getaddresses([value]): - self.add(name, formataddr(pair)) - - -def generate_summaries(*log_args): - """Generate a brief summary for each revision requested. - - log_args are strings that will be passed directly to "git log" as - revision selectors. Iterate over (sha1_short, subject) for each - commit specified by log_args (subject is the first line of the - commit message as a string without EOLs).""" - - cmd = [ - 'log', '--abbrev', '--format=%h %s', - ] + list(log_args) + ['--'] - for line in read_git_lines(cmd): - yield tuple(line.split(' ', 1)) - - -def limit_lines(lines, max_lines): - for (index, line) in enumerate(lines): - if index < max_lines: - yield line - - if index >= max_lines: - yield '... %d lines suppressed ...\n' % (index + 1 - max_lines,) - - -def limit_linelength(lines, max_linelength): - for line in lines: - # Don't forget that lines always include a trailing newline. - if len(line) > max_linelength + 1: - line = line[:max_linelength - 7] + ' [...]\n' - yield line - - -class CommitSet(object): - """A (constant) set of object names. - - The set should be initialized with full SHA1 object names. The - __contains__() method returns True iff its argument is an - abbreviation of any the names in the set.""" - - def __init__(self, names): - self._names = sorted(names) - - def __len__(self): - return len(self._names) - - def __contains__(self, sha1_abbrev): - """Return True iff this set contains sha1_abbrev (which might be abbreviated).""" - - i = bisect.bisect_left(self._names, sha1_abbrev) - return i < len(self) and self._names[i].startswith(sha1_abbrev) - - -class GitObject(object): - def __init__(self, sha1, type=None): - if sha1 == ZEROS: - self.sha1 = self.type = self.commit_sha1 = None - else: - self.sha1 = sha1 - self.type = type or read_git_output(['cat-file', '-t', self.sha1]) - - if self.type == 'commit': - self.commit_sha1 = self.sha1 - elif self.type == 'tag': - try: - self.commit_sha1 = read_git_output( - ['rev-parse', '--verify', '%s^0' % (self.sha1,)] - ) - except CommandError: - # Cannot deref tag to determine commit_sha1 - self.commit_sha1 = None - else: - self.commit_sha1 = None - - self.short = read_git_output(['rev-parse', '--short', sha1]) - - def get_summary(self): - """Return (sha1_short, subject) for this commit.""" - - if not self.sha1: - raise ValueError('Empty commit has no summary') - - return next(iter(generate_summaries('--no-walk', self.sha1))) - - def __eq__(self, other): - return isinstance(other, GitObject) and self.sha1 == other.sha1 - - def __ne__(self, other): - return not self == other - - def __hash__(self): - return hash(self.sha1) - - def __nonzero__(self): - return bool(self.sha1) - - def __bool__(self): - """Python 2 backward compatibility""" - return self.__nonzero__() - - def __str__(self): - return self.sha1 or ZEROS - - -class Change(object): - """A Change that has been made to the Git repository. - - Abstract class from which both Revisions and ReferenceChanges are - derived. A Change knows how to generate a notification email - describing itself.""" - - def __init__(self, environment): - self.environment = environment - self._values = None - self._contains_html_diff = False - - def _contains_diff(self): - # We do contain a diff, should it be rendered in HTML? - if self.environment.commit_email_format == "html": - self._contains_html_diff = True - - def _compute_values(self): - """Return a dictionary {keyword: expansion} for this Change. - - Derived classes overload this method to add more entries to - the return value. This method is used internally by - get_values(). The return value should always be a new - dictionary.""" - - values = self.environment.get_values() - fromaddr = self.environment.get_fromaddr(change=self) - if fromaddr is not None: - values['fromaddr'] = fromaddr - values['multimail_version'] = get_version() - return values - - # Aliases usable in template strings. Tuple of pairs (destination, - # source). - VALUES_ALIAS = ( - ("id", "newrev"), - ) - - def get_values(self, **extra_values): - """Return a dictionary {keyword: expansion} for this Change. - - Return a dictionary mapping keywords to the values that they - should be expanded to for this Change (used when interpolating - template strings). If any keyword arguments are supplied, add - those to the return value as well. The return value is always - a new dictionary.""" - - if self._values is None: - self._values = self._compute_values() - - values = self._values.copy() - if extra_values: - values.update(extra_values) - - for alias, val in self.VALUES_ALIAS: - values[alias] = values[val] - return values - - def expand(self, template, **extra_values): - """Expand template. - - Expand the template (which should be a string) using string - interpolation of the values for this Change. If any keyword - arguments are provided, also include those in the keywords - available for interpolation.""" - - return template % self.get_values(**extra_values) - - def expand_lines(self, template, html_escape_val=False, **extra_values): - """Break template into lines and expand each line.""" - - values = self.get_values(**extra_values) - if html_escape_val: - for k in values: - if is_string(values[k]): - values[k] = html_escape(values[k]) - for line in template.splitlines(True): - yield line % values - - def expand_header_lines(self, template, **extra_values): - """Break template into lines and expand each line as an RFC 2822 header. - - Encode values and split up lines that are too long. Silently - skip lines that contain references to unknown variables.""" - - values = self.get_values(**extra_values) - if self._contains_html_diff: - self._content_type = 'html' - else: - self._content_type = 'plain' - values['contenttype'] = self._content_type - - for line in template.splitlines(): - (name, value) = line.split(': ', 1) - - try: - value = value % values - except KeyError: - t, e, traceback = sys.exc_info() - if DEBUG: - self.environment.log_warning( - 'Warning: unknown variable %r in the following line; line skipped:\n' - ' %s\n' - % (e.args[0], line,) - ) - else: - if name.lower() in ADDR_HEADERS: - value = addr_header_encode(value, name) - else: - value = header_encode(value, name) - for splitline in ('%s: %s\n' % (name, value)).splitlines(True): - yield splitline - - def generate_email_header(self): - """Generate the RFC 2822 email headers for this Change, a line at a time. - - The output should not include the trailing blank line.""" - - raise NotImplementedError() - - def generate_browse_link(self, base_url): - """Generate a link to an online repository browser.""" - return iter(()) - - def generate_email_intro(self, html_escape_val=False): - """Generate the email intro for this Change, a line at a time. - - The output will be used as the standard boilerplate at the top - of the email body.""" - - raise NotImplementedError() - - def generate_email_body(self, push): - """Generate the main part of the email body, a line at a time. - - The text in the body might be truncated after a specified - number of lines (see multimailhook.emailmaxlines).""" - - raise NotImplementedError() - - def generate_email_footer(self, html_escape_val): - """Generate the footer of the email, a line at a time. - - The footer is always included, irrespective of - multimailhook.emailmaxlines.""" - - raise NotImplementedError() - - def _wrap_for_html(self, lines): - """Wrap the lines in HTML 
 tag when using HTML format.
-
-        Escape special HTML characters and add 
 and 
 tags around
-        the given lines if we should be generating HTML as indicated by
-        self._contains_html_diff being set to true.
-        """
-        if self._contains_html_diff:
-            yield "
\n"
-
-            for line in lines:
-                yield html_escape(line)
-
-            yield '
\n'
-        else:
-            for line in lines:
-                yield line
-
-    def generate_email(self, push, body_filter=None, extra_header_values={}):
-        """Generate an email describing this change.
-
-        Iterate over the lines (including the header lines) of an
-        email describing this change.  If body_filter is not None,
-        then use it to filter the lines that are intended for the
-        email body.
-
-        The extra_header_values field is received as a dict and not as
-        **kwargs, to allow passing other keyword arguments in the
-        future (e.g. passing extra values to generate_email_intro()"""
-
-        for line in self.generate_email_header(**extra_header_values):
-            yield line
-        yield '\n'
-        html_escape_val = (self.environment.html_in_intro and
-                           self._contains_html_diff)
-        intro = self.generate_email_intro(html_escape_val)
-        if not self.environment.html_in_intro:
-            intro = self._wrap_for_html(intro)
-        for line in intro:
-            yield line
-
-        if self.environment.commitBrowseURL:
-            for line in self.generate_browse_link(self.environment.commitBrowseURL):
-                yield line
-
-        body = self.generate_email_body(push)
-        if body_filter is not None:
-            body = body_filter(body)
-
-        diff_started = False
-        if self._contains_html_diff:
-            # "white-space: pre" is the default, but we need to
-            # specify it again in case the message is viewed in a
-            # webmail which wraps it in an element setting white-space
-            # to something else (Zimbra does this and sets
-            # white-space: pre-line).
-            yield '
'
-        for line in body:
-            if self._contains_html_diff:
-                # This is very, very naive. It would be much better to really
-                # parse the diff, i.e. look at how many lines do we have in
-                # the hunk headers instead of blindly highlighting everything
-                # that looks like it might be part of a diff.
-                bgcolor = ''
-                fgcolor = ''
-                if line.startswith('--- a/'):
-                    diff_started = True
-                    bgcolor = 'e0e0ff'
-                elif line.startswith('diff ') or line.startswith('index '):
-                    diff_started = True
-                    fgcolor = '808080'
-                elif diff_started:
-                    if line.startswith('+++ '):
-                        bgcolor = 'e0e0ff'
-                    elif line.startswith('@@'):
-                        bgcolor = 'e0e0e0'
-                    elif line.startswith('+'):
-                        bgcolor = 'e0ffe0'
-                    elif line.startswith('-'):
-                        bgcolor = 'ffe0e0'
-                elif line.startswith('commit '):
-                    fgcolor = '808000'
-                elif line.startswith('    '):
-                    fgcolor = '404040'
-
-                # Chop the trailing LF, we don't want it inside 
.
-                line = html_escape(line[:-1])
-
-                if bgcolor or fgcolor:
-                    style = 'display:block; white-space:pre;'
-                    if bgcolor:
-                        style += 'background:#' + bgcolor + ';'
-                    if fgcolor:
-                        style += 'color:#' + fgcolor + ';'
-                    # Use a %s\n" % (style, line)
-                else:
-                    line = line + '\n'
-
-            yield line
-        if self._contains_html_diff:
-            yield '
'
-        html_escape_val = (self.environment.html_in_footer and
-                           self._contains_html_diff)
-        footer = self.generate_email_footer(html_escape_val)
-        if not self.environment.html_in_footer:
-            footer = self._wrap_for_html(footer)
-        for line in footer:
-            yield line
-
-    def get_specific_fromaddr(self):
-        """For kinds of Changes which specify it, return the kind-specific
-        From address to use."""
-        return None
-
-
-class Revision(Change):
-    """A Change consisting of a single git commit."""
-
-    CC_RE = re.compile(r'^\s*C[Cc]:\s*(?P[^#]+@[^\s#]*)\s*(#.*)?$')
-
-    def __init__(self, reference_change, rev, num, tot):
-        Change.__init__(self, reference_change.environment)
-        self.reference_change = reference_change
-        self.rev = rev
-        self.change_type = self.reference_change.change_type
-        self.refname = self.reference_change.refname
-        self.num = num
-        self.tot = tot
-        self.author = read_git_output(['log', '--no-walk', '--format=%aN <%aE>', self.rev.sha1])
-        self.recipients = self.environment.get_revision_recipients(self)
-
-        # -s is short for --no-patch, but -s works on older git's (e.g. 1.7)
-        self.parents = read_git_lines(['show', '-s', '--format=%P',
-                                      self.rev.sha1])[0].split()
-
-        self.cc_recipients = ''
-        if self.environment.get_scancommitforcc():
-            self.cc_recipients = ', '.join(to.strip() for to in self._cc_recipients())
-            if self.cc_recipients:
-                self.environment.log_msg(
-                    'Add %s to CC for %s' % (self.cc_recipients, self.rev.sha1))
-
-    def _cc_recipients(self):
-        cc_recipients = []
-        message = read_git_output(['log', '--no-walk', '--format=%b', self.rev.sha1])
-        lines = message.strip().split('\n')
-        for line in lines:
-            m = re.match(self.CC_RE, line)
-            if m:
-                cc_recipients.append(m.group('to'))
-
-        return cc_recipients
-
-    def _compute_values(self):
-        values = Change._compute_values(self)
-
-        oneline = read_git_output(
-            ['log', '--format=%s', '--no-walk', self.rev.sha1]
-            )
-
-        max_subject_length = self.environment.get_max_subject_length()
-        if max_subject_length > 0 and len(oneline) > max_subject_length:
-            oneline = oneline[:max_subject_length - 6] + ' [...]'
-
-        values['rev'] = self.rev.sha1
-        values['parents'] = ' '.join(self.parents)
-        values['rev_short'] = self.rev.short
-        values['change_type'] = self.change_type
-        values['refname'] = self.refname
-        values['newrev'] = self.rev.sha1
-        values['short_refname'] = self.reference_change.short_refname
-        values['refname_type'] = self.reference_change.refname_type
-        values['reply_to_msgid'] = self.reference_change.msgid
-        values['thread_index'] = self.reference_change.thread_index
-        values['num'] = self.num
-        values['tot'] = self.tot
-        values['recipients'] = self.recipients
-        if self.cc_recipients:
-            values['cc_recipients'] = self.cc_recipients
-        values['oneline'] = oneline
-        values['author'] = self.author
-
-        reply_to = self.environment.get_reply_to_commit(self)
-        if reply_to:
-            values['reply_to'] = reply_to
-
-        return values
-
-    def generate_email_header(self, **extra_values):
-        for line in self.expand_header_lines(
-                REVISION_HEADER_TEMPLATE, **extra_values
-                ):
-            yield line
-
-    def generate_browse_link(self, base_url):
-        if '%(' not in base_url:
-            base_url += '%(id)s'
-        url = "".join(self.expand_lines(base_url))
-        if self._content_type == 'html':
-            for line in self.expand_lines(LINK_HTML_TEMPLATE,
-                                          html_escape_val=True,
-                                          browse_url=url):
-                yield line
-        elif self._content_type == 'plain':
-            for line in self.expand_lines(LINK_TEXT_TEMPLATE,
-                                          html_escape_val=False,
-                                          browse_url=url):
-                yield line
-        else:
-            raise NotImplementedError("Content-type %s unsupported. Please report it as a bug.")
-
-    def generate_email_intro(self, html_escape_val=False):
-        for line in self.expand_lines(REVISION_INTRO_TEMPLATE,
-                                      html_escape_val=html_escape_val):
-            yield line
-
-    def generate_email_body(self, push):
-        """Show this revision."""
-
-        for line in read_git_lines(
-                ['log'] + self.environment.commitlogopts + ['-1', self.rev.sha1],
-                keepends=True,
-                errors='replace'):
-            if line.startswith('Date:   ') and self.environment.date_substitute:
-                yield self.environment.date_substitute + line[len('Date:   '):]
-            else:
-                yield line
-
-    def generate_email_footer(self, html_escape_val):
-        return self.expand_lines(REVISION_FOOTER_TEMPLATE,
-                                 html_escape_val=html_escape_val)
-
-    def generate_email(self, push, body_filter=None, extra_header_values={}):
-        self._contains_diff()
-        return Change.generate_email(self, push, body_filter, extra_header_values)
-
-    def get_specific_fromaddr(self):
-        return self.environment.from_commit
-
-
-class ReferenceChange(Change):
-    """A Change to a Git reference.
-
-    An abstract class representing a create, update, or delete of a
-    Git reference.  Derived classes handle specific types of reference
-    (e.g., tags vs. branches).  These classes generate the main
-    reference change email summarizing the reference change and
-    whether it caused any any commits to be added or removed.
-
-    ReferenceChange objects are usually created using the static
-    create() method, which has the logic to decide which derived class
-    to instantiate."""
-
-    REF_RE = re.compile(r'^refs\/(?P[^\/]+)\/(?P.*)$')
-
-    @staticmethod
-    def create(environment, oldrev, newrev, refname):
-        """Return a ReferenceChange object representing the change.
-
-        Return an object that represents the type of change that is being
-        made. oldrev and newrev should be SHA1s or ZEROS."""
-
-        old = GitObject(oldrev)
-        new = GitObject(newrev)
-        rev = new or old
-
-        # The revision type tells us what type the commit is, combined with
-        # the location of the ref we can decide between
-        #  - working branch
-        #  - tracking branch
-        #  - unannotated tag
-        #  - annotated tag
-        m = ReferenceChange.REF_RE.match(refname)
-        if m:
-            area = m.group('area')
-            short_refname = m.group('shortname')
-        else:
-            area = ''
-            short_refname = refname
-
-        if rev.type == 'tag':
-            # Annotated tag:
-            klass = AnnotatedTagChange
-        elif rev.type == 'commit':
-            if area == 'tags':
-                # Non-annotated tag:
-                klass = NonAnnotatedTagChange
-            elif area == 'heads':
-                # Branch:
-                klass = BranchChange
-            elif area == 'remotes':
-                # Tracking branch:
-                environment.log_warning(
-                    '*** Push-update of tracking branch %r\n'
-                    '***  - incomplete email generated.'
-                    % (refname,)
-                    )
-                klass = OtherReferenceChange
-            else:
-                # Some other reference namespace:
-                environment.log_warning(
-                    '*** Push-update of strange reference %r\n'
-                    '***  - incomplete email generated.'
-                    % (refname,)
-                    )
-                klass = OtherReferenceChange
-        else:
-            # Anything else (is there anything else?)
-            environment.log_warning(
-                '*** Unknown type of update to %r (%s)\n'
-                '***  - incomplete email generated.'
-                % (refname, rev.type,)
-                )
-            klass = OtherReferenceChange
-
-        return klass(
-            environment,
-            refname=refname, short_refname=short_refname,
-            old=old, new=new, rev=rev,
-            )
-
-    @staticmethod
-    def make_thread_index():
-        """Return a string appropriate for the Thread-Index header,
-        needed by MS Outlook to get threading right.
-
-        The format is (base64-encoded):
-        - 1 byte must be 1
-        - 5 bytes encode a date (hardcoded here)
-        - 16 bytes for a globally unique identifier
-
-        FIXME: Unfortunately, even with the Thread-Index field, MS
-        Outlook doesn't seem to do the threading reliably (see
-        https://github.com/git-multimail/git-multimail/pull/194).
-        """
-        thread_index = b'\x01\x00\x00\x12\x34\x56' + uuid.uuid4().bytes
-        return base64.standard_b64encode(thread_index).decode('ascii')
-
-    def __init__(self, environment, refname, short_refname, old, new, rev):
-        Change.__init__(self, environment)
-        self.change_type = {
-            (False, True): 'create',
-            (True, True): 'update',
-            (True, False): 'delete',
-            }[bool(old), bool(new)]
-        self.refname = refname
-        self.short_refname = short_refname
-        self.old = old
-        self.new = new
-        self.rev = rev
-        self.msgid = make_msgid()
-        self.thread_index = self.make_thread_index()
-        self.diffopts = environment.diffopts
-        self.graphopts = environment.graphopts
-        self.logopts = environment.logopts
-        self.commitlogopts = environment.commitlogopts
-        self.showgraph = environment.refchange_showgraph
-        self.showlog = environment.refchange_showlog
-
-        self.header_template = REFCHANGE_HEADER_TEMPLATE
-        self.intro_template = REFCHANGE_INTRO_TEMPLATE
-        self.footer_template = FOOTER_TEMPLATE
-
-    def _compute_values(self):
-        values = Change._compute_values(self)
-
-        values['change_type'] = self.change_type
-        values['refname_type'] = self.refname_type
-        values['refname'] = self.refname
-        values['short_refname'] = self.short_refname
-        values['msgid'] = self.msgid
-        values['thread_index'] = self.thread_index
-        values['recipients'] = self.recipients
-        values['oldrev'] = str(self.old)
-        values['oldrev_short'] = self.old.short
-        values['newrev'] = str(self.new)
-        values['newrev_short'] = self.new.short
-
-        if self.old:
-            values['oldrev_type'] = self.old.type
-        if self.new:
-            values['newrev_type'] = self.new.type
-
-        reply_to = self.environment.get_reply_to_refchange(self)
-        if reply_to:
-            values['reply_to'] = reply_to
-
-        return values
-
-    def send_single_combined_email(self, known_added_sha1s):
-        """Determine if a combined refchange/revision email should be sent
-
-        If there is only a single new (non-merge) commit added by a
-        change, it is useful to combine the ReferenceChange and
-        Revision emails into one.  In such a case, return the single
-        revision; otherwise, return None.
-
-        This method is overridden in BranchChange."""
-
-        return None
-
-    def generate_combined_email(self, push, revision, body_filter=None, extra_header_values={}):
-        """Generate an email describing this change AND specified revision.
-
-        Iterate over the lines (including the header lines) of an
-        email describing this change.  If body_filter is not None,
-        then use it to filter the lines that are intended for the
-        email body.
-
-        The extra_header_values field is received as a dict and not as
-        **kwargs, to allow passing other keyword arguments in the
-        future (e.g. passing extra values to generate_email_intro()
-
-        This method is overridden in BranchChange."""
-
-        raise NotImplementedError
-
-    def get_subject(self):
-        template = {
-            'create': REF_CREATED_SUBJECT_TEMPLATE,
-            'update': REF_UPDATED_SUBJECT_TEMPLATE,
-            'delete': REF_DELETED_SUBJECT_TEMPLATE,
-            }[self.change_type]
-        return self.expand(template)
-
-    def generate_email_header(self, **extra_values):
-        if 'subject' not in extra_values:
-            extra_values['subject'] = self.get_subject()
-
-        for line in self.expand_header_lines(
-                self.header_template, **extra_values
-                ):
-            yield line
-
-    def generate_email_intro(self, html_escape_val=False):
-        for line in self.expand_lines(self.intro_template,
-                                      html_escape_val=html_escape_val):
-            yield line
-
-    def generate_email_body(self, push):
-        """Call the appropriate body-generation routine.
-
-        Call one of generate_create_summary() /
-        generate_update_summary() / generate_delete_summary()."""
-
-        change_summary = {
-            'create': self.generate_create_summary,
-            'delete': self.generate_delete_summary,
-            'update': self.generate_update_summary,
-            }[self.change_type](push)
-        for line in change_summary:
-            yield line
-
-        for line in self.generate_revision_change_summary(push):
-            yield line
-
-    def generate_email_footer(self, html_escape_val):
-        return self.expand_lines(self.footer_template,
-                                 html_escape_val=html_escape_val)
-
-    def generate_revision_change_graph(self, push):
-        if self.showgraph:
-            args = ['--graph'] + self.graphopts
-            for newold in ('new', 'old'):
-                has_newold = False
-                spec = push.get_commits_spec(newold, self)
-                for line in git_log(spec, args=args, keepends=True):
-                    if not has_newold:
-                        has_newold = True
-                        yield '\n'
-                        yield 'Graph of %s commits:\n\n' % (
-                            {'new': 'new', 'old': 'discarded'}[newold],)
-                    yield '  ' + line
-                if has_newold:
-                    yield '\n'
-
-    def generate_revision_change_log(self, new_commits_list):
-        if self.showlog:
-            yield '\n'
-            yield 'Detailed log of new commits:\n\n'
-            for line in read_git_lines(
-                    ['log', '--no-walk'] +
-                    self.logopts +
-                    new_commits_list +
-                    ['--'],
-                    keepends=True,
-                    ):
-                yield line
-
-    def generate_new_revision_summary(self, tot, new_commits_list, push):
-        for line in self.expand_lines(NEW_REVISIONS_TEMPLATE, tot=tot):
-            yield line
-        for line in self.generate_revision_change_graph(push):
-            yield line
-        for line in self.generate_revision_change_log(new_commits_list):
-            yield line
-
-    def generate_revision_change_summary(self, push):
-        """Generate a summary of the revisions added/removed by this change."""
-
-        if self.new.commit_sha1 and not self.old.commit_sha1:
-            # A new reference was created.  List the new revisions
-            # brought by the new reference (i.e., those revisions that
-            # were not in the repository before this reference
-            # change).
-            sha1s = list(push.get_new_commits(self))
-            sha1s.reverse()
-            tot = len(sha1s)
-            new_revisions = [
-                Revision(self, GitObject(sha1), num=i + 1, tot=tot)
-                for (i, sha1) in enumerate(sha1s)
-                ]
-
-            if new_revisions:
-                yield self.expand('This %(refname_type)s includes the following new commits:\n')
-                yield '\n'
-                for r in new_revisions:
-                    (sha1, subject) = r.rev.get_summary()
-                    yield r.expand(
-                        BRIEF_SUMMARY_TEMPLATE, action='new', text=subject,
-                        )
-                yield '\n'
-                for line in self.generate_new_revision_summary(
-                        tot, [r.rev.sha1 for r in new_revisions], push):
-                    yield line
-            else:
-                for line in self.expand_lines(NO_NEW_REVISIONS_TEMPLATE):
-                    yield line
-
-        elif self.new.commit_sha1 and self.old.commit_sha1:
-            # A reference was changed to point at a different commit.
-            # List the revisions that were removed and/or added *from
-            # that reference* by this reference change, along with a
-            # diff between the trees for its old and new values.
-
-            # List of the revisions that were added to the branch by
-            # this update.  Note this list can include revisions that
-            # have already had notification emails; we want such
-            # revisions in the summary even though we will not send
-            # new notification emails for them.
-            adds = list(generate_summaries(
-                '--topo-order', '--reverse', '%s..%s'
-                % (self.old.commit_sha1, self.new.commit_sha1,)
-                ))
-
-            # List of the revisions that were removed from the branch
-            # by this update.  This will be empty except for
-            # non-fast-forward updates.
-            discards = list(generate_summaries(
-                '%s..%s' % (self.new.commit_sha1, self.old.commit_sha1,)
-                ))
-
-            if adds:
-                new_commits_list = push.get_new_commits(self)
-            else:
-                new_commits_list = []
-            new_commits = CommitSet(new_commits_list)
-
-            if discards:
-                discarded_commits = CommitSet(push.get_discarded_commits(self))
-            else:
-                discarded_commits = CommitSet([])
-
-            if discards and adds:
-                for (sha1, subject) in discards:
-                    if sha1 in discarded_commits:
-                        action = 'discard'
-                    else:
-                        action = 'omit'
-                    yield self.expand(
-                        BRIEF_SUMMARY_TEMPLATE, action=action,
-                        rev_short=sha1, text=subject,
-                        )
-                for (sha1, subject) in adds:
-                    if sha1 in new_commits:
-                        action = 'new'
-                    else:
-                        action = 'add'
-                    yield self.expand(
-                        BRIEF_SUMMARY_TEMPLATE, action=action,
-                        rev_short=sha1, text=subject,
-                        )
-                yield '\n'
-                for line in self.expand_lines(NON_FF_TEMPLATE):
-                    yield line
-
-            elif discards:
-                for (sha1, subject) in discards:
-                    if sha1 in discarded_commits:
-                        action = 'discard'
-                    else:
-                        action = 'omit'
-                    yield self.expand(
-                        BRIEF_SUMMARY_TEMPLATE, action=action,
-                        rev_short=sha1, text=subject,
-                        )
-                yield '\n'
-                for line in self.expand_lines(REWIND_ONLY_TEMPLATE):
-                    yield line
-
-            elif adds:
-                (sha1, subject) = self.old.get_summary()
-                yield self.expand(
-                    BRIEF_SUMMARY_TEMPLATE, action='from',
-                    rev_short=sha1, text=subject,
-                    )
-                for (sha1, subject) in adds:
-                    if sha1 in new_commits:
-                        action = 'new'
-                    else:
-                        action = 'add'
-                    yield self.expand(
-                        BRIEF_SUMMARY_TEMPLATE, action=action,
-                        rev_short=sha1, text=subject,
-                        )
-
-            yield '\n'
-
-            if new_commits:
-                for line in self.generate_new_revision_summary(
-                        len(new_commits), new_commits_list, push):
-                    yield line
-            else:
-                for line in self.expand_lines(NO_NEW_REVISIONS_TEMPLATE):
-                    yield line
-                for line in self.generate_revision_change_graph(push):
-                    yield line
-
-            # The diffstat is shown from the old revision to the new
-            # revision.  This is to show the truth of what happened in
-            # this change.  There's no point showing the stat from the
-            # base to the new revision because the base is effectively a
-            # random revision at this point - the user will be interested
-            # in what this revision changed - including the undoing of
-            # previous revisions in the case of non-fast-forward updates.
-            yield '\n'
-            yield 'Summary of changes:\n'
-            for line in read_git_lines(
-                    ['diff-tree'] +
-                    self.diffopts +
-                    ['%s..%s' % (self.old.commit_sha1, self.new.commit_sha1,)],
-                    keepends=True,
-                    ):
-                yield line
-
-        elif self.old.commit_sha1 and not self.new.commit_sha1:
-            # A reference was deleted.  List the revisions that were
-            # removed from the repository by this reference change.
-
-            sha1s = list(push.get_discarded_commits(self))
-            tot = len(sha1s)
-            discarded_revisions = [
-                Revision(self, GitObject(sha1), num=i + 1, tot=tot)
-                for (i, sha1) in enumerate(sha1s)
-                ]
-
-            if discarded_revisions:
-                for line in self.expand_lines(DISCARDED_REVISIONS_TEMPLATE):
-                    yield line
-                yield '\n'
-                for r in discarded_revisions:
-                    (sha1, subject) = r.rev.get_summary()
-                    yield r.expand(
-                        BRIEF_SUMMARY_TEMPLATE, action='discard', text=subject,
-                        )
-                for line in self.generate_revision_change_graph(push):
-                    yield line
-            else:
-                for line in self.expand_lines(NO_DISCARDED_REVISIONS_TEMPLATE):
-                    yield line
-
-        elif not self.old.commit_sha1 and not self.new.commit_sha1:
-            for line in self.expand_lines(NON_COMMIT_UPDATE_TEMPLATE):
-                yield line
-
-    def generate_create_summary(self, push):
-        """Called for the creation of a reference."""
-
-        # This is a new reference and so oldrev is not valid
-        (sha1, subject) = self.new.get_summary()
-        yield self.expand(
-            BRIEF_SUMMARY_TEMPLATE, action='at',
-            rev_short=sha1, text=subject,
-            )
-        yield '\n'
-
-    def generate_update_summary(self, push):
-        """Called for the change of a pre-existing branch."""
-
-        return iter([])
-
-    def generate_delete_summary(self, push):
-        """Called for the deletion of any type of reference."""
-
-        (sha1, subject) = self.old.get_summary()
-        yield self.expand(
-            BRIEF_SUMMARY_TEMPLATE, action='was',
-            rev_short=sha1, text=subject,
-            )
-        yield '\n'
-
-    def get_specific_fromaddr(self):
-        return self.environment.from_refchange
-
-
-class BranchChange(ReferenceChange):
-    refname_type = 'branch'
-
-    def __init__(self, environment, refname, short_refname, old, new, rev):
-        ReferenceChange.__init__(
-            self, environment,
-            refname=refname, short_refname=short_refname,
-            old=old, new=new, rev=rev,
-            )
-        self.recipients = environment.get_refchange_recipients(self)
-        self._single_revision = None
-
-    def send_single_combined_email(self, known_added_sha1s):
-        if not self.environment.combine_when_single_commit:
-            return None
-
-        # In the sadly-all-too-frequent usecase of people pushing only
-        # one of their commits at a time to a repository, users feel
-        # the reference change summary emails are noise rather than
-        # important signal.  This is because, in this particular
-        # usecase, there is a reference change summary email for each
-        # new commit, and all these summaries do is point out that
-        # there is one new commit (which can readily be inferred by
-        # the existence of the individual revision email that is also
-        # sent).  In such cases, our users prefer there to be a combined
-        # reference change summary/new revision email.
-        #
-        # So, if the change is an update and it doesn't discard any
-        # commits, and it adds exactly one non-merge commit (gerrit
-        # forces a workflow where every commit is individually merged
-        # and the git-multimail hook fired off for just this one
-        # change), then we send a combined refchange/revision email.
-        try:
-            # If this change is a reference update that doesn't discard
-            # any commits...
-            if self.change_type != 'update':
-                return None
-
-            if read_git_lines(
-                    ['merge-base', self.old.sha1, self.new.sha1]
-                    ) != [self.old.sha1]:
-                return None
-
-            # Check if this update introduced exactly one non-merge
-            # commit:
-
-            def split_line(line):
-                """Split line into (sha1, [parent,...])."""
-
-                words = line.split()
-                return (words[0], words[1:])
-
-            # Get the new commits introduced by the push as a list of
-            # (sha1, [parent,...])
-            new_commits = [
-                split_line(line)
-                for line in read_git_lines(
-                    [
-                        'log', '-3', '--format=%H %P',
-                        '%s..%s' % (self.old.sha1, self.new.sha1),
-                        ]
-                    )
-                ]
-
-            if not new_commits:
-                return None
-
-            # If the newest commit is a merge, save it for a later check
-            # but otherwise ignore it
-            merge = None
-            tot = len(new_commits)
-            if len(new_commits[0][1]) > 1:
-                merge = new_commits[0][0]
-                del new_commits[0]
-
-            # Our primary check: we can't combine if more than one commit
-            # is introduced.  We also currently only combine if the new
-            # commit is a non-merge commit, though it may make sense to
-            # combine if it is a merge as well.
-            if not (
-                    len(new_commits) == 1 and
-                    len(new_commits[0][1]) == 1 and
-                    new_commits[0][0] in known_added_sha1s
-                    ):
-                return None
-
-            # We do not want to combine revision and refchange emails if
-            # those go to separate locations.
-            rev = Revision(self, GitObject(new_commits[0][0]), 1, tot)
-            if rev.recipients != self.recipients:
-                return None
-
-            # We ignored the newest commit if it was just a merge of the one
-            # commit being introduced.  But we don't want to ignore that
-            # merge commit it it involved conflict resolutions.  Check that.
-            if merge and merge != read_git_output(['diff-tree', '--cc', merge]):
-                return None
-
-            # We can combine the refchange and one new revision emails
-            # into one.  Return the Revision that a combined email should
-            # be sent about.
-            return rev
-        except CommandError:
-            # Cannot determine number of commits in old..new or new..old;
-            # don't combine reference/revision emails:
-            return None
-
-    def generate_combined_email(self, push, revision, body_filter=None, extra_header_values={}):
-        values = revision.get_values()
-        if extra_header_values:
-            values.update(extra_header_values)
-        if 'subject' not in extra_header_values:
-            values['subject'] = self.expand(COMBINED_REFCHANGE_REVISION_SUBJECT_TEMPLATE, **values)
-
-        self._single_revision = revision
-        self._contains_diff()
-        self.header_template = COMBINED_HEADER_TEMPLATE
-        self.intro_template = COMBINED_INTRO_TEMPLATE
-        self.footer_template = COMBINED_FOOTER_TEMPLATE
-
-        def revision_gen_link(base_url):
-            # revision is used only to generate the body, and
-            # _content_type is set while generating headers. Get it
-            # from the BranchChange object.
-            revision._content_type = self._content_type
-            return revision.generate_browse_link(base_url)
-        self.generate_browse_link = revision_gen_link
-        for line in self.generate_email(push, body_filter, values):
-            yield line
-
-    def generate_email_body(self, push):
-        '''Call the appropriate body generation routine.
-
-        If this is a combined refchange/revision email, the special logic
-        for handling this combined email comes from this function.  For
-        other cases, we just use the normal handling.'''
-
-        # If self._single_revision isn't set; don't override
-        if not self._single_revision:
-            for line in super(BranchChange, self).generate_email_body(push):
-                yield line
-            return
-
-        # This is a combined refchange/revision email; we first provide
-        # some info from the refchange portion, and then call the revision
-        # generate_email_body function to handle the revision portion.
-        adds = list(generate_summaries(
-            '--topo-order', '--reverse', '%s..%s'
-            % (self.old.commit_sha1, self.new.commit_sha1,)
-            ))
-
-        yield self.expand("The following commit(s) were added to %(refname)s by this push:\n")
-        for (sha1, subject) in adds:
-            yield self.expand(
-                BRIEF_SUMMARY_TEMPLATE, action='new',
-                rev_short=sha1, text=subject,
-                )
-
-        yield self._single_revision.rev.short + " is described below\n"
-        yield '\n'
-
-        for line in self._single_revision.generate_email_body(push):
-            yield line
-
-
-class AnnotatedTagChange(ReferenceChange):
-    refname_type = 'annotated tag'
-
-    def __init__(self, environment, refname, short_refname, old, new, rev):
-        ReferenceChange.__init__(
-            self, environment,
-            refname=refname, short_refname=short_refname,
-            old=old, new=new, rev=rev,
-            )
-        self.recipients = environment.get_announce_recipients(self)
-        self.show_shortlog = environment.announce_show_shortlog
-
-    ANNOTATED_TAG_FORMAT = (
-        '%(*objectname)\n'
-        '%(*objecttype)\n'
-        '%(taggername)\n'
-        '%(taggerdate)'
-        )
-
-    def describe_tag(self, push):
-        """Describe the new value of an annotated tag."""
-
-        # Use git for-each-ref to pull out the individual fields from
-        # the tag
-        [tagobject, tagtype, tagger, tagged] = read_git_lines(
-            ['for-each-ref', '--format=%s' % (self.ANNOTATED_TAG_FORMAT,), self.refname],
-            )
-
-        yield self.expand(
-            BRIEF_SUMMARY_TEMPLATE, action='tagging',
-            rev_short=tagobject, text='(%s)' % (tagtype,),
-            )
-        if tagtype == 'commit':
-            # If the tagged object is a commit, then we assume this is a
-            # release, and so we calculate which tag this tag is
-            # replacing
-            try:
-                prevtag = read_git_output(['describe', '--abbrev=0', '%s^' % (self.new,)])
-            except CommandError:
-                prevtag = None
-            if prevtag:
-                yield ' replaces %s\n' % (prevtag,)
-        else:
-            prevtag = None
-            yield '  length %s bytes\n' % (read_git_output(['cat-file', '-s', tagobject]),)
-
-        yield '      by %s\n' % (tagger,)
-        yield '      on %s\n' % (tagged,)
-        yield '\n'
-
-        # Show the content of the tag message; this might contain a
-        # change log or release notes so is worth displaying.
-        yield LOGBEGIN
-        contents = list(read_git_lines(['cat-file', 'tag', self.new.sha1], keepends=True))
-        contents = contents[contents.index('\n') + 1:]
-        if contents and contents[-1][-1:] != '\n':
-            contents.append('\n')
-        for line in contents:
-            yield line
-
-        if self.show_shortlog and tagtype == 'commit':
-            # Only commit tags make sense to have rev-list operations
-            # performed on them
-            yield '\n'
-            if prevtag:
-                # Show changes since the previous release
-                revlist = read_git_output(
-                    ['rev-list', '--pretty=short', '%s..%s' % (prevtag, self.new,)],
-                    keepends=True,
-                    )
-            else:
-                # No previous tag, show all the changes since time
-                # began
-                revlist = read_git_output(
-                    ['rev-list', '--pretty=short', '%s' % (self.new,)],
-                    keepends=True,
-                    )
-            for line in read_git_lines(['shortlog'], input=revlist, keepends=True):
-                yield line
-
-        yield LOGEND
-        yield '\n'
-
-    def generate_create_summary(self, push):
-        """Called for the creation of an annotated tag."""
-
-        for line in self.expand_lines(TAG_CREATED_TEMPLATE):
-            yield line
-
-        for line in self.describe_tag(push):
-            yield line
-
-    def generate_update_summary(self, push):
-        """Called for the update of an annotated tag.
-
-        This is probably a rare event and may not even be allowed."""
-
-        for line in self.expand_lines(TAG_UPDATED_TEMPLATE):
-            yield line
-
-        for line in self.describe_tag(push):
-            yield line
-
-    def generate_delete_summary(self, push):
-        """Called when a non-annotated reference is updated."""
-
-        for line in self.expand_lines(TAG_DELETED_TEMPLATE):
-            yield line
-
-        yield self.expand('   tag was  %(oldrev_short)s\n')
-        yield '\n'
-
-
-class NonAnnotatedTagChange(ReferenceChange):
-    refname_type = 'tag'
-
-    def __init__(self, environment, refname, short_refname, old, new, rev):
-        ReferenceChange.__init__(
-            self, environment,
-            refname=refname, short_refname=short_refname,
-            old=old, new=new, rev=rev,
-            )
-        self.recipients = environment.get_refchange_recipients(self)
-
-    def generate_create_summary(self, push):
-        """Called for the creation of an annotated tag."""
-
-        for line in self.expand_lines(TAG_CREATED_TEMPLATE):
-            yield line
-
-    def generate_update_summary(self, push):
-        """Called when a non-annotated reference is updated."""
-
-        for line in self.expand_lines(TAG_UPDATED_TEMPLATE):
-            yield line
-
-    def generate_delete_summary(self, push):
-        """Called when a non-annotated reference is updated."""
-
-        for line in self.expand_lines(TAG_DELETED_TEMPLATE):
-            yield line
-
-        for line in ReferenceChange.generate_delete_summary(self, push):
-            yield line
-
-
-class OtherReferenceChange(ReferenceChange):
-    refname_type = 'reference'
-
-    def __init__(self, environment, refname, short_refname, old, new, rev):
-        # We use the full refname as short_refname, because otherwise
-        # the full name of the reference would not be obvious from the
-        # text of the email.
-        ReferenceChange.__init__(
-            self, environment,
-            refname=refname, short_refname=refname,
-            old=old, new=new, rev=rev,
-            )
-        self.recipients = environment.get_refchange_recipients(self)
-
-
-class Mailer(object):
-    """An object that can send emails."""
-
-    def __init__(self, environment):
-        self.environment = environment
-
-    def close(self):
-        pass
-
-    def send(self, lines, to_addrs):
-        """Send an email consisting of lines.
-
-        lines must be an iterable over the lines constituting the
-        header and body of the email.  to_addrs is a list of recipient
-        addresses (can be needed even if lines already contains a
-        "To:" field).  It can be either a string (comma-separated list
-        of email addresses) or a Python list of individual email
-        addresses.
-
-        """
-
-        raise NotImplementedError()
-
-
-class SendMailer(Mailer):
-    """Send emails using 'sendmail -oi -t'."""
-
-    SENDMAIL_CANDIDATES = [
-        '/usr/sbin/sendmail',
-        '/usr/lib/sendmail',
-        ]
-
-    @staticmethod
-    def find_sendmail():
-        for path in SendMailer.SENDMAIL_CANDIDATES:
-            if os.access(path, os.X_OK):
-                return path
-        else:
-            raise ConfigurationException(
-                'No sendmail executable found.  '
-                'Try setting multimailhook.sendmailCommand.'
-                )
-
-    def __init__(self, environment, command=None, envelopesender=None):
-        """Construct a SendMailer instance.
-
-        command should be the command and arguments used to invoke
-        sendmail, as a list of strings.  If an envelopesender is
-        provided, it will also be passed to the command, via '-f
-        envelopesender'."""
-        super(SendMailer, self).__init__(environment)
-        if command:
-            self.command = command[:]
-        else:
-            self.command = [self.find_sendmail(), '-oi', '-t']
-
-        if envelopesender:
-            self.command.extend(['-f', envelopesender])
-
-    def send(self, lines, to_addrs):
-        try:
-            p = subprocess.Popen(self.command, stdin=subprocess.PIPE)
-        except OSError:
-            self.environment.get_logger().error(
-                '*** Cannot execute command: %s\n' % ' '.join(self.command) +
-                '*** %s\n' % sys.exc_info()[1] +
-                '*** Try setting multimailhook.mailer to "smtp"\n' +
-                '*** to send emails without using the sendmail command.\n'
-                )
-            sys.exit(1)
-        try:
-            lines = (str_to_bytes(line) for line in lines)
-            p.stdin.writelines(lines)
-        except Exception:
-            self.environment.get_logger().error(
-                '*** Error while generating commit email\n'
-                '***  - mail sending aborted.\n'
-                )
-            if hasattr(p, 'terminate'):
-                # subprocess.terminate() is not available in Python 2.4
-                p.terminate()
-            else:
-                import signal
-                os.kill(p.pid, signal.SIGTERM)
-            raise
-        else:
-            p.stdin.close()
-            retcode = p.wait()
-            if retcode:
-                raise CommandError(self.command, retcode)
-
-
-class SMTPMailer(Mailer):
-    """Send emails using Python's smtplib."""
-
-    def __init__(self, environment,
-                 envelopesender, smtpserver,
-                 smtpservertimeout=10.0, smtpserverdebuglevel=0,
-                 smtpencryption='none',
-                 smtpuser='', smtppass='',
-                 smtpcacerts=''
-                 ):
-        super(SMTPMailer, self).__init__(environment)
-        if not envelopesender:
-            self.environment.get_logger().error(
-                'fatal: git_multimail: cannot use SMTPMailer without a sender address.\n'
-                'please set either multimailhook.envelopeSender or user.email\n'
-                )
-            sys.exit(1)
-        if smtpencryption == 'ssl' and not (smtpuser and smtppass):
-            raise ConfigurationException(
-                'Cannot use SMTPMailer with security option ssl '
-                'without options username and password.'
-                )
-        self.envelopesender = envelopesender
-        self.smtpserver = smtpserver
-        self.smtpservertimeout = smtpservertimeout
-        self.smtpserverdebuglevel = smtpserverdebuglevel
-        self.security = smtpencryption
-        self.username = smtpuser
-        self.password = smtppass
-        self.smtpcacerts = smtpcacerts
-        self.loggedin = False
-        try:
-            def call(klass, server, timeout):
-                try:
-                    return klass(server, timeout=timeout)
-                except TypeError:
-                    # Old Python versions do not have timeout= argument.
-                    return klass(server)
-            if self.security == 'none':
-                self.smtp = call(smtplib.SMTP, self.smtpserver, timeout=self.smtpservertimeout)
-            elif self.security == 'ssl':
-                if self.smtpcacerts:
-                    raise smtplib.SMTPException(
-                        "Checking certificate is not supported for ssl, prefer starttls"
-                        )
-                self.smtp = call(smtplib.SMTP_SSL, self.smtpserver, timeout=self.smtpservertimeout)
-            elif self.security == 'tls':
-                if 'ssl' not in sys.modules:
-                    self.environment.get_logger().error(
-                        '*** Your Python version does not have the ssl library installed\n'
-                        '*** smtpEncryption=tls is not available.\n'
-                        '*** Either upgrade Python to 2.6 or later\n'
-                        '    or use git_multimail.py version 1.2.\n')
-                if ':' not in self.smtpserver:
-                    self.smtpserver += ':587'  # default port for TLS
-                self.smtp = call(smtplib.SMTP, self.smtpserver, timeout=self.smtpservertimeout)
-                # start: ehlo + starttls
-                # equivalent to
-                #     self.smtp.ehlo()
-                #     self.smtp.starttls()
-                # with access to the ssl layer
-                self.smtp.ehlo()
-                if not self.smtp.has_extn("starttls"):
-                    raise smtplib.SMTPException("STARTTLS extension not supported by server")
-                resp, reply = self.smtp.docmd("STARTTLS")
-                if resp != 220:
-                    raise smtplib.SMTPException("Wrong answer to the STARTTLS command")
-                if self.smtpcacerts:
-                    self.smtp.sock = ssl.wrap_socket(
-                        self.smtp.sock,
-                        ca_certs=self.smtpcacerts,
-                        cert_reqs=ssl.CERT_REQUIRED
-                        )
-                else:
-                    self.smtp.sock = ssl.wrap_socket(
-                        self.smtp.sock,
-                        cert_reqs=ssl.CERT_NONE
-                        )
-                    self.environment.get_logger().error(
-                        '*** Warning, the server certificate is not verified (smtp) ***\n'
-                        '***          set the option smtpCACerts                   ***\n'
-                        )
-                if not hasattr(self.smtp.sock, "read"):
-                    # using httplib.FakeSocket with Python 2.5.x or earlier
-                    self.smtp.sock.read = self.smtp.sock.recv
-                self.smtp.file = smtplib.SSLFakeFile(self.smtp.sock)
-                self.smtp.helo_resp = None
-                self.smtp.ehlo_resp = None
-                self.smtp.esmtp_features = {}
-                self.smtp.does_esmtp = 0
-                # end:   ehlo + starttls
-                self.smtp.ehlo()
-            else:
-                sys.stdout.write('*** Error: Control reached an invalid option. ***')
-                sys.exit(1)
-            if self.smtpserverdebuglevel > 0:
-                sys.stdout.write(
-                    "*** Setting debug on for SMTP server connection (%s) ***\n"
-                    % self.smtpserverdebuglevel)
-                self.smtp.set_debuglevel(self.smtpserverdebuglevel)
-        except Exception:
-            self.environment.get_logger().error(
-                '*** Error establishing SMTP connection to %s ***\n'
-                '*** %s\n'
-                % (self.smtpserver, sys.exc_info()[1]))
-            sys.exit(1)
-
-    def close(self):
-        if hasattr(self, 'smtp'):
-            self.smtp.quit()
-            del self.smtp
-
-    def __del__(self):
-        self.close()
-
-    def send(self, lines, to_addrs):
-        try:
-            if self.username or self.password:
-                if not self.loggedin:
-                    self.smtp.login(self.username, self.password)
-                    self.loggedin = True
-            msg = ''.join(lines)
-            # turn comma-separated list into Python list if needed.
-            if is_string(to_addrs):
-                to_addrs = [email for (name, email) in getaddresses([to_addrs])]
-            self.smtp.sendmail(self.envelopesender, to_addrs, msg)
-        except socket.timeout:
-            self.environment.get_logger().error(
-                '*** Error sending email ***\n'
-                '*** SMTP server timed out (timeout is %s)\n'
-                % self.smtpservertimeout)
-        except smtplib.SMTPResponseException:
-            err = sys.exc_info()[1]
-            self.environment.get_logger().error(
-                '*** Error sending email ***\n'
-                '*** Error %d: %s\n'
-                % (err.smtp_code, bytes_to_str(err.smtp_error)))
-            try:
-                smtp = self.smtp
-                # delete the field before quit() so that in case of
-                # error, self.smtp is deleted anyway.
-                del self.smtp
-                smtp.quit()
-            except:
-                self.environment.get_logger().error(
-                    '*** Error closing the SMTP connection ***\n'
-                    '*** Exiting anyway ... ***\n'
-                    '*** %s\n' % sys.exc_info()[1])
-            sys.exit(1)
-
-
-class OutputMailer(Mailer):
-    """Write emails to an output stream, bracketed by lines of '=' characters.
-
-    This is intended for debugging purposes."""
-
-    SEPARATOR = '=' * 75 + '\n'
-
-    def __init__(self, f, environment=None):
-        super(OutputMailer, self).__init__(environment=environment)
-        self.f = f
-
-    def send(self, lines, to_addrs):
-        write_str(self.f, self.SEPARATOR)
-        for line in lines:
-            write_str(self.f, line)
-        write_str(self.f, self.SEPARATOR)
-
-
-def get_git_dir():
-    """Determine GIT_DIR.
-
-    Determine GIT_DIR either from the GIT_DIR environment variable or
-    from the working directory, using Git's usual rules."""
-
-    try:
-        return read_git_output(['rev-parse', '--git-dir'])
-    except CommandError:
-        sys.stderr.write('fatal: git_multimail: not in a git directory\n')
-        sys.exit(1)
-
-
-class Environment(object):
-    """Describes the environment in which the push is occurring.
-
-    An Environment object encapsulates information about the local
-    environment.  For example, it knows how to determine:
-
-    * the name of the repository to which the push occurred
-
-    * what user did the push
-
-    * what users want to be informed about various types of changes.
-
-    An Environment object is expected to have the following methods:
-
-        get_repo_shortname()
-
-            Return a short name for the repository, for display
-            purposes.
-
-        get_repo_path()
-
-            Return the absolute path to the Git repository.
-
-        get_emailprefix()
-
-            Return a string that will be prefixed to every email's
-            subject.
-
-        get_pusher()
-
-            Return the username of the person who pushed the changes.
-            This value is used in the email body to indicate who
-            pushed the change.
-
-        get_pusher_email() (may return None)
-
-            Return the email address of the person who pushed the
-            changes.  The value should be a single RFC 2822 email
-            address as a string; e.g., "Joe User "
-            if available, otherwise "user@example.com".  If set, the
-            value is used as the Reply-To address for refchange
-            emails.  If it is impossible to determine the pusher's
-            email, this attribute should be set to None (in which case
-            no Reply-To header will be output).
-
-        get_sender()
-
-            Return the address to be used as the 'From' email address
-            in the email envelope.
-
-        get_fromaddr(change=None)
-
-            Return the 'From' email address used in the email 'From:'
-            headers.  If the change is known when this function is
-            called, it is passed in as the 'change' parameter.  (May
-            be a full RFC 2822 email address like 'Joe User
-            '.)
-
-        get_administrator()
-
-            Return the name and/or email of the repository
-            administrator.  This value is used in the footer as the
-            person to whom requests to be removed from the
-            notification list should be sent.  Ideally, it should
-            include a valid email address.
-
-        get_reply_to_refchange()
-        get_reply_to_commit()
-
-            Return the address to use in the email "Reply-To" header,
-            as a string.  These can be an RFC 2822 email address, or
-            None to omit the "Reply-To" header.
-            get_reply_to_refchange() is used for refchange emails;
-            get_reply_to_commit() is used for individual commit
-            emails.
-
-        get_ref_filter_regex()
-
-            Return a tuple -- a compiled regex, and a boolean indicating
-            whether the regex picks refs to include (if False, the regex
-            matches on refs to exclude).
-
-        get_default_ref_ignore_regex()
-
-            Return a regex that should be ignored for both what emails
-            to send and when computing what commits are considered new
-            to the repository.  Default is "^refs/notes/".
-
-        get_max_subject_length()
-
-            Return an int giving the maximal length for the subject
-            (git log --oneline).
-
-    They should also define the following attributes:
-
-        announce_show_shortlog (bool)
-
-            True iff announce emails should include a shortlog.
-
-        commit_email_format (string)
-
-            If "html", generate commit emails in HTML instead of plain text
-            used by default.
-
-        html_in_intro (bool)
-        html_in_footer (bool)
-
-            When generating HTML emails, the introduction (respectively,
-            the footer) will be HTML-escaped iff html_in_intro (respectively,
-            the footer) is true. When false, only the values used to expand
-            the template are escaped.
-
-        refchange_showgraph (bool)
-
-            True iff refchanges emails should include a detailed graph.
-
-        refchange_showlog (bool)
-
-            True iff refchanges emails should include a detailed log.
-
-        diffopts (list of strings)
-
-            The options that should be passed to 'git diff' for the
-            summary email.  The value should be a list of strings
-            representing words to be passed to the command.
-
-        graphopts (list of strings)
-
-            Analogous to diffopts, but contains options passed to
-            'git log --graph' when generating the detailed graph for
-            a set of commits (see refchange_showgraph)
-
-        logopts (list of strings)
-
-            Analogous to diffopts, but contains options passed to
-            'git log' when generating the detailed log for a set of
-            commits (see refchange_showlog)
-
-        commitlogopts (list of strings)
-
-            The options that should be passed to 'git log' for each
-            commit mail.  The value should be a list of strings
-            representing words to be passed to the command.
-
-        date_substitute (string)
-
-            String to be used in substitution for 'Date:' at start of
-            line in the output of 'git log'.
-
-        quiet (bool)
-            On success do not write to stderr
-
-        stdout (bool)
-            Write email to stdout rather than emailing. Useful for debugging
-
-        combine_when_single_commit (bool)
-
-            True if a combined email should be produced when a single
-            new commit is pushed to a branch, False otherwise.
-
-        from_refchange, from_commit (strings)
-
-            Addresses to use for the From: field for refchange emails
-            and commit emails respectively.  Set from
-            multimailhook.fromRefchange and multimailhook.fromCommit
-            by ConfigEnvironmentMixin.
-
-        log_file, error_log_file, debug_log_file (string)
-
-            Name of a file to which logs should be sent.
-
-        verbose (int)
-
-            How verbose the system should be.
-            - 0 (default): show info, errors, ...
-            - 1 : show basic debug info
-    """
-
-    REPO_NAME_RE = re.compile(r'^(?P.+?)(?:\.git)$')
-
-    def __init__(self, osenv=None):
-        self.osenv = osenv or os.environ
-        self.announce_show_shortlog = False
-        self.commit_email_format = "text"
-        self.html_in_intro = False
-        self.html_in_footer = False
-        self.commitBrowseURL = None
-        self.maxcommitemails = 500
-        self.excludemergerevisions = False
-        self.diffopts = ['--stat', '--summary', '--find-copies-harder']
-        self.graphopts = ['--oneline', '--decorate']
-        self.logopts = []
-        self.refchange_showgraph = False
-        self.refchange_showlog = False
-        self.commitlogopts = ['-C', '--stat', '-p', '--cc']
-        self.date_substitute = 'AuthorDate: '
-        self.quiet = False
-        self.stdout = False
-        self.combine_when_single_commit = True
-        self.logger = None
-
-        self.COMPUTED_KEYS = [
-            'administrator',
-            'charset',
-            'emailprefix',
-            'pusher',
-            'pusher_email',
-            'repo_path',
-            'repo_shortname',
-            'sender',
-            ]
-
-        self._values = None
-
-    def get_logger(self):
-        """Get (possibly creates) the logger associated to this environment."""
-        if self.logger is None:
-            self.logger = Logger(self)
-        return self.logger
-
-    def get_repo_shortname(self):
-        """Use the last part of the repo path, with ".git" stripped off if present."""
-
-        basename = os.path.basename(os.path.abspath(self.get_repo_path()))
-        m = self.REPO_NAME_RE.match(basename)
-        if m:
-            return m.group('name')
-        else:
-            return basename
-
-    def get_pusher(self):
-        raise NotImplementedError()
-
-    def get_pusher_email(self):
-        return None
-
-    def get_fromaddr(self, change=None):
-        config = Config('user')
-        fromname = config.get('name', default='')
-        fromemail = config.get('email', default='')
-        if fromemail:
-            return formataddr([fromname, fromemail])
-        return self.get_sender()
-
-    def get_administrator(self):
-        return 'the administrator of this repository'
-
-    def get_emailprefix(self):
-        return ''
-
-    def get_repo_path(self):
-        if read_git_output(['rev-parse', '--is-bare-repository']) == 'true':
-            path = get_git_dir()
-        else:
-            path = read_git_output(['rev-parse', '--show-toplevel'])
-        return os.path.abspath(path)
-
-    def get_charset(self):
-        return CHARSET
-
-    def get_values(self):
-        """Return a dictionary {keyword: expansion} for this Environment.
-
-        This method is called by Change._compute_values().  The keys
-        in the returned dictionary are available to be used in any of
-        the templates.  The dictionary is created by calling
-        self.get_NAME() for each of the attributes named in
-        COMPUTED_KEYS and recording those that do not return None.
-        The return value is always a new dictionary."""
-
-        if self._values is None:
-            values = {'': ''}  # %()s expands to the empty string.
-
-            for key in self.COMPUTED_KEYS:
-                value = getattr(self, 'get_%s' % (key,))()
-                if value is not None:
-                    values[key] = value
-
-            self._values = values
-
-        return self._values.copy()
-
-    def get_refchange_recipients(self, refchange):
-        """Return the recipients for notifications about refchange.
-
-        Return the list of email addresses to which notifications
-        about the specified ReferenceChange should be sent."""
-
-        raise NotImplementedError()
-
-    def get_announce_recipients(self, annotated_tag_change):
-        """Return the recipients for notifications about annotated_tag_change.
-
-        Return the list of email addresses to which notifications
-        about the specified AnnotatedTagChange should be sent."""
-
-        raise NotImplementedError()
-
-    def get_reply_to_refchange(self, refchange):
-        return self.get_pusher_email()
-
-    def get_revision_recipients(self, revision):
-        """Return the recipients for messages about revision.
-
-        Return the list of email addresses to which notifications
-        about the specified Revision should be sent.  This method
-        could be overridden, for example, to take into account the
-        contents of the revision when deciding whom to notify about
-        it.  For example, there could be a scheme for users to express
-        interest in particular files or subdirectories, and only
-        receive notification emails for revisions that affecting those
-        files."""
-
-        raise NotImplementedError()
-
-    def get_reply_to_commit(self, revision):
-        return revision.author
-
-    def get_default_ref_ignore_regex(self):
-        # The commit messages of git notes are essentially meaningless
-        # and "filenames" in git notes commits are an implementational
-        # detail that might surprise users at first.  As such, we
-        # would need a completely different method for handling emails
-        # of git notes in order for them to be of benefit for users,
-        # which we simply do not have right now.
-        return "^refs/notes/"
-
-    def get_max_subject_length(self):
-        """Return the maximal subject line (git log --oneline) length.
-        Longer subject lines will be truncated."""
-        raise NotImplementedError()
-
-    def filter_body(self, lines):
-        """Filter the lines intended for an email body.
-
-        lines is an iterable over the lines that would go into the
-        email body.  Filter it (e.g., limit the number of lines, the
-        line length, character set, etc.), returning another iterable.
-        See FilterLinesEnvironmentMixin and MaxlinesEnvironmentMixin
-        for classes implementing this functionality."""
-
-        return lines
-
-    def log_msg(self, msg):
-        """Write the string msg on a log file or on stderr.
-
-        Sends the text to stderr by default, override to change the behavior."""
-        self.get_logger().info(msg)
-
-    def log_warning(self, msg):
-        """Write the string msg on a log file or on stderr.
-
-        Sends the text to stderr by default, override to change the behavior."""
-        self.get_logger().warning(msg)
-
-    def log_error(self, msg):
-        """Write the string msg on a log file or on stderr.
-
-        Sends the text to stderr by default, override to change the behavior."""
-        self.get_logger().error(msg)
-
-    def check(self):
-        pass
-
-
-class ConfigEnvironmentMixin(Environment):
-    """A mixin that sets self.config to its constructor's config argument.
-
-    This class's constructor consumes the "config" argument.
-
-    Mixins that need to inspect the config should inherit from this
-    class (1) to make sure that "config" is still in the constructor
-    arguments with its own constructor runs and/or (2) to be sure that
-    self.config is set after construction."""
-
-    def __init__(self, config, **kw):
-        super(ConfigEnvironmentMixin, self).__init__(**kw)
-        self.config = config
-
-
-class ConfigOptionsEnvironmentMixin(ConfigEnvironmentMixin):
-    """An Environment that reads most of its information from "git config"."""
-
-    @staticmethod
-    def forbid_field_values(name, value, forbidden):
-        for forbidden_val in forbidden:
-            if value is not None and value.lower() == forbidden:
-                raise ConfigurationException(
-                    '"%s" is not an allowed setting for %s' % (value, name)
-                    )
-
-    def __init__(self, config, **kw):
-        super(ConfigOptionsEnvironmentMixin, self).__init__(
-            config=config, **kw
-            )
-
-        for var, cfg in (
-                ('announce_show_shortlog', 'announceshortlog'),
-                ('refchange_showgraph', 'refchangeShowGraph'),
-                ('refchange_showlog', 'refchangeshowlog'),
-                ('quiet', 'quiet'),
-                ('stdout', 'stdout'),
-                ):
-            val = config.get_bool(cfg)
-            if val is not None:
-                setattr(self, var, val)
-
-        commit_email_format = config.get('commitEmailFormat')
-        if commit_email_format is not None:
-            if commit_email_format != "html" and commit_email_format != "text":
-                self.log_warning(
-                    '*** Unknown value for multimailhook.commitEmailFormat: %s\n' %
-                    commit_email_format +
-                    '*** Expected either "text" or "html".  Ignoring.\n'
-                    )
-            else:
-                self.commit_email_format = commit_email_format
-
-        html_in_intro = config.get_bool('htmlInIntro')
-        if html_in_intro is not None:
-            self.html_in_intro = html_in_intro
-
-        html_in_footer = config.get_bool('htmlInFooter')
-        if html_in_footer is not None:
-            self.html_in_footer = html_in_footer
-
-        self.commitBrowseURL = config.get('commitBrowseURL')
-
-        self.excludemergerevisions = config.get('excludeMergeRevisions')
-
-        maxcommitemails = config.get('maxcommitemails')
-        if maxcommitemails is not None:
-            try:
-                self.maxcommitemails = int(maxcommitemails)
-            except ValueError:
-                self.log_warning(
-                    '*** Malformed value for multimailhook.maxCommitEmails: %s\n'
-                    % maxcommitemails +
-                    '*** Expected a number.  Ignoring.\n'
-                    )
-
-        diffopts = config.get('diffopts')
-        if diffopts is not None:
-            self.diffopts = shlex.split(diffopts)
-
-        graphopts = config.get('graphOpts')
-        if graphopts is not None:
-            self.graphopts = shlex.split(graphopts)
-
-        logopts = config.get('logopts')
-        if logopts is not None:
-            self.logopts = shlex.split(logopts)
-
-        commitlogopts = config.get('commitlogopts')
-        if commitlogopts is not None:
-            self.commitlogopts = shlex.split(commitlogopts)
-
-        date_substitute = config.get('dateSubstitute')
-        if date_substitute == 'none':
-            self.date_substitute = None
-        elif date_substitute is not None:
-            self.date_substitute = date_substitute
-
-        reply_to = config.get('replyTo')
-        self.__reply_to_refchange = config.get('replyToRefchange', default=reply_to)
-        self.forbid_field_values('replyToRefchange',
-                                 self.__reply_to_refchange,
-                                 ['author'])
-        self.__reply_to_commit = config.get('replyToCommit', default=reply_to)
-
-        self.from_refchange = config.get('fromRefchange')
-        self.forbid_field_values('fromRefchange',
-                                 self.from_refchange,
-                                 ['author', 'none'])
-        self.from_commit = config.get('fromCommit')
-        self.forbid_field_values('fromCommit',
-                                 self.from_commit,
-                                 ['none'])
-
-        combine = config.get_bool('combineWhenSingleCommit')
-        if combine is not None:
-            self.combine_when_single_commit = combine
-
-        self.log_file = config.get('logFile', default=None)
-        self.error_log_file = config.get('errorLogFile', default=None)
-        self.debug_log_file = config.get('debugLogFile', default=None)
-        if config.get_bool('Verbose', default=False):
-            self.verbose = 1
-        else:
-            self.verbose = 0
-
-    def get_administrator(self):
-        return (
-            self.config.get('administrator') or
-            self.get_sender() or
-            super(ConfigOptionsEnvironmentMixin, self).get_administrator()
-            )
-
-    def get_repo_shortname(self):
-        return (
-            self.config.get('reponame') or
-            super(ConfigOptionsEnvironmentMixin, self).get_repo_shortname()
-            )
-
-    def get_emailprefix(self):
-        emailprefix = self.config.get('emailprefix')
-        if emailprefix is not None:
-            emailprefix = emailprefix.strip()
-            if emailprefix:
-                emailprefix += ' '
-        else:
-            emailprefix = '[%(repo_shortname)s] '
-        short_name = self.get_repo_shortname()
-        try:
-            return emailprefix % {'repo_shortname': short_name}
-        except:
-            self.get_logger().error(
-                '*** Invalid multimailhook.emailPrefix: %s\n' % emailprefix +
-                '*** %s\n' % sys.exc_info()[1] +
-                "*** Only the '%(repo_shortname)s' placeholder is allowed\n"
-                )
-            raise ConfigurationException(
-                '"%s" is not an allowed setting for emailPrefix' % emailprefix
-                )
-
-    def get_sender(self):
-        return self.config.get('envelopesender')
-
-    def process_addr(self, addr, change):
-        if addr.lower() == 'author':
-            if hasattr(change, 'author'):
-                return change.author
-            else:
-                return None
-        elif addr.lower() == 'pusher':
-            return self.get_pusher_email()
-        elif addr.lower() == 'none':
-            return None
-        else:
-            return addr
-
-    def get_fromaddr(self, change=None):
-        fromaddr = self.config.get('from')
-        if change:
-            specific_fromaddr = change.get_specific_fromaddr()
-            if specific_fromaddr:
-                fromaddr = specific_fromaddr
-        if fromaddr:
-            fromaddr = self.process_addr(fromaddr, change)
-        if fromaddr:
-            return fromaddr
-        return super(ConfigOptionsEnvironmentMixin, self).get_fromaddr(change)
-
-    def get_reply_to_refchange(self, refchange):
-        if self.__reply_to_refchange is None:
-            return super(ConfigOptionsEnvironmentMixin, self).get_reply_to_refchange(refchange)
-        else:
-            return self.process_addr(self.__reply_to_refchange, refchange)
-
-    def get_reply_to_commit(self, revision):
-        if self.__reply_to_commit is None:
-            return super(ConfigOptionsEnvironmentMixin, self).get_reply_to_commit(revision)
-        else:
-            return self.process_addr(self.__reply_to_commit, revision)
-
-    def get_scancommitforcc(self):
-        return self.config.get('scancommitforcc')
-
-
-class FilterLinesEnvironmentMixin(Environment):
-    """Handle encoding and maximum line length of body lines.
-
-        email_max_line_length (int or None)
-
-            The maximum length of any single line in the email body.
-            Longer lines are truncated at that length with ' [...]'
-            appended.
-
-        strict_utf8 (bool)
-
-            If this field is set to True, then the email body text is
-            expected to be UTF-8.  Any invalid characters are
-            converted to U+FFFD, the Unicode replacement character
-            (encoded as UTF-8, of course).
-
-    """
-
-    def __init__(self, strict_utf8=True,
-                 email_max_line_length=500, max_subject_length=500,
-                 **kw):
-        super(FilterLinesEnvironmentMixin, self).__init__(**kw)
-        self.__strict_utf8 = strict_utf8
-        self.__email_max_line_length = email_max_line_length
-        self.__max_subject_length = max_subject_length
-
-    def filter_body(self, lines):
-        lines = super(FilterLinesEnvironmentMixin, self).filter_body(lines)
-        if self.__strict_utf8:
-            if not PYTHON3:
-                lines = (line.decode(ENCODING, 'replace') for line in lines)
-            # Limit the line length in Unicode-space to avoid
-            # splitting characters:
-            if self.__email_max_line_length > 0:
-                lines = limit_linelength(lines, self.__email_max_line_length)
-            if not PYTHON3:
-                lines = (line.encode(ENCODING, 'replace') for line in lines)
-        elif self.__email_max_line_length:
-            lines = limit_linelength(lines, self.__email_max_line_length)
-
-        return lines
-
-    def get_max_subject_length(self):
-        return self.__max_subject_length
-
-
-class ConfigFilterLinesEnvironmentMixin(
-        ConfigEnvironmentMixin,
-        FilterLinesEnvironmentMixin,
-        ):
-    """Handle encoding and maximum line length based on config."""
-
-    def __init__(self, config, **kw):
-        strict_utf8 = config.get_bool('emailstrictutf8', default=None)
-        if strict_utf8 is not None:
-            kw['strict_utf8'] = strict_utf8
-
-        email_max_line_length = config.get('emailmaxlinelength')
-        if email_max_line_length is not None:
-            kw['email_max_line_length'] = int(email_max_line_length)
-
-        max_subject_length = config.get('subjectMaxLength', default=email_max_line_length)
-        if max_subject_length is not None:
-            kw['max_subject_length'] = int(max_subject_length)
-
-        super(ConfigFilterLinesEnvironmentMixin, self).__init__(
-            config=config, **kw
-            )
-
-
-class MaxlinesEnvironmentMixin(Environment):
-    """Limit the email body to a specified number of lines."""
-
-    def __init__(self, emailmaxlines, **kw):
-        super(MaxlinesEnvironmentMixin, self).__init__(**kw)
-        self.__emailmaxlines = emailmaxlines
-
-    def filter_body(self, lines):
-        lines = super(MaxlinesEnvironmentMixin, self).filter_body(lines)
-        if self.__emailmaxlines > 0:
-            lines = limit_lines(lines, self.__emailmaxlines)
-        return lines
-
-
-class ConfigMaxlinesEnvironmentMixin(
-        ConfigEnvironmentMixin,
-        MaxlinesEnvironmentMixin,
-        ):
-    """Limit the email body to the number of lines specified in config."""
-
-    def __init__(self, config, **kw):
-        emailmaxlines = int(config.get('emailmaxlines', default='0'))
-        super(ConfigMaxlinesEnvironmentMixin, self).__init__(
-            config=config,
-            emailmaxlines=emailmaxlines,
-            **kw
-            )
-
-
-class FQDNEnvironmentMixin(Environment):
-    """A mixin that sets the host's FQDN to its constructor argument."""
-
-    def __init__(self, fqdn, **kw):
-        super(FQDNEnvironmentMixin, self).__init__(**kw)
-        self.COMPUTED_KEYS += ['fqdn']
-        self.__fqdn = fqdn
-
-    def get_fqdn(self):
-        """Return the fully-qualified domain name for this host.
-
-        Return None if it is unavailable or unwanted."""
-
-        return self.__fqdn
-
-
-class ConfigFQDNEnvironmentMixin(
-        ConfigEnvironmentMixin,
-        FQDNEnvironmentMixin,
-        ):
-    """Read the FQDN from the config."""
-
-    def __init__(self, config, **kw):
-        fqdn = config.get('fqdn')
-        super(ConfigFQDNEnvironmentMixin, self).__init__(
-            config=config,
-            fqdn=fqdn,
-            **kw
-            )
-
-
-class ComputeFQDNEnvironmentMixin(FQDNEnvironmentMixin):
-    """Get the FQDN by calling socket.getfqdn()."""
-
-    def __init__(self, **kw):
-        super(ComputeFQDNEnvironmentMixin, self).__init__(
-            fqdn=socket.getfqdn(),
-            **kw
-            )
-
-
-class PusherDomainEnvironmentMixin(ConfigEnvironmentMixin):
-    """Deduce pusher_email from pusher by appending an emaildomain."""
-
-    def __init__(self, **kw):
-        super(PusherDomainEnvironmentMixin, self).__init__(**kw)
-        self.__emaildomain = self.config.get('emaildomain')
-
-    def get_pusher_email(self):
-        if self.__emaildomain:
-            # Derive the pusher's full email address in the default way:
-            return '%s@%s' % (self.get_pusher(), self.__emaildomain)
-        else:
-            return super(PusherDomainEnvironmentMixin, self).get_pusher_email()
-
-
-class StaticRecipientsEnvironmentMixin(Environment):
-    """Set recipients statically based on constructor parameters."""
-
-    def __init__(
-            self,
-            refchange_recipients, announce_recipients, revision_recipients, scancommitforcc,
-            **kw
-            ):
-        super(StaticRecipientsEnvironmentMixin, self).__init__(**kw)
-
-        # The recipients for various types of notification emails, as
-        # RFC 2822 email addresses separated by commas (or the empty
-        # string if no recipients are configured).  Although there is
-        # a mechanism to choose the recipient lists based on on the
-        # actual *contents* of the change being reported, we only
-        # choose based on the *type* of the change.  Therefore we can
-        # compute them once and for all:
-        self.__refchange_recipients = refchange_recipients
-        self.__announce_recipients = announce_recipients
-        self.__revision_recipients = revision_recipients
-
-    def check(self):
-        if not (self.get_refchange_recipients(None) or
-                self.get_announce_recipients(None) or
-                self.get_revision_recipients(None) or
-                self.get_scancommitforcc()):
-            raise ConfigurationException('No email recipients configured!')
-        super(StaticRecipientsEnvironmentMixin, self).check()
-
-    def get_refchange_recipients(self, refchange):
-        if self.__refchange_recipients is None:
-            return super(StaticRecipientsEnvironmentMixin,
-                         self).get_refchange_recipients(refchange)
-        return self.__refchange_recipients
-
-    def get_announce_recipients(self, annotated_tag_change):
-        if self.__announce_recipients is None:
-            return super(StaticRecipientsEnvironmentMixin,
-                         self).get_refchange_recipients(annotated_tag_change)
-        return self.__announce_recipients
-
-    def get_revision_recipients(self, revision):
-        if self.__revision_recipients is None:
-            return super(StaticRecipientsEnvironmentMixin,
-                         self).get_refchange_recipients(revision)
-        return self.__revision_recipients
-
-
-class CLIRecipientsEnvironmentMixin(Environment):
-    """Mixin storing recipients information coming from the
-    command-line."""
-
-    def __init__(self, cli_recipients=None, **kw):
-        super(CLIRecipientsEnvironmentMixin, self).__init__(**kw)
-        self.__cli_recipients = cli_recipients
-
-    def get_refchange_recipients(self, refchange):
-        if self.__cli_recipients is None:
-            return super(CLIRecipientsEnvironmentMixin,
-                         self).get_refchange_recipients(refchange)
-        return self.__cli_recipients
-
-    def get_announce_recipients(self, annotated_tag_change):
-        if self.__cli_recipients is None:
-            return super(CLIRecipientsEnvironmentMixin,
-                         self).get_announce_recipients(annotated_tag_change)
-        return self.__cli_recipients
-
-    def get_revision_recipients(self, revision):
-        if self.__cli_recipients is None:
-            return super(CLIRecipientsEnvironmentMixin,
-                         self).get_revision_recipients(revision)
-        return self.__cli_recipients
-
-
-class ConfigRecipientsEnvironmentMixin(
-        ConfigEnvironmentMixin,
-        StaticRecipientsEnvironmentMixin
-        ):
-    """Determine recipients statically based on config."""
-
-    def __init__(self, config, **kw):
-        super(ConfigRecipientsEnvironmentMixin, self).__init__(
-            config=config,
-            refchange_recipients=self._get_recipients(
-                config, 'refchangelist', 'mailinglist',
-                ),
-            announce_recipients=self._get_recipients(
-                config, 'announcelist', 'refchangelist', 'mailinglist',
-                ),
-            revision_recipients=self._get_recipients(
-                config, 'commitlist', 'mailinglist',
-                ),
-            scancommitforcc=config.get('scancommitforcc'),
-            **kw
-            )
-
-    def _get_recipients(self, config, *names):
-        """Return the recipients for a particular type of message.
-
-        Return the list of email addresses to which a particular type
-        of notification email should be sent, by looking at the config
-        value for "multimailhook.$name" for each of names.  Use the
-        value from the first name that is configured.  The return
-        value is a (possibly empty) string containing RFC 2822 email
-        addresses separated by commas.  If no configuration could be
-        found, raise a ConfigurationException."""
-
-        for name in names:
-            lines = config.get_all(name)
-            if lines is not None:
-                lines = [line.strip() for line in lines]
-                # Single "none" is a special value equivalen to empty string.
-                if lines == ['none']:
-                    lines = ['']
-                return ', '.join(lines)
-        else:
-            return ''
-
-
-class StaticRefFilterEnvironmentMixin(Environment):
-    """Set branch filter statically based on constructor parameters."""
-
-    def __init__(self, ref_filter_incl_regex, ref_filter_excl_regex,
-                 ref_filter_do_send_regex, ref_filter_dont_send_regex,
-                 **kw):
-        super(StaticRefFilterEnvironmentMixin, self).__init__(**kw)
-
-        if ref_filter_incl_regex and ref_filter_excl_regex:
-            raise ConfigurationException(
-                "Cannot specify both a ref inclusion and exclusion regex.")
-        self.__is_inclusion_filter = bool(ref_filter_incl_regex)
-        default_exclude = self.get_default_ref_ignore_regex()
-        if ref_filter_incl_regex:
-            ref_filter_regex = ref_filter_incl_regex
-        elif ref_filter_excl_regex:
-            ref_filter_regex = ref_filter_excl_regex + '|' + default_exclude
-        else:
-            ref_filter_regex = default_exclude
-        try:
-            self.__compiled_regex = re.compile(ref_filter_regex)
-        except Exception:
-            raise ConfigurationException(
-                'Invalid Ref Filter Regex "%s": %s' % (ref_filter_regex, sys.exc_info()[1]))
-
-        if ref_filter_do_send_regex and ref_filter_dont_send_regex:
-            raise ConfigurationException(
-                "Cannot specify both a ref doSend and dontSend regex.")
-        self.__is_do_send_filter = bool(ref_filter_do_send_regex)
-        if ref_filter_do_send_regex:
-            ref_filter_send_regex = ref_filter_do_send_regex
-        elif ref_filter_dont_send_regex:
-            ref_filter_send_regex = ref_filter_dont_send_regex
-        else:
-            ref_filter_send_regex = '.*'
-            self.__is_do_send_filter = True
-        try:
-            self.__send_compiled_regex = re.compile(ref_filter_send_regex)
-        except Exception:
-            raise ConfigurationException(
-                'Invalid Ref Filter Regex "%s": %s' %
-                (ref_filter_send_regex, sys.exc_info()[1]))
-
-    def get_ref_filter_regex(self, send_filter=False):
-        if send_filter:
-            return self.__send_compiled_regex, self.__is_do_send_filter
-        else:
-            return self.__compiled_regex, self.__is_inclusion_filter
-
-
-class ConfigRefFilterEnvironmentMixin(
-        ConfigEnvironmentMixin,
-        StaticRefFilterEnvironmentMixin
-        ):
-    """Determine branch filtering statically based on config."""
-
-    def _get_regex(self, config, key):
-        """Get a list of whitespace-separated regex. The refFilter* config
-        variables are multivalued (hence the use of get_all), and we
-        allow each entry to be a whitespace-separated list (hence the
-        split on each line). The whole thing is glued into a single regex."""
-        values = config.get_all(key)
-        if values is None:
-            return values
-        items = []
-        for line in values:
-            for i in line.split():
-                items.append(i)
-        if items == []:
-            return None
-        return '|'.join(items)
-
-    def __init__(self, config, **kw):
-        super(ConfigRefFilterEnvironmentMixin, self).__init__(
-            config=config,
-            ref_filter_incl_regex=self._get_regex(config, 'refFilterInclusionRegex'),
-            ref_filter_excl_regex=self._get_regex(config, 'refFilterExclusionRegex'),
-            ref_filter_do_send_regex=self._get_regex(config, 'refFilterDoSendRegex'),
-            ref_filter_dont_send_regex=self._get_regex(config, 'refFilterDontSendRegex'),
-            **kw
-            )
-
-
-class ProjectdescEnvironmentMixin(Environment):
-    """Make a "projectdesc" value available for templates.
-
-    By default, it is set to the first line of $GIT_DIR/description
-    (if that file is present and appears to be set meaningfully)."""
-
-    def __init__(self, **kw):
-        super(ProjectdescEnvironmentMixin, self).__init__(**kw)
-        self.COMPUTED_KEYS += ['projectdesc']
-
-    def get_projectdesc(self):
-        """Return a one-line description of the project."""
-
-        git_dir = get_git_dir()
-        try:
-            projectdesc = open(os.path.join(git_dir, 'description')).readline().strip()
-            if projectdesc and not projectdesc.startswith('Unnamed repository'):
-                return projectdesc
-        except IOError:
-            pass
-
-        return 'UNNAMED PROJECT'
-
-
-class GenericEnvironmentMixin(Environment):
-    def get_pusher(self):
-        return self.osenv.get('USER', self.osenv.get('USERNAME', 'unknown user'))
-
-
-class GitoliteEnvironmentHighPrecMixin(Environment):
-    def get_pusher(self):
-        return self.osenv.get('GL_USER', 'unknown user')
-
-
-class GitoliteEnvironmentLowPrecMixin(
-        ConfigEnvironmentMixin,
-        Environment):
-
-    def get_repo_shortname(self):
-        # The gitolite environment variable $GL_REPO is a pretty good
-        # repo_shortname (though it's probably not as good as a value
-        # the user might have explicitly put in his config).
-        return (
-            self.osenv.get('GL_REPO', None) or
-            super(GitoliteEnvironmentLowPrecMixin, self).get_repo_shortname()
-            )
-
-    @staticmethod
-    def _compile_regex(re_template):
-        return (
-            re.compile(re_template % x)
-            for x in (
-                r'BEGIN\s+USER\s+EMAILS',
-                r'([^\s]+)\s+(.*)',
-                r'END\s+USER\s+EMAILS',
-                ))
-
-    def get_fromaddr(self, change=None):
-        GL_USER = self.osenv.get('GL_USER')
-        if GL_USER is not None:
-            # Find the path to gitolite.conf.  Note that gitolite v3
-            # did away with the GL_ADMINDIR and GL_CONF environment
-            # variables (they are now hard-coded).
-            GL_ADMINDIR = self.osenv.get(
-                'GL_ADMINDIR',
-                os.path.expanduser(os.path.join('~', '.gitolite')))
-            GL_CONF = self.osenv.get(
-                'GL_CONF',
-                os.path.join(GL_ADMINDIR, 'conf', 'gitolite.conf'))
-
-            mailaddress_map = self.config.get('MailaddressMap')
-            # If relative, consider relative to GL_CONF:
-            if mailaddress_map:
-                mailaddress_map = os.path.join(os.path.dirname(GL_CONF),
-                                               mailaddress_map)
-                if os.path.isfile(mailaddress_map):
-                    f = open(mailaddress_map, 'rU')
-                    try:
-                        # Leading '#' is optional
-                        re_begin, re_user, re_end = self._compile_regex(
-                            r'^(?:\s*#)?\s*%s\s*$')
-                        for l in f:
-                            l = l.rstrip('\n')
-                            if re_begin.match(l) or re_end.match(l):
-                                continue  # Ignore these lines
-                            m = re_user.match(l)
-                            if m:
-                                if m.group(1) == GL_USER:
-                                    return m.group(2)
-                                else:
-                                    continue  # Not this user, but not an error
-                            raise ConfigurationException(
-                                "Syntax error in mail address map.\n"
-                                "Check file {}.\n"
-                                "Line: {}".format(mailaddress_map, l))
-
-                    finally:
-                        f.close()
-
-            if os.path.isfile(GL_CONF):
-                f = open(GL_CONF, 'rU')
-                try:
-                    in_user_emails_section = False
-                    re_begin, re_user, re_end = self._compile_regex(
-                        r'^\s*#\s*%s\s*$')
-                    for l in f:
-                        l = l.rstrip('\n')
-                        if not in_user_emails_section:
-                            if re_begin.match(l):
-                                in_user_emails_section = True
-                            continue
-                        if re_end.match(l):
-                            break
-                        m = re_user.match(l)
-                        if m and m.group(1) == GL_USER:
-                            return m.group(2)
-                finally:
-                    f.close()
-        return super(GitoliteEnvironmentLowPrecMixin, self).get_fromaddr(change)
-
-
-class IncrementalDateTime(object):
-    """Simple wrapper to give incremental date/times.
-
-    Each call will result in a date/time a second later than the
-    previous call.  This can be used to falsify email headers, to
-    increase the likelihood that email clients sort the emails
-    correctly."""
-
-    def __init__(self):
-        self.time = time.time()
-        self.next = self.__next__  # Python 2 backward compatibility
-
-    def __next__(self):
-        formatted = formatdate(self.time, True)
-        self.time += 1
-        return formatted
-
-
-class StashEnvironmentHighPrecMixin(Environment):
-    def __init__(self, user=None, repo=None, **kw):
-        super(StashEnvironmentHighPrecMixin,
-              self).__init__(user=user, repo=repo, **kw)
-        self.__user = user
-        self.__repo = repo
-
-    def get_pusher(self):
-        return re.match(r'(.*?)\s*<', self.__user).group(1)
-
-    def get_pusher_email(self):
-        return self.__user
-
-
-class StashEnvironmentLowPrecMixin(Environment):
-    def __init__(self, user=None, repo=None, **kw):
-        super(StashEnvironmentLowPrecMixin, self).__init__(**kw)
-        self.__repo = repo
-        self.__user = user
-
-    def get_repo_shortname(self):
-        return self.__repo
-
-    def get_fromaddr(self, change=None):
-        return self.__user
-
-
-class GerritEnvironmentHighPrecMixin(Environment):
-    def __init__(self, project=None, submitter=None, update_method=None, **kw):
-        super(GerritEnvironmentHighPrecMixin,
-              self).__init__(submitter=submitter, project=project, **kw)
-        self.__project = project
-        self.__submitter = submitter
-        self.__update_method = update_method
-        "Make an 'update_method' value available for templates."
-        self.COMPUTED_KEYS += ['update_method']
-
-    def get_pusher(self):
-        if self.__submitter:
-            if self.__submitter.find('<') != -1:
-                # Submitter has a configured email, we transformed
-                # __submitter into an RFC 2822 string already.
-                return re.match(r'(.*?)\s*<', self.__submitter).group(1)
-            else:
-                # Submitter has no configured email, it's just his name.
-                return self.__submitter
-        else:
-            # If we arrive here, this means someone pushed "Submit" from
-            # the gerrit web UI for the CR (or used one of the programmatic
-            # APIs to do the same, such as gerrit review) and the
-            # merge/push was done by the Gerrit user.  It was technically
-            # triggered by someone else, but sadly we have no way of
-            # determining who that someone else is at this point.
-            return 'Gerrit'  # 'unknown user'?
-
-    def get_pusher_email(self):
-        if self.__submitter:
-            return self.__submitter
-        else:
-            return super(GerritEnvironmentHighPrecMixin, self).get_pusher_email()
-
-    def get_default_ref_ignore_regex(self):
-        default = super(GerritEnvironmentHighPrecMixin, self).get_default_ref_ignore_regex()
-        return default + '|^refs/changes/|^refs/cache-automerge/|^refs/meta/'
-
-    def get_revision_recipients(self, revision):
-        # Merge commits created by Gerrit when users hit "Submit this patchset"
-        # in the Web UI (or do equivalently with REST APIs or the gerrit review
-        # command) are not something users want to see an individual email for.
-        # Filter them out.
-        committer = read_git_output(['log', '--no-walk', '--format=%cN',
-                                     revision.rev.sha1])
-        if committer == 'Gerrit Code Review':
-            return []
-        else:
-            return super(GerritEnvironmentHighPrecMixin, self).get_revision_recipients(revision)
-
-    def get_update_method(self):
-        return self.__update_method
-
-
-class GerritEnvironmentLowPrecMixin(Environment):
-    def __init__(self, project=None, submitter=None, **kw):
-        super(GerritEnvironmentLowPrecMixin, self).__init__(**kw)
-        self.__project = project
-        self.__submitter = submitter
-
-    def get_repo_shortname(self):
-        return self.__project
-
-    def get_fromaddr(self, change=None):
-        if self.__submitter and self.__submitter.find('<') != -1:
-            return self.__submitter
-        else:
-            return super(GerritEnvironmentLowPrecMixin, self).get_fromaddr(change)
-
-
-class Push(object):
-    """Represent an entire push (i.e., a group of ReferenceChanges).
-
-    It is easy to figure out what commits were added to a *branch* by
-    a Reference change:
-
-        git rev-list change.old..change.new
-
-    or removed from a *branch*:
-
-        git rev-list change.new..change.old
-
-    But it is not quite so trivial to determine which entirely new
-    commits were added to the *repository* by a push and which old
-    commits were discarded by a push.  A big part of the job of this
-    class is to figure out these things, and to make sure that new
-    commits are only detailed once even if they were added to multiple
-    references.
-
-    The first step is to determine the "other" references--those
-    unaffected by the current push.  They are computed by listing all
-    references then removing any affected by this push.  The results
-    are stored in Push._other_ref_sha1s.
-
-    The commits contained in the repository before this push were
-
-        git rev-list other1 other2 other3 ... change1.old change2.old ...
-
-    Where "changeN.old" is the old value of one of the references
-    affected by this push.
-
-    The commits contained in the repository after this push are
-
-        git rev-list other1 other2 other3 ... change1.new change2.new ...
-
-    The commits added by this push are the difference between these
-    two sets, which can be written
-
-        git rev-list \
-            ^other1 ^other2 ... \
-            ^change1.old ^change2.old ... \
-            change1.new change2.new ...
-
-    The commits removed by this push can be computed by
-
-        git rev-list \
-            ^other1 ^other2 ... \
-            ^change1.new ^change2.new ... \
-            change1.old change2.old ...
-
-    The last point is that it is possible that other pushes are
-    occurring simultaneously to this one, so reference values can
-    change at any time.  It is impossible to eliminate all race
-    conditions, but we reduce the window of time during which problems
-    can occur by translating reference names to SHA1s as soon as
-    possible and working with SHA1s thereafter (because SHA1s are
-    immutable)."""
-
-    # A map {(changeclass, changetype): integer} specifying the order
-    # that reference changes will be processed if multiple reference
-    # changes are included in a single push.  The order is significant
-    # mostly because new commit notifications are threaded together
-    # with the first reference change that includes the commit.  The
-    # following order thus causes commits to be grouped with branch
-    # changes (as opposed to tag changes) if possible.
-    SORT_ORDER = dict(
-        (value, i) for (i, value) in enumerate([
-            (BranchChange, 'update'),
-            (BranchChange, 'create'),
-            (AnnotatedTagChange, 'update'),
-            (AnnotatedTagChange, 'create'),
-            (NonAnnotatedTagChange, 'update'),
-            (NonAnnotatedTagChange, 'create'),
-            (BranchChange, 'delete'),
-            (AnnotatedTagChange, 'delete'),
-            (NonAnnotatedTagChange, 'delete'),
-            (OtherReferenceChange, 'update'),
-            (OtherReferenceChange, 'create'),
-            (OtherReferenceChange, 'delete'),
-            ])
-        )
-
-    def __init__(self, environment, changes, ignore_other_refs=False):
-        self.changes = sorted(changes, key=self._sort_key)
-        self.__other_ref_sha1s = None
-        self.__cached_commits_spec = {}
-        self.environment = environment
-
-        if ignore_other_refs:
-            self.__other_ref_sha1s = set()
-
-    @classmethod
-    def _sort_key(klass, change):
-        return (klass.SORT_ORDER[change.__class__, change.change_type], change.refname,)
-
-    @property
-    def _other_ref_sha1s(self):
-        """The GitObjects referred to by references unaffected by this push.
-        """
-        if self.__other_ref_sha1s is None:
-            # The refnames being changed by this push:
-            updated_refs = set(
-                change.refname
-                for change in self.changes
-                )
-
-            # The SHA-1s of commits referred to by all references in this
-            # repository *except* updated_refs:
-            sha1s = set()
-            fmt = (
-                '%(objectname) %(objecttype) %(refname)\n'
-                '%(*objectname) %(*objecttype) %(refname)'
-                )
-            ref_filter_regex, is_inclusion_filter = \
-                self.environment.get_ref_filter_regex()
-            for line in read_git_lines(
-                    ['for-each-ref', '--format=%s' % (fmt,)]):
-                (sha1, type, name) = line.split(' ', 2)
-                if (sha1 and type == 'commit' and
-                        name not in updated_refs and
-                        include_ref(name, ref_filter_regex, is_inclusion_filter)):
-                    sha1s.add(sha1)
-
-            self.__other_ref_sha1s = sha1s
-
-        return self.__other_ref_sha1s
-
-    def _get_commits_spec_incl(self, new_or_old, reference_change=None):
-        """Get new or old SHA-1 from one or each of the changed refs.
-
-        Return a list of SHA-1 commit identifier strings suitable as
-        arguments to 'git rev-list' (or 'git log' or ...).  The
-        returned identifiers are either the old or new values from one
-        or all of the changed references, depending on the values of
-        new_or_old and reference_change.
-
-        new_or_old is either the string 'new' or the string 'old'.  If
-        'new', the returned SHA-1 identifiers are the new values from
-        each changed reference.  If 'old', the SHA-1 identifiers are
-        the old values from each changed reference.
-
-        If reference_change is specified and not None, only the new or
-        old reference from the specified reference is included in the
-        return value.
-
-        This function returns None if there are no matching revisions
-        (e.g., because a branch was deleted and new_or_old is 'new').
-        """
-
-        if not reference_change:
-            incl_spec = sorted(
-                getattr(change, new_or_old).sha1
-                for change in self.changes
-                if getattr(change, new_or_old)
-                )
-            if not incl_spec:
-                incl_spec = None
-        elif not getattr(reference_change, new_or_old).commit_sha1:
-            incl_spec = None
-        else:
-            incl_spec = [getattr(reference_change, new_or_old).commit_sha1]
-        return incl_spec
-
-    def _get_commits_spec_excl(self, new_or_old):
-        """Get exclusion revisions for determining new or discarded commits.
-
-        Return a list of strings suitable as arguments to 'git
-        rev-list' (or 'git log' or ...) that will exclude all
-        commits that, depending on the value of new_or_old, were
-        either previously in the repository (useful for determining
-        which commits are new to the repository) or currently in the
-        repository (useful for determining which commits were
-        discarded from the repository).
-
-        new_or_old is either the string 'new' or the string 'old'.  If
-        'new', the commits to be excluded are those that were in the
-        repository before the push.  If 'old', the commits to be
-        excluded are those that are currently in the repository.  """
-
-        old_or_new = {'old': 'new', 'new': 'old'}[new_or_old]
-        excl_revs = self._other_ref_sha1s.union(
-            getattr(change, old_or_new).sha1
-            for change in self.changes
-            if getattr(change, old_or_new).type in ['commit', 'tag']
-            )
-        return ['^' + sha1 for sha1 in sorted(excl_revs)]
-
-    def get_commits_spec(self, new_or_old, reference_change=None):
-        """Get rev-list arguments for added or discarded commits.
-
-        Return a list of strings suitable as arguments to 'git
-        rev-list' (or 'git log' or ...) that select those commits
-        that, depending on the value of new_or_old, are either new to
-        the repository or were discarded from the repository.
-
-        new_or_old is either the string 'new' or the string 'old'.  If
-        'new', the returned list is used to select commits that are
-        new to the repository.  If 'old', the returned value is used
-        to select the commits that have been discarded from the
-        repository.
-
-        If reference_change is specified and not None, the new or
-        discarded commits are limited to those that are reachable from
-        the new or old value of the specified reference.
-
-        This function returns None if there are no added (or discarded)
-        revisions.
-        """
-        key = (new_or_old, reference_change)
-        if key not in self.__cached_commits_spec:
-            ret = self._get_commits_spec_incl(new_or_old, reference_change)
-            if ret is not None:
-                ret.extend(self._get_commits_spec_excl(new_or_old))
-            self.__cached_commits_spec[key] = ret
-        return self.__cached_commits_spec[key]
-
-    def get_new_commits(self, reference_change=None):
-        """Return a list of commits added by this push.
-
-        Return a list of the object names of commits that were added
-        by the part of this push represented by reference_change.  If
-        reference_change is None, then return a list of *all* commits
-        added by this push."""
-
-        spec = self.get_commits_spec('new', reference_change)
-        return git_rev_list(spec)
-
-    def get_discarded_commits(self, reference_change):
-        """Return a list of commits discarded by this push.
-
-        Return a list of the object names of commits that were
-        entirely discarded from the repository by the part of this
-        push represented by reference_change."""
-
-        spec = self.get_commits_spec('old', reference_change)
-        return git_rev_list(spec)
-
-    def send_emails(self, mailer, body_filter=None):
-        """Use send all of the notification emails needed for this push.
-
-        Use send all of the notification emails (including reference
-        change emails and commit emails) needed for this push.  Send
-        the emails using mailer.  If body_filter is not None, then use
-        it to filter the lines that are intended for the email
-        body."""
-
-        # The sha1s of commits that were introduced by this push.
-        # They will be removed from this set as they are processed, to
-        # guarantee that one (and only one) email is generated for
-        # each new commit.
-        unhandled_sha1s = set(self.get_new_commits())
-        send_date = IncrementalDateTime()
-        for change in self.changes:
-            sha1s = []
-            for sha1 in reversed(list(self.get_new_commits(change))):
-                if sha1 in unhandled_sha1s:
-                    sha1s.append(sha1)
-                    unhandled_sha1s.remove(sha1)
-
-            # Check if we've got anyone to send to
-            if not change.recipients:
-                change.environment.log_warning(
-                    '*** no recipients configured so no email will be sent\n'
-                    '*** for %r update %s->%s'
-                    % (change.refname, change.old.sha1, change.new.sha1,)
-                    )
-            else:
-                if not change.environment.quiet:
-                    change.environment.log_msg(
-                        'Sending notification emails to: %s' % (change.recipients,))
-                extra_values = {'send_date': next(send_date)}
-
-                rev = change.send_single_combined_email(sha1s)
-                if rev:
-                    mailer.send(
-                        change.generate_combined_email(self, rev, body_filter, extra_values),
-                        rev.recipients,
-                        )
-                    # This change is now fully handled; no need to handle
-                    # individual revisions any further.
-                    continue
-                else:
-                    mailer.send(
-                        change.generate_email(self, body_filter, extra_values),
-                        change.recipients,
-                        )
-
-            max_emails = change.environment.maxcommitemails
-            if max_emails and len(sha1s) > max_emails:
-                change.environment.log_warning(
-                    '*** Too many new commits (%d), not sending commit emails.\n' % len(sha1s) +
-                    '*** Try setting multimailhook.maxCommitEmails to a greater value\n' +
-                    '*** Currently, multimailhook.maxCommitEmails=%d' % max_emails
-                    )
-                return
-
-            for (num, sha1) in enumerate(sha1s):
-                rev = Revision(change, GitObject(sha1), num=num + 1, tot=len(sha1s))
-                if len(rev.parents) > 1 and change.environment.excludemergerevisions:
-                    # skipping a merge commit
-                    continue
-                if not rev.recipients and rev.cc_recipients:
-                    change.environment.log_msg('*** Replacing Cc: with To:')
-                    rev.recipients = rev.cc_recipients
-                    rev.cc_recipients = None
-                if rev.recipients:
-                    extra_values = {'send_date': next(send_date)}
-                    mailer.send(
-                        rev.generate_email(self, body_filter, extra_values),
-                        rev.recipients,
-                        )
-
-        # Consistency check:
-        if unhandled_sha1s:
-            change.environment.log_error(
-                'ERROR: No emails were sent for the following new commits:\n'
-                '    %s'
-                % ('\n    '.join(sorted(unhandled_sha1s)),)
-                )
-
-
-def include_ref(refname, ref_filter_regex, is_inclusion_filter):
-    does_match = bool(ref_filter_regex.search(refname))
-    if is_inclusion_filter:
-        return does_match
-    else:  # exclusion filter -- we include the ref if the regex doesn't match
-        return not does_match
-
-
-def run_as_post_receive_hook(environment, mailer):
-    environment.check()
-    send_filter_regex, send_is_inclusion_filter = environment.get_ref_filter_regex(True)
-    ref_filter_regex, is_inclusion_filter = environment.get_ref_filter_regex(False)
-    changes = []
-    while True:
-        line = read_line(sys.stdin)
-        if line == '':
-            break
-        (oldrev, newrev, refname) = line.strip().split(' ', 2)
-        environment.get_logger().debug(
-            "run_as_post_receive_hook: oldrev=%s, newrev=%s, refname=%s" %
-            (oldrev, newrev, refname))
-
-        if not include_ref(refname, ref_filter_regex, is_inclusion_filter):
-            continue
-        if not include_ref(refname, send_filter_regex, send_is_inclusion_filter):
-            continue
-        changes.append(
-            ReferenceChange.create(environment, oldrev, newrev, refname)
-            )
-    if not changes:
-        mailer.close()
-        return
-    push = Push(environment, changes)
-    try:
-        push.send_emails(mailer, body_filter=environment.filter_body)
-    finally:
-        mailer.close()
-
-
-def run_as_update_hook(environment, mailer, refname, oldrev, newrev, force_send=False):
-    environment.check()
-    send_filter_regex, send_is_inclusion_filter = environment.get_ref_filter_regex(True)
-    ref_filter_regex, is_inclusion_filter = environment.get_ref_filter_regex(False)
-    if not include_ref(refname, ref_filter_regex, is_inclusion_filter):
-        return
-    if not include_ref(refname, send_filter_regex, send_is_inclusion_filter):
-        return
-    changes = [
-        ReferenceChange.create(
-            environment,
-            read_git_output(['rev-parse', '--verify', oldrev]),
-            read_git_output(['rev-parse', '--verify', newrev]),
-            refname,
-            ),
-        ]
-    if not changes:
-        mailer.close()
-        return
-    push = Push(environment, changes, force_send)
-    try:
-        push.send_emails(mailer, body_filter=environment.filter_body)
-    finally:
-        mailer.close()
-
-
-def check_ref_filter(environment):
-    send_filter_regex, send_is_inclusion = environment.get_ref_filter_regex(True)
-    ref_filter_regex, ref_is_inclusion = environment.get_ref_filter_regex(False)
-
-    def inc_exc_lusion(b):
-        if b:
-            return 'inclusion'
-        else:
-            return 'exclusion'
-
-    if send_filter_regex:
-        sys.stdout.write("DoSend/DontSend filter regex (" +
-                         (inc_exc_lusion(send_is_inclusion)) +
-                         '): ' + send_filter_regex.pattern +
-                         '\n')
-    if send_filter_regex:
-        sys.stdout.write("Include/Exclude filter regex (" +
-                         (inc_exc_lusion(ref_is_inclusion)) +
-                         '): ' + ref_filter_regex.pattern +
-                         '\n')
-    sys.stdout.write(os.linesep)
-
-    sys.stdout.write(
-        "Refs marked as EXCLUDE are excluded by either refFilterInclusionRegex\n"
-        "or refFilterExclusionRegex. No emails will be sent for commits included\n"
-        "in these refs.\n"
-        "Refs marked as DONT-SEND are excluded by either refFilterDoSendRegex or\n"
-        "refFilterDontSendRegex, but not by either refFilterInclusionRegex or\n"
-        "refFilterExclusionRegex. Emails will be sent for commits included in these\n"
-        "refs only when the commit reaches a ref which isn't excluded.\n"
-        "Refs marked as DO-SEND are not excluded by any filter. Emails will\n"
-        "be sent normally for commits included in these refs.\n")
-
-    sys.stdout.write(os.linesep)
-
-    for refname in read_git_lines(['for-each-ref', '--format', '%(refname)']):
-        sys.stdout.write(refname)
-        if not include_ref(refname, ref_filter_regex, ref_is_inclusion):
-            sys.stdout.write(' EXCLUDE')
-        elif not include_ref(refname, send_filter_regex, send_is_inclusion):
-            sys.stdout.write(' DONT-SEND')
-        else:
-            sys.stdout.write(' DO-SEND')
-
-        sys.stdout.write(os.linesep)
-
-
-def show_env(environment, out):
-    out.write('Environment values:\n')
-    for (k, v) in sorted(environment.get_values().items()):
-        if k:  # Don't show the {'' : ''} pair.
-            out.write('    %s : %r\n' % (k, v))
-    out.write('\n')
-    # Flush to avoid interleaving with further log output
-    out.flush()
-
-
-def check_setup(environment):
-    environment.check()
-    show_env(environment, sys.stdout)
-    sys.stdout.write("Now, checking that git-multimail's standard input "
-                     "is properly set ..." + os.linesep)
-    sys.stdout.write("Please type some text and then press Return" + os.linesep)
-    stdin = sys.stdin.readline()
-    sys.stdout.write("You have just entered:" + os.linesep)
-    sys.stdout.write(stdin)
-    sys.stdout.write("git-multimail seems properly set up." + os.linesep)
-
-
-def choose_mailer(config, environment):
-    mailer = config.get('mailer', default='sendmail')
-
-    if mailer == 'smtp':
-        smtpserver = config.get('smtpserver', default='localhost')
-        smtpservertimeout = float(config.get('smtpservertimeout', default=10.0))
-        smtpserverdebuglevel = int(config.get('smtpserverdebuglevel', default=0))
-        smtpencryption = config.get('smtpencryption', default='none')
-        smtpuser = config.get('smtpuser', default='')
-        smtppass = config.get('smtppass', default='')
-        smtpcacerts = config.get('smtpcacerts', default='')
-        mailer = SMTPMailer(
-            environment,
-            envelopesender=(environment.get_sender() or environment.get_fromaddr()),
-            smtpserver=smtpserver, smtpservertimeout=smtpservertimeout,
-            smtpserverdebuglevel=smtpserverdebuglevel,
-            smtpencryption=smtpencryption,
-            smtpuser=smtpuser,
-            smtppass=smtppass,
-            smtpcacerts=smtpcacerts
-            )
-    elif mailer == 'sendmail':
-        command = config.get('sendmailcommand')
-        if command:
-            command = shlex.split(command)
-        mailer = SendMailer(environment,
-                            command=command, envelopesender=environment.get_sender())
-    else:
-        environment.log_error(
-            'fatal: multimailhook.mailer is set to an incorrect value: "%s"\n' % mailer +
-            'please use one of "smtp" or "sendmail".'
-            )
-        sys.exit(1)
-    return mailer
-
-
-KNOWN_ENVIRONMENTS = {
-    'generic': {'highprec': GenericEnvironmentMixin},
-    'gitolite': {'highprec': GitoliteEnvironmentHighPrecMixin,
-                 'lowprec': GitoliteEnvironmentLowPrecMixin},
-    'stash': {'highprec': StashEnvironmentHighPrecMixin,
-              'lowprec': StashEnvironmentLowPrecMixin},
-    'gerrit': {'highprec': GerritEnvironmentHighPrecMixin,
-               'lowprec': GerritEnvironmentLowPrecMixin},
-    }
-
-
-def choose_environment(config, osenv=None, env=None, recipients=None,
-                       hook_info=None):
-    env_name = choose_environment_name(config, env, osenv)
-    environment_klass = build_environment_klass(env_name)
-    env = build_environment(environment_klass, env_name, config,
-                            osenv, recipients, hook_info)
-    return env
-
-
-def choose_environment_name(config, env, osenv):
-    if not osenv:
-        osenv = os.environ
-
-    if not env:
-        env = config.get('environment')
-
-    if not env:
-        if 'GL_USER' in osenv and 'GL_REPO' in osenv:
-            env = 'gitolite'
-        else:
-            env = 'generic'
-    return env
-
-
-COMMON_ENVIRONMENT_MIXINS = [
-    ConfigRecipientsEnvironmentMixin,
-    CLIRecipientsEnvironmentMixin,
-    ConfigRefFilterEnvironmentMixin,
-    ProjectdescEnvironmentMixin,
-    ConfigMaxlinesEnvironmentMixin,
-    ComputeFQDNEnvironmentMixin,
-    ConfigFilterLinesEnvironmentMixin,
-    PusherDomainEnvironmentMixin,
-    ConfigOptionsEnvironmentMixin,
-    ]
-
-
-def build_environment_klass(env_name):
-    if 'class' in KNOWN_ENVIRONMENTS[env_name]:
-        return KNOWN_ENVIRONMENTS[env_name]['class']
-
-    environment_mixins = []
-    known_env = KNOWN_ENVIRONMENTS[env_name]
-    if 'highprec' in known_env:
-        high_prec_mixin = known_env['highprec']
-        environment_mixins.append(high_prec_mixin)
-    environment_mixins = environment_mixins + COMMON_ENVIRONMENT_MIXINS
-    if 'lowprec' in known_env:
-        low_prec_mixin = known_env['lowprec']
-        environment_mixins.append(low_prec_mixin)
-    environment_mixins.append(Environment)
-    klass_name = env_name.capitalize() + 'Environment'
-    environment_klass = type(
-        klass_name,
-        tuple(environment_mixins),
-        {},
-        )
-    KNOWN_ENVIRONMENTS[env_name]['class'] = environment_klass
-    return environment_klass
-
-
-GerritEnvironment = build_environment_klass('gerrit')
-StashEnvironment = build_environment_klass('stash')
-GitoliteEnvironment = build_environment_klass('gitolite')
-GenericEnvironment = build_environment_klass('generic')
-
-
-def build_environment(environment_klass, env, config,
-                      osenv, recipients, hook_info):
-    environment_kw = {
-        'osenv': osenv,
-        'config': config,
-        }
-
-    if env == 'stash':
-        environment_kw['user'] = hook_info['stash_user']
-        environment_kw['repo'] = hook_info['stash_repo']
-    elif env == 'gerrit':
-        environment_kw['project'] = hook_info['project']
-        environment_kw['submitter'] = hook_info['submitter']
-        environment_kw['update_method'] = hook_info['update_method']
-
-    environment_kw['cli_recipients'] = recipients
-
-    return environment_klass(**environment_kw)
-
-
-def get_version():
-    oldcwd = os.getcwd()
-    try:
-        try:
-            os.chdir(os.path.dirname(os.path.realpath(__file__)))
-            git_version = read_git_output(['describe', '--tags', 'HEAD'])
-            if git_version == __version__:
-                return git_version
-            else:
-                return '%s (%s)' % (__version__, git_version)
-        except:
-            pass
-    finally:
-        os.chdir(oldcwd)
-    return __version__
-
-
-def compute_gerrit_options(options, args, required_gerrit_options,
-                           raw_refname):
-    if None in required_gerrit_options:
-        raise SystemExit("Error: Specify all of --oldrev, --newrev, --refname, "
-                         "and --project; or none of them.")
-
-    if options.environment not in (None, 'gerrit'):
-        raise SystemExit("Non-gerrit environments incompatible with --oldrev, "
-                         "--newrev, --refname, and --project")
-    options.environment = 'gerrit'
-
-    if args:
-        raise SystemExit("Error: Positional parameters not allowed with "
-                         "--oldrev, --newrev, and --refname.")
-
-    # Gerrit oddly omits 'refs/heads/' in the refname when calling
-    # ref-updated hook; put it back.
-    git_dir = get_git_dir()
-    if (not os.path.exists(os.path.join(git_dir, raw_refname)) and
-        os.path.exists(os.path.join(git_dir, 'refs', 'heads',
-                                    raw_refname))):
-        options.refname = 'refs/heads/' + options.refname
-
-    # New revisions can appear in a gerrit repository either due to someone
-    # pushing directly (in which case options.submitter will be set), or they
-    # can press "Submit this patchset" in the web UI for some CR (in which
-    # case options.submitter will not be set and gerrit will not have provided
-    # us the information about who pressed the button).
-    #
-    # Note for the nit-picky: I'm lumping in REST API calls and the ssh
-    # gerrit review command in with "Submit this patchset" button, since they
-    # have the same effect.
-    if options.submitter:
-        update_method = 'pushed'
-        # The submitter argument is almost an RFC 2822 email address; change it
-        # from 'User Name (email@domain)' to 'User Name ' so it is
-        options.submitter = options.submitter.replace('(', '<').replace(')', '>')
-    else:
-        update_method = 'submitted'
-        # Gerrit knew who submitted this patchset, but threw that information
-        # away when it invoked this hook.  However, *IF* Gerrit created a
-        # merge to bring the patchset in (project 'Submit Type' is either
-        # "Always Merge", or is "Merge if Necessary" and happens to be
-        # necessary for this particular CR), then it will have the committer
-        # of that merge be 'Gerrit Code Review' and the author will be the
-        # person who requested the submission of the CR.  Since this is fairly
-        # likely for most gerrit installations (of a reasonable size), it's
-        # worth the extra effort to try to determine the actual submitter.
-        rev_info = read_git_lines(['log', '--no-walk', '--merges',
-                                   '--format=%cN%n%aN <%aE>', options.newrev])
-        if rev_info and rev_info[0] == 'Gerrit Code Review':
-            options.submitter = rev_info[1]
-
-    # We pass back refname, oldrev, newrev as args because then the
-    # gerrit ref-updated hook is much like the git update hook
-    return (options,
-            [options.refname, options.oldrev, options.newrev],
-            {'project': options.project, 'submitter': options.submitter,
-             'update_method': update_method})
-
-
-def check_hook_specific_args(options, args):
-    raw_refname = options.refname
-    # Convert each string option unicode for Python3.
-    if PYTHON3:
-        opts = ['environment', 'recipients', 'oldrev', 'newrev', 'refname',
-                'project', 'submitter', 'stash_user', 'stash_repo']
-        for opt in opts:
-            if not hasattr(options, opt):
-                continue
-            obj = getattr(options, opt)
-            if obj:
-                enc = obj.encode('utf-8', 'surrogateescape')
-                dec = enc.decode('utf-8', 'replace')
-                setattr(options, opt, dec)
-
-    # First check for stash arguments
-    if (options.stash_user is None) != (options.stash_repo is None):
-        raise SystemExit("Error: Specify both of --stash-user and "
-                         "--stash-repo or neither.")
-    if options.stash_user:
-        options.environment = 'stash'
-        return options, args, {'stash_user': options.stash_user,
-                               'stash_repo': options.stash_repo}
-
-    # Finally, check for gerrit specific arguments
-    required_gerrit_options = (options.oldrev, options.newrev, options.refname,
-                               options.project)
-    if required_gerrit_options != (None,) * 4:
-        return compute_gerrit_options(options, args, required_gerrit_options,
-                                      raw_refname)
-
-    # No special options in use, just return what we started with
-    return options, args, {}
-
-
-class Logger(object):
-    def parse_verbose(self, verbose):
-        if verbose > 0:
-            return logging.DEBUG
-        else:
-            return logging.INFO
-
-    def create_log_file(self, environment, name, path, verbosity):
-        log_file = logging.getLogger(name)
-        file_handler = logging.FileHandler(path)
-        log_fmt = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
-        file_handler.setFormatter(log_fmt)
-        log_file.addHandler(file_handler)
-        log_file.setLevel(verbosity)
-        return log_file
-
-    def __init__(self, environment):
-        self.environment = environment
-        self.loggers = []
-        stderr_log = logging.getLogger('git_multimail.stderr')
-
-        class EncodedStderr(object):
-            def write(self, x):
-                write_str(sys.stderr, x)
-
-            def flush(self):
-                sys.stderr.flush()
-
-        stderr_handler = logging.StreamHandler(EncodedStderr())
-        stderr_log.addHandler(stderr_handler)
-        stderr_log.setLevel(self.parse_verbose(environment.verbose))
-        self.loggers.append(stderr_log)
-
-        if environment.debug_log_file is not None:
-            debug_log_file = self.create_log_file(
-                environment, 'git_multimail.debug', environment.debug_log_file, logging.DEBUG)
-            self.loggers.append(debug_log_file)
-
-        if environment.log_file is not None:
-            log_file = self.create_log_file(
-                environment, 'git_multimail.file', environment.log_file, logging.INFO)
-            self.loggers.append(log_file)
-
-        if environment.error_log_file is not None:
-            error_log_file = self.create_log_file(
-                environment, 'git_multimail.error', environment.error_log_file, logging.ERROR)
-            self.loggers.append(error_log_file)
-
-    def info(self, msg, *args, **kwargs):
-        for l in self.loggers:
-            l.info(msg, *args, **kwargs)
-
-    def debug(self, msg, *args, **kwargs):
-        for l in self.loggers:
-            l.debug(msg, *args, **kwargs)
-
-    def warning(self, msg, *args, **kwargs):
-        for l in self.loggers:
-            l.warning(msg, *args, **kwargs)
-
-    def error(self, msg, *args, **kwargs):
-        for l in self.loggers:
-            l.error(msg, *args, **kwargs)
-
-
-def main(args):
-    parser = optparse.OptionParser(
-        description=__doc__,
-        usage='%prog [OPTIONS]\n   or: %prog [OPTIONS] REFNAME OLDREV NEWREV',
-        )
-
-    parser.add_option(
-        '--environment', '--env', action='store', type='choice',
-        choices=list(KNOWN_ENVIRONMENTS.keys()), default=None,
-        help=(
-            'Choose type of environment is in use.  Default is taken from '
-            'multimailhook.environment if set; otherwise "generic".'
-            ),
-        )
-    parser.add_option(
-        '--stdout', action='store_true', default=False,
-        help='Output emails to stdout rather than sending them.',
-        )
-    parser.add_option(
-        '--recipients', action='store', default=None,
-        help='Set list of email recipients for all types of emails.',
-        )
-    parser.add_option(
-        '--show-env', action='store_true', default=False,
-        help=(
-            'Write to stderr the values determined for the environment '
-            '(intended for debugging purposes), then proceed normally.'
-            ),
-        )
-    parser.add_option(
-        '--force-send', action='store_true', default=False,
-        help=(
-            'Force sending refchange email when using as an update hook. '
-            'This is useful to work around the unreliable new commits '
-            'detection in this mode.'
-            ),
-        )
-    parser.add_option(
-        '-c', metavar="=", action='append',
-        help=(
-            'Pass a configuration parameter through to git.  The value given '
-            'will override values from configuration files.  See the -c option '
-            'of git(1) for more details.  (Only works with git >= 1.7.3)'
-            ),
-        )
-    parser.add_option(
-        '--version', '-v', action='store_true', default=False,
-        help=(
-            "Display git-multimail's version"
-            ),
-        )
-
-    parser.add_option(
-        '--python-version', action='store_true', default=False,
-        help=(
-            "Display the version of Python used by git-multimail"
-            ),
-        )
-
-    parser.add_option(
-        '--check-ref-filter', action='store_true', default=False,
-        help=(
-            'List refs and show information on how git-multimail '
-            'will process them.'
-            )
-        )
-
-    # The following options permit this script to be run as a gerrit
-    # ref-updated hook.  See e.g.
-    # code.google.com/p/gerrit/source/browse/Documentation/config-hooks.txt
-    # We suppress help for these items, since these are specific to gerrit,
-    # and we don't want users directly using them any way other than how the
-    # gerrit ref-updated hook is called.
-    parser.add_option('--oldrev', action='store', help=optparse.SUPPRESS_HELP)
-    parser.add_option('--newrev', action='store', help=optparse.SUPPRESS_HELP)
-    parser.add_option('--refname', action='store', help=optparse.SUPPRESS_HELP)
-    parser.add_option('--project', action='store', help=optparse.SUPPRESS_HELP)
-    parser.add_option('--submitter', action='store', help=optparse.SUPPRESS_HELP)
-
-    # The following allow this to be run as a stash asynchronous post-receive
-    # hook (almost identical to a git post-receive hook but triggered also for
-    # merges of pull requests from the UI).  We suppress help for these items,
-    # since these are specific to stash.
-    parser.add_option('--stash-user', action='store', help=optparse.SUPPRESS_HELP)
-    parser.add_option('--stash-repo', action='store', help=optparse.SUPPRESS_HELP)
-
-    (options, args) = parser.parse_args(args)
-    (options, args, hook_info) = check_hook_specific_args(options, args)
-
-    if options.version:
-        sys.stdout.write('git-multimail version ' + get_version() + '\n')
-        return
-
-    if options.python_version:
-        sys.stdout.write('Python version ' + sys.version + '\n')
-        return
-
-    if options.c:
-        Config.add_config_parameters(options.c)
-
-    config = Config('multimailhook')
-
-    environment = None
-    try:
-        environment = choose_environment(
-            config, osenv=os.environ,
-            env=options.environment,
-            recipients=options.recipients,
-            hook_info=hook_info,
-            )
-
-        if options.show_env:
-            show_env(environment, sys.stderr)
-
-        if options.stdout or environment.stdout:
-            mailer = OutputMailer(sys.stdout, environment)
-        else:
-            mailer = choose_mailer(config, environment)
-
-        must_check_setup = os.environ.get('GIT_MULTIMAIL_CHECK_SETUP')
-        if must_check_setup == '':
-            must_check_setup = False
-        if options.check_ref_filter:
-            check_ref_filter(environment)
-        elif must_check_setup:
-            check_setup(environment)
-        # Dual mode: if arguments were specified on the command line, run
-        # like an update hook; otherwise, run as a post-receive hook.
-        elif args:
-            if len(args) != 3:
-                parser.error('Need zero or three non-option arguments')
-            (refname, oldrev, newrev) = args
-            environment.get_logger().debug(
-                "run_as_update_hook: refname=%s, oldrev=%s, newrev=%s, force_send=%s" %
-                (refname, oldrev, newrev, options.force_send))
-            run_as_update_hook(environment, mailer, refname, oldrev, newrev, options.force_send)
-        else:
-            run_as_post_receive_hook(environment, mailer)
-    except ConfigurationException:
-        sys.exit(sys.exc_info()[1])
-    except SystemExit:
-        raise
-    except Exception:
-        t, e, tb = sys.exc_info()
-        import traceback
-        sys.stderr.write('\n')  # Avoid mixing message with previous output
-        msg = (
-            'Exception \'' + t.__name__ +
-            '\' raised. Please report this as a bug to\n'
-            'https://github.com/git-multimail/git-multimail/issues\n'
-            'with the information below:\n\n'
-            'git-multimail version ' + get_version() + '\n'
-            'Python version ' + sys.version + '\n' +
-            traceback.format_exc())
-        try:
-            environment.get_logger().error(msg)
-        except:
-            sys.stderr.write(msg)
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main(sys.argv[1:])
diff --git a/contrib/hooks/multimail/migrate-mailhook-config b/contrib/hooks/multimail/migrate-mailhook-config
deleted file mode 100755
index 241ba22fa3..0000000000
--- a/contrib/hooks/multimail/migrate-mailhook-config
+++ /dev/null
@@ -1,274 +0,0 @@
-#! /usr/bin/env python
-
-"""Migrate a post-receive-email configuration to be usable with git_multimail.py.
-
-See README.migrate-from-post-receive-email for more information.
-
-"""
-
-import sys
-import optparse
-
-from git_multimail import CommandError
-from git_multimail import Config
-from git_multimail import read_output
-
-
-OLD_NAMES = [
-    'mailinglist',
-    'announcelist',
-    'envelopesender',
-    'emailprefix',
-    'showrev',
-    'emailmaxlines',
-    'diffopts',
-    'scancommitforcc',
-    ]
-
-NEW_NAMES = [
-    'environment',
-    'reponame',
-    'mailinglist',
-    'refchangelist',
-    'commitlist',
-    'announcelist',
-    'announceshortlog',
-    'envelopesender',
-    'administrator',
-    'emailprefix',
-    'emailmaxlines',
-    'diffopts',
-    'emaildomain',
-    'scancommitforcc',
-    ]
-
-
-INFO = """\
-
-SUCCESS!
-
-Your post-receive-email configuration has been converted to
-git-multimail format.  Please see README and
-README.migrate-from-post-receive-email to learn about other
-git-multimail configuration possibilities.
-
-For example, git-multimail has the following new options with no
-equivalent in post-receive-email.  You might want to read about them
-to see if they would be useful in your situation:
-
-"""
-
-
-def _check_old_config_exists(old):
-    """Check that at least one old configuration value is set."""
-
-    for name in OLD_NAMES:
-        if name in old:
-            return True
-
-    return False
-
-
-def _check_new_config_clear(new):
-    """Check that none of the new configuration names are set."""
-
-    retval = True
-    for name in NEW_NAMES:
-        if name in new:
-            if retval:
-                sys.stderr.write('INFO: The following configuration values already exist:\n\n')
-            sys.stderr.write('    "%s.%s"\n' % (new.section, name))
-            retval = False
-
-    return retval
-
-
-def erase_values(config, names):
-    for name in names:
-        if name in config:
-            try:
-                sys.stderr.write('...unsetting "%s.%s"\n' % (config.section, name))
-                config.unset_all(name)
-            except CommandError:
-                sys.stderr.write(
-                    '\nWARNING: could not unset "%s.%s".  '
-                    'Perhaps it is not set at the --local level?\n\n'
-                    % (config.section, name)
-                    )
-
-
-def is_section_empty(section, local):
-    """Return True iff the specified configuration section is empty.
-
-    Iff local is True, use the --local option when invoking 'git
-    config'."""
-
-    if local:
-        local_option = ['--local']
-    else:
-        local_option = []
-
-    try:
-        read_output(
-            ['git', 'config'] +
-            local_option +
-            ['--get-regexp', '^%s\.' % (section,)]
-            )
-    except CommandError:
-        t, e, traceback = sys.exc_info()
-        if e.retcode == 1:
-            # This means that no settings were found.
-            return True
-        else:
-            raise
-    else:
-        return False
-
-
-def remove_section_if_empty(section):
-    """If the specified configuration section is empty, delete it."""
-
-    try:
-        empty = is_section_empty(section, local=True)
-    except CommandError:
-        # Older versions of git do not support the --local option, so
-        # if the first attempt fails, try without --local.
-        try:
-            empty = is_section_empty(section, local=False)
-        except CommandError:
-            sys.stderr.write(
-                '\nINFO: If configuration section "%s.*" is empty, you might want '
-                'to delete it.\n\n'
-                % (section,)
-                )
-            return
-
-    if empty:
-        sys.stderr.write('...removing section "%s.*"\n' % (section,))
-        read_output(['git', 'config', '--remove-section', section])
-    else:
-        sys.stderr.write(
-            '\nINFO: Configuration section "%s.*" still has contents.  '
-            'It will not be deleted.\n\n'
-            % (section,)
-            )
-
-
-def migrate_config(strict=False, retain=False, overwrite=False):
-    old = Config('hooks')
-    new = Config('multimailhook')
-    if not _check_old_config_exists(old):
-        sys.exit(
-            'Your repository has no post-receive-email configuration.  '
-            'Nothing to do.'
-            )
-    if not _check_new_config_clear(new):
-        if overwrite:
-            sys.stderr.write('\nWARNING: Erasing the above values...\n\n')
-            erase_values(new, NEW_NAMES)
-        else:
-            sys.exit(
-                '\nERROR: Refusing to overwrite existing values.  Use the --overwrite\n'
-                'option to continue anyway.'
-                )
-
-    name = 'showrev'
-    if name in old:
-        msg = 'git-multimail does not support "%s.%s"' % (old.section, name,)
-        if strict:
-            sys.exit(
-                'ERROR: %s.\n'
-                'Please unset that value then try again, or run without --strict.'
-                % (msg,)
-                )
-        else:
-            sys.stderr.write('\nWARNING: %s (ignoring).\n\n' % (msg,))
-
-    for name in ['mailinglist', 'announcelist']:
-        if name in old:
-            sys.stderr.write(
-                '...copying "%s.%s" to "%s.%s"\n' % (old.section, name, new.section, name)
-                )
-            old_recipients = old.get_all(name, default=None)
-            old_recipients = ', '.join(o.strip() for o in old_recipients)
-            new.set_recipients(name, old_recipients)
-
-    if strict:
-        sys.stderr.write(
-            '...setting "%s.commitlist" to the empty string\n' % (new.section,)
-            )
-        new.set_recipients('commitlist', '')
-        sys.stderr.write(
-            '...setting "%s.announceshortlog" to "true"\n' % (new.section,)
-            )
-        new.set('announceshortlog', 'true')
-
-    for name in ['envelopesender', 'emailmaxlines', 'diffopts', 'scancommitforcc']:
-        if name in old:
-            sys.stderr.write(
-                '...copying "%s.%s" to "%s.%s"\n' % (old.section, name, new.section, name)
-                )
-            new.set(name, old.get(name))
-
-    name = 'emailprefix'
-    if name in old:
-        sys.stderr.write(
-            '...copying "%s.%s" to "%s.%s"\n' % (old.section, name, new.section, name)
-            )
-        new.set(name, old.get(name))
-    elif strict:
-        sys.stderr.write(
-            '...setting "%s.%s" to "[SCM]" to preserve old subject lines\n'
-            % (new.section, name)
-            )
-        new.set(name, '[SCM]')
-
-    if not retain:
-        erase_values(old, OLD_NAMES)
-        remove_section_if_empty(old.section)
-
-    sys.stderr.write(INFO)
-    for name in NEW_NAMES:
-        if name not in OLD_NAMES:
-            sys.stderr.write('    "%s.%s"\n' % (new.section, name,))
-    sys.stderr.write('\n')
-
-
-def main(args):
-    parser = optparse.OptionParser(
-        description=__doc__,
-        usage='%prog [OPTIONS]',
-        )
-
-    parser.add_option(
-        '--strict', action='store_true', default=False,
-        help=(
-            'Slavishly configure git-multimail as closely as possible to '
-            'the post-receive-email configuration.  Default is to turn '
-            'on some new features that have no equivalent in post-receive-email.'
-            ),
-        )
-    parser.add_option(
-        '--retain', action='store_true', default=False,
-        help=(
-            'Retain the post-receive-email configuration values.  '
-            'Default is to delete them after the new values are set.'
-            ),
-        )
-    parser.add_option(
-        '--overwrite', action='store_true', default=False,
-        help=(
-            'Overwrite any existing git-multimail configuration settings.  '
-            'Default is to abort if such settings already exist.'
-            ),
-        )
-
-    (options, args) = parser.parse_args(args)
-
-    if args:
-        parser.error('Unexpected arguments: %s' % (' '.join(args),))
-
-    migrate_config(strict=options.strict, retain=options.retain, overwrite=options.overwrite)
-
-
-main(sys.argv[1:])
diff --git a/contrib/hooks/multimail/post-receive.example b/contrib/hooks/multimail/post-receive.example
deleted file mode 100755
index 0f98c5a23d..0000000000
--- a/contrib/hooks/multimail/post-receive.example
+++ /dev/null
@@ -1,101 +0,0 @@
-#! /usr/bin/env python
-
-"""Example post-receive hook based on git-multimail.
-
-The simplest way to use git-multimail is to use the script
-git_multimail.py directly as a post-receive hook, and to configure it
-using Git's configuration files and command-line parameters.  You can
-also write your own Python wrapper for more advanced configurability,
-using git_multimail.py as a Python module.
-
-This script is a simple example of such a post-receive hook.  It is
-intended to be customized before use; see the comments in the script
-to help you get started.
-
-Using git-multimail as a Python module as done here provides more
-flexibility.  It has the following advantages:
-
-* The tool's behavior can be customized using arbitrary Python code,
-  without having to edit git_multimail.py.
-
-* Configuration settings can be read from other sources; for example,
-  user names and email addresses could be read from LDAP or from a
-  database.  Or the settings can even be hardcoded in the importing
-  Python script, if this is preferred.
-
-This script is a very basic example of how to use git_multimail.py as
-a module.  The comments below explain some of the points at which the
-script's behavior could be changed or customized.
-
-"""
-
-import sys
-
-# If necessary, add the path to the directory containing
-# git_multimail.py to the Python path as follows.  (This is not
-# necessary if git_multimail.py is in the same directory as this
-# script):
-
-#LIBDIR = 'path/to/directory/containing/module'
-#sys.path.insert(0, LIBDIR)
-
-import git_multimail
-
-# It is possible to modify the output templates here; e.g.:
-
-#git_multimail.FOOTER_TEMPLATE = """\
-#
-#-- \n\
-#This email was generated by the wonderful git-multimail tool.
-#"""
-
-
-# Specify which "git config" section contains the configuration for
-# git-multimail:
-config = git_multimail.Config('multimailhook')
-
-# Set some Git configuration variables. Equivalent to passing var=val
-# to "git -c var=val" each time git is called, or to adding the
-# configuration in .git/config (must come before instantiating the
-# environment) :
-#git_multimail.Config.add_config_parameters('multimailhook.commitEmailFormat=html')
-#git_multimail.Config.add_config_parameters(('user.name=foo', 'user.email=foo@example.com'))
-
-# Select the type of environment:
-try:
-    environment = git_multimail.GenericEnvironment(config=config)
-    #environment = git_multimail.GitoliteEnvironment(config=config)
-except git_multimail.ConfigurationException:
-    sys.stderr.write('*** %s\n' % sys.exc_info()[1])
-    sys.exit(1)
-
-
-# Choose the method of sending emails based on the git config:
-mailer = git_multimail.choose_mailer(config, environment)
-
-# Alternatively, you may hardcode the mailer using code like one of
-# the following:
-
-# Use "/usr/sbin/sendmail -oi -t" to send emails.  The envelopesender
-# argument is optional:
-#mailer = git_multimail.SendMailer(
-#    command=['/usr/sbin/sendmail', '-oi', '-t'],
-#    envelopesender='git-repo@example.com',
-#    )
-
-# Use Python's smtplib to send emails.  Both arguments are required.
-#mailer = git_multimail.SMTPMailer(
-#    environment=environment,
-#    envelopesender='git-repo@example.com',
-#    # The smtpserver argument can also include a port number; e.g.,
-#    #     smtpserver='mail.example.com:25'
-#    smtpserver='mail.example.com',
-#    )
-
-# OutputMailer is intended only for testing; it writes the emails to
-# the specified file stream.
-#mailer = git_multimail.OutputMailer(sys.stdout)
-
-
-# Read changes from stdin and send notification emails:
-git_multimail.run_as_post_receive_hook(environment, mailer)

From cd0a852981162fb77f9fac68f231e3d41a99c993 Mon Sep 17 00:00:00 2001
From: Matthew Rogers 
Date: Sun, 6 Jun 2021 12:02:52 +0000
Subject: [PATCH 224/397] cmake: add knob to disable vcpkg

When building on windows users have the option to use vcpkg to provide
the dependencies needed to compile.  Previously, this was used only when
using the Visual Studio generator which was not ideal because:

  - Not all users who want to use vcpkg use the Visual Studio
    generators.

  - Some versions of Visual Studio 2019 moved away from using the
    VS 2019  generator by default, making it impossible for Visual
    Studio to configure the project in the likely event that it couldn't
    find the dependencies.

  - Inexperienced users of CMake are very likely to get tripped up by
    the errors caused by a lack of vcpkg, making the above bullet point
    both annoying and hard to debug.

As such, let's make using vcpkg the default on windows.  Users who want
to avoid using vcpkg can disable it by passing -DNO_VCPKG=TRUE.

Signed-off-by: Matthew Rogers 
Acked-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 contrib/buildsystems/CMakeLists.txt | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt
index ac3dbc079a..083a3bf31f 100644
--- a/contrib/buildsystems/CMakeLists.txt
+++ b/contrib/buildsystems/CMakeLists.txt
@@ -43,14 +43,23 @@ NOTE: By default CMake uses Makefile as the build tool on Linux and Visual Studi
 to use another tool say `ninja` add this to the command line when configuring.
 `-G Ninja`
 
+NOTE: By default CMake will install vcpkg locally to your source tree on configuration,
+to avoid this, add `-DNO_VCPKG=TRUE` to the command line when configuring.
+
 ]]
 cmake_minimum_required(VERSION 3.14)
 
 #set the source directory to root of git
 set(CMAKE_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/../..)
-if(WIN32)
+
+option(USE_VCPKG "Whether or not to use vcpkg for obtaining dependencies.  Only applicable to Windows platforms" ON)
+if(NOT WIN32)
+	set(USE_VCPKG OFF CACHE BOOL FORCE)
+endif()
+
+if(USE_VCPKG)
 	set(VCPKG_DIR "${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg")
-	if(MSVC AND NOT EXISTS ${VCPKG_DIR})
+	if(NOT EXISTS ${VCPKG_DIR})
 		message("Initializing vcpkg and building the Git's dependencies (this will take a while...)")
 		execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat)
 	endif()
@@ -174,7 +183,9 @@ endif()
 
 find_program(MSGFMT_EXE msgfmt)
 if(NOT MSGFMT_EXE)
-	set(MSGFMT_EXE ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg/downloads/tools/msys2/msys64/usr/bin/msgfmt.exe)
+	if (USE_VCPKG)
+		set(MSGFMT_EXE ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg/downloads/tools/msys2/msys64/usr/bin/msgfmt.exe)
+	endif()
 	if(NOT EXISTS ${MSGFMT_EXE})
 		message(WARNING "Text Translations won't be built")
 		unset(MSGFMT_EXE)
@@ -956,7 +967,7 @@ file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "X='${EXE_EXTENSION}'\n")
 file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_GETTEXT='${NO_GETTEXT}'\n")
 file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "RUNTIME_PREFIX='${RUNTIME_PREFIX}'\n")
 file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "NO_PYTHON='${NO_PYTHON}'\n")
-if(WIN32)
+if(USE_VCPKG)
 	file(APPEND ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/x64-windows/bin\"\n")
 endif()
 

From 409047a2b3fabb6a5f3fdbb28d93a5db3a7de28c Mon Sep 17 00:00:00 2001
From: Matthew Rogers 
Date: Sun, 6 Jun 2021 12:02:53 +0000
Subject: [PATCH 225/397] cmake: create compile_commands.json by default

Some users have expressed interest in a more "batteries included" way of
building via CMake[1], and a big part of that is providing easier access
to tooling external tools.

A straightforward way to accomplish this is to make it as simple as
possible is to enable the generation of the compile_commands.json file,
which is supported by many tools such as: clang-tidy, clang-format,
sourcetrail, etc.

This does come with a small run-time overhead during the configuration
step (~6 seconds on my machine):

    Time to configure with CMAKE_EXPORT_COMPILE_COMMANDS=TRUE

    real    1m9.840s
    user    0m0.031s
    sys     0m0.031s

    Time to configure with CMAKE_EXPORT_COMPILE_COMMANDS=FALSE

    real    1m3.195s
    user    0m0.015s
    sys     0m0.015s

This seems like a small enough price to pay to make the project more
accessible to newer users.  Additionally there are other large projects
like llvm [2] which has had this enabled by default for >6 years at the
time of this writing, and no real negative consequences that I can find
with my search-skills.

NOTE: That the compile_commands.json is currently produced only when
using the Ninja and Makefile generators.  See The CMake documentation[3]
for more info.

1: https://lore.kernel.org/git/CAOjrSZusMSvs7AS-ZDsV8aQUgsF2ZA754vSDjgFKMRgi_oZAWw@mail.gmail.com/
2: https://github.com/llvm/llvm-project/commit/2c5712051b31b316a9fc972f692579bd8efa6e67
3: https://cmake.org/cmake/help/latest/variable/CMAKE_EXPORT_COMPILE_COMMANDS.html

Signed-off-by: Matthew Rogers 
Acked-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 contrib/buildsystems/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt
index 083a3bf31f..67b722ffd4 100644
--- a/contrib/buildsystems/CMakeLists.txt
+++ b/contrib/buildsystems/CMakeLists.txt
@@ -57,6 +57,10 @@ if(NOT WIN32)
 	set(USE_VCPKG OFF CACHE BOOL FORCE)
 endif()
 
+if(NOT DEFINED CMAKE_EXPORT_COMPILE_COMMANDS)
+	set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
+endif()
+
 if(USE_VCPKG)
 	set(VCPKG_DIR "${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg")
 	if(NOT EXISTS ${VCPKG_DIR})

From ce24797d388ab0fc13605d25239ae08bcb9c6738 Mon Sep 17 00:00:00 2001
From: Matthew Rogers 
Date: Sun, 6 Jun 2021 12:02:54 +0000
Subject: [PATCH 226/397] cmake: add warning for ignored MSGFMT_EXE

It does not make sense to attempt to set MSGFMT_EXE when NO_GETTEXT is
configured, as such add a check for NO_GETTEXT before attempting to set
it.

Suggested-by: Johannes Schindelin 
Signed-off-by: Matthew Rogers 
Acked-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 contrib/buildsystems/CMakeLists.txt | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt
index 67b722ffd4..675e2ae6fb 100644
--- a/contrib/buildsystems/CMakeLists.txt
+++ b/contrib/buildsystems/CMakeLists.txt
@@ -185,14 +185,18 @@ if(WIN32 AND NOT MSVC)#not required for visual studio builds
 	endif()
 endif()
 
-find_program(MSGFMT_EXE msgfmt)
-if(NOT MSGFMT_EXE)
-	if (USE_VCPKG)
-		set(MSGFMT_EXE ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg/downloads/tools/msys2/msys64/usr/bin/msgfmt.exe)
-	endif()
-	if(NOT EXISTS ${MSGFMT_EXE})
-		message(WARNING "Text Translations won't be built")
-		unset(MSGFMT_EXE)
+if(NO_GETTEXT)
+	message(STATUS "msgfmt not used under NO_GETTEXT")
+else()
+	find_program(MSGFMT_EXE msgfmt)
+	if(NOT MSGFMT_EXE)
+		if(USE_VCPKG)
+			set(MSGFMT_EXE ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg/downloads/tools/msys2/msys64/usr/bin/msgfmt.exe)
+		endif()
+		if(NOT EXISTS ${MSGFMT_EXE})
+			message(WARNING "Text Translations won't be built")
+			unset(MSGFMT_EXE)
+		endif()
 	endif()
 endif()
 

From abcb66c614c574cfa1afccb230bf22cbde4d5557 Mon Sep 17 00:00:00 2001
From: Andrei Rybak 
Date: Fri, 11 Jun 2021 13:18:50 +0200
Subject: [PATCH 227/397] *: fix typos which duplicate a word

Fix typos in documentation, code comments, and RelNotes which repeat
various words.  In trivial cases, just delete the duplicated word and
rewrap text, if needed.  Reword the affected sentence in
Documentation/RelNotes/1.8.4.txt for it to make sense.

Signed-off-by: Andrei Rybak 
Signed-off-by: Junio C Hamano 
---
 Documentation/RelNotes/1.6.0.3.txt                   | 2 +-
 Documentation/RelNotes/1.8.4.txt                     | 2 +-
 Documentation/RelNotes/2.29.0.txt                    | 4 ++--
 Documentation/RelNotes/2.8.0.txt                     | 2 +-
 Documentation/technical/hash-function-transition.txt | 2 +-
 Documentation/technical/protocol-v2.txt              | 2 +-
 attr.c                                               | 2 +-
 builtin/log.c                                        | 3 +--
 git-compat-util.h                                    | 2 +-
 git-cvsserver.perl                                   | 2 +-
 merge-recursive.c                                    | 2 +-
 remote.c                                             | 2 +-
 t/t5505-remote.sh                                    | 2 +-
 t/t9100-git-svn-basic.sh                             | 2 +-
 14 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/Documentation/RelNotes/1.6.0.3.txt b/Documentation/RelNotes/1.6.0.3.txt
index ae0577836a..ad36c0f0b7 100644
--- a/Documentation/RelNotes/1.6.0.3.txt
+++ b/Documentation/RelNotes/1.6.0.3.txt
@@ -50,7 +50,7 @@ Fixes since v1.6.0.2
   if the working tree is currently dirty.
 
 * "git for-each-ref --format=%(subject)" fixed for commits with no
-  no newline in the message body.
+  newline in the message body.
 
 * "git remote" fixed to protect printf from user input.
 
diff --git a/Documentation/RelNotes/1.8.4.txt b/Documentation/RelNotes/1.8.4.txt
index 255e185af6..2e7529928b 100644
--- a/Documentation/RelNotes/1.8.4.txt
+++ b/Documentation/RelNotes/1.8.4.txt
@@ -365,7 +365,7 @@ details).
    (merge 2fbd4f9 mh/maint-lockfile-overflow later to maint).
 
  * Invocations of "git checkout" used internally by "git rebase" were
-   counted as "checkout", and affected later "git checkout -" to the
+   counted as "checkout", and affected later "git checkout -", which took
    the user to an unexpected place.
    (merge 3bed291 rr/rebase-checkout-reflog later to maint).
 
diff --git a/Documentation/RelNotes/2.29.0.txt b/Documentation/RelNotes/2.29.0.txt
index 06ba2f803f..1f41302ebb 100644
--- a/Documentation/RelNotes/2.29.0.txt
+++ b/Documentation/RelNotes/2.29.0.txt
@@ -184,8 +184,8 @@ Performance, Internal Implementation, Development Support etc.
    the ref backend in use, as its format is much richer than the
    normal refs, and written directly by "git fetch" as a plain file..
 
- * An unused binary has been discarded, and and a bunch of commands
-   have been turned into into built-in.
+ * An unused binary has been discarded, and a bunch of commands
+   have been turned into built-in.
 
  * A handful of places in in-tree code still relied on being able to
    execute the git subcommands, especially built-ins, in "git-foo"
diff --git a/Documentation/RelNotes/2.8.0.txt b/Documentation/RelNotes/2.8.0.txt
index 27320b6a9f..38453281b8 100644
--- a/Documentation/RelNotes/2.8.0.txt
+++ b/Documentation/RelNotes/2.8.0.txt
@@ -377,7 +377,7 @@ notes for details).
    on that order.
 
  * "git show 'HEAD:Foo[BAR]Baz'" did not interpret the argument as a
-   rev, i.e. the object named by the the pathname with wildcard
+   rev, i.e. the object named by the pathname with wildcard
    characters in a tree object.
    (merge aac4fac nd/dwim-wildcards-as-pathspecs later to maint).
 
diff --git a/Documentation/technical/hash-function-transition.txt b/Documentation/technical/hash-function-transition.txt
index 7c1630bf83..260224b033 100644
--- a/Documentation/technical/hash-function-transition.txt
+++ b/Documentation/technical/hash-function-transition.txt
@@ -599,7 +599,7 @@ supports four different modes of operation:
     convert any object names written to output to SHA-1, but store
     objects using SHA-256.  This allows users to test the code with no
     visible behavior change except for performance.  This allows
-    allows running even tests that assume the SHA-1 hash function, to
+    running even tests that assume the SHA-1 hash function, to
     sanity-check the behavior of the new mode.
 
  2. ("early transition") Allow both SHA-1 and SHA-256 object names in
diff --git a/Documentation/technical/protocol-v2.txt b/Documentation/technical/protocol-v2.txt
index a1e31367f4..1040d85319 100644
--- a/Documentation/technical/protocol-v2.txt
+++ b/Documentation/technical/protocol-v2.txt
@@ -540,7 +540,7 @@ An `object-info` request takes the following arguments:
 	Indicates to the server an object which the client wants to obtain
 	information for.
 
-The response of `object-info` is a list of the the requested object ids
+The response of `object-info` is a list of the requested object ids
 and associated requested information, each separated by a single space.
 
 	output = info flush-pkt
diff --git a/attr.c b/attr.c
index 9e897e43f5..d029e681f2 100644
--- a/attr.c
+++ b/attr.c
@@ -685,7 +685,7 @@ static struct attr_stack *read_attr_from_array(const char **list)
  * Callers into the attribute system assume there is a single, system-wide
  * global state where attributes are read from and when the state is flipped by
  * calling git_attr_set_direction(), the stack frames that have been
- * constructed need to be discarded so so that subsequent calls into the
+ * constructed need to be discarded so that subsequent calls into the
  * attribute system will lazily read from the right place.  Since changing
  * direction causes a global paradigm shift, it should not ever be called while
  * another thread could potentially be calling into the attribute system.
diff --git a/builtin/log.c b/builtin/log.c
index 6102893fcc..516a1142dd 100644
--- a/builtin/log.c
+++ b/builtin/log.c
@@ -1968,8 +1968,7 @@ int cmd_format_patch(int argc, const char **argv, const char *prefix)
 	} else if (rev.diffopt.close_file) {
 		/*
 		 * The diff code parsed --output; it has already opened the
-		 * file, but but we must instruct it not to close after each
-		 * diff.
+		 * file, but we must instruct it not to close after each diff.
 		 */
 		rev.diffopt.no_free = 1;
 	} else {
diff --git a/git-compat-util.h b/git-compat-util.h
index a508dbe5a3..df7dae9be1 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -1368,7 +1368,7 @@ static inline void *container_of_or_null_offset(void *ptr, size_t offset)
 	(type *)container_of_or_null_offset(ptr, offsetof(type, member))
 
 /*
- * like offsetof(), but takes a pointer to a a variable of type which
+ * like offsetof(), but takes a pointer to a variable of type which
  * contains @member, instead of a specified type.
  * @ptr is subject to multiple evaluation since we can't rely on __typeof__
  * everywhere.
diff --git a/git-cvsserver.perl b/git-cvsserver.perl
index f6f3fc192c..ed035f32c2 100755
--- a/git-cvsserver.perl
+++ b/git-cvsserver.perl
@@ -2149,7 +2149,7 @@ sub req_diff
                    ( $meta2->{revision} or "workingcopy" ));
 
         # TODO: Use --label instead of -L because -L is no longer
-        #  documented and may go away someday.  Not sure if there there are
+        #  documented and may go away someday.  Not sure if there are
         #  versions that only support -L, which would make this change risky?
         #  http://osdir.com/ml/bug-gnu-utils-gnu/2010-12/msg00060.html
         #    ("man diff" should actually document the best migration strategy,
diff --git a/merge-recursive.c b/merge-recursive.c
index d146bb116f..4327e0cfa3 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -2152,7 +2152,7 @@ static char *handle_path_level_conflicts(struct merge_options *opt,
  *      implicit renaming of files that should be left in place.  (See
  *      testcase 6b in t6043 for details.)
  *   2. Prune directory renames if there are still files left in the
- *      the original directory.  These represent a partial directory rename,
+ *      original directory.  These represent a partial directory rename,
  *      i.e. a rename where only some of the files within the directory
  *      were renamed elsewhere.  (Technically, this could be done earlier
  *      in get_directory_renames(), except that would prevent us from
diff --git a/remote.c b/remote.c
index 6d1e8d02df..dfb863d808 100644
--- a/remote.c
+++ b/remote.c
@@ -1592,7 +1592,7 @@ void set_ref_status_for_push(struct ref *remote_refs, int send_mirror,
 			else
 				/*
 				 * If the ref isn't stale, and is reachable
-				 * from from one of the reflog entries of
+				 * from one of the reflog entries of
 				 * the local branch, force the update.
 				 */
 				force_ref_update = 1;
diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh
index c7b392794b..e6e3c8f552 100755
--- a/t/t5505-remote.sh
+++ b/t/t5505-remote.sh
@@ -182,7 +182,7 @@ test_expect_success 'rename errors out early when deleting non-existent branch'
 	)
 '
 
-test_expect_success 'rename errors out early when when new name is invalid' '
+test_expect_success 'rename errors out early when new name is invalid' '
 	test_config remote.foo.vcs bar &&
 	echo "fatal: '\''invalid...name'\'' is not a valid remote name" >expect &&
 	test_must_fail git remote rename foo invalid...name 2>actual &&
diff --git a/t/t9100-git-svn-basic.sh b/t/t9100-git-svn-basic.sh
index 1d3fdcc997..ef35a54885 100755
--- a/t/t9100-git-svn-basic.sh
+++ b/t/t9100-git-svn-basic.sh
@@ -330,7 +330,7 @@ test_expect_success 'git-svn works in a bare repository' '
 	git svn fetch ) &&
 	rm -rf bare-repo
 	'
-test_expect_success 'git-svn works in in a repository with a gitdir: link' '
+test_expect_success 'git-svn works in a repository with a gitdir: link' '
 	mkdir worktree gitdir &&
 	( cd worktree &&
 	git svn init "$svnrepo" &&

From 670b81a890388c60b7032a4f5b879f2ece8c4558 Mon Sep 17 00:00:00 2001
From: Junio C Hamano 
Date: Mon, 14 Jun 2021 13:23:28 +0900
Subject: [PATCH 228/397] The second batch

Signed-off-by: Junio C Hamano 
---
 Documentation/RelNotes/2.33.0.txt | 50 +++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/Documentation/RelNotes/2.33.0.txt b/Documentation/RelNotes/2.33.0.txt
index 6795a2734f..57443c7466 100644
--- a/Documentation/RelNotes/2.33.0.txt
+++ b/Documentation/RelNotes/2.33.0.txt
@@ -1,15 +1,49 @@
 Git 2.33 Release Notes
 ======================
 
+Backward compatibility notes
+----------------------------
+
+ * The "-m" option in "git log -m" that does not specify which format,
+   if any, of diff is desired did not have any visible effect; it now
+   implies some form of diff (by default "--patch") is produced.
+
+   You can disable the diff output with "git log -m --no-patch", but
+   then there probably isn't much point in passing "-m" in the first
+   place ;-).
+
+
 Updates since Git 2.32
 ----------------------
 
 UI, Workflows & Features
 
+ * "git send-email" learned the "--sendmail-cmd" command line option
+   and the "sendemail.sendmailCmd" configuration variable, which is a
+   more sensible approach than the current way of repurposing the
+   "smtp-server" that is meant to name the server to instead name the
+   command to talk to the server.
+
+ * The "-m" option in "git log -m" that does not specify which format,
+   if any, of diff is desired did not have any visible effect; it now
+   implies some form of diff (by default "--patch") is produced.
+
+
 Performance, Internal Implementation, Development Support etc.
 
+ * The code to handle the "--format" option in "for-each-ref" and
+   friends made too many string comparisons on %(atom)s used in the
+   format string, which has been corrected by converting them into
+   enum when the format string is parsed.
 
-Fixes since v2.31
+ * Use the hashfile API in the codepath that writes the index file to
+   reduce code duplication.
+
+ * Repeated rename detections in a sequence of mergy operations have
+   been optimize out.
+
+
+Fixes since v2.32
 -----------------
 
  * We historically rejected a very short string as an author name
@@ -19,14 +53,26 @@ Fixes since v2.31
  * The parallel checkout codepath did not initialize object ID field
    used to talk to the worker processes in a futureproof way.
 
- * Rewrite code that triggers undefined behaiour warning.
+ * Rewrite code that triggers undefined behaviour warning.
    (merge aafa5df0df jn/size-t-casted-to-off-t-fix later to maint).
 
  * The description of "fast-forward" in the glossary has been updated.
    (merge e22f2daed0 ry/clarify-fast-forward-in-glossary later to maint).
 
+ * Recent "git clone" left a temporary directory behind when the
+   transport layer returned an failure.
+   (merge 6aacb7d861 jk/clone-clean-upon-transport-error later to maint).
+
+ * "git fetch" over protocol v2 left its side of the socket open after
+   it finished speaking, which unnecessarily wasted the resource on
+   the other side.
+   (merge ae1a7eefff jk/fetch-pack-v2-half-close-early later to maint).
+
  * Other code cleanup, docfix, build fix, etc.
    (merge bfe35a6165 ah/doc-describe later to maint).
    (merge f302c1e4aa jc/clarify-revision-range later to maint).
    (merge 3127ff90ea tl/fix-packfile-uri-doc later to maint).
    (merge a84216c684 jk/doc-color-pager later to maint).
+   (merge 4e0a64a713 ab/trace2-squelch-gcc-warning later to maint).
+   (merge 225f7fa847 ps/rev-list-object-type-filter later to maint).
+   (merge 5317dfeaed dd/honor-users-tar-in-tests later to maint).

From aa9ad6fee54898b9965f4fd26b3035fdd7b20f37 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Mon, 14 Jun 2021 08:05:44 -0400
Subject: [PATCH 229/397] bitmaps: don't recurse into trees already in the
 bitmap

If an object is already mentioned in a reachability bitmap we are
building, then by definition so are all of the objects it can reach. We
have an optimization to stop traversing commits when we see they are
already in the bitmap, but we don't do the same for trees.

It's generally unavoidable to recurse into trees for commits not yet
covered by bitmaps (since most commits generally do have unique
top-level trees). But they usually have subtrees that are shared with
other commits (i.e., all of the subtrees the commit _didn't_ touch). And
some of those commits (and their trees) may be covered by the bitmap.

Usually this isn't _too_ big a deal, because we'll visit those subtrees
only once in total for the whole walk. But if you have a large number of
unbitmapped commits, and if your tree is big, then you may end up
opening a lot of sub-trees for no good reason.

We can use the same optimization we do for commits here: when we are
about to open a tree, see if it's in the bitmap (either the one we are
building, or the "seen" bitmap which covers the UNINTERESTING side of
the bitmap when doing a set-difference).

This works especially well because we'll visit all commits before
hitting any trees. So even in a history like:

  A -- B

if "A" has a bitmap on disk but "B" doesn't, we'll already have OR-ed in
the results from A before looking at B's tree (so we really will only
look at trees touched by B).

For most repositories, the timings produced by p5310 are unspectacular.
Here's linux.git:

  Test                         HEAD^             HEAD
  --------------------------------------------------------------------
  5310.4: simulated clone      6.00(5.90+0.10)   5.98(5.90+0.08) -0.3%
  5310.5: simulated fetch      2.98(5.45+0.18)   2.85(5.31+0.18) -4.4%
  5310.7: rev-list (commits)   0.32(0.29+0.03)   0.33(0.30+0.03) +3.1%
  5310.8: rev-list (objects)   1.48(1.44+0.03)   1.49(1.44+0.05) +0.7%

Any improvement there is within the noise (the +3.1% on test 7 has to be
noise, since we are not recursing into trees, and thus the new code
isn't even run). The results for git.git are likewise uninteresting.

But here are numbers from some other real-world repositories (that are
not public). This one's tree is comparable in size to linux.git, but has
~16k refs (and so less complete bitmap coverage):

  Test                         HEAD^               HEAD
  -------------------------------------------------------------------------
  5310.4: simulated clone      38.34(39.86+0.74)   33.95(35.53+0.76) -11.5%
  5310.5: simulated fetch      2.29(6.31+0.35)     2.20(5.97+0.41) -3.9%
  5310.7: rev-list (commits)   0.99(0.86+0.13)     0.96(0.85+0.11) -3.0%
  5310.8: rev-list (objects)   11.32(11.04+0.27)   6.59(6.37+0.21) -41.8%

And here's another with a very large tree (~340k entries), and a fairly
large number of refs (~10k):

  Test                         HEAD^               HEAD
  -------------------------------------------------------------------------
  5310.3: simulated clone      53.83(54.71+1.54)   39.77(40.76+1.50) -26.1%
  5310.4: simulated fetch      19.91(20.11+0.56)   19.79(19.98+0.67) -0.6%
  5310.6: rev-list (commits)   0.54(0.44+0.11)     0.51(0.43+0.07) -5.6%
  5310.7: rev-list (objects)   24.32(23.59+0.73)   9.85(9.49+0.36) -59.5%

This patch provides substantial improvements in these larger cases, and
have any drawbacks for smaller ones (the cost of the bitmap check is
quite small compared to an actual tree traversal).

Note that we have to add a version of revision.c's include_check
callback which handles non-commits. We could possibly consolidate this
into a single callback for all objects types, as there's only one user
of the feature which would need converted (pack-bitmap.c:should_include).
That would in theory let us avoid duplicating any logic. But when I
tried it, the code ended up much worse to read, with lots of repeated
"if it's a commit do this, otherwise do that". Having two separate
callbacks splits that naturally, and matches the existing split of
show_commit/show_object callbacks.

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 list-objects.c |  3 +++
 pack-bitmap.c  | 18 ++++++++++++++++++
 revision.h     |  1 +
 3 files changed, 22 insertions(+)

diff --git a/list-objects.c b/list-objects.c
index 7f404677d5..473a332416 100644
--- a/list-objects.c
+++ b/list-objects.c
@@ -164,6 +164,9 @@ static void process_tree(struct traversal_context *ctx,
 		die("bad tree object");
 	if (obj->flags & (UNINTERESTING | SEEN))
 		return;
+	if (revs->include_check_obj &&
+	    !revs->include_check_obj(&tree->object, revs->include_check_data))
+		return;
 
 	failed_parse = parse_tree_gently(tree, 1);
 	if (failed_parse) {
diff --git a/pack-bitmap.c b/pack-bitmap.c
index d90e1d9d8c..bfc10148f5 100644
--- a/pack-bitmap.c
+++ b/pack-bitmap.c
@@ -525,6 +525,22 @@ static int should_include(struct commit *commit, void *_data)
 	return 1;
 }
 
+static int should_include_obj(struct object *obj, void *_data)
+{
+	struct include_data *data = _data;
+	int bitmap_pos;
+
+	bitmap_pos = bitmap_position(data->bitmap_git, &obj->oid);
+	if (bitmap_pos < 0)
+		return 1;
+	if ((data->seen && bitmap_get(data->seen, bitmap_pos)) ||
+	     bitmap_get(data->base, bitmap_pos)) {
+		obj->flags |= SEEN;
+		return 0;
+	}
+	return 1;
+}
+
 static int add_commit_to_bitmap(struct bitmap_index *bitmap_git,
 				struct bitmap **base,
 				struct commit *commit)
@@ -620,6 +636,7 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
 		incdata.seen = seen;
 
 		revs->include_check = should_include;
+		revs->include_check_obj = should_include_obj;
 		revs->include_check_data = &incdata;
 
 		if (prepare_revision_walk(revs))
@@ -633,6 +650,7 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
 					      &show_data, NULL);
 
 		revs->include_check = NULL;
+		revs->include_check_obj = NULL;
 		revs->include_check_data = NULL;
 	}
 
diff --git a/revision.h b/revision.h
index 93aa012f51..7bb5fa4e7c 100644
--- a/revision.h
+++ b/revision.h
@@ -262,6 +262,7 @@ struct rev_info {
 	int min_parents;
 	int max_parents;
 	int (*include_check)(struct commit *, void *);
+	int (*include_check_obj)(struct object *obj, void *);
 	void *include_check_data;
 
 	/* diff info for patches and for paths limiting */

From f7ee88f1d09ce47b8e12b52a2f19c7867ae10b29 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Mon, 14 Jun 2021 12:41:52 +0000
Subject: [PATCH 230/397] subtree: fix the GIT_EXEC_PATH sanity check to work
 on Windows

In 22d550749361 (subtree: don't fuss with PATH, 2021-04-27), `git
subtree` was broken thoroughly on Windows.

The reason is that it assumes Unix semantics, where `PATH` is
colon-separated, and it assumes that `$GIT_EXEC_PATH:` is a verbatim
prefix of `$PATH`. Neither are true, the latter in particular because
`GIT_EXEC_PATH` is a Windows-style path, while `PATH` is a Unix-style
path list.

Let's make extra certain that `$GIT_EXEC_PATH` and the first component
of `$PATH` refer to different entities before erroring out.

We do that by using the `test  -ef ` command that verifies
that the inode of `` and of `` is the same.

Sadly, this construct is non-portable, according to
https://pubs.opengroup.org/onlinepubs/009695399/utilities/test.html.
However, it does not matter in practice because we still first look
whether `$GIT_EXEC_PREFIX` is string-identical to the first component of
`$PATH`. This will give us the expected result everywhere but in Git for
Windows, and Git for Windows' own Bash _does_ handle the `-ef` operator.

Just in case that we _do_ need to show the error message _and_ are
running in a shell that lacks support for `-ef`, we simply suppress the
error output for that part.

This fixes https://github.com/git-for-windows/git/issues/3260

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 contrib/subtree/git-subtree.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/contrib/subtree/git-subtree.sh b/contrib/subtree/git-subtree.sh
index b06782bc79..3935cea7dd 100755
--- a/contrib/subtree/git-subtree.sh
+++ b/contrib/subtree/git-subtree.sh
@@ -5,7 +5,10 @@
 # Copyright (C) 2009 Avery Pennarun 
 #
 
-if test -z "$GIT_EXEC_PATH" || test "${PATH#"${GIT_EXEC_PATH}:"}" = "$PATH" || ! test -f "$GIT_EXEC_PATH/git-sh-setup"
+if test -z "$GIT_EXEC_PATH" || ! test -f "$GIT_EXEC_PATH/git-sh-setup" || {
+	test "${PATH#"${GIT_EXEC_PATH}:"}" = "$PATH" &&
+	test ! "$GIT_EXEC_PATH" -ef "${PATH%%:*}" 2>/dev/null
+}
 then
 	echo >&2 'It looks like either your git installation or your'
 	echo >&2 'git-subtree installation is broken.'

From 77f37de39fa96e4e5d59cda1e7e85c4784b48ee9 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Mon, 14 Jun 2021 12:41:53 +0000
Subject: [PATCH 231/397] subtree: fix assumption about the directory separator

On Windows, both forward and backslash are valid separators. In
22d550749361 (subtree: don't fuss with PATH, 2021-04-27), however, we
added code that assumes that it can only be the forward slash.

Let's fix that.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 contrib/subtree/git-subtree.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/contrib/subtree/git-subtree.sh b/contrib/subtree/git-subtree.sh
index 3935cea7dd..7f767b5c38 100755
--- a/contrib/subtree/git-subtree.sh
+++ b/contrib/subtree/git-subtree.sh
@@ -10,6 +10,7 @@ if test -z "$GIT_EXEC_PATH" || ! test -f "$GIT_EXEC_PATH/git-sh-setup" || {
 	test ! "$GIT_EXEC_PATH" -ef "${PATH%%:*}" 2>/dev/null
 }
 then
+	basename=${0##*[/\\]}
 	echo >&2 'It looks like either your git installation or your'
 	echo >&2 'git-subtree installation is broken.'
 	echo >&2
@@ -17,10 +18,10 @@ then
 	echo >&2 " - If \`git --exec-path\` does not print the correct path to"
 	echo >&2 "   your git install directory, then set the GIT_EXEC_PATH"
 	echo >&2 "   environment variable to the correct directory."
-	echo >&2 " - Make sure that your \`${0##*/}\` file is either in your"
+	echo >&2 " - Make sure that your \`$basename\` file is either in your"
 	echo >&2 "   PATH or in your git exec path (\`$(git --exec-path)\`)."
-	echo >&2 " - You should run git-subtree as \`git ${0##*/git-}\`,"
-	echo >&2 "   not as \`${0##*/}\`." >&2
+	echo >&2 " - You should run git-subtree as \`git ${basename#git-}\`,"
+	echo >&2 "   not as \`$basename\`." >&2
 	exit 126
 fi
 

From fce3b089df2edf3a0b3825aef62d243b8a2e0146 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Mon, 14 Jun 2021 19:28:18 +0200
Subject: [PATCH 232/397] mktag tests: parse out options in helper
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change check_verify_failure() helper to parse out options from
$@. This makes it easier to add new options in the future. See
06ce79152be (mktag: add a --[no-]strict option, 2021-01-06) for the
initial implementation.

Let's also replace "" quotes with '' for the test body, the varables
we need are eval'd into the body, so there's no need for the quoting
confusion.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3800-mktag.sh | 43 +++++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/t/t3800-mktag.sh b/t/t3800-mktag.sh
index 6275c98523..e9008744e3 100755
--- a/t/t3800-mktag.sh
+++ b/t/t3800-mktag.sh
@@ -12,15 +12,29 @@ test_description='git mktag: tag object verify test'
 # given in the expect.pat file.
 
 check_verify_failure () {
-	test_expect_success "$1" "
-		test_must_fail git mktag message &&
-		grep '$2' message &&
-		if test '$3' != '--no-strict'
+	subject=$1 &&
+	message=$2 &&
+	shift 2 &&
+
+	no_strict= &&
+	while test $# != 0
+	do
+		case "$1" in
+		--no-strict)
+			no_strict=yes
+			;;
+		esac &&
+		shift
+	done &&
+
+	test_expect_success "fail with [--[no-]strict]: $subject" '
+		test_must_fail git mktag err &&
+		if test -z "$no_strict"
 		then
-			test_must_fail git mktag --no-strict message.no-strict &&
-			grep '$2' message.no-strict
+			test_must_fail git mktag err2 &&
+			test_cmp err err2
 		fi
-	"
+	'
 }
 
 test_expect_mktag_success() {
@@ -243,7 +257,8 @@ tagger . <> 0 +0000
 EOF
 
 check_verify_failure 'verify tag-name check' \
-	'^error:.* badTagName:' '--no-strict'
+	'^error:.* badTagName:' \
+	--no-strict
 
 ############################################################
 # 11. tagger line label check #1
@@ -257,7 +272,8 @@ This is filler
 EOF
 
 check_verify_failure '"tagger" line label check #1' \
-	'^error:.* missingTaggerEntry:' '--no-strict'
+	'^error:.* missingTaggerEntry:' \
+	--no-strict
 
 ############################################################
 # 12. tagger line label check #2
@@ -272,7 +288,8 @@ This is filler
 EOF
 
 check_verify_failure '"tagger" line label check #2' \
-	'^error:.* missingTaggerEntry:' '--no-strict'
+	'^error:.* missingTaggerEntry:' \
+	--no-strict
 
 ############################################################
 # 13. allow missing tag author name like fsck
@@ -301,7 +318,8 @@ tagger T A Gger <
 EOF
 
 check_verify_failure 'disallow malformed tagger' \
-	'^error:.* badEmail:' '--no-strict'
+	'^error:.* badEmail:' \
+	--no-strict
 
 ############################################################
 # 15. allow empty tag email
@@ -425,7 +443,8 @@ this line should not be here
 EOF
 
 check_verify_failure 'detect invalid header entry' \
-	'^error:.* extraHeaderEntry:' '--no-strict'
+	'^error:.* extraHeaderEntry:' \
+	--no-strict
 
 test_expect_success 'invalid header entry config & fsck' '
 	test_must_fail git mktag 
Date: Mon, 14 Jun 2021 19:28:19 +0200
Subject: [PATCH 233/397] mktag tests: invert --no-strict test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change the mktag --no-strict test to actually test success under
--no-strict, that test was added in 06ce79152be (mktag: add a
--[no-]strict option, 2021-01-06).

It doesn't make sense to check that we have the same failure except
when we want --no-strict, by doing that we're assuming that the
behavior will be different under --no-strict, bun nothing was testing
for that.

We should instead assert that --strict is the same as --no-strict,
except in the cases where we've declared that it's not.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3800-mktag.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/t/t3800-mktag.sh b/t/t3800-mktag.sh
index e9008744e3..951e6d39c2 100755
--- a/t/t3800-mktag.sh
+++ b/t/t3800-mktag.sh
@@ -33,6 +33,8 @@ check_verify_failure () {
 		then
 			test_must_fail git mktag err2 &&
 			test_cmp err err2
+		else
+			git mktag --no-strict 
Date: Mon, 14 Jun 2021 15:51:15 +0000
Subject: [PATCH 234/397] split-index: use oideq instead of memcmp to compare
 object_id's

cache_entry contains an object_id, and compare_ce_content() would
include that field when calling memcmp on a subset of the cache_entry.
Depending on which hashing algorithm is being used, only part of
object_id.hash is actually being used, therefore including it in a
memcmp() is incorrect. Instead we choose to exclude the object_id when
calling memcmp(), and call oideq() separately.

This issue was found when running t1700-split-index with MSAN, see MSAN
output below (on my machine, offset 76 corresponds to 4 bytes after the
start of object_id.hash).

Uninitialized bytes in MemcmpInterceptorCommon at offset 76 inside [0x7f60e7c00118, 92)
==27914==WARNING: MemorySanitizer: use-of-uninitialized-value
    #0 0x4524ee in memcmp /home/abuild/rpmbuild/BUILD/llvm-11.0.0.src/build/../projects/compiler-rt/lib/msan/../sanitizer_common/sanitizer_common_interceptors.inc:873:10
    #1 0xc867ae in compare_ce_content /home/ahunt/git/git/split-index.c:208:8
    #2 0xc859fb in prepare_to_write_split_index /home/ahunt/git/git/split-index.c:336:9
    #3 0xb4bbca in write_split_index /home/ahunt/git/git/read-cache.c:3107:2
    #4 0xb42b4d in write_locked_index /home/ahunt/git/git/read-cache.c:3295:8
    #5 0x638058 in try_merge_strategy /home/ahunt/git/git/builtin/merge.c:758:7
    #6 0x63057f in cmd_merge /home/ahunt/git/git/builtin/merge.c:1663:9
    #7 0x4a1e76 in run_builtin /home/ahunt/git/git/git.c:461:11
    #8 0x49e1e7 in handle_builtin /home/ahunt/git/git/git.c:714:3
    #9 0x4a0c08 in run_argv /home/ahunt/git/git/git.c:781:4
    #10 0x49d5a8 in cmd_main /home/ahunt/git/git/git.c:912:19
    #11 0x7974da in main /home/ahunt/git/git/common-main.c:52:11
    #12 0x7f60e928e349 in __libc_start_main (/lib64/libc.so.6+0x24349)
    #13 0x421bd9 in _start /home/abuild/rpmbuild/BUILD/glibc-2.26/csu/../sysdeps/x86_64/start.S:120

  Uninitialized value was stored to memory at
    #0 0x447eb9 in __msan_memcpy /home/abuild/rpmbuild/BUILD/llvm-11.0.0.src/build/../projects/compiler-rt/lib/msan/msan_interceptors.cpp:1558:3
    #1 0xb4d1e6 in dup_cache_entry /home/ahunt/git/git/read-cache.c:3457:2
    #2 0xd214fa in add_entry /home/ahunt/git/git/unpack-trees.c:215:18
    #3 0xd1fae0 in keep_entry /home/ahunt/git/git/unpack-trees.c:2276:2
    #4 0xd1ff9e in twoway_merge /home/ahunt/git/git/unpack-trees.c:2504:11
    #5 0xd27028 in call_unpack_fn /home/ahunt/git/git/unpack-trees.c:593:12
    #6 0xd2443d in unpack_nondirectories /home/ahunt/git/git/unpack-trees.c:1106:12
    #7 0xd19435 in unpack_callback /home/ahunt/git/git/unpack-trees.c:1306:6
    #8 0xd0d7ff in traverse_trees /home/ahunt/git/git/tree-walk.c:532:17
    #9 0xd1773a in unpack_trees /home/ahunt/git/git/unpack-trees.c:1683:9
    #10 0xdc6370 in checkout /home/ahunt/git/git/merge-ort.c:3590:8
    #11 0xdc51c3 in merge_switch_to_result /home/ahunt/git/git/merge-ort.c:3728:7
    #12 0xa195a9 in merge_ort_recursive /home/ahunt/git/git/merge-ort-wrappers.c:58:2
    #13 0x637fff in try_merge_strategy /home/ahunt/git/git/builtin/merge.c:751:12
    #14 0x63057f in cmd_merge /home/ahunt/git/git/builtin/merge.c:1663:9
    #15 0x4a1e76 in run_builtin /home/ahunt/git/git/git.c:461:11
    #16 0x49e1e7 in handle_builtin /home/ahunt/git/git/git.c:714:3
    #17 0x4a0c08 in run_argv /home/ahunt/git/git/git.c:781:4
    #18 0x49d5a8 in cmd_main /home/ahunt/git/git/git.c:912:19
    #19 0x7974da in main /home/ahunt/git/git/common-main.c:52:11

  Uninitialized value was created by a heap allocation
    #0 0x44e73d in malloc /home/abuild/rpmbuild/BUILD/llvm-11.0.0.src/build/../projects/compiler-rt/lib/msan/msan_interceptors.cpp:901:3
    #1 0xd592f6 in do_xmalloc /home/ahunt/git/git/wrapper.c:41:8
    #2 0xd59248 in xmalloc /home/ahunt/git/git/wrapper.c:62:9
    #3 0xa17088 in mem_pool_alloc_block /home/ahunt/git/git/mem-pool.c:22:6
    #4 0xa16f78 in mem_pool_init /home/ahunt/git/git/mem-pool.c:44:3
    #5 0xb481b8 in load_all_cache_entries /home/ahunt/git/git/read-cache.c
    #6 0xb44d40 in do_read_index /home/ahunt/git/git/read-cache.c:2298:17
    #7 0xb48a1b in read_index_from /home/ahunt/git/git/read-cache.c:2389:8
    #8 0xbd5a0b in repo_read_index /home/ahunt/git/git/repository.c:276:8
    #9 0xb4bcaf in repo_read_index_unmerged /home/ahunt/git/git/read-cache.c:3326:2
    #10 0x62ed26 in cmd_merge /home/ahunt/git/git/builtin/merge.c:1362:6
    #11 0x4a1e76 in run_builtin /home/ahunt/git/git/git.c:461:11
    #12 0x49e1e7 in handle_builtin /home/ahunt/git/git/git.c:714:3
    #13 0x4a0c08 in run_argv /home/ahunt/git/git/git.c:781:4
    #14 0x49d5a8 in cmd_main /home/ahunt/git/git/git.c:912:19
    #15 0x7974da in main /home/ahunt/git/git/common-main.c:52:11
    #16 0x7f60e928e349 in __libc_start_main (/lib64/libc.so.6+0x24349)

SUMMARY: MemorySanitizer: use-of-uninitialized-value /home/abuild/rpmbuild/BUILD/llvm-11.0.0.src/build/../projects/compiler-rt/lib/msan/../sanitizer_common/sanitizer_common_interceptors.inc:873:10 in memcmp
Exiting

Signed-off-by: Andrzej Hunt 
Signed-off-by: Junio C Hamano 
---
 split-index.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/split-index.c b/split-index.c
index 4d6e52d46f..8e52e891c3 100644
--- a/split-index.c
+++ b/split-index.c
@@ -207,7 +207,8 @@ static int compare_ce_content(struct cache_entry *a, struct cache_entry *b)
 	b->ce_flags &= ondisk_flags;
 	ret = memcmp(&a->ce_stat_data, &b->ce_stat_data,
 		     offsetof(struct cache_entry, name) -
-		     offsetof(struct cache_entry, ce_stat_data));
+		     offsetof(struct cache_entry, oid)) ||
+		!oideq(&a->oid, &b->oid);
 	a->ce_flags = ce_flags;
 	b->ce_flags = base_flags;
 

From 4dbc55e87da408e746888d0e65064a87bf8eb8b5 Mon Sep 17 00:00:00 2001
From: Andrzej Hunt 
Date: Mon, 14 Jun 2021 15:51:16 +0000
Subject: [PATCH 235/397] builtin/checkout--worker: zero-initialise struct to
 avoid MSAN complaints

report_result() sends a struct to the parent process, but that struct
would contain uninitialised padding bytes. Running this code under MSAN
rightly triggers a warning - but we don't particularly care about this
warning because we control the receiving code, and we therefore know
that those padding bytes won't be read on the receiving end.

We could simply suppress this warning under MSAN with the approporiate
ifdef'd attributes, but a less intrusive solution is to 0-initialise the
struct, which guarantees that the padding will also be initialised.

Interestingly, in the error-case branch, we only try to copy the first
two members of pc_item_result, by copying only PC_ITEM_RESULT_BASE_SIZE
bytes. However PC_ITEM_RESULT_BASE_SIZE is defined as
'offsetof(the_last_member)', which means that we're copying padding bytes
after the end of the second last member. We could avoid doing this by
redefining PC_ITEM_RESULT_BASE_SIZE as
'offsetof(second_last_member) + sizeof(second_last_member)', but there's
no huge benefit to doing so (and this patch silences the MSAN warning in
this scenario either way).

MSAN output from t2080 (partially interleaved due to the
parallel work :) ):

Uninitialized bytes in __interceptor_write at offset 12 inside [0x7fff37d83408, 160)
==23279==WARNING: MemorySanitizer: use-of-uninitialized-value
Uninitialized bytes in __interceptor_write at offset 12 inside [0x7ffdb8a07ec8, 160)
==23280==WARNING: MemorySanitizer: use-of-uninitialized-value
    #0 0xd5ac28 in xwrite /home/ahunt/git/git/wrapper.c:256:8
    #1 0xd5b327 in write_in_full /home/ahunt/git/git/wrapper.c:311:21
    #2 0xb0a8c4 in do_packet_write /home/ahunt/git/git/pkt-line.c:221:6
    #3 0xb0a5fd in packet_write /home/ahunt/git/git/pkt-line.c:242:6
    #4 0x4f7441 in report_result /home/ahunt/git/git/builtin/checkout--worker.c:69:2
    #5 0x4f6be6 in worker_loop /home/ahunt/git/git/builtin/checkout--worker.c:100:3
    #6 0x4f68d3 in cmd_checkout__worker /home/ahunt/git/git/builtin/checkout--worker.c:143:2
    #7 0x4a1e76 in run_builtin /home/ahunt/git/git/git.c:461:11
    #8 0x49e1e7 in handle_builtin /home/ahunt/git/git/git.c:714:3
    #9 0x4a0c08 in run_argv /home/ahunt/git/git/git.c:781:4
    #10 0x49d5a8 in cmd_main /home/ahunt/git/git/git.c:912:19
    #11 0x7974da in main /home/ahunt/git/git/common-main.c:52:11
    #12 0x7f8778114349 in __libc_start_main (/lib64/libc.so.6+0x24349)
    #13 0x421bd9 in _start /home/abuild/rpmbuild/BUILD/glibc-2.26/csu/../sysdeps/x86_64/start.S:120

  Uninitialized value was created by an allocation of 'res' in the stack frame of function 'report_result'
    #0 0x4f72c0 in report_result /home/ahunt/git/git/builtin/checkout--worker.c:55

SUMMARY: MemorySanitizer: use-of-uninitialized-value /home/ahunt/git/git/wrapper.c:256:8 in xwrite
Exiting
    #0 0xd5ac28 in xwrite /home/ahunt/git/git/wrapper.c:256:8
    #1 0xd5b327 in write_in_full /home/ahunt/git/git/wrapper.c:311:21
    #2 0xb0a8c4 in do_packet_write /home/ahunt/git/git/pkt-line.c:221:6
    #3 0xb0a5fd in packet_write /home/ahunt/git/git/pkt-line.c:242:6
    #4 0x4f7441 in report_result /home/ahunt/git/git/builtin/checkout--worker.c:69:2
    #5 0x4f6be6 in worker_loop /home/ahunt/git/git/builtin/checkout--worker.c:100:3
    #6 0x4f68d3 in cmd_checkout__worker /home/ahunt/git/git/builtin/checkout--worker.c:143:2
    #7 0x4a1e76 in run_builtin /home/ahunt/git/git/git.c:461:11
    #8 0x49e1e7 in handle_builtin /home/ahunt/git/git/git.c:714:3
    #9 0x4a0c08 in run_argv /home/ahunt/git/git/git.c:781:4
    #10 0x49d5a8 in cmd_main /home/ahunt/git/git/git.c:912:19
    #11 0x7974da in main /home/ahunt/git/git/common-main.c:52:11
    #12 0x7f2749a0e349 in __libc_start_main (/lib64/libc.so.6+0x24349)
    #13 0x421bd9 in _start /home/abuild/rpmbuild/BUILD/glibc-2.26/csu/../sysdeps/x86_64/start.S:120

  Uninitialized value was created by an allocation of 'res' in the stack frame of function 'report_result'
    #0 0x4f72c0 in report_result /home/ahunt/git/git/builtin/checkout--worker.c:55

SUMMARY: MemorySanitizer: use-of-uninitialized-value /home/ahunt/git/git/wrapper.c:256:8 in xwrite

Signed-off-by: Andrzej Hunt 
Signed-off-by: Junio C Hamano 
---
 builtin/checkout--worker.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/builtin/checkout--worker.c b/builtin/checkout--worker.c
index 289a9b8f89..fb9fd13b73 100644
--- a/builtin/checkout--worker.c
+++ b/builtin/checkout--worker.c
@@ -53,7 +53,7 @@ static void packet_to_pc_item(const char *buffer, int len,
 
 static void report_result(struct parallel_checkout_item *pc_item)
 {
-	struct pc_item_result res;
+	struct pc_item_result res = { 0 };
 	size_t size;
 
 	res.id = pc_item->id;

From 4f5ce122ac05bdd24d1e0da95a7f878fabc78b20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Mon, 14 Jun 2021 19:18:07 +0200
Subject: [PATCH 236/397] show-branch tests: rename the one "show-branch" test
 file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Rename the only *show-branch* test file to indicate that more tests
belong it in than just the one-off octopus test it now contains.

The test was initially added in ce567d1867a (Add test to show that
show-branch misses out the 8th column, 2008-07-23) and
11ee57bc4c4 (sort_in_topological_order(): avoid setting a commit flag,
2008-07-23). Those two add almost the same content, one with a
test_expect_success and the other a test_expect_failure (a bug being
tested for was fixed on one of the branches).

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/{t3202-show-branch-octopus.sh => t3202-show-branch.sh} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename t/{t3202-show-branch-octopus.sh => t3202-show-branch.sh} (95%)

diff --git a/t/t3202-show-branch-octopus.sh b/t/t3202-show-branch.sh
similarity index 95%
rename from t/t3202-show-branch-octopus.sh
rename to t/t3202-show-branch.sh
index 5cb0126cfe..8cfbbf79c1 100755
--- a/t/t3202-show-branch-octopus.sh
+++ b/t/t3202-show-branch.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-test_description='test show-branch with more than 8 heads'
+test_description='test show-branch'
 
 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME

From 9b6e74a9c0133c4e34f6ec56cc93a6a83eba2c3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Mon, 14 Jun 2021 19:18:08 +0200
Subject: [PATCH 237/397] show-branch tests: modernize test code
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Modernize test code added in ce567d1867a (Add test to show that
show-branch misses out the 8th column, 2008-07-23) and
11ee57bc4c4 (sort_in_topological_order(): avoid setting a commit flag,
2008-07-23) to use test helpers.

I'm renaming "out" to "actual" for consistency with other tests, and
introducing a "branches.sorted" file in the setup, to make it clear
that it's important that the list be sorted in this particular way.

The "show-branch" output is indented with spaces, which would cause
complaints under "git show --check" with an indented here-doc
block. Let's prefix the lines with "> " to work around that, and to
make it clear that the leading whitespace is important.

We can also get rid of the hardcoding of "main" added here in
334afbc76fb (tests: mark tests relying on the current default for
`init.defaultBranch`, 2020-11-18). For this test we're setting up an
"initial" commit anyway, and now that we've moved over to test_commit
we can reference that instead.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3202-show-branch.sh | 92 ++++++++++++++++++------------------------
 1 file changed, 40 insertions(+), 52 deletions(-)

diff --git a/t/t3202-show-branch.sh b/t/t3202-show-branch.sh
index 8cfbbf79c1..7b06048905 100755
--- a/t/t3202-show-branch.sh
+++ b/t/t3202-show-branch.sh
@@ -2,69 +2,57 @@
 
 test_description='test show-branch'
 
-GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
-export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
-
 . ./test-lib.sh
 
-numbers="1 2 3 4 5 6 7 8 9 10"
-
 test_expect_success 'setup' '
-
-	> file &&
-	git add file &&
-	test_tick &&
-	git commit -m initial &&
-
-	for i in $numbers
+	test_commit initial &&
+	for i in $(test_seq 1 10)
 	do
-		git checkout -b branch$i main &&
-		> file$i &&
-		git add file$i &&
-		test_tick &&
-		git commit -m branch$i || return 1
-	done
-
+		git checkout -b branch$i initial &&
+		test_commit --no-tag branch$i
+	done &&
+	git for-each-ref \
+		--sort=version:refname \
+		--format="%(refname:strip=2)" \
+		"refs/heads/branch*" >branches.sorted &&
+	sed "s/^> //" >expect <<-\EOF
+	> ! [branch1] branch1
+	>  ! [branch2] branch2
+	>   ! [branch3] branch3
+	>    ! [branch4] branch4
+	>     ! [branch5] branch5
+	>      ! [branch6] branch6
+	>       ! [branch7] branch7
+	>        ! [branch8] branch8
+	>         ! [branch9] branch9
+	>          * [branch10] branch10
+	> ----------
+	>          * [branch10] branch10
+	>         +  [branch9] branch9
+	>        +   [branch8] branch8
+	>       +    [branch7] branch7
+	>      +     [branch6] branch6
+	>     +      [branch5] branch5
+	>    +       [branch4] branch4
+	>   +        [branch3] branch3
+	>  +         [branch2] branch2
+	> +          [branch1] branch1
+	> +++++++++* [branch10^] initial
+	EOF
 '
 
-cat > expect << EOF
-! [branch1] branch1
- ! [branch2] branch2
-  ! [branch3] branch3
-   ! [branch4] branch4
-    ! [branch5] branch5
-     ! [branch6] branch6
-      ! [branch7] branch7
-       ! [branch8] branch8
-        ! [branch9] branch9
-         * [branch10] branch10
-----------
-         * [branch10] branch10
-        +  [branch9] branch9
-       +   [branch8] branch8
-      +    [branch7] branch7
-     +     [branch6] branch6
-    +      [branch5] branch5
-   +       [branch4] branch4
-  +        [branch3] branch3
- +         [branch2] branch2
-+          [branch1] branch1
-+++++++++* [branch10^] initial
-EOF
-
 test_expect_success 'show-branch with more than 8 branches' '
-
-	git show-branch $(for i in $numbers; do echo branch$i; done) > out &&
-	test_cmp expect out
-
+	git show-branch $(cat branches.sorted) >actual &&
+	test_cmp expect actual
 '
 
 test_expect_success 'show-branch with showbranch.default' '
-	for i in $numbers; do
-		git config --add showbranch.default branch$i
+	for branch in $(cat branches.sorted)
+	do
+		test_config showbranch.default $branch --add
 	done &&
-	git show-branch >out &&
-	test_cmp expect out
+	git show-branch >actual &&
+	test_cmp expect actual
 '
 
 test_done

From 98538307876ab9565c24d45bce6b5302f99fec0b Mon Sep 17 00:00:00 2001
From: Alex Henrie 
Date: Sat, 12 Jun 2021 12:41:44 -0600
Subject: [PATCH 238/397] graph: improve grammar of "invalid color" error
 message

Without the "d", it sounds like a command, not an error, and is liable
to be translated incorrectly.

Signed-off-by: Alex Henrie 
Signed-off-by: Junio C Hamano 
---
 graph.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/graph.c b/graph.c
index c128ad0cce..e3828eb8f2 100644
--- a/graph.c
+++ b/graph.c
@@ -95,7 +95,7 @@ static void parse_graph_colors_config(struct strvec *colors, const char *string)
 		if (!color_parse_mem(start, comma - start, color))
 			strvec_push(colors, color);
 		else
-			warning(_("ignore invalid color '%.*s' in log.graphColors"),
+			warning(_("ignored invalid color '%.*s' in log.graphColors"),
 				(int)(comma - start), start);
 		start = comma + 1;
 	}

From aac578492d1dbfa80b15caab06e3786c6cc654cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Mon, 14 Jun 2021 12:37:35 +0200
Subject: [PATCH 239/397] pre-commit hook tests: don't leave "actual"
 nonexisting on failure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Start by creating an "actual" file in a core.hooksPath test that has
the hook echoing to the "actual" file.

We later test_cmp that file to see what hooks were run. If we fail to
run our hook(s) we'll have an empty list of hooks for the test_cmp
instead of a nonexisting file. For the logic of this test that makes more sense.

See 867ad08a261 (hooks: allow customizing where the hook directory is,
2016-05-04) for the commit that added these tests.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t1350-config-hooks-path.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/t/t1350-config-hooks-path.sh b/t/t1350-config-hooks-path.sh
index f1f9aee9f5..fa9647a7c0 100755
--- a/t/t1350-config-hooks-path.sh
+++ b/t/t1350-config-hooks-path.sh
@@ -5,6 +5,7 @@ test_description='Test the core.hooksPath configuration variable'
 . ./test-lib.sh
 
 test_expect_success 'set up a pre-commit hook in core.hooksPath' '
+	>actual &&
 	mkdir -p .git/custom-hooks .git/hooks &&
 	write_script .git/custom-hooks/pre-commit <<-\EOF &&
 	echo CUSTOM >>actual

From 9eb542f2eea88234d1ae8f84f40ef0f8daf1e87e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Mon, 14 Jun 2021 12:39:25 +0200
Subject: [PATCH 240/397] gc tests: add a test for the "pre-auto-gc" hook
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a missing test for the behavior of the pre-auto-gc hook added in
0b85d92661e (Documentation/hooks: add pre-auto-gc hook, 2008-04-02).

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t6500-gc.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/t/t6500-gc.sh b/t/t6500-gc.sh
index 60d961b526..10c7ae7f09 100755
--- a/t/t6500-gc.sh
+++ b/t/t6500-gc.sh
@@ -95,6 +95,52 @@ test_expect_success 'gc --keep-largest-pack' '
 	)
 '
 
+test_expect_success 'pre-auto-gc hook can stop auto gc' '
+	cat >err.expect <<-\EOF &&
+	no gc for you
+	EOF
+
+	git init pre-auto-gc-hook &&
+	(
+		cd pre-auto-gc-hook &&
+		write_script ".git/hooks/pre-auto-gc" <<-\EOF &&
+		echo >&2 no gc for you &&
+		exit 1
+		EOF
+
+		git config gc.auto 3 &&
+		git config gc.autoDetach false &&
+
+		# We need to create two object whose sha1s start with 17
+		# since this is what git gc counts.  As it happens, these
+		# two blobs will do so.
+		test_commit "$(test_oid obj1)" &&
+		test_commit "$(test_oid obj2)" &&
+
+		git gc --auto >../out.actual 2>../err.actual
+	) &&
+	test_must_be_empty out.actual &&
+	test_cmp err.expect err.actual &&
+
+	cat >err.expect <<-\EOF &&
+	will gc for you
+	Auto packing the repository for optimum performance.
+	See "git help gc" for manual housekeeping.
+	EOF
+
+	(
+		cd pre-auto-gc-hook &&
+		write_script ".git/hooks/pre-auto-gc" <<-\EOF &&
+		echo >&2 will gc for you &&
+		exit 0
+		EOF
+		git gc --auto >../out.actual 2>../err.actual
+	) &&
+
+	test_must_be_empty out.actual &&
+	test_cmp err.expect err.actual
+'
+
 test_expect_success 'auto gc with too many loose objects does not attempt to create bitmaps' '
 	test_config gc.auto 3 &&
 	test_config gc.autodetach false &&

From 69b3367f6c987b36b98937d8f446f75c5dbc000a Mon Sep 17 00:00:00 2001
From: Felipe Contreras 
Date: Tue, 15 Jun 2021 14:11:09 +0000
Subject: [PATCH 241/397] doc: avoid using the gender of other people

Using gendered pronouns for an anonymous person applies a gender where
none is known and further excludes readers who do not use gendered
pronouns. Avoid such examples in the documentation by using "they" or
passive voice to avoid the need for a pronoun.

Inspired-by: Derrick Stolee 
Signed-off-by: Felipe Contreras 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 Documentation/SubmittingPatches | 5 ++---
 Documentation/git-push.txt      | 4 ++--
 Documentation/user-manual.txt   | 2 +-
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches
index 0452db2e67..1d9f06b798 100644
--- a/Documentation/SubmittingPatches
+++ b/Documentation/SubmittingPatches
@@ -370,9 +370,8 @@ If you like, you can put extra tags at the end:
 . `Acked-by:` says that the person who is more familiar with the area
   the patch attempts to modify liked the patch.
 . `Reviewed-by:`, unlike the other tags, can only be offered by the
-  reviewer and means that she is completely satisfied that the patch
-  is ready for application.  It is usually offered only after a
-  detailed review.
+  reviewers themselves when they are completely satisfied with the
+  patch after a detailed analysis.
 . `Tested-by:` is used to indicate that the person applied the patch
   and found it to have the desired effect.
 
diff --git a/Documentation/git-push.txt b/Documentation/git-push.txt
index ab103c82cf..48cf37b5dc 100644
--- a/Documentation/git-push.txt
+++ b/Documentation/git-push.txt
@@ -244,8 +244,8 @@ Imagine that you have to rebase what you have already published.
 You will have to bypass the "must fast-forward" rule in order to
 replace the history you originally published with the rebased history.
 If somebody else built on top of your original history while you are
-rebasing, the tip of the branch at the remote may advance with her
-commit, and blindly pushing with `--force` will lose her work.
+rebasing, the tip of the branch at the remote may advance with their
+commit, and blindly pushing with `--force` will lose their work.
 +
 This option allows you to say that you expect the history you are
 updating is what you rebased and want to replace. If the remote ref
diff --git a/Documentation/user-manual.txt b/Documentation/user-manual.txt
index fd480b8645..2983e3bd60 100644
--- a/Documentation/user-manual.txt
+++ b/Documentation/user-manual.txt
@@ -2789,7 +2789,7 @@ A fast-forward looks something like this:
 
 In some cases it is possible that the new head will *not* actually be
 a descendant of the old head.  For example, the developer may have
-realized she made a serious mistake, and decided to backtrack,
+realized a serious mistake was made and decided to backtrack,
 resulting in a situation like:
 
 ................................................

From 0e20b229eea63716cdd1bea916b7e416f450278a Mon Sep 17 00:00:00 2001
From: Felipe Contreras 
Date: Tue, 15 Jun 2021 14:11:10 +0000
Subject: [PATCH 242/397] comments: avoid using the gender of our users
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We generally avoid specifying the gender of our users in order to be
more inclusive, but sometimes a few slip by due to habit.

Since by doing a little bit of rewording we can avoid this irrelevant
detail, let's do so.

Inspired-by: Derrick Stolee 
Helped-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Felipe Contreras 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 commit.c    | 2 +-
 config.h    | 4 ++--
 date.c      | 2 +-
 pathspec.h  | 2 +-
 strbuf.h    | 4 ++--
 wt-status.c | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/commit.c b/commit.c
index 42f8117b51..d8889601cc 100644
--- a/commit.c
+++ b/commit.c
@@ -1164,7 +1164,7 @@ static void handle_signed_tag(struct commit *parent, struct commit_extra_header
 	/*
 	 * We could verify this signature and either omit the tag when
 	 * it does not validate, but the integrator may not have the
-	 * public key of the signer of the tag he is merging, while a
+	 * public key of the signer of the tag being merged, while a
 	 * later auditor may have it while auditing, so let's not run
 	 * verify-signed-buffer here for now...
 	 *
diff --git a/config.h b/config.h
index 19a9adbaa9..11d53dc8fc 100644
--- a/config.h
+++ b/config.h
@@ -448,8 +448,8 @@ void git_configset_init(struct config_set *cs);
 /**
  * Parses the file and adds the variable-value pairs to the `config_set`,
  * dies if there is an error in parsing the file. Returns 0 on success, or
- * -1 if the file does not exist or is inaccessible. The user has to decide
- * if he wants to free the incomplete configset or continue using it when
+ * -1 if the file does not exist or is inaccessible. The caller decides
+ * whether to free the incomplete configset or continue using it when
  * the function returns -1.
  */
 int git_configset_add_file(struct config_set *cs, const char *filename);
diff --git a/date.c b/date.c
index f9ea807b3a..c55ea47e96 100644
--- a/date.c
+++ b/date.c
@@ -908,7 +908,7 @@ int parse_expiry_date(const char *date, timestamp_t *timestamp)
 		/*
 		 * We take over "now" here, which usually translates
 		 * to the current timestamp.  This is because the user
-		 * really means to expire everything she has done in
+		 * really means to expire everything that was done in
 		 * the past, and by definition reflogs are the record
 		 * of the past, and there is nothing from the future
 		 * to be kept.
diff --git a/pathspec.h b/pathspec.h
index 454ce364fa..dfb0b212cd 100644
--- a/pathspec.h
+++ b/pathspec.h
@@ -108,7 +108,7 @@ struct pathspec {
  *
  * A similar process is applied when a new pathspec magic is added. The designer
  * lifts the GUARD_PATHSPEC restriction in the functions that support the new
- * magic. At the same time (s)he has to make sure this new feature will be
+ * magic while at the same time making sure this new feature will be
  * caught at parse_pathspec() in commands that cannot handle the new magic in
  * some cases. grepping parse_pathspec() should help.
  */
diff --git a/strbuf.h b/strbuf.h
index 223ee2094a..a86dcaaf44 100644
--- a/strbuf.h
+++ b/strbuf.h
@@ -337,8 +337,8 @@ const char *strbuf_join_argv(struct strbuf *buf, int argc,
  * placeholder is unknown, then the percent sign is copied, too.
  *
  * In order to facilitate caching and to make it possible to give
- * parameters to the callback, `strbuf_expand()` passes a context pointer,
- * which can be used by the programmer of the callback as she sees fit.
+ * parameters to the callback, `strbuf_expand()` passes a context
+ * pointer with any kind of data.
  */
 typedef size_t (*expand_fn_t) (struct strbuf *sb,
 			       const char *placeholder,
diff --git a/wt-status.c b/wt-status.c
index 1aed68c43c..805ed35cd3 100644
--- a/wt-status.c
+++ b/wt-status.c
@@ -638,7 +638,7 @@ static void wt_status_collect_changes_index(struct wt_status *s)
 		 * mode by passing a command line option we do not ignore any
 		 * changed submodule SHA-1s when comparing index and HEAD, no
 		 * matter what is configured. Otherwise the user won't be
-		 * shown any submodules she manually added (and which are
+		 * shown any submodules manually added (and which are
 		 * staged to be committed), which would be really confusing.
 		 */
 		handle_ignore_submodules_arg(&rev.diffopt, "dirty");

From 46a237f42fe24545e6a2a72488ad04d836f91717 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Tue, 15 Jun 2021 14:11:11 +0000
Subject: [PATCH 243/397] *: fix typos

These typos were found while searching the codebase for gendered
pronouns. In the case of t9300-fast-import.sh, remove a confusing
comment that is unnecessary to the understanding of the test.

Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 config.c               | 2 +-
 t/t9300-fast-import.sh | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/config.c b/config.c
index 6428393a41..c008987db3 100644
--- a/config.c
+++ b/config.c
@@ -2834,7 +2834,7 @@ static void maybe_remove_section(struct config_store_data *store,
 	begin = store->parsed[i].begin;
 
 	/*
-	 * Next, make sure that we are removing he last key(s) in the section,
+	 * Next, make sure that we are removing the last key(s) in the section,
 	 * and that there are no comments that are possibly about the current
 	 * section.
 	 */
diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh
index 5c47ac4465..7607ef65cb 100755
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@ -1538,7 +1538,6 @@ test_expect_success 'O: comments are all skipped' '
 	commit refs/heads/O1
 	# -- ignore all of this text
 	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	# $GIT_COMMITTER_NAME has inserted here for his benefit.
 	data <
Date: Tue, 15 Jun 2021 22:41:42 +0000
Subject: [PATCH 244/397] promisor-remote: output trace2 statistics for number
 of objects fetched

Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 promisor-remote.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/promisor-remote.c b/promisor-remote.c
index da3f2ca261..d465377d7d 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -12,7 +12,8 @@ void set_repository_format_partial_clone(char *partial_clone)
 	repository_format_partial_clone = xstrdup_or_null(partial_clone);
 }
 
-static int fetch_objects(const char *remote_name,
+static int fetch_objects(struct repository *repo,
+			 const char *remote_name,
 			 const struct object_id *oids,
 			 int oid_nr)
 {
@@ -30,6 +31,8 @@ static int fetch_objects(const char *remote_name,
 		die(_("promisor-remote: unable to fork off fetch subprocess"));
 	child_in = xfdopen(child.in, "w");
 
+	trace2_data_intmax("promisor", repo, "fetch_count", oid_nr);
+
 	for (i = 0; i < oid_nr; i++) {
 		if (fputs(oid_to_hex(&oids[i]), child_in) < 0)
 			die_errno(_("promisor-remote: could not write to fetch subprocess"));
@@ -238,7 +241,7 @@ int promisor_remote_get_direct(struct repository *repo,
 	promisor_remote_init();
 
 	for (r = promisors; r; r = r->next) {
-		if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) {
+		if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
 			if (remaining_nr == 1)
 				continue;
 			remaining_nr = remove_fetched_oids(repo, &remaining_oids,

From c4e317814f611d13d78981fc51bba1da8d1f0be6 Mon Sep 17 00:00:00 2001
From: Julian Verdurmen 
Date: Tue, 2 Mar 2021 00:58:09 +0000
Subject: [PATCH 245/397] userdiff: add support for C# record types

Records are added in C# 9

Code example :

    public record Person(string FirstName, string LastName);

For more information, see:
* https://docs.microsoft.com/en-us/dotnet/csharp/whats-new/csharp-9

Signed-off-by: Julian Verdurmen 
Reviewed-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 userdiff.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/userdiff.c b/userdiff.c
index 3c3bbe38b0..d9b2ba752f 100644
--- a/userdiff.c
+++ b/userdiff.c
@@ -65,7 +65,7 @@ PATTERNS("csharp",
 	 /* Properties */
 	 "^[ \t]*(((static|public|internal|private|protected|new|virtual|sealed|override|unsafe)[ \t]+)*[][<>@.~_[:alnum:]]+[ \t]+[@._[:alnum:]]+)[ \t]*$\n"
 	 /* Type definitions */
-	 "^[ \t]*(((static|public|internal|private|protected|new|unsafe|sealed|abstract|partial)[ \t]+)*(class|enum|interface|struct)[ \t]+.*)$\n"
+	 "^[ \t]*(((static|public|internal|private|protected|new|unsafe|sealed|abstract|partial)[ \t]+)*(class|enum|interface|struct|record)[ \t]+.*)$\n"
 	 /* Namespace */
 	 "^[ \t]*(namespace[ \t]+.*)$",
 	 /* -- */

From eb87c6f559f0d7550a5f078a5b78159c290b0256 Mon Sep 17 00:00:00 2001
From: Jiang Xin 
Date: Thu, 17 Jun 2021 11:14:11 +0800
Subject: [PATCH 246/397] t6020: fix incompatible parameter expansion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Ævar reported that the function `make_user_friendly_and_stable_output()`
failed on a i386 box (gcc45) in the gcc farm boxes with error:

    sed: couldn't re-allocate memory

It turns out that older versions of bash (4.3) or dash (0.5.7) cannot
evaluate expression like `${A%${A#???????}}` used to get the leading 7
characters of variable A.

Replace the incompatible parameter expansion so that t6020 works on
older version of bash or dash.

Reported-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Jiang Xin 
Signed-off-by: Junio C Hamano 
---
 t/t6020-bundle-misc.sh | 50 ++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 19 deletions(-)

diff --git a/t/t6020-bundle-misc.sh b/t/t6020-bundle-misc.sh
index 881f72fd44..3140ca4fdc 100755
--- a/t/t6020-bundle-misc.sh
+++ b/t/t6020-bundle-misc.sh
@@ -80,30 +80,42 @@ test_commit_setvar () {
 	eval $var=$oid
 }
 
+get_abbrev_oid () {
+	oid=$1 &&
+	suffix=${oid#???????} &&
+	oid=${oid%$suffix} &&
+	if test -n "$oid"
+	then
+		echo "$oid"
+	else
+		echo "undefined-oid"
+	fi
+}
+
 # Format the output of git commands to make a user-friendly and stable
 # text.  We can easily prepare the expect text without having to worry
 # about future changes of the commit ID and spaces of the output.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/${A%${A#???????}}[0-9a-f]*//g" \
-		-e "s/${B%${B#???????}}[0-9a-f]*//g" \
-		-e "s/${C%${C#???????}}[0-9a-f]*//g" \
-		-e "s/${D%${D#???????}}[0-9a-f]*//g" \
-		-e "s/${E%${E#???????}}[0-9a-f]*//g" \
-		-e "s/${F%${F#???????}}[0-9a-f]*//g" \
-		-e "s/${G%${G#???????}}[0-9a-f]*//g" \
-		-e "s/${H%${H#???????}}[0-9a-f]*//g" \
-		-e "s/${I%${I#???????}}[0-9a-f]*//g" \
-		-e "s/${J%${J#???????}}[0-9a-f]*//g" \
-		-e "s/${K%${K#???????}}[0-9a-f]*//g" \
-		-e "s/${L%${L#???????}}[0-9a-f]*//g" \
-		-e "s/${M%${M#???????}}[0-9a-f]*//g" \
-		-e "s/${N%${N#???????}}[0-9a-f]*//g" \
-		-e "s/${O%${O#???????}}[0-9a-f]*//g" \
-		-e "s/${P%${P#???????}}[0-9a-f]*//g" \
-		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*//g" \
-		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*//g" \
-		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $A)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $B)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $C)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $D)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $E)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $F)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $G)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $H)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $I)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $J)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $K)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $L)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $M)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $N)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $O)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $P)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $TAG1)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $TAG2)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $TAG3)[0-9a-f]*//g" \
 		-e "s/ *\$//"
 }
 

From 5210225f256d01938960d439bff9d809c2ff1809 Mon Sep 17 00:00:00 2001
From: Jiang Xin 
Date: Thu, 17 Jun 2021 11:17:24 +0800
Subject: [PATCH 247/397] sideband: don't lose clear-to-eol at packet boundary

When "demultiplex_sideband()" sees a nonempty message ending with CR or
LF on the sideband #2, it adds "suffix" string to clear to the end of
the current line, which helps when relaying a progress display whose
records are terminated with CRs.  But if it sees a single LF, no
clear-to-end suffix should be appended, because this single LF is used
to end the progress display by moving to the next line, and the final
progress display above should be preserved.

However, the code forgot that depending on the length of the payload
line, such a CR may fall exactly at the packet boundary and the
number of bytes before the CR from the beginning of the packet could
be zero.  In such a case, the message that was terminated by the CR
were leftover in the "scratch" buffer in the previous call to the
function and we still need to clear to the end of the current line.

Helped-by: Junio C Hamano 
Helped-by: Nicolas Pitre 
Signed-off-by: Jiang Xin 
Signed-off-by: Junio C Hamano 
---
 sideband.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/sideband.c b/sideband.c
index 6f9e026732..85bddfdcd4 100644
--- a/sideband.c
+++ b/sideband.c
@@ -183,8 +183,31 @@ int demultiplex_sideband(const char *me, int status,
 		while ((brk = strpbrk(b, "\n\r"))) {
 			int linelen = brk - b;
 
+			/*
+			 * For message accross packet boundary, there would have
+			 * a nonempty "scratch" buffer from last call of this
+			 * function, and there may have a leading CR/LF in "buf".
+			 * For this case we should add a clear-to-eol suffix to
+			 * clean leftover letters we previously have written on
+			 * the same line.
+			 */
+			if (scratch->len && !linelen)
+				strbuf_addstr(scratch, suffix);
+
 			if (!scratch->len)
 				strbuf_addstr(scratch, DISPLAY_PREFIX);
+
+			/*
+			 * A use case that we should not add clear-to-eol suffix
+			 * to empty lines:
+			 *
+			 * For progress reporting we may receive a bunch of
+			 * percentage updates followed by '\r' to remain on the
+			 * same line, and at the end receive a single '\n' to
+			 * move to the next line. We should preserve the final
+			 * status report line by not appending clear-to-eol
+			 * suffix to this single line break.
+			 */
 			if (linelen > 0) {
 				maybe_colorize_sideband(scratch, b, linelen);
 				strbuf_addstr(scratch, suffix);

From 2bafb3d702d4cd77de0d3e68f13188980e0de734 Mon Sep 17 00:00:00 2001
From: Jiang Xin 
Date: Thu, 17 Jun 2021 11:17:25 +0800
Subject: [PATCH 248/397] test: compare raw output, not mangle tabs and spaces
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Before comparing with the expect file, we used to call function
"make_user_friendly_and_stable_output" to filter out trailing spaces in
output.  Ævar recommends using pattern "s/Z$//" to prepare expect file,
and then compare it with raw output.

Since we have fixed the issue of occasionally missing the clear-to-eol
suffix when displaying sideband #2 messages, it is safe and stable to
test against raw output.

Suggested-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Jiang Xin 
Signed-off-by: Junio C Hamano 
---
 t/t5411/common-functions.sh                   |   9 +-
 t/t5411/test-0000-standard-git-push.sh        |  82 +++----
 .../test-0001-standard-git-push--porcelain.sh |  90 ++++----
 ...st-0003-pre-receive-declined--porcelain.sh |   8 +-
 t/t5411/test-0011-no-hook-error.sh            |  40 ++--
 t/t5411/test-0012-no-hook-error--porcelain.sh |  42 ++--
 t/t5411/test-0013-bad-protocol.sh             |  62 +++---
 t/t5411/test-0014-bad-protocol--porcelain.sh  |  80 +++----
 t/t5411/test-0020-report-ng.sh                |  32 +--
 t/t5411/test-0021-report-ng--porcelain.sh     |  36 ++--
 t/t5411/test-0022-report-unexpect-ref.sh      |  26 +--
 ...est-0023-report-unexpect-ref--porcelain.sh |  28 +--
 t/t5411/test-0024-report-unknown-ref.sh       |  18 +-
 ...test-0025-report-unknown-ref--porcelain.sh |  20 +-
 t/t5411/test-0026-push-options.sh             |  58 ++---
 t/t5411/test-0027-push-options--porcelain.sh  |  62 +++---
 t/t5411/test-0030-report-ok.sh                |  20 +-
 t/t5411/test-0031-report-ok--porcelain.sh     |  22 +-
 t/t5411/test-0032-report-with-options.sh      | 186 ++++++++--------
 ...est-0033-report-with-options--porcelain.sh | 200 +++++++++---------
 t/t5411/test-0034-report-ft.sh                |  22 +-
 t/t5411/test-0035-report-ft--porcelain.sh     |  24 +--
 ...t-0036-report-multi-rewrite-for-one-ref.sh | 132 ++++++------
 ...rt-multi-rewrite-for-one-ref--porcelain.sh | 138 ++++++------
 t/t5411/test-0038-report-mixed-refs.sh        |  74 +++----
 .../test-0039-report-mixed-refs--porcelain.sh |  76 +++----
 t/t5411/test-0040-process-all-refs.sh         |  80 +++----
 .../test-0041-process-all-refs--porcelain.sh  |  82 +++----
 ...t-0050-proc-receive-refs-with-modifiers.sh |  90 ++++----
 t/t5548-push-porcelain.sh                     |  54 ++---
 t/t6020-bundle-misc.sh                        |  45 ++--
 31 files changed, 972 insertions(+), 966 deletions(-)

diff --git a/t/t5411/common-functions.sh b/t/t5411/common-functions.sh
index 6694858e18..6c347b4caa 100644
--- a/t/t5411/common-functions.sh
+++ b/t/t5411/common-functions.sh
@@ -33,17 +33,14 @@ create_commits_in () {
 
 # Format the output of git-push, git-show-ref and other commands to make a
 # user-friendly and stable text.  We can easily prepare the expect text
-# without having to worry about future changes of the commit ID and spaces
+# without having to worry about changes of the commit ID (full or abbrev.)
 # of the output.  Single quotes are replaced with double quotes, because
 # it is boring to prepare unquoted single quotes in expect text.  We also
 # remove some locale error messages. The emitted human-readable errors are
 # redundant to the more machine-readable output the tests already assert.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/  *\$//" \
-		-e "s/  */ /g" \
 		-e "s/'/\"/g" \
-		-e "s/	/    /g" \
 		-e "s/$A//g" \
 		-e "s/$B//g" \
 		-e "s/$TAG//g" \
@@ -59,6 +56,10 @@ filter_out_user_friendly_and_stable_output () {
 		sed -n ${1+"$@"}
 }
 
+format_and_save_expect () {
+	sed -e 's/^> //' -e 's/Z$//' >expect
+}
+
 test_cmp_refs () {
 	indir=
 	if test "$1" = "-C"
diff --git a/t/t5411/test-0000-standard-git-push.sh b/t/t5411/test-0000-standard-git-push.sh
index e1e0175c12..d8aafc235c 100644
--- a/t/t5411/test-0000-standard-git-push.sh
+++ b/t/t5411/test-0000-standard-git-push.sh
@@ -7,16 +7,16 @@ test_expect_success "git-push ($PROTOCOL)" '
 		HEAD:refs/heads/next \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/next
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/heads/next
-	To 
-	 ..  -> main
-	 * [new branch] HEAD -> next
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	>    ..   -> main
+	>  * [new branch]      HEAD -> next
 	EOF
 	test_cmp expect actual &&
 
@@ -38,10 +38,10 @@ test_expect_success "git-push --atomic ($PROTOCOL)" '
 		-e "/^To / { p; }" \
 		-e "/^ ! / { p; }" \
 		actual &&
-	cat >expect <<-EOF &&
-	To 
-	 ! [rejected] main -> main (non-fast-forward)
-	 ! [rejected]  -> next (atomic push failed)
+	format_and_save_expect <<-EOF &&
+	> To 
+	>  ! [rejected]        main -> main (non-fast-forward)
+	>  ! [rejected]         -> next (atomic push failed)
 	EOF
 	test_cmp expect actual &&
 
@@ -63,14 +63,14 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL)" '
 		$B:refs/heads/next \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	To 
-	 ..  -> next
-	 ! [rejected] main -> main (non-fast-forward)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	>    ..   -> next
+	>  ! [rejected]        main -> main (non-fast-forward)
 	EOF
 	test_cmp expect actual &&
 
@@ -92,25 +92,25 @@ test_expect_success "git-push -f ($PROTOCOL)" '
 		HEAD:refs/heads/a/b/c \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/tags/v123
-	remote: pre-receive<   refs/review/main/topic
-	remote: pre-receive<   refs/heads/a/b/c
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/heads/next
-	remote: post-receive<   refs/tags/v123
-	remote: post-receive<   refs/review/main/topic
-	remote: post-receive<   refs/heads/a/b/c
-	To 
-	 + ... main -> main (forced update)
-	 - [deleted] next
-	 * [new tag] v123 -> v123
-	 * [new reference] main -> refs/review/main/topic
-	 * [new branch] HEAD -> a/b/c
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/tags/v123        Z
+	> remote: pre-receive<   refs/review/main/topic        Z
+	> remote: pre-receive<   refs/heads/a/b/c        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> remote: post-receive<   refs/tags/v123        Z
+	> remote: post-receive<   refs/review/main/topic        Z
+	> remote: post-receive<   refs/heads/a/b/c        Z
+	> To 
+	>  + ... main -> main (forced update)
+	>  - [deleted]         next
+	>  * [new tag]         v123 -> v123
+	>  * [new reference]   main -> refs/review/main/topic
+	>  * [new branch]      HEAD -> a/b/c
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0001-standard-git-push--porcelain.sh b/t/t5411/test-0001-standard-git-push--porcelain.sh
index bcbda72341..2078d0a027 100644
--- a/t/t5411/test-0001-standard-git-push--porcelain.sh
+++ b/t/t5411/test-0001-standard-git-push--porcelain.sh
@@ -7,17 +7,17 @@ test_expect_success "git-push ($PROTOCOL/porcelain)" '
 		HEAD:refs/heads/next \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/next
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/heads/next
-	To 
-	     :refs/heads/main    ..
-	*    HEAD:refs/heads/next    [new branch]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	>  	:refs/heads/main	..
+	> *	HEAD:refs/heads/next	[new branch]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -38,12 +38,12 @@ test_expect_success "git-push --atomic ($PROTOCOL/porcelain)" '
 	filter_out_user_friendly_and_stable_output \
 		-e "s/^# GETTEXT POISON #//" \
 		-e "/^To / { p; }" \
-		-e "/^! / { p; }" \
+		-e "/^!/ { p; }" \
 		actual &&
-	cat >expect <<-EOF &&
-	To 
-	!    refs/heads/main:refs/heads/main    [rejected] (non-fast-forward)
-	!    :refs/heads/next    [rejected] (atomic push failed)
+	format_and_save_expect <<-EOF &&
+	> To 
+	> !	refs/heads/main:refs/heads/main	[rejected] (non-fast-forward)
+	> !	:refs/heads/next	[rejected] (atomic push failed)
 	EOF
 	test_cmp expect actual &&
 
@@ -65,15 +65,15 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL/porcelain)" '
 		$B:refs/heads/next \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	To 
-	     :refs/heads/next    ..
-	!    refs/heads/main:refs/heads/main    [rejected] (non-fast-forward)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	>  	:refs/heads/next	..
+	> !	refs/heads/main:refs/heads/main	[rejected] (non-fast-forward)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -95,26 +95,26 @@ test_expect_success "git-push -f ($PROTOCOL/porcelain)" '
 		HEAD:refs/heads/a/b/c \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/tags/v123
-	remote: pre-receive<   refs/review/main/topic
-	remote: pre-receive<   refs/heads/a/b/c
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/heads/next
-	remote: post-receive<   refs/tags/v123
-	remote: post-receive<   refs/review/main/topic
-	remote: post-receive<   refs/heads/a/b/c
-	To 
-	+    refs/heads/main:refs/heads/main    ... (forced update)
-	-    :refs/heads/next    [deleted]
-	*    refs/tags/v123:refs/tags/v123    [new tag]
-	*    refs/heads/main:refs/review/main/topic    [new reference]
-	*    HEAD:refs/heads/a/b/c    [new branch]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/tags/v123        Z
+	> remote: pre-receive<   refs/review/main/topic        Z
+	> remote: pre-receive<   refs/heads/a/b/c        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> remote: post-receive<   refs/tags/v123        Z
+	> remote: post-receive<   refs/review/main/topic        Z
+	> remote: post-receive<   refs/heads/a/b/c        Z
+	> To 
+	> +	refs/heads/main:refs/heads/main	... (forced update)
+	> -	:refs/heads/next	[deleted]
+	> *	refs/tags/v123:refs/tags/v123	[new tag]
+	> *	refs/heads/main:refs/review/main/topic	[new reference]
+	> *	HEAD:refs/heads/a/b/c	[new branch]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0003-pre-receive-declined--porcelain.sh b/t/t5411/test-0003-pre-receive-declined--porcelain.sh
index e9c9db5d1f..2393b04ad9 100644
--- a/t/t5411/test-0003-pre-receive-declined--porcelain.sh
+++ b/t/t5411/test-0003-pre-receive-declined--porcelain.sh
@@ -14,10 +14,10 @@ test_expect_success "git-push is declined ($PROTOCOL/porcelain)" '
 		HEAD:refs/heads/next \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	To 
-	!    :refs/heads/main    [remote rejected] (pre-receive hook declined)
-	!    HEAD:refs/heads/next    [remote rejected] (pre-receive hook declined)
+	format_and_save_expect <<-EOF &&
+	> To 
+	> !	:refs/heads/main	[remote rejected] (pre-receive hook declined)
+	> !	HEAD:refs/heads/next	[remote rejected] (pre-receive hook declined)
 	Done
 	EOF
 	test_cmp expect actual &&
diff --git a/t/t5411/test-0011-no-hook-error.sh b/t/t5411/test-0011-no-hook-error.sh
index 3ef136e6ef..d35002b1f0 100644
--- a/t/t5411/test-0011-no-hook-error.sh
+++ b/t/t5411/test-0011-no-hook-error.sh
@@ -7,16 +7,16 @@ test_expect_success "proc-receive: no hook, fail to push special ref ($PROTOCOL)
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: error: cannot find hook "proc-receive"
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	To 
-	 * [new branch] HEAD -> next
-	 ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: error: cannot find hook "proc-receive"        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	>  * [new branch]      HEAD -> next
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
 	EOF
 	test_cmp expect actual &&
 
@@ -41,16 +41,16 @@ test_expect_success "proc-receive: no hook, all failed for atomic push ($PROTOCO
 		HEAD:next \
 		HEAD:refs/for/main/topic >out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: error: cannot find hook "proc-receive"
-	To 
-	 ! [remote rejected]  -> main (fail to run proc-receive hook)
-	 ! [remote rejected] HEAD -> next (fail to run proc-receive hook)
-	 ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: error: cannot find hook "proc-receive"        Z
+	> To 
+	>  ! [remote rejected]  -> main (fail to run proc-receive hook)
+	>  ! [remote rejected] HEAD -> next (fail to run proc-receive hook)
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (fail to run proc-receive hook)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0012-no-hook-error--porcelain.sh b/t/t5411/test-0012-no-hook-error--porcelain.sh
index 19f66fbd7d..04468b5018 100644
--- a/t/t5411/test-0012-no-hook-error--porcelain.sh
+++ b/t/t5411/test-0012-no-hook-error--porcelain.sh
@@ -7,16 +7,16 @@ test_expect_success "proc-receive: no hook, fail to push special ref ($PROTOCOL/
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: error: cannot find hook "proc-receive"
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	To 
-	*    HEAD:refs/heads/next    [new branch]
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: error: cannot find hook "proc-receive"        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	> *	HEAD:refs/heads/next	[new branch]
+	> !	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -42,17 +42,17 @@ test_expect_success "proc-receive: no hook, all failed for atomic push ($PROTOCO
 		HEAD:next \
 		HEAD:refs/for/main/topic >out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: error: cannot find hook "proc-receive"
-	To 
-	!    :refs/heads/main    [remote rejected] (fail to run proc-receive hook)
-	!    HEAD:refs/heads/next    [remote rejected] (fail to run proc-receive hook)
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: error: cannot find hook "proc-receive"        Z
+	> To 
+	> !	:refs/heads/main	[remote rejected] (fail to run proc-receive hook)
+	> !	HEAD:refs/heads/next	[remote rejected] (fail to run proc-receive hook)
+	> !	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0013-bad-protocol.sh b/t/t5411/test-0013-bad-protocol.sh
index 095e613f6f..c08a00ded2 100644
--- a/t/t5411/test-0013-bad-protocol.sh
+++ b/t/t5411/test-0013-bad-protocol.sh
@@ -29,8 +29,8 @@ test_expect_success "proc-receive: bad protocol (unknown version, $PROTOCOL)" '
 	# message ("remote: fatal: the remote end hung up unexpectedly") which
 	# is different from the remote HTTP server with different locale settings.
 	grep "^remote: error:" actual-error &&
-	cat >expect <<-EOF &&
-	remote: error: proc-receive version "2" is not supported
+	format_and_save_expect <<-EOF &&
+	> remote: error: proc-receive version "2" is not supported        Z
 	EOF
 	test_cmp expect actual-error &&
 
@@ -208,17 +208,17 @@ test_expect_success "proc-receive: bad protocol (no report, $PROTOCOL)" '
 		HEAD:refs/heads/next \
 		HEAD:refs/for/main/topic >out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	To 
-	 * [new branch] HEAD -> next
-	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	>  * [new branch]      HEAD -> next
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
@@ -251,15 +251,15 @@ test_expect_success "proc-receive: bad protocol (no ref, $PROTOCOL)" '
 		HEAD:refs/for/main/topic\
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok
-	remote: error: proc-receive reported incomplete status line: "ok"
-	To 
-	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok        Z
+	> remote: error: proc-receive reported incomplete status line: "ok"        Z
+	> To 
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
@@ -284,15 +284,15 @@ test_expect_success "proc-receive: bad protocol (unknown status, $PROTOCOL)" '
 			HEAD:refs/for/main/topic \
 			>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> xx refs/for/main/topic
-	remote: error: proc-receive reported bad status "xx" on ref "refs/for/main/topic"
-	To 
-	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> xx refs/for/main/topic        Z
+	> remote: error: proc-receive reported bad status "xx" on ref "refs/for/main/topic"        Z
+	> To 
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0014-bad-protocol--porcelain.sh b/t/t5411/test-0014-bad-protocol--porcelain.sh
index a44649789c..3eaa597e0f 100644
--- a/t/t5411/test-0014-bad-protocol--porcelain.sh
+++ b/t/t5411/test-0014-bad-protocol--porcelain.sh
@@ -20,7 +20,7 @@ test_expect_success "proc-receive: bad protocol (unknown version, $PROTOCOL/porc
 		actual-report &&
 	cat >expect <<-EOF &&
 	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	!	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual-report &&
@@ -29,8 +29,8 @@ test_expect_success "proc-receive: bad protocol (unknown version, $PROTOCOL/porc
 	# message ("remote: fatal: the remote end hung up unexpectedly") which
 	# is different from the remote HTTP server with different locale settings.
 	grep "^remote: error:" actual-error &&
-	cat >expect <<-EOF &&
-	remote: error: proc-receive version "2" is not supported
+	format_and_save_expect <<-EOF &&
+	> remote: error: proc-receive version "2" is not supported        Z
 	EOF
 	test_cmp expect actual-error &&
 
@@ -58,7 +58,7 @@ test_expect_success "proc-receive: bad protocol (hook --die-read-version, $PROTO
 		actual &&
 	cat >expect <<-EOF &&
 	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	!	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -89,7 +89,7 @@ test_expect_success "proc-receive: bad protocol (hook --die-write-version, $PROT
 		actual &&
 	cat >expect <<-EOF &&
 	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	!	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -120,7 +120,7 @@ test_expect_success "proc-receive: bad protocol (hook --die-read-commands, $PROT
 		actual &&
 	cat >expect <<-EOF &&
 	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	!	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -152,7 +152,7 @@ test_expect_success "proc-receive: bad protocol (hook --die-read-push-options, $
 		actual &&
 	cat >expect <<-EOF &&
 	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	!	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -182,7 +182,7 @@ test_expect_success "proc-receive: bad protocol (hook --die-write-report, $PROTO
 		actual &&
 	cat >expect <<-EOF &&
 	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (fail to run proc-receive hook)
+	!	HEAD:refs/for/main/topic	[remote rejected] (fail to run proc-receive hook)
 	Done
 	EOF
 	test_cmp expect actual &&
@@ -208,18 +208,18 @@ test_expect_success "proc-receive: bad protocol (no report, $PROTOCOL/porcelain)
 		HEAD:refs/heads/next \
 		HEAD:refs/for/main/topic >out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	To 
-	*    HEAD:refs/heads/next    [new branch]
-	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> To 
+	> *	HEAD:refs/heads/next	[new branch]
+	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -251,16 +251,16 @@ test_expect_success "proc-receive: bad protocol (no ref, $PROTOCOL/porcelain)" '
 		HEAD:refs/for/main/topic\
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok
-	remote: error: proc-receive reported incomplete status line: "ok"
-	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok        Z
+	> remote: error: proc-receive reported incomplete status line: "ok"        Z
+	> To 
+	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -285,16 +285,16 @@ test_expect_success "proc-receive: bad protocol (unknown status, $PROTOCOL/porce
 			HEAD:refs/for/main/topic \
 			>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> xx refs/for/main/topic
-	remote: error: proc-receive reported bad status "xx" on ref "refs/for/main/topic"
-	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> xx refs/for/main/topic        Z
+	> remote: error: proc-receive reported bad status "xx" on ref "refs/for/main/topic"        Z
+	> To 
+	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0020-report-ng.sh b/t/t5411/test-0020-report-ng.sh
index ad2c8f6535..e915dbc28d 100644
--- a/t/t5411/test-0020-report-ng.sh
+++ b/t/t5411/test-0020-report-ng.sh
@@ -14,14 +14,14 @@ test_expect_success "proc-receive: fail to update (ng, no message, $PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic
-	To 
-	 ! [remote rejected] HEAD -> refs/for/main/topic (failed)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic        Z
+	> To 
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (failed)
 	EOF
 	test_cmp expect actual &&
 
@@ -46,14 +46,14 @@ test_expect_success "proc-receive: fail to update (ng, with message, $PROTOCOL)"
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic error msg
-	To 
-	 ! [remote rejected] HEAD -> refs/for/main/topic (error msg)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic error msg        Z
+	> To 
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (error msg)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0021-report-ng--porcelain.sh b/t/t5411/test-0021-report-ng--porcelain.sh
index d8ae9d3414..2a392e099b 100644
--- a/t/t5411/test-0021-report-ng--porcelain.sh
+++ b/t/t5411/test-0021-report-ng--porcelain.sh
@@ -14,15 +14,15 @@ test_expect_success "proc-receive: fail to update (ng, no message, $PROTOCOL/por
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic
-	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (failed)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic        Z
+	> To 
+	> !	HEAD:refs/for/main/topic	[remote rejected] (failed)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -47,15 +47,15 @@ test_expect_success "proc-receive: fail to update (ng, with message, $PROTOCOL/p
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ng refs/for/main/topic error msg
-	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (error msg)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ng refs/for/main/topic error msg        Z
+	> To 
+	> !	HEAD:refs/for/main/topic	[remote rejected] (error msg)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0022-report-unexpect-ref.sh b/t/t5411/test-0022-report-unexpect-ref.sh
index dbed467186..a2ce7ebdf6 100644
--- a/t/t5411/test-0022-report-unexpect-ref.sh
+++ b/t/t5411/test-0022-report-unexpect-ref.sh
@@ -15,19 +15,19 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/heads/main
-	remote: error: proc-receive reported status on unexpected ref: refs/heads/main
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	To 
-	 ..  -> main
-	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: error: proc-receive reported status on unexpected ref: refs/heads/main        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> To 
+	>    ..   -> main
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0023-report-unexpect-ref--porcelain.sh b/t/t5411/test-0023-report-unexpect-ref--porcelain.sh
index e89096fa13..4e56b163f8 100644
--- a/t/t5411/test-0023-report-unexpect-ref--porcelain.sh
+++ b/t/t5411/test-0023-report-unexpect-ref--porcelain.sh
@@ -15,20 +15,20 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL/porcelain)"
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/heads/main
-	remote: error: proc-receive reported status on unexpected ref: refs/heads/main
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	To 
-	     :refs/heads/main    ..
-	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: error: proc-receive reported status on unexpected ref: refs/heads/main        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> To 
+	>  	:refs/heads/main	..
+	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0024-report-unknown-ref.sh b/t/t5411/test-0024-report-unknown-ref.sh
index 77204244b8..af055aa086 100644
--- a/t/t5411/test-0024-report-unknown-ref.sh
+++ b/t/t5411/test-0024-report-unknown-ref.sh
@@ -14,15 +14,15 @@ test_expect_success "proc-receive: report unknown reference ($PROTOCOL)" '
 		HEAD:refs/for/a/b/c/my/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/a/b/c/my/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/a/b/c/my/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: error: proc-receive reported status on unknown ref: refs/for/main/topic
-	To 
-	 ! [remote rejected] HEAD -> refs/for/a/b/c/my/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/a/b/c/my/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/a/b/c/my/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: error: proc-receive reported status on unknown ref: refs/for/main/topic        Z
+	> To 
+	>  ! [remote rejected] HEAD -> refs/for/a/b/c/my/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0025-report-unknown-ref--porcelain.sh b/t/t5411/test-0025-report-unknown-ref--porcelain.sh
index eeb1ce6b2c..99601ca321 100644
--- a/t/t5411/test-0025-report-unknown-ref--porcelain.sh
+++ b/t/t5411/test-0025-report-unknown-ref--porcelain.sh
@@ -14,16 +14,16 @@ test_expect_success "proc-receive: report unknown reference ($PROTOCOL/porcelain
 		HEAD:refs/for/a/b/c/my/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/a/b/c/my/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/a/b/c/my/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: error: proc-receive reported status on unknown ref: refs/for/main/topic
-	To 
-	!    HEAD:refs/for/a/b/c/my/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/a/b/c/my/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/a/b/c/my/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: error: proc-receive reported status on unknown ref: refs/for/main/topic        Z
+	> To 
+	> !	HEAD:refs/for/a/b/c/my/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0026-push-options.sh b/t/t5411/test-0026-push-options.sh
index 1ec2cb95bc..fec5f95793 100644
--- a/t/t5411/test-0026-push-options.sh
+++ b/t/t5411/test-0026-push-options.sh
@@ -52,19 +52,19 @@ test_expect_success "proc-receive: ignore push-options for version 0 ($PROTOCOL)
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 * [new branch] HEAD -> next
-	 * [new reference] HEAD -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  * [new branch]      HEAD -> next
+	>  * [new reference]   HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual &&
 
@@ -101,22 +101,22 @@ test_expect_success "proc-receive: push with options ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive: atomic push_options
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive< issue=123
-	remote: proc-receive< reviewer=user1
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 * [new branch] HEAD -> next
-	 * [new reference] HEAD -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive: atomic push_options        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive< issue=123        Z
+	> remote: proc-receive< reviewer=user1        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  * [new branch]      HEAD -> next
+	>  * [new reference]   HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0027-push-options--porcelain.sh b/t/t5411/test-0027-push-options--porcelain.sh
index 447fbfec0c..8fb75a8789 100644
--- a/t/t5411/test-0027-push-options--porcelain.sh
+++ b/t/t5411/test-0027-push-options--porcelain.sh
@@ -54,20 +54,20 @@ test_expect_success "proc-receive: ignore push-options for version 0 ($PROTOCOL/
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	remote: post-receive<   refs/for/main/topic
-	To 
-	*    HEAD:refs/heads/next    [new branch]
-	*    HEAD:refs/for/main/topic    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	> *	HEAD:refs/heads/next	[new branch]
+	> *	HEAD:refs/for/main/topic	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -105,23 +105,23 @@ test_expect_success "proc-receive: push with options ($PROTOCOL/porcelain)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/next
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive: atomic push_options
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive< issue=123
-	remote: proc-receive< reviewer=user1
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/next
-	remote: post-receive<   refs/for/main/topic
-	To 
-	*    HEAD:refs/heads/next    [new branch]
-	*    HEAD:refs/for/main/topic    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive: atomic push_options        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive< issue=123        Z
+	> remote: proc-receive< reviewer=user1        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/next        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	> *	HEAD:refs/heads/next	[new branch]
+	> *	HEAD:refs/for/main/topic	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0030-report-ok.sh b/t/t5411/test-0030-report-ok.sh
index 8acb4f204f..a3a6278213 100644
--- a/t/t5411/test-0030-report-ok.sh
+++ b/t/t5411/test-0030-report-ok.sh
@@ -14,16 +14,16 @@ test_expect_success "proc-receive: ok ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 * [new reference] HEAD -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  * [new reference]   HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0031-report-ok--porcelain.sh b/t/t5411/test-0031-report-ok--porcelain.sh
index a967718046..0e175388b6 100644
--- a/t/t5411/test-0031-report-ok--porcelain.sh
+++ b/t/t5411/test-0031-report-ok--porcelain.sh
@@ -14,17 +14,17 @@ test_expect_success "proc-receive: ok ($PROTOCOL/porcelain)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	*    HEAD:refs/for/main/topic    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	> *	HEAD:refs/for/main/topic	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0032-report-with-options.sh b/t/t5411/test-0032-report-with-options.sh
index 437ade012d..cd4f213a7d 100644
--- a/t/t5411/test-0032-report-with-options.sh
+++ b/t/t5411/test-0032-report-with-options.sh
@@ -15,16 +15,16 @@ test_expect_success "proc-receive: report option without matching ok ($PROTOCOL)
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: error: proc-receive reported "option" without a matching "ok/ng" directive
-	To 
-	 ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: error: proc-receive reported "option" without a matching "ok/ng" directive        Z
+	> To 
+	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual
 '
@@ -46,17 +46,17 @@ test_expect_success "proc-receive: report option refname ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	To 
-	 * [new reference] HEAD -> refs/pull/123/head
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> To 
+	>  * [new reference]   HEAD -> refs/pull/123/head
 	EOF
 	test_cmp expect actual
 '
@@ -78,18 +78,18 @@ test_expect_success "proc-receive: report option refname and forced-update ($PRO
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option forced-update
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	To 
-	 * [new reference] HEAD -> refs/pull/123/head
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> To 
+	>  * [new reference]   HEAD -> refs/pull/123/head
 	EOF
 	test_cmp expect actual
 '
@@ -112,18 +112,18 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL)
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	To 
-	 .. HEAD -> refs/pull/123/head
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> To 
+	>    ..  HEAD -> refs/pull/123/head
 	EOF
 	test_cmp expect actual
 '
@@ -145,17 +145,17 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 .. HEAD -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>    ..  HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual
 '
@@ -178,18 +178,18 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL)
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 .. HEAD -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>    ..  HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual
 '
@@ -219,31 +219,31 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/next/topic
-	remote: pre-receive<   refs/for/a/b/c/topic
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/next/topic
-	remote: proc-receive<   refs/for/a/b/c/topic
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/a/b/c/topic
-	remote: proc-receive> ok refs/for/next/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/124/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option forced-update
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	remote: post-receive<   refs/for/a/b/c/topic
-	remote: post-receive<   refs/pull/124/head
-	To 
-	 * [new reference] HEAD -> refs/pull/123/head
-	 * [new reference] HEAD -> refs/for/a/b/c/topic
-	 + ... HEAD -> refs/pull/124/head (forced update)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/next/topic        Z
+	> remote: pre-receive<   refs/for/a/b/c/topic        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/next/topic        Z
+	> remote: proc-receive<   refs/for/a/b/c/topic        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/a/b/c/topic        Z
+	> remote: proc-receive> ok refs/for/next/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/124/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> remote: post-receive<   refs/for/a/b/c/topic        Z
+	> remote: post-receive<   refs/pull/124/head        Z
+	> To 
+	>  * [new reference]   HEAD -> refs/pull/123/head
+	>  * [new reference]   HEAD -> refs/for/a/b/c/topic
+	>  + ... HEAD -> refs/pull/124/head (forced update)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0033-report-with-options--porcelain.sh b/t/t5411/test-0033-report-with-options--porcelain.sh
index 11486720ee..7433870e40 100644
--- a/t/t5411/test-0033-report-with-options--porcelain.sh
+++ b/t/t5411/test-0033-report-with-options--porcelain.sh
@@ -15,17 +15,17 @@ test_expect_success "proc-receive: report option without matching ok ($PROTOCOL/
 		HEAD:refs/for/main/topic \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: error: proc-receive reported "option" without a matching "ok/ng" directive
-	To 
-	!    HEAD:refs/for/main/topic    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: error: proc-receive reported "option" without a matching "ok/ng" directive        Z
+	> To 
+	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual
 '
@@ -47,18 +47,18 @@ test_expect_success "proc-receive: report option refname ($PROTOCOL/porcelain)"
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	To 
-	*    HEAD:refs/pull/123/head    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> To 
+	> *	HEAD:refs/pull/123/head	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual
 '
@@ -81,19 +81,19 @@ test_expect_success "proc-receive: report option refname and forced-update ($PRO
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option forced-update
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	To 
-	*    HEAD:refs/pull/123/head    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> To 
+	> *	HEAD:refs/pull/123/head	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual
 '
@@ -116,19 +116,19 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL/
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	To 
-	     HEAD:refs/pull/123/head    ..
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> To 
+	>  	HEAD:refs/pull/123/head	..
+	> Done
 	EOF
 	test_cmp expect actual
 '
@@ -150,18 +150,18 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL/porcelain)"
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	     HEAD:refs/for/main/topic    ..
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  	HEAD:refs/for/main/topic	..
+	> Done
 	EOF
 	test_cmp expect actual
 '
@@ -184,19 +184,19 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL/
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	     HEAD:refs/for/main/topic    ..
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  	HEAD:refs/for/main/topic	..
+	> Done
 	EOF
 	test_cmp expect actual
 '
@@ -227,32 +227,32 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL/porc
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/next/topic
-	remote: pre-receive<   refs/for/a/b/c/topic
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/next/topic
-	remote: proc-receive<   refs/for/a/b/c/topic
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/a/b/c/topic
-	remote: proc-receive> ok refs/for/next/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/124/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option forced-update
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	remote: post-receive<   refs/for/a/b/c/topic
-	remote: post-receive<   refs/pull/124/head
-	To 
-	*    HEAD:refs/pull/123/head    [new reference]
-	*    HEAD:refs/for/a/b/c/topic    [new reference]
-	+    HEAD:refs/pull/124/head    ... (forced update)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/next/topic        Z
+	> remote: pre-receive<   refs/for/a/b/c/topic        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/next/topic        Z
+	> remote: proc-receive<   refs/for/a/b/c/topic        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/a/b/c/topic        Z
+	> remote: proc-receive> ok refs/for/next/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/124/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> remote: post-receive<   refs/for/a/b/c/topic        Z
+	> remote: post-receive<   refs/pull/124/head        Z
+	> To 
+	> *	HEAD:refs/pull/123/head	[new reference]
+	> *	HEAD:refs/for/a/b/c/topic	[new reference]
+	> +	HEAD:refs/pull/124/head	... (forced update)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0034-report-ft.sh b/t/t5411/test-0034-report-ft.sh
index 6e0d08b327..73a47d1ffd 100644
--- a/t/t5411/test-0034-report-ft.sh
+++ b/t/t5411/test-0034-report-ft.sh
@@ -15,17 +15,17 @@ test_expect_success "proc-receive: fall throught, let receive-pack to execute ($
 		$B:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option fall-through
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 * [new reference]  -> refs/for/main/topic
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  * [new reference]    -> refs/for/main/topic
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0035-report-ft--porcelain.sh b/t/t5411/test-0035-report-ft--porcelain.sh
index 81bae9f2ec..c350201107 100644
--- a/t/t5411/test-0035-report-ft--porcelain.sh
+++ b/t/t5411/test-0035-report-ft--porcelain.sh
@@ -15,18 +15,18 @@ test_expect_success "proc-receive: fall throught, let receive-pack to execute ($
 		$B:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option fall-through
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	To 
-	*    :refs/for/main/topic    [new reference]
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	> *	:refs/for/main/topic	[new reference]
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh b/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh
index be9b18b2b6..e3f456ca5a 100644
--- a/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh
+++ b/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh
@@ -39,30 +39,30 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/24/124/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/25/125/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	remote: post-receive<   refs/changes/24/124/1
-	remote: post-receive<   refs/changes/25/125/1
-	To 
-	 .. HEAD -> refs/for/main/topic
-	 * [new reference] HEAD -> refs/changes/24/124/1
-	 .. HEAD -> refs/changes/25/125/1
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/24/124/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/25/125/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> remote: post-receive<   refs/changes/24/124/1        Z
+	> remote: post-receive<   refs/changes/25/125/1        Z
+	> To 
+	>    ..  HEAD -> refs/for/main/topic
+	>  * [new reference]   HEAD -> refs/changes/24/124/1
+	>    ..  HEAD -> refs/changes/25/125/1
 	EOF
 	test_cmp expect actual &&
 
@@ -113,31 +113,31 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/24/124/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/25/125/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> option forced-update
-	remote: # post-receive hook
-	remote: post-receive<   refs/changes/24/124/1
-	remote: post-receive<   refs/for/main/topic
-	remote: post-receive<   refs/changes/25/125/1
-	To 
-	 * [new reference] HEAD -> refs/changes/24/124/1
-	 .. HEAD -> refs/for/main/topic
-	 + ... HEAD -> refs/changes/25/125/1 (forced update)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/24/124/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/25/125/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/changes/24/124/1        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> remote: post-receive<   refs/changes/25/125/1        Z
+	> To 
+	>  * [new reference]   HEAD -> refs/changes/24/124/1
+	>    ..  HEAD -> refs/for/main/topic
+	>  + ... HEAD -> refs/changes/25/125/1 (forced update)
 	EOF
 	test_cmp expect actual &&
 
@@ -182,23 +182,23 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL)" '
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/23/123/1
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/24/124/2
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/changes/23/123/1
-	remote: post-receive<   refs/changes/24/124/2
-	To 
-	 * [new reference] HEAD -> refs/changes/23/123/1
-	 .. HEAD -> refs/changes/24/124/2
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/23/123/1        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/24/124/2        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/changes/23/123/1        Z
+	> remote: post-receive<   refs/changes/24/124/2        Z
+	> To 
+	>  * [new reference]   HEAD -> refs/changes/23/123/1
+	>    ..  HEAD -> refs/changes/24/124/2
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh b/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh
index 95fb89c031..7786079ba5 100644
--- a/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh
+++ b/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh
@@ -24,31 +24,31 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/24/124/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/25/125/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/for/main/topic
-	remote: post-receive<   refs/changes/24/124/1
-	remote: post-receive<   refs/changes/25/125/1
-	To 
-	     HEAD:refs/for/main/topic    ..
-	*    HEAD:refs/changes/24/124/1    [new reference]
-	     HEAD:refs/changes/25/125/1    ..
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/24/124/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/25/125/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> remote: post-receive<   refs/changes/24/124/1        Z
+	> remote: post-receive<   refs/changes/25/125/1        Z
+	> To 
+	>  	HEAD:refs/for/main/topic	..
+	> *	HEAD:refs/changes/24/124/1	[new reference]
+	>  	HEAD:refs/changes/25/125/1	..
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -84,32 +84,32 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/24/124/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/25/125/1
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> option forced-update
-	remote: # post-receive hook
-	remote: post-receive<   refs/changes/24/124/1
-	remote: post-receive<   refs/for/main/topic
-	remote: post-receive<   refs/changes/25/125/1
-	To 
-	*    HEAD:refs/changes/24/124/1    [new reference]
-	     HEAD:refs/for/main/topic    ..
-	+    HEAD:refs/changes/25/125/1    ... (forced update)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/24/124/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/25/125/1        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/changes/24/124/1        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> remote: post-receive<   refs/changes/25/125/1        Z
+	> To 
+	> *	HEAD:refs/changes/24/124/1	[new reference]
+	>  	HEAD:refs/for/main/topic	..
+	> +	HEAD:refs/changes/25/125/1	... (forced update)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
@@ -139,24 +139,24 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL/porc
 		HEAD:refs/for/main/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/for/main/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/23/123/1
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/changes/24/124/2
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/changes/23/123/1
-	remote: post-receive<   refs/changes/24/124/2
-	To 
-	*    HEAD:refs/changes/23/123/1    [new reference]
-	     HEAD:refs/changes/24/124/2    ..
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/23/123/1        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/changes/24/124/2        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/changes/23/123/1        Z
+	> remote: post-receive<   refs/changes/24/124/2        Z
+	> To 
+	> *	HEAD:refs/changes/23/123/1	[new reference]
+	>  	HEAD:refs/changes/24/124/2	..
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0038-report-mixed-refs.sh b/t/t5411/test-0038-report-mixed-refs.sh
index 5e005299cc..0990a3f76a 100644
--- a/t/t5411/test-0038-report-mixed-refs.sh
+++ b/t/t5411/test-0038-report-mixed-refs.sh
@@ -26,43 +26,43 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL)" '
 		HEAD:refs/for/next/topic3 \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/bar
-	remote: pre-receive<   refs/heads/baz
-	remote: pre-receive<   refs/for/next/topic2
-	remote: pre-receive<   refs/for/next/topic1
-	remote: pre-receive<   refs/heads/foo
-	remote: pre-receive<   refs/for/main/topic
-	remote: pre-receive<   refs/for/next/topic3
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/next/topic2
-	remote: proc-receive<   refs/for/next/topic1
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive<   refs/for/next/topic3
-	remote: proc-receive> ok refs/for/next/topic2
-	remote: proc-receive> ng refs/for/next/topic1 fail to call Web API
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/heads/bar
-	remote: post-receive<   refs/heads/baz
-	remote: post-receive<   refs/for/next/topic2
-	remote: post-receive<   refs/heads/foo
-	remote: post-receive<   refs/for/main/topic
-	To 
-	 ..  -> main
-	 * [new branch] HEAD -> bar
-	 * [new branch] HEAD -> baz
-	 * [new reference] HEAD -> refs/for/next/topic2
-	 * [new branch] HEAD -> foo
-	 .. HEAD -> refs/for/main/topic
-	 ! [remote rejected] HEAD -> refs/for/next/topic1 (fail to call Web API)
-	 ! [remote rejected] HEAD -> refs/for/next/topic3 (proc-receive failed to report status)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/bar        Z
+	> remote: pre-receive<   refs/heads/baz        Z
+	> remote: pre-receive<   refs/for/next/topic2        Z
+	> remote: pre-receive<   refs/for/next/topic1        Z
+	> remote: pre-receive<   refs/heads/foo        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: pre-receive<   refs/for/next/topic3        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/next/topic2        Z
+	> remote: proc-receive<   refs/for/next/topic1        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive<   refs/for/next/topic3        Z
+	> remote: proc-receive> ok refs/for/next/topic2        Z
+	> remote: proc-receive> ng refs/for/next/topic1 fail to call Web API        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/heads/bar        Z
+	> remote: post-receive<   refs/heads/baz        Z
+	> remote: post-receive<   refs/for/next/topic2        Z
+	> remote: post-receive<   refs/heads/foo        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>    ..   -> main
+	>  * [new branch]      HEAD -> bar
+	>  * [new branch]      HEAD -> baz
+	>  * [new reference]   HEAD -> refs/for/next/topic2
+	>  * [new branch]      HEAD -> foo
+	>    ..  HEAD -> refs/for/main/topic
+	>  ! [remote rejected] HEAD -> refs/for/next/topic1 (fail to call Web API)
+	>  ! [remote rejected] HEAD -> refs/for/next/topic3 (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0039-report-mixed-refs--porcelain.sh b/t/t5411/test-0039-report-mixed-refs--porcelain.sh
index 8f891c5385..7e4d08a939 100644
--- a/t/t5411/test-0039-report-mixed-refs--porcelain.sh
+++ b/t/t5411/test-0039-report-mixed-refs--porcelain.sh
@@ -26,44 +26,44 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL/porcel
 		HEAD:refs/for/next/topic3 \
 		>out-$test_count 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/bar
-	remote: pre-receive<   refs/heads/baz
-	remote: pre-receive<   refs/for/next/topic2
-	remote: pre-receive<   refs/for/next/topic1
-	remote: pre-receive<   refs/heads/foo
-	remote: pre-receive<   refs/for/main/topic
-	remote: pre-receive<   refs/for/next/topic3
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/for/next/topic2
-	remote: proc-receive<   refs/for/next/topic1
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive<   refs/for/next/topic3
-	remote: proc-receive> ok refs/for/next/topic2
-	remote: proc-receive> ng refs/for/next/topic1 fail to call Web API
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/for/main/topic
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/heads/bar
-	remote: post-receive<   refs/heads/baz
-	remote: post-receive<   refs/for/next/topic2
-	remote: post-receive<   refs/heads/foo
-	remote: post-receive<   refs/for/main/topic
-	To 
-	     :refs/heads/main    ..
-	*    HEAD:refs/heads/bar    [new branch]
-	*    HEAD:refs/heads/baz    [new branch]
-	*    HEAD:refs/for/next/topic2    [new reference]
-	*    HEAD:refs/heads/foo    [new branch]
-	     HEAD:refs/for/main/topic    ..
-	!    HEAD:refs/for/next/topic1    [remote rejected] (fail to call Web API)
-	!    HEAD:refs/for/next/topic3    [remote rejected] (proc-receive failed to report status)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/bar        Z
+	> remote: pre-receive<   refs/heads/baz        Z
+	> remote: pre-receive<   refs/for/next/topic2        Z
+	> remote: pre-receive<   refs/for/next/topic1        Z
+	> remote: pre-receive<   refs/heads/foo        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: pre-receive<   refs/for/next/topic3        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/for/next/topic2        Z
+	> remote: proc-receive<   refs/for/next/topic1        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive<   refs/for/next/topic3        Z
+	> remote: proc-receive> ok refs/for/next/topic2        Z
+	> remote: proc-receive> ng refs/for/next/topic1 fail to call Web API        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/for/main/topic        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/heads/bar        Z
+	> remote: post-receive<   refs/heads/baz        Z
+	> remote: post-receive<   refs/for/next/topic2        Z
+	> remote: post-receive<   refs/heads/foo        Z
+	> remote: post-receive<   refs/for/main/topic        Z
+	> To 
+	>  	:refs/heads/main	..
+	> *	HEAD:refs/heads/bar	[new branch]
+	> *	HEAD:refs/heads/baz	[new branch]
+	> *	HEAD:refs/for/next/topic2	[new reference]
+	> *	HEAD:refs/heads/foo	[new branch]
+	>  	HEAD:refs/for/main/topic	..
+	> !	HEAD:refs/for/next/topic1	[remote rejected] (fail to call Web API)
+	> !	HEAD:refs/for/next/topic3	[remote rejected] (proc-receive failed to report status)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0040-process-all-refs.sh b/t/t5411/test-0040-process-all-refs.sh
index fdcdcc7c2e..f67cb2a7b3 100644
--- a/t/t5411/test-0040-process-all-refs.sh
+++ b/t/t5411/test-0040-process-all-refs.sh
@@ -50,46 +50,46 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL)" '
 		HEAD:refs/for/next/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/bar
-	remote: pre-receive<   refs/heads/foo
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/for/main/topic
-	remote: pre-receive<   refs/for/next/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/heads/bar
-	remote: proc-receive<   refs/heads/foo
-	remote: proc-receive<   refs/heads/main
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive<   refs/for/next/topic
-	remote: proc-receive> ok refs/heads/main
-	remote: proc-receive> option fall-through
-	remote: proc-receive> ok refs/heads/foo
-	remote: proc-receive> option fall-through
-	remote: proc-receive> ok refs/heads/bar
-	remote: proc-receive> option fall-through
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/next/topic
-	remote: proc-receive> option refname refs/pull/124/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> option forced-update
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/bar
-	remote: post-receive<   refs/heads/foo
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/pull/123/head
-	remote: post-receive<   refs/pull/124/head
-	To 
-	 ..  -> bar
-	 - [deleted] foo
-	 + ... HEAD -> main (forced update)
-	 .. HEAD -> refs/pull/123/head
-	 + ... HEAD -> refs/pull/124/head (forced update)
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/bar        Z
+	> remote: pre-receive<   refs/heads/foo        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: pre-receive<   refs/for/next/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/heads/bar        Z
+	> remote: proc-receive<   refs/heads/foo        Z
+	> remote: proc-receive<   refs/heads/main        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive<   refs/for/next/topic        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: proc-receive> ok refs/heads/foo        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: proc-receive> ok refs/heads/bar        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/next/topic        Z
+	> remote: proc-receive> option refname refs/pull/124/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/bar        Z
+	> remote: post-receive<   refs/heads/foo        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> remote: post-receive<   refs/pull/124/head        Z
+	> To 
+	>    ..   -> bar
+	>  - [deleted]         foo
+	>  + ... HEAD -> main (forced update)
+	>    ..  HEAD -> refs/pull/123/head
+	>  + ... HEAD -> refs/pull/124/head (forced update)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0041-process-all-refs--porcelain.sh b/t/t5411/test-0041-process-all-refs--porcelain.sh
index 73b35fe0aa..7d1a99d3a2 100644
--- a/t/t5411/test-0041-process-all-refs--porcelain.sh
+++ b/t/t5411/test-0041-process-all-refs--porcelain.sh
@@ -50,47 +50,47 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL/porcelain)" '
 		HEAD:refs/for/next/topic \
 		>out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/bar
-	remote: pre-receive<   refs/heads/foo
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/for/main/topic
-	remote: pre-receive<   refs/for/next/topic
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/heads/bar
-	remote: proc-receive<   refs/heads/foo
-	remote: proc-receive<   refs/heads/main
-	remote: proc-receive<   refs/for/main/topic
-	remote: proc-receive<   refs/for/next/topic
-	remote: proc-receive> ok refs/heads/main
-	remote: proc-receive> option fall-through
-	remote: proc-receive> ok refs/heads/foo
-	remote: proc-receive> option fall-through
-	remote: proc-receive> ok refs/heads/bar
-	remote: proc-receive> option fall-through
-	remote: proc-receive> ok refs/for/main/topic
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/for/next/topic
-	remote: proc-receive> option refname refs/pull/124/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> option forced-update
-	remote: # post-receive hook
-	remote: post-receive<   refs/heads/bar
-	remote: post-receive<   refs/heads/foo
-	remote: post-receive<   refs/heads/main
-	remote: post-receive<   refs/pull/123/head
-	remote: post-receive<   refs/pull/124/head
-	To 
-	     :refs/heads/bar    ..
-	-    :refs/heads/foo    [deleted]
-	+    HEAD:refs/heads/main    ... (forced update)
-	     HEAD:refs/pull/123/head    ..
-	+    HEAD:refs/pull/124/head    ... (forced update)
-	Done
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/bar        Z
+	> remote: pre-receive<   refs/heads/foo        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/for/main/topic        Z
+	> remote: pre-receive<   refs/for/next/topic        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/heads/bar        Z
+	> remote: proc-receive<   refs/heads/foo        Z
+	> remote: proc-receive<   refs/heads/main        Z
+	> remote: proc-receive<   refs/for/main/topic        Z
+	> remote: proc-receive<   refs/for/next/topic        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: proc-receive> ok refs/heads/foo        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: proc-receive> ok refs/heads/bar        Z
+	> remote: proc-receive> option fall-through        Z
+	> remote: proc-receive> ok refs/for/main/topic        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/for/next/topic        Z
+	> remote: proc-receive> option refname refs/pull/124/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> option forced-update        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/heads/bar        Z
+	> remote: post-receive<   refs/heads/foo        Z
+	> remote: post-receive<   refs/heads/main        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> remote: post-receive<   refs/pull/124/head        Z
+	> To 
+	>  	:refs/heads/bar	..
+	> -	:refs/heads/foo	[deleted]
+	> +	HEAD:refs/heads/main	... (forced update)
+	>  	HEAD:refs/pull/123/head	..
+	> +	HEAD:refs/pull/124/head	... (forced update)
+	> Done
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh b/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh
index 7214647ada..dba544162c 100644
--- a/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh
+++ b/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh
@@ -29,25 +29,25 @@ test_expect_success "proc-receive: update branch and new tag ($PROTOCOL)" '
 		$B:refs/heads/main \
 		v123 >out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/tags/v123
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/heads/main
-	remote: proc-receive<   refs/tags/v123
-	remote: proc-receive> ok refs/heads/main
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/tags/v123
-	remote: proc-receive> option refname refs/pull/124/head
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	remote: post-receive<   refs/pull/124/head
-	To 
-	 ..  -> refs/pull/123/head
-	 * [new reference] v123 -> refs/pull/124/head
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/tags/v123        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/heads/main        Z
+	> remote: proc-receive<   refs/tags/v123        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/tags/v123         Z
+	> remote: proc-receive> option refname refs/pull/124/head        Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> remote: post-receive<   refs/pull/124/head        Z
+	> To 
+	>    ..   -> refs/pull/123/head
+	>  * [new reference]   v123 -> refs/pull/124/head
 	EOF
 	test_cmp expect actual &&
 
@@ -93,32 +93,32 @@ test_expect_success "proc-receive: create/delete branch, and delete tag ($PROTOC
 		$A:refs/heads/next \
 		:refs/tags/v123 >out 2>&1 &&
 	make_user_friendly_and_stable_output actual &&
-	cat >expect <<-EOF &&
-	remote: # pre-receive hook
-	remote: pre-receive<   refs/heads/main
-	remote: pre-receive<   refs/heads/topic
-	remote: pre-receive<   refs/tags/v123
-	remote: pre-receive<   refs/heads/next
-	remote: # proc-receive hook
-	remote: proc-receive<   refs/heads/main
-	remote: proc-receive<   refs/heads/next
-	remote: proc-receive> ok refs/heads/main
-	remote: proc-receive> option refname refs/pull/123/head
-	remote: proc-receive> option old-oid 
-	remote: proc-receive> option new-oid 
-	remote: proc-receive> ok refs/heads/next
-	remote: proc-receive> option refname refs/pull/124/head
-	remote: proc-receive> option new-oid 
-	remote: # post-receive hook
-	remote: post-receive<   refs/pull/123/head
-	remote: post-receive<   refs/heads/topic
-	remote: post-receive<   refs/tags/v123
-	remote: post-receive<   refs/pull/124/head
-	To 
-	 - [deleted] refs/pull/123/head
-	 ..  -> topic
-	 - [deleted] v123
-	 * [new reference]  -> refs/pull/124/head
+	format_and_save_expect <<-EOF &&
+	> remote: # pre-receive hook        Z
+	> remote: pre-receive<   refs/heads/main        Z
+	> remote: pre-receive<   refs/heads/topic        Z
+	> remote: pre-receive<   refs/tags/v123        Z
+	> remote: pre-receive<   refs/heads/next        Z
+	> remote: # proc-receive hook        Z
+	> remote: proc-receive<   refs/heads/main        Z
+	> remote: proc-receive<   refs/heads/next        Z
+	> remote: proc-receive> ok refs/heads/main        Z
+	> remote: proc-receive> option refname refs/pull/123/head        Z
+	> remote: proc-receive> option old-oid         Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: proc-receive> ok refs/heads/next        Z
+	> remote: proc-receive> option refname refs/pull/124/head        Z
+	> remote: proc-receive> option new-oid         Z
+	> remote: # post-receive hook        Z
+	> remote: post-receive<   refs/pull/123/head        Z
+	> remote: post-receive<   refs/heads/topic        Z
+	> remote: post-receive<   refs/tags/v123        Z
+	> remote: post-receive<   refs/pull/124/head        Z
+	> To 
+	>  - [deleted]         refs/pull/123/head
+	>    ..   -> topic
+	>  - [deleted]         v123
+	>  * [new reference]    -> refs/pull/124/head
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5548-push-porcelain.sh b/t/t5548-push-porcelain.sh
index 5a761f3642..2ab7c3cf5a 100755
--- a/t/t5548-push-porcelain.sh
+++ b/t/t5548-push-porcelain.sh
@@ -45,9 +45,6 @@ create_commits_in () {
 # of the output.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/  *\$//" \
-		-e "s/   */ /g" \
-		-e "s/	/    /g" \
 		-e "s/$A//g" \
 		-e "s/$B//g" \
 		-e "s/$ZERO_OID//g" \
@@ -56,6 +53,10 @@ make_user_friendly_and_stable_output () {
 		-e "s#To $URL_PREFIX/upstream.git#To #"
 }
 
+format_and_save_expect () {
+	sed -e 's/^> //' -e 's/Z$//' >expect
+}
+
 setup_upstream_and_workbench () {
 	# Upstream  after setup : main(B)  foo(A)  bar(A)  baz(A)
 	# Workbench after setup : main(A)
@@ -111,14 +112,14 @@ run_git_push_porcelain_output_test() {
 				next
 		) >out &&
 		make_user_friendly_and_stable_output actual &&
-		cat >expect <<-EOF &&
-		To 
-		=    refs/heads/baz:refs/heads/baz    [up to date]
-		     :refs/heads/bar    ..
-		-    :refs/heads/foo    [deleted]
-		+    refs/heads/main:refs/heads/main    ... (forced update)
-		*    refs/heads/next:refs/heads/next    [new branch]
-		Done
+		format_and_save_expect <<-EOF &&
+		> To 
+		> =	refs/heads/baz:refs/heads/baz	[up to date]
+		>  	:refs/heads/bar	..
+		> -	:refs/heads/foo	[deleted]
+		> +	refs/heads/main:refs/heads/main	... (forced update)
+		> *	refs/heads/next:refs/heads/next	[new branch]
+		> Done
 		EOF
 		test_cmp expect actual &&
 
@@ -148,12 +149,12 @@ run_git_push_porcelain_output_test() {
 				next
 		) >out &&
 		make_user_friendly_and_stable_output actual &&
-		cat >expect <<-EOF &&
+		format_and_save_expect <<-EOF &&
 		To 
-		=    refs/heads/next:refs/heads/next    [up to date]
-		!    refs/heads/bar:refs/heads/bar    [rejected] (non-fast-forward)
-		!    (delete):refs/heads/baz    [rejected] (atomic push failed)
-		!    refs/heads/main:refs/heads/main    [rejected] (atomic push failed)
+		> =	refs/heads/next:refs/heads/next	[up to date]
+		> !	refs/heads/bar:refs/heads/bar	[rejected] (non-fast-forward)
+		> !	(delete):refs/heads/baz	[rejected] (atomic push failed)
+		> !	refs/heads/main:refs/heads/main	[rejected] (atomic push failed)
 		Done
 		EOF
 		test_cmp expect actual &&
@@ -168,6 +169,7 @@ run_git_push_porcelain_output_test() {
 		EOF
 		test_cmp expect actual
 	'
+
 	test_expect_success "prepare pre-receive hook ($PROTOCOL)" '
 		write_script "$upstream/hooks/pre-receive" <<-EOF
 		exit 1
@@ -189,12 +191,12 @@ run_git_push_porcelain_output_test() {
 				next
 		) >out &&
 		make_user_friendly_and_stable_output actual &&
-		cat >expect <<-EOF &&
+		format_and_save_expect <<-EOF &&
 		To 
-		=    refs/heads/next:refs/heads/next    [up to date]
-		!    refs/heads/bar:refs/heads/bar    [remote rejected] (pre-receive hook declined)
-		!    :refs/heads/baz    [remote rejected] (pre-receive hook declined)
-		!    refs/heads/main:refs/heads/main    [remote rejected] (pre-receive hook declined)
+		> =	refs/heads/next:refs/heads/next	[up to date]
+		> !	refs/heads/bar:refs/heads/bar	[remote rejected] (pre-receive hook declined)
+		> !	:refs/heads/baz	[remote rejected] (pre-receive hook declined)
+		> !	refs/heads/main:refs/heads/main	[remote rejected] (pre-receive hook declined)
 		Done
 		EOF
 		test_cmp expect actual &&
@@ -227,12 +229,12 @@ run_git_push_porcelain_output_test() {
 				next
 		) >out &&
 		make_user_friendly_and_stable_output actual &&
-		cat >expect <<-EOF &&
+		format_and_save_expect <<-EOF &&
 		To 
-		=    refs/heads/next:refs/heads/next    [up to date]
-		-    :refs/heads/baz    [deleted]
-		     refs/heads/main:refs/heads/main    ..
-		!    refs/heads/bar:refs/heads/bar    [rejected] (non-fast-forward)
+		> =	refs/heads/next:refs/heads/next	[up to date]
+		> -	:refs/heads/baz	[deleted]
+		>  	refs/heads/main:refs/heads/main	..
+		> !	refs/heads/bar:refs/heads/bar	[rejected] (non-fast-forward)
 		Done
 		EOF
 		test_cmp expect actual &&
diff --git a/t/t6020-bundle-misc.sh b/t/t6020-bundle-misc.sh
index 881f72fd44..cf13bcc3c8 100755
--- a/t/t6020-bundle-misc.sh
+++ b/t/t6020-bundle-misc.sh
@@ -82,7 +82,7 @@ test_commit_setvar () {
 
 # Format the output of git commands to make a user-friendly and stable
 # text.  We can easily prepare the expect text without having to worry
-# about future changes of the commit ID and spaces of the output.
+# about future changes of the commit ID.
 make_user_friendly_and_stable_output () {
 	sed \
 		-e "s/${A%${A#???????}}[0-9a-f]*//g" \
@@ -103,8 +103,11 @@ make_user_friendly_and_stable_output () {
 		-e "s/${P%${P#???????}}[0-9a-f]*//g" \
 		-e "s/${TAG1%${TAG1#???????}}[0-9a-f]*//g" \
 		-e "s/${TAG2%${TAG2#???????}}[0-9a-f]*//g" \
-		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*//g" \
-		-e "s/ *\$//"
+		-e "s/${TAG3%${TAG3#???????}}[0-9a-f]*//g"
+}
+
+format_and_save_expect () {
+	sed -e 's/Z$//' >expect
 }
 
 #            (C)   (D, pull/1/head, topic/1)
@@ -179,11 +182,11 @@ test_expect_success 'create bundle from special rev: main^!' '
 
 	git bundle verify special-rev.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
+	format_and_save_expect <<-\EOF &&
 	The bundle contains this ref:
 	 refs/heads/main
 	The bundle requires this ref:
-	
+	 Z
 	EOF
 	test_cmp expect actual &&
 
@@ -200,12 +203,12 @@ test_expect_success 'create bundle with --max-count option' '
 
 	git bundle verify max-count.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
+	format_and_save_expect <<-\EOF &&
 	The bundle contains these 2 refs:
 	 refs/heads/main
 	 refs/tags/v1
 	The bundle requires this ref:
-	
+	 Z
 	EOF
 	test_cmp expect actual &&
 
@@ -225,7 +228,7 @@ test_expect_success 'create bundle with --since option' '
 
 	git bundle verify since.bdl |
 		make_user_friendly_and_stable_output >actual &&
-	cat >expect <<-\EOF &&
+	format_and_save_expect <<-\EOF &&
 	The bundle contains these 5 refs:
 	 refs/heads/main
 	 refs/heads/release
@@ -233,8 +236,8 @@ test_expect_success 'create bundle with --since option' '
 	 refs/tags/v3
 	 HEAD
 	The bundle requires these 2 refs:
-	
-	
+	 Z
+	 Z
 	EOF
 	test_cmp expect actual &&
 
@@ -293,13 +296,13 @@ test_expect_success 'create bundle 2 - has prerequisites' '
 		--stdin \
 		release expect <<-\EOF &&
+	format_and_save_expect <<-\EOF &&
 	The bundle contains this ref:
 	 refs/heads/release
 	The bundle requires these 3 refs:
-	
-	
-	
+	 Z
+	 Z
+	 Z
 	EOF
 
 	git bundle verify 2.bdl |
@@ -317,11 +320,11 @@ test_expect_success 'create bundle 2 - has prerequisites' '
 test_expect_success 'fail to verify bundle without prerequisites' '
 	git init --bare test1.git &&
 
-	cat >expect <<-\EOF &&
+	format_and_save_expect <<-\EOF &&
 	error: Repository lacks these prerequisite commits:
-	error: 
-	error: 
-	error: 
+	error:  Z
+	error:  Z
+	error:  Z
 	EOF
 
 	test_must_fail git -C test1.git bundle verify ../2.bdl 2>&1 |
@@ -352,13 +355,13 @@ test_expect_success 'create bundle 3 - two refs, same object' '
 		--stdin \
 		main HEAD expect <<-\EOF &&
+	format_and_save_expect <<-\EOF &&
 	The bundle contains these 2 refs:
 	 refs/heads/main
 	 HEAD
 	The bundle requires these 2 refs:
-	
-	
+	 Z
+	 Z
 	EOF
 
 	git bundle verify 3.bdl |

From 3c06a583398f6630c7162b463ff1acbdf0110f83 Mon Sep 17 00:00:00 2001
From: Jiang Xin 
Date: Thu, 17 Jun 2021 11:17:26 +0800
Subject: [PATCH 249/397] test: refactor to use "test_commit" to create commits

Refactor function "create_commits_in" to use "test_commit" to create
commit.

Signed-off-by: Jiang Xin 
Signed-off-by: Junio C Hamano 
---
 t/t5411/common-functions.sh | 25 ++++++-------------------
 t/t5548-push-porcelain.sh   | 25 ++++++-------------------
 2 files changed, 12 insertions(+), 38 deletions(-)

diff --git a/t/t5411/common-functions.sh b/t/t5411/common-functions.sh
index 6c347b4caa..b9c6adecf5 100644
--- a/t/t5411/common-functions.sh
+++ b/t/t5411/common-functions.sh
@@ -6,29 +6,16 @@
 # NOTE: Never calling this function from a subshell since variable
 # assignments will disappear when subshell exits.
 create_commits_in () {
-	repo="$1" &&
-	if ! parent=$(git -C "$repo" rev-parse HEAD^{} --)
-	then
-		parent=
-	fi &&
-	T=$(git -C "$repo" write-tree) &&
+	repo="$1" && test -d "$repo" ||
+	error "Repository $repo does not exist."
 	shift &&
 	while test $# -gt 0
 	do
 		name=$1 &&
-		test_tick &&
-		if test -z "$parent"
-		then
-			oid=$(echo $name | git -C "$repo" commit-tree $T)
-		else
-			oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
-		fi &&
-		eval $name=$oid &&
-		parent=$oid &&
-		shift ||
-		return 1
-	done &&
-	git -C "$repo" update-ref refs/heads/main $oid
+		shift &&
+		test_commit -C "$repo" --no-tag "$name" &&
+		eval $name=$(git -C "$repo" rev-parse HEAD)
+	done
 }
 
 # Format the output of git-push, git-show-ref and other commands to make a
diff --git a/t/t5548-push-porcelain.sh b/t/t5548-push-porcelain.sh
index 2ab7c3cf5a..c8f18a7a12 100755
--- a/t/t5548-push-porcelain.sh
+++ b/t/t5548-push-porcelain.sh
@@ -14,29 +14,16 @@ test_description='Test git push porcelain output'
 # NOTE: Never calling this function from a subshell since variable
 # assignments will disappear when subshell exits.
 create_commits_in () {
-	repo="$1" &&
-	if ! parent=$(git -C "$repo" rev-parse HEAD^{} --)
-	then
-		parent=
-	fi &&
-	T=$(git -C "$repo" write-tree) &&
+	repo="$1" && test -d "$repo" ||
+	error "Repository $repo does not exist."
 	shift &&
 	while test $# -gt 0
 	do
 		name=$1 &&
-		test_tick &&
-		if test -z "$parent"
-		then
-			oid=$(echo $name | git -C "$repo" commit-tree $T)
-		else
-			oid=$(echo $name | git -C "$repo" commit-tree -p $parent $T)
-		fi &&
-		eval $name=$oid &&
-		parent=$oid &&
-		shift ||
-		return 1
-	done &&
-	git -C "$repo" update-ref refs/heads/main $oid
+		shift &&
+		test_commit -C "$repo" --no-tag "$name" &&
+		eval $name=$(git -C "$repo" rev-parse HEAD)
+	done
 }
 
 # Format the output of git-push, git-show-ref and other commands to make a

From 12d6991cf417465d567c38ba8ae7d59bda7a202f Mon Sep 17 00:00:00 2001
From: Jiang Xin 
Date: Thu, 17 Jun 2021 11:17:27 +0800
Subject: [PATCH 250/397] test: refactor to use "get_abbrev_oid" to get abbrev
 oid

Add new function "get_abbrev_oid" to get abbrev object ID.  This
function has a default value which helps to prepare a nonempty replace
pattern for sed command.  An empty replace pattern may cause sed fail
to allocate memory.

Refactor function "make_user_friendly_and_stable_output" to use
"get_abbrev_oid" to get abbrev object ID.

Signed-off-by: Jiang Xin 
Signed-off-by: Junio C Hamano 
---
 t/t5411/common-functions.sh                   | 20 ++++++++++++----
 t/t5411/test-0000-standard-git-push.sh        |  6 ++---
 .../test-0001-standard-git-push--porcelain.sh |  6 ++---
 t/t5411/test-0022-report-unexpect-ref.sh      |  2 +-
 ...est-0023-report-unexpect-ref--porcelain.sh |  2 +-
 t/t5411/test-0032-report-with-options.sh      |  8 +++----
 ...est-0033-report-with-options--porcelain.sh |  8 +++----
 ...t-0036-report-multi-rewrite-for-one-ref.sh | 10 ++++----
 ...rt-multi-rewrite-for-one-ref--porcelain.sh | 10 ++++----
 t/t5411/test-0038-report-mixed-refs.sh        |  4 ++--
 .../test-0039-report-mixed-refs--porcelain.sh |  4 ++--
 t/t5411/test-0040-process-all-refs.sh         |  8 +++----
 .../test-0041-process-all-refs--porcelain.sh  |  8 +++----
 ...t-0050-proc-receive-refs-with-modifiers.sh |  4 ++--
 t/t5548-push-porcelain.sh                     | 24 +++++++++++++------
 15 files changed, 72 insertions(+), 52 deletions(-)

diff --git a/t/t5411/common-functions.sh b/t/t5411/common-functions.sh
index b9c6adecf5..3c747782c1 100644
--- a/t/t5411/common-functions.sh
+++ b/t/t5411/common-functions.sh
@@ -18,6 +18,18 @@ create_commits_in () {
 	done
 }
 
+get_abbrev_oid () {
+	oid=$1 &&
+	suffix=${oid#???????} &&
+	oid=${oid%$suffix} &&
+	if test -n "$oid"
+	then
+		echo "$oid"
+	else
+		echo "undefined-oid"
+	fi
+}
+
 # Format the output of git-push, git-show-ref and other commands to make a
 # user-friendly and stable text.  We can easily prepare the expect text
 # without having to worry about changes of the commit ID (full or abbrev.)
@@ -28,12 +40,10 @@ create_commits_in () {
 make_user_friendly_and_stable_output () {
 	sed \
 		-e "s/'/\"/g" \
-		-e "s/$A//g" \
-		-e "s/$B//g" \
-		-e "s/$TAG//g" \
+		-e "s/$(get_abbrev_oid $A)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $B)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $TAG)[0-9a-f]*//g" \
 		-e "s/$ZERO_OID//g" \
-		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*//g" \
-		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*//g" \
 		-e "s#To $URL_PREFIX/upstream.git#To #" \
 		-e "/^error: / d"
 }
diff --git a/t/t5411/test-0000-standard-git-push.sh b/t/t5411/test-0000-standard-git-push.sh
index d8aafc235c..ce64bb660b 100644
--- a/t/t5411/test-0000-standard-git-push.sh
+++ b/t/t5411/test-0000-standard-git-push.sh
@@ -15,7 +15,7 @@ test_expect_success "git-push ($PROTOCOL)" '
 	> remote: post-receive<   refs/heads/main        Z
 	> remote: post-receive<   refs/heads/next        Z
 	> To 
-	>    ..   -> main
+	>    ..   -> main
 	>  * [new branch]      HEAD -> next
 	EOF
 	test_cmp expect actual &&
@@ -69,7 +69,7 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL)" '
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/heads/next        Z
 	> To 
-	>    ..   -> next
+	>    ..   -> next
 	>  ! [rejected]        main -> main (non-fast-forward)
 	EOF
 	test_cmp expect actual &&
@@ -106,7 +106,7 @@ test_expect_success "git-push -f ($PROTOCOL)" '
 	> remote: post-receive<   refs/review/main/topic        Z
 	> remote: post-receive<   refs/heads/a/b/c        Z
 	> To 
-	>  + ... main -> main (forced update)
+	>  + ... main -> main (forced update)
 	>  - [deleted]         next
 	>  * [new tag]         v123 -> v123
 	>  * [new reference]   main -> refs/review/main/topic
diff --git a/t/t5411/test-0001-standard-git-push--porcelain.sh b/t/t5411/test-0001-standard-git-push--porcelain.sh
index 2078d0a027..373ec3d865 100644
--- a/t/t5411/test-0001-standard-git-push--porcelain.sh
+++ b/t/t5411/test-0001-standard-git-push--porcelain.sh
@@ -15,7 +15,7 @@ test_expect_success "git-push ($PROTOCOL/porcelain)" '
 	> remote: post-receive<   refs/heads/main        Z
 	> remote: post-receive<   refs/heads/next        Z
 	> To 
-	>  	:refs/heads/main	..
+	>  	:refs/heads/main	..
 	> *	HEAD:refs/heads/next	[new branch]
 	> Done
 	EOF
@@ -71,7 +71,7 @@ test_expect_success "non-fast-forward git-push ($PROTOCOL/porcelain)" '
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/heads/next        Z
 	> To 
-	>  	:refs/heads/next	..
+	>  	:refs/heads/next	..
 	> !	refs/heads/main:refs/heads/main	[rejected] (non-fast-forward)
 	> Done
 	EOF
@@ -109,7 +109,7 @@ test_expect_success "git-push -f ($PROTOCOL/porcelain)" '
 	> remote: post-receive<   refs/review/main/topic        Z
 	> remote: post-receive<   refs/heads/a/b/c        Z
 	> To 
-	> +	refs/heads/main:refs/heads/main	... (forced update)
+	> +	refs/heads/main:refs/heads/main	... (forced update)
 	> -	:refs/heads/next	[deleted]
 	> *	refs/tags/v123:refs/tags/v123	[new tag]
 	> *	refs/heads/main:refs/review/main/topic	[new reference]
diff --git a/t/t5411/test-0022-report-unexpect-ref.sh b/t/t5411/test-0022-report-unexpect-ref.sh
index a2ce7ebdf6..f7a494bdb9 100644
--- a/t/t5411/test-0022-report-unexpect-ref.sh
+++ b/t/t5411/test-0022-report-unexpect-ref.sh
@@ -26,7 +26,7 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL)" '
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/heads/main        Z
 	> To 
-	>    ..   -> main
+	>    ..   -> main
 	>  ! [remote rejected] HEAD -> refs/for/main/topic (proc-receive failed to report status)
 	EOF
 	test_cmp expect actual &&
diff --git a/t/t5411/test-0023-report-unexpect-ref--porcelain.sh b/t/t5411/test-0023-report-unexpect-ref--porcelain.sh
index 4e56b163f8..63c479e975 100644
--- a/t/t5411/test-0023-report-unexpect-ref--porcelain.sh
+++ b/t/t5411/test-0023-report-unexpect-ref--porcelain.sh
@@ -26,7 +26,7 @@ test_expect_success "proc-receive: report unexpected ref ($PROTOCOL/porcelain)"
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/heads/main        Z
 	> To 
-	>  	:refs/heads/main	..
+	>  	:refs/heads/main	..
 	> !	HEAD:refs/for/main/topic	[remote rejected] (proc-receive failed to report status)
 	> Done
 	EOF
diff --git a/t/t5411/test-0032-report-with-options.sh b/t/t5411/test-0032-report-with-options.sh
index cd4f213a7d..988a4302a6 100644
--- a/t/t5411/test-0032-report-with-options.sh
+++ b/t/t5411/test-0032-report-with-options.sh
@@ -123,7 +123,7 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL)
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/pull/123/head        Z
 	> To 
-	>    ..  HEAD -> refs/pull/123/head
+	>    ..  HEAD -> refs/pull/123/head
 	EOF
 	test_cmp expect actual
 '
@@ -155,7 +155,7 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL)" '
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/for/main/topic        Z
 	> To 
-	>    ..  HEAD -> refs/for/main/topic
+	>    ..  HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual
 '
@@ -189,7 +189,7 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL)
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/for/main/topic        Z
 	> To 
-	>    ..  HEAD -> refs/for/main/topic
+	>    ..  HEAD -> refs/for/main/topic
 	EOF
 	test_cmp expect actual
 '
@@ -243,7 +243,7 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL)" '
 	> To 
 	>  * [new reference]   HEAD -> refs/pull/123/head
 	>  * [new reference]   HEAD -> refs/for/a/b/c/topic
-	>  + ... HEAD -> refs/pull/124/head (forced update)
+	>  + ... HEAD -> refs/pull/124/head (forced update)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0033-report-with-options--porcelain.sh b/t/t5411/test-0033-report-with-options--porcelain.sh
index 7433870e40..daacb3d69d 100644
--- a/t/t5411/test-0033-report-with-options--porcelain.sh
+++ b/t/t5411/test-0033-report-with-options--porcelain.sh
@@ -127,7 +127,7 @@ test_expect_success "proc-receive: report option refname and old-oid ($PROTOCOL/
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/pull/123/head        Z
 	> To 
-	>  	HEAD:refs/pull/123/head	..
+	>  	HEAD:refs/pull/123/head	..
 	> Done
 	EOF
 	test_cmp expect actual
@@ -160,7 +160,7 @@ test_expect_success "proc-receive: report option old-oid ($PROTOCOL/porcelain)"
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/for/main/topic        Z
 	> To 
-	>  	HEAD:refs/for/main/topic	..
+	>  	HEAD:refs/for/main/topic	..
 	> Done
 	EOF
 	test_cmp expect actual
@@ -195,7 +195,7 @@ test_expect_success "proc-receive: report option old-oid and new-oid ($PROTOCOL/
 	> remote: # post-receive hook        Z
 	> remote: post-receive<   refs/for/main/topic        Z
 	> To 
-	>  	HEAD:refs/for/main/topic	..
+	>  	HEAD:refs/for/main/topic	..
 	> Done
 	EOF
 	test_cmp expect actual
@@ -251,7 +251,7 @@ test_expect_success "proc-receive: report with multiple rewrites ($PROTOCOL/porc
 	> To 
 	> *	HEAD:refs/pull/123/head	[new reference]
 	> *	HEAD:refs/for/a/b/c/topic	[new reference]
-	> +	HEAD:refs/pull/124/head	... (forced update)
+	> +	HEAD:refs/pull/124/head	... (forced update)
 	> Done
 	EOF
 	test_cmp expect actual &&
diff --git a/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh b/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh
index e3f456ca5a..8c8a6c16e1 100644
--- a/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh
+++ b/t/t5411/test-0036-report-multi-rewrite-for-one-ref.sh
@@ -60,9 +60,9 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for
 	> remote: post-receive<   refs/changes/24/124/1        Z
 	> remote: post-receive<   refs/changes/25/125/1        Z
 	> To 
-	>    ..  HEAD -> refs/for/main/topic
+	>    ..  HEAD -> refs/for/main/topic
 	>  * [new reference]   HEAD -> refs/changes/24/124/1
-	>    ..  HEAD -> refs/changes/25/125/1
+	>    ..  HEAD -> refs/changes/25/125/1
 	EOF
 	test_cmp expect actual &&
 
@@ -136,8 +136,8 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for
 	> remote: post-receive<   refs/changes/25/125/1        Z
 	> To 
 	>  * [new reference]   HEAD -> refs/changes/24/124/1
-	>    ..  HEAD -> refs/for/main/topic
-	>  + ... HEAD -> refs/changes/25/125/1 (forced update)
+	>    ..  HEAD -> refs/for/main/topic
+	>  + ... HEAD -> refs/changes/25/125/1 (forced update)
 	EOF
 	test_cmp expect actual &&
 
@@ -198,7 +198,7 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL)" '
 	> remote: post-receive<   refs/changes/24/124/2        Z
 	> To 
 	>  * [new reference]   HEAD -> refs/changes/23/123/1
-	>    ..  HEAD -> refs/changes/24/124/2
+	>    ..  HEAD -> refs/changes/24/124/2
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh b/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh
index 7786079ba5..bc44810f33 100644
--- a/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh
+++ b/t/t5411/test-0037-report-multi-rewrite-for-one-ref--porcelain.sh
@@ -45,9 +45,9 @@ test_expect_success "proc-receive: multiple rewrite for one ref, no refname for
 	> remote: post-receive<   refs/changes/24/124/1        Z
 	> remote: post-receive<   refs/changes/25/125/1        Z
 	> To 
-	>  	HEAD:refs/for/main/topic	..
+	>  	HEAD:refs/for/main/topic	..
 	> *	HEAD:refs/changes/24/124/1	[new reference]
-	>  	HEAD:refs/changes/25/125/1	..
+	>  	HEAD:refs/changes/25/125/1	..
 	> Done
 	EOF
 	test_cmp expect actual &&
@@ -107,8 +107,8 @@ test_expect_success "proc-receive: multiple rewrites for one ref, no refname for
 	> remote: post-receive<   refs/changes/25/125/1        Z
 	> To 
 	> *	HEAD:refs/changes/24/124/1	[new reference]
-	>  	HEAD:refs/for/main/topic	..
-	> +	HEAD:refs/changes/25/125/1	... (forced update)
+	>  	HEAD:refs/for/main/topic	..
+	> +	HEAD:refs/changes/25/125/1	... (forced update)
 	> Done
 	EOF
 	test_cmp expect actual &&
@@ -155,7 +155,7 @@ test_expect_success "proc-receive: multiple rewrites for one ref ($PROTOCOL/porc
 	> remote: post-receive<   refs/changes/24/124/2        Z
 	> To 
 	> *	HEAD:refs/changes/23/123/1	[new reference]
-	>  	HEAD:refs/changes/24/124/2	..
+	>  	HEAD:refs/changes/24/124/2	..
 	> Done
 	EOF
 	test_cmp expect actual &&
diff --git a/t/t5411/test-0038-report-mixed-refs.sh b/t/t5411/test-0038-report-mixed-refs.sh
index 0990a3f76a..e63fe7ba11 100644
--- a/t/t5411/test-0038-report-mixed-refs.sh
+++ b/t/t5411/test-0038-report-mixed-refs.sh
@@ -55,12 +55,12 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL)" '
 	> remote: post-receive<   refs/heads/foo        Z
 	> remote: post-receive<   refs/for/main/topic        Z
 	> To 
-	>    ..   -> main
+	>    ..   -> main
 	>  * [new branch]      HEAD -> bar
 	>  * [new branch]      HEAD -> baz
 	>  * [new reference]   HEAD -> refs/for/next/topic2
 	>  * [new branch]      HEAD -> foo
-	>    ..  HEAD -> refs/for/main/topic
+	>    ..  HEAD -> refs/for/main/topic
 	>  ! [remote rejected] HEAD -> refs/for/next/topic1 (fail to call Web API)
 	>  ! [remote rejected] HEAD -> refs/for/next/topic3 (proc-receive failed to report status)
 	EOF
diff --git a/t/t5411/test-0039-report-mixed-refs--porcelain.sh b/t/t5411/test-0039-report-mixed-refs--porcelain.sh
index 7e4d08a939..99d17b73af 100644
--- a/t/t5411/test-0039-report-mixed-refs--porcelain.sh
+++ b/t/t5411/test-0039-report-mixed-refs--porcelain.sh
@@ -55,12 +55,12 @@ test_expect_success "proc-receive: report update of mixed refs ($PROTOCOL/porcel
 	> remote: post-receive<   refs/heads/foo        Z
 	> remote: post-receive<   refs/for/main/topic        Z
 	> To 
-	>  	:refs/heads/main	..
+	>  	:refs/heads/main	..
 	> *	HEAD:refs/heads/bar	[new branch]
 	> *	HEAD:refs/heads/baz	[new branch]
 	> *	HEAD:refs/for/next/topic2	[new reference]
 	> *	HEAD:refs/heads/foo	[new branch]
-	>  	HEAD:refs/for/main/topic	..
+	>  	HEAD:refs/for/main/topic	..
 	> !	HEAD:refs/for/next/topic1	[remote rejected] (fail to call Web API)
 	> !	HEAD:refs/for/next/topic3	[remote rejected] (proc-receive failed to report status)
 	> Done
diff --git a/t/t5411/test-0040-process-all-refs.sh b/t/t5411/test-0040-process-all-refs.sh
index f67cb2a7b3..2f405adefa 100644
--- a/t/t5411/test-0040-process-all-refs.sh
+++ b/t/t5411/test-0040-process-all-refs.sh
@@ -85,11 +85,11 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL)" '
 	> remote: post-receive<   refs/pull/123/head        Z
 	> remote: post-receive<   refs/pull/124/head        Z
 	> To 
-	>    ..   -> bar
+	>    ..   -> bar
 	>  - [deleted]         foo
-	>  + ... HEAD -> main (forced update)
-	>    ..  HEAD -> refs/pull/123/head
-	>  + ... HEAD -> refs/pull/124/head (forced update)
+	>  + ... HEAD -> main (forced update)
+	>    ..  HEAD -> refs/pull/123/head
+	>  + ... HEAD -> refs/pull/124/head (forced update)
 	EOF
 	test_cmp expect actual &&
 
diff --git a/t/t5411/test-0041-process-all-refs--porcelain.sh b/t/t5411/test-0041-process-all-refs--porcelain.sh
index 7d1a99d3a2..c88405792e 100644
--- a/t/t5411/test-0041-process-all-refs--porcelain.sh
+++ b/t/t5411/test-0041-process-all-refs--porcelain.sh
@@ -85,11 +85,11 @@ test_expect_success "proc-receive: process all refs ($PROTOCOL/porcelain)" '
 	> remote: post-receive<   refs/pull/123/head        Z
 	> remote: post-receive<   refs/pull/124/head        Z
 	> To 
-	>  	:refs/heads/bar	..
+	>  	:refs/heads/bar	..
 	> -	:refs/heads/foo	[deleted]
-	> +	HEAD:refs/heads/main	... (forced update)
-	>  	HEAD:refs/pull/123/head	..
-	> +	HEAD:refs/pull/124/head	... (forced update)
+	> +	HEAD:refs/heads/main	... (forced update)
+	>  	HEAD:refs/pull/123/head	..
+	> +	HEAD:refs/pull/124/head	... (forced update)
 	> Done
 	EOF
 	test_cmp expect actual &&
diff --git a/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh b/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh
index dba544162c..31989f0185 100644
--- a/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh
+++ b/t/t5411/test-0050-proc-receive-refs-with-modifiers.sh
@@ -46,7 +46,7 @@ test_expect_success "proc-receive: update branch and new tag ($PROTOCOL)" '
 	> remote: post-receive<   refs/pull/123/head        Z
 	> remote: post-receive<   refs/pull/124/head        Z
 	> To 
-	>    ..   -> refs/pull/123/head
+	>    ..   -> refs/pull/123/head
 	>  * [new reference]   v123 -> refs/pull/124/head
 	EOF
 	test_cmp expect actual &&
@@ -116,7 +116,7 @@ test_expect_success "proc-receive: create/delete branch, and delete tag ($PROTOC
 	> remote: post-receive<   refs/pull/124/head        Z
 	> To 
 	>  - [deleted]         refs/pull/123/head
-	>    ..   -> topic
+	>    ..   -> topic
 	>  - [deleted]         v123
 	>  * [new reference]    -> refs/pull/124/head
 	EOF
diff --git a/t/t5548-push-porcelain.sh b/t/t5548-push-porcelain.sh
index c8f18a7a12..f11ff57e54 100755
--- a/t/t5548-push-porcelain.sh
+++ b/t/t5548-push-porcelain.sh
@@ -26,17 +26,27 @@ create_commits_in () {
 	done
 }
 
+get_abbrev_oid () {
+	oid=$1 &&
+	suffix=${oid#???????} &&
+	oid=${oid%$suffix} &&
+	if test -n "$oid"
+	then
+		echo "$oid"
+	else
+		echo "undefined-oid"
+	fi
+}
+
 # Format the output of git-push, git-show-ref and other commands to make a
 # user-friendly and stable text.  We can easily prepare the expect text
 # without having to worry about future changes of the commit ID and spaces
 # of the output.
 make_user_friendly_and_stable_output () {
 	sed \
-		-e "s/$A//g" \
-		-e "s/$B//g" \
+		-e "s/$(get_abbrev_oid $A)[0-9a-f]*//g" \
+		-e "s/$(get_abbrev_oid $B)[0-9a-f]*//g" \
 		-e "s/$ZERO_OID//g" \
-		-e "s/$(echo $A | cut -c1-7)[0-9a-f]*//g" \
-		-e "s/$(echo $B | cut -c1-7)[0-9a-f]*//g" \
 		-e "s#To $URL_PREFIX/upstream.git#To #"
 }
 
@@ -102,9 +112,9 @@ run_git_push_porcelain_output_test() {
 		format_and_save_expect <<-EOF &&
 		> To 
 		> =	refs/heads/baz:refs/heads/baz	[up to date]
-		>  	:refs/heads/bar	..
+		>  	:refs/heads/bar	..
 		> -	:refs/heads/foo	[deleted]
-		> +	refs/heads/main:refs/heads/main	... (forced update)
+		> +	refs/heads/main:refs/heads/main	... (forced update)
 		> *	refs/heads/next:refs/heads/next	[new branch]
 		> Done
 		EOF
@@ -220,7 +230,7 @@ run_git_push_porcelain_output_test() {
 		To 
 		> =	refs/heads/next:refs/heads/next	[up to date]
 		> -	:refs/heads/baz	[deleted]
-		>  	refs/heads/main:refs/heads/main	..
+		>  	refs/heads/main:refs/heads/main	..
 		> !	refs/heads/bar:refs/heads/bar	[rejected] (non-fast-forward)
 		Done
 		EOF

From 7c0afdf23c1cb331eab0068d70ad5c0c156f216a Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Fri, 18 Jun 2021 12:32:22 -0400
Subject: [PATCH 251/397] t: use portable wrapper for readlink(1)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Not all systems have a readlink program available for use by the shell.
This causes t3210 to fail on at least AIX. Let's provide a perl
one-liner to do the same thing, and use it there.

I also updated calls in t9802. Nobody reported failure there, but it's
the same issue. Presumably nobody actually tests with p4 on AIX in the
first place (if it is even available there).

I left the use of readlink in the "--valgrind" setup in test-lib.sh, as
valgrind isn't available on exotic platforms anyway (and I didn't want
to increase dependencies between test-lib.sh and test-lib-functions.sh).

There's one other curious case. Commit d2addc3b96 (t7800: readlink may
not be available, 2016-05-31) fixed a similar case. We can't use our
wrapper function there, though, as it's inside a sub-script triggered by
Git. It uses a slightly different technique ("ls" piped to "sed"). I
chose not to use that here as it gives confusing "ls -l" output if the
file is unexpectedly not a symlink (which is OK for its limited use, but
potentially confusing for general use within the test suite). The perl
version emits the empty string.

Reported-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 t/t3210-pack-refs.sh       | 2 +-
 t/t9802-git-p4-filetype.sh | 4 ++--
 t/test-lib-functions.sh    | 6 ++++++
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/t/t3210-pack-refs.sh b/t/t3210-pack-refs.sh
index 3b7cdc56ec..577f32dc71 100755
--- a/t/t3210-pack-refs.sh
+++ b/t/t3210-pack-refs.sh
@@ -253,7 +253,7 @@ test_expect_success SYMLINKS 'pack symlinked packed-refs' '
 	git for-each-ref >all-refs-packed &&
 	test_cmp all-refs-before all-refs-packed &&
 	test -h .git/packed-refs &&
-	test "$(readlink .git/packed-refs)" = "my-deviant-packed-refs"
+	test "$(test_readlink .git/packed-refs)" = "my-deviant-packed-refs"
 '
 
 test_done
diff --git a/t/t9802-git-p4-filetype.sh b/t/t9802-git-p4-filetype.sh
index 94edebe272..19073c6e9f 100755
--- a/t/t9802-git-p4-filetype.sh
+++ b/t/t9802-git-p4-filetype.sh
@@ -263,7 +263,7 @@ test_expect_success SYMLINKS 'ensure p4 symlink parsed correctly' '
 	(
 		cd "$git" &&
 		test -L symlink &&
-		test $(readlink symlink) = symlink-target
+		test $(test_readlink symlink) = symlink-target
 	)
 '
 
@@ -329,7 +329,7 @@ test_expect_success SYMLINKS 'empty symlink target' '
 	git p4 clone --dest="$git" //depot@all &&
 	(
 		cd "$git" &&
-		test $(readlink empty-symlink) = target2
+		test $(test_readlink empty-symlink) = target2
 	)
 '
 
diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh
index b823c14027..661f376077 100644
--- a/t/test-lib-functions.sh
+++ b/t/test-lib-functions.sh
@@ -1692,3 +1692,9 @@ test_region () {
 
 	return 0
 }
+
+# Print the destination of symlink(s) provided as arguments. Basically
+# the same as the readlink command, but it's not available everywhere.
+test_readlink () {
+	perl -le 'print readlink($_) for @ARGV' "$@"
+}

From cea232194d4da32696df26f9b5ad00d5624621db Mon Sep 17 00:00:00 2001
From: Fabian Wermelinger 
Date: Fri, 18 Jun 2021 16:02:57 +0200
Subject: [PATCH 252/397] completion: bash: fix late declaration of
 __git_cmd_idx

A recent update to contrib/completion/git-completion.bash causes bash to fail
auto complete custom commands that are wrapped with __git_func_wrap. Declaring
__git_cmd_idx=0 inside __git_func_wrap resolves the issue.

Signed-off-by: Fabian Wermelinger 
Signed-off-by: Junio C Hamano 
---
 contrib/completion/git-completion.bash | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash
index 7bce9a0112..7b1bbb90ce 100644
--- a/contrib/completion/git-completion.bash
+++ b/contrib/completion/git-completion.bash
@@ -3506,6 +3506,7 @@ fi
 __git_func_wrap ()
 {
 	local cur words cword prev
+	local __git_cmd_idx=0
 	_get_comp_words_by_ref -n =: cur words cword prev
 	$1
 }

From 340062243a474b8fa56f4c3f6572bf015392f7f9 Mon Sep 17 00:00:00 2001
From: Felipe Contreras 
Date: Thu, 17 Jun 2021 11:17:08 -0500
Subject: [PATCH 253/397] pull: cleanup autostash check

Currently "git pull --rebase" takes a shortcut in the case a
fast-forward merge is possible; run_merge() is called with --ff-only.

However, "git merge" didn't have an --autostash option, so, when "git
pull --rebase --autostash" was called *and* the fast-forward merge
shortcut was taken, then the pull failed.

This was fixed in commit f15e7cf5cc (pull: ff --rebase --autostash
works in dirty repo, 2017-06-01) by simply skipping the fast-forward
merge shortcut.

Later on "git merge" learned the --autostash option [a03b55530a
(merge: teach --autostash option, 2020-04-07)], and so did "git pull"
[d9f15d37f1 (pull: pass --autostash to merge, 2020-04-07)].

Therefore it's not necessary to skip the fast-forward merge shortcut
anymore when called with --rebase --autostash.

Let's always take the fast-forward merge shortcut by essentially
reverting f15e7cf5cc.

Signed-off-by: Felipe Contreras 
Signed-off-by: Junio C Hamano 
---
 builtin/pull.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/builtin/pull.c b/builtin/pull.c
index e8927fc2ff..a22293b7db 100644
--- a/builtin/pull.c
+++ b/builtin/pull.c
@@ -947,7 +947,6 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 	struct oid_array merge_heads = OID_ARRAY_INIT;
 	struct object_id orig_head, curr_head;
 	struct object_id rebase_fork_point;
-	int autostash;
 	int rebase_unspecified = 0;
 	int can_ff;
 
@@ -982,8 +981,8 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 	if (get_oid("HEAD", &orig_head))
 		oidclr(&orig_head);
 
-	autostash = config_autostash;
 	if (opt_rebase) {
+		int autostash = config_autostash;
 		if (opt_autostash != -1)
 			autostash = opt_autostash;
 
@@ -1065,13 +1064,12 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 		     recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND) &&
 		    submodule_touches_in_range(the_repository, &upstream, &curr_head))
 			die(_("cannot rebase with locally recorded submodule modifications"));
-		if (!autostash) {
-			if (can_ff) {
-				/* we can fast-forward this without invoking rebase */
-				opt_ff = "--ff-only";
-				ran_ff = 1;
-				ret = run_merge();
-			}
+
+		if (can_ff) {
+			/* we can fast-forward this without invoking rebase */
+			opt_ff = "--ff-only";
+			ran_ff = 1;
+			ret = run_merge();
 		}
 		if (!ran_ff)
 			ret = run_rebase(&newbase, &upstream);

From a751e0296f6771e4dec6258d8c3f9bf658da30de Mon Sep 17 00:00:00 2001
From: Felipe Contreras 
Date: Thu, 17 Jun 2021 11:17:09 -0500
Subject: [PATCH 254/397] pull: trivial cleanup

There's no need to store ran_ff. Now it's obvious from the conditionals.

Signed-off-by: Felipe Contreras 
Signed-off-by: Junio C Hamano 
---
 builtin/pull.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/builtin/pull.c b/builtin/pull.c
index a22293b7db..80e2f55cbc 100644
--- a/builtin/pull.c
+++ b/builtin/pull.c
@@ -1053,7 +1053,6 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 
 	if (opt_rebase) {
 		int ret = 0;
-		int ran_ff = 0;
 
 		struct object_id newbase;
 		struct object_id upstream;
@@ -1068,11 +1067,10 @@ int cmd_pull(int argc, const char **argv, const char *prefix)
 		if (can_ff) {
 			/* we can fast-forward this without invoking rebase */
 			opt_ff = "--ff-only";
-			ran_ff = 1;
 			ret = run_merge();
-		}
-		if (!ran_ff)
+		} else {
 			ret = run_rebase(&newbase, &upstream);
+		}
 
 		if (!ret && (recurse_submodules == RECURSE_SUBMODULES_ON ||
 			     recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND))

From a7d18a11098b5f9cc8b1511ecaef5378c1030f97 Mon Sep 17 00:00:00 2001
From: Felipe Contreras 
Date: Thu, 17 Jun 2021 11:17:10 -0500
Subject: [PATCH 255/397] pull: trivial whitespace style fix

Two spaces unaligned to anything is not part of the coding-style. A
single tab is.

Signed-off-by: Felipe Contreras 
Signed-off-by: Junio C Hamano 
---
 builtin/pull.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/builtin/pull.c b/builtin/pull.c
index 80e2f55cbc..3e13f81084 100644
--- a/builtin/pull.c
+++ b/builtin/pull.c
@@ -126,9 +126,9 @@ static struct option pull_options[] = {
 	/* Options passed to git-merge or git-rebase */
 	OPT_GROUP(N_("Options related to merging")),
 	OPT_CALLBACK_F('r', "rebase", &opt_rebase,
-	  "(false|true|merges|preserve|interactive)",
-	  N_("incorporate changes by rebasing rather than merging"),
-	  PARSE_OPT_OPTARG, parse_opt_rebase),
+		"(false|true|merges|preserve|interactive)",
+		N_("incorporate changes by rebasing rather than merging"),
+		PARSE_OPT_OPTARG, parse_opt_rebase),
 	OPT_PASSTHRU('n', NULL, &opt_diffstat, NULL,
 		N_("do not show a diffstat at the end of the merge"),
 		PARSE_OPT_NOARG | PARSE_OPT_NONEG),

From c75c42395267f3b484f47404ae746d323586a929 Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Tue, 22 Jun 2021 08:04:38 +0000
Subject: [PATCH 256/397] t6421: add tests checking for excessive object
 downloads during merge

Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 t/t6421-merge-partial-clone.sh | 440 +++++++++++++++++++++++++++++++++
 1 file changed, 440 insertions(+)
 create mode 100755 t/t6421-merge-partial-clone.sh

diff --git a/t/t6421-merge-partial-clone.sh b/t/t6421-merge-partial-clone.sh
new file mode 100755
index 0000000000..3dcffc15f8
--- /dev/null
+++ b/t/t6421-merge-partial-clone.sh
@@ -0,0 +1,440 @@
+#!/bin/sh
+
+test_description="limiting blob downloads when merging with partial clones"
+# Uses a methodology similar to
+#   t6042: corner cases with renames but not criss-cross merges
+#   t6036: corner cases with both renames and criss-cross merges
+#   t6423: directory rename detection
+#
+# The setup for all of them, pictorially, is:
+#
+#      A
+#      o
+#     / \
+#  O o   ?
+#     \ /
+#      o
+#      B
+#
+# To help make it easier to follow the flow of tests, they have been
+# divided into sections and each test will start with a quick explanation
+# of what commits O, A, and B contain.
+#
+# Notation:
+#    z/{b,c}   means  files z/b and z/c both exist
+#    x/d_1     means  file x/d exists with content d1.  (Purpose of the
+#                     underscore notation is to differentiate different
+#                     files that might be renamed into each other's paths.)
+
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/lib-merge.sh
+
+test_setup_repo () {
+	test -d server && return
+	test_create_repo server &&
+	(
+		cd server &&
+
+		git config uploadpack.allowfilter 1 &&
+		git config uploadpack.allowanysha1inwant 1 &&
+
+		mkdir -p general &&
+		test_seq 2 9 >general/leap1 &&
+		cp general/leap1 general/leap2 &&
+		echo leap2 >>general/leap2 &&
+
+		mkdir -p basename &&
+		cp general/leap1 basename/numbers &&
+		cp general/leap1 basename/sequence &&
+		cp general/leap1 basename/values &&
+		echo numbers >>basename/numbers &&
+		echo sequence >>basename/sequence &&
+		echo values >>basename/values &&
+
+		mkdir -p dir/unchanged &&
+		mkdir -p dir/subdir/tweaked &&
+		echo a >dir/subdir/a &&
+		echo b >dir/subdir/b &&
+		echo c >dir/subdir/c &&
+		echo d >dir/subdir/d &&
+		echo e >dir/subdir/e &&
+		cp general/leap1 dir/subdir/Makefile &&
+		echo toplevel makefile >>dir/subdir/Makefile &&
+		echo f >dir/subdir/tweaked/f &&
+		echo g >dir/subdir/tweaked/g &&
+		echo h >dir/subdir/tweaked/h &&
+		echo subdirectory makefile >dir/subdir/tweaked/Makefile &&
+		for i in $(test_seq 1 88)
+		do
+			echo content $i >dir/unchanged/file_$i
+		done &&
+		git add . &&
+		git commit -m "O" &&
+
+		git branch O &&
+		git branch A &&
+		git branch B-single &&
+		git branch B-dir &&
+		git branch B-many &&
+
+		git switch A &&
+
+		git rm general/leap* &&
+		mkdir general/ &&
+		test_seq 1 9 >general/jump1 &&
+		cp general/jump1 general/jump2 &&
+		echo leap2 >>general/jump2 &&
+
+		rm basename/numbers basename/sequence basename/values &&
+		mkdir -p basename/subdir/
+		cp general/jump1 basename/subdir/numbers &&
+		cp general/jump1 basename/subdir/sequence &&
+		cp general/jump1 basename/subdir/values &&
+		echo numbers >>basename/subdir/numbers &&
+		echo sequence >>basename/subdir/sequence &&
+		echo values >>basename/subdir/values &&
+
+		git rm dir/subdir/tweaked/f &&
+		echo more >>dir/subdir/e &&
+		echo more >>dir/subdir/Makefile &&
+		echo more >>dir/subdir/tweaked/Makefile &&
+		mkdir dir/subdir/newsubdir &&
+		echo rust code >dir/subdir/newsubdir/newfile.rs &&
+		git mv dir/subdir/e dir/subdir/newsubdir/ &&
+		git mv dir folder &&
+		git add . &&
+		git commit -m "A" &&
+
+		git switch B-single &&
+		echo new first line >dir/subdir/Makefile &&
+		cat general/leap1 >>dir/subdir/Makefile &&
+		echo toplevel makefile >>dir/subdir/Makefile &&
+		echo perl code >general/newfile.pl &&
+		git add . &&
+		git commit -m "B-single" &&
+
+		git switch B-dir &&
+		echo java code >dir/subdir/newfile.java &&
+		echo scala code >dir/subdir/newfile.scala &&
+		echo groovy code >dir/subdir/newfile.groovy &&
+		git add . &&
+		git commit -m "B-dir" &&
+
+		git switch B-many &&
+		test_seq 2 10 >general/leap1 &&
+		rm general/leap2 &&
+		cp general/leap1 general/leap2 &&
+		echo leap2 >>general/leap2 &&
+
+		rm basename/numbers basename/sequence basename/values &&
+		mkdir -p basename/subdir/
+		cp general/leap1 basename/subdir/numbers &&
+		cp general/leap1 basename/subdir/sequence &&
+		cp general/leap1 basename/subdir/values &&
+		echo numbers >>basename/subdir/numbers &&
+		echo sequence >>basename/subdir/sequence &&
+		echo values >>basename/subdir/values &&
+
+		mkdir dir/subdir/newsubdir/ &&
+		echo c code >dir/subdir/newfile.c &&
+		echo python code >dir/subdir/newsubdir/newfile.py &&
+		git add . &&
+		git commit -m "B-many" &&
+
+		git switch A
+	)
+}
+
+# Testcase: Objects downloaded for single relevant rename
+#   Commit O:
+#              general/{leap1_O, leap2_O}
+#              basename/{numbers_O, sequence_O, values_O}
+#              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
+#              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
+#              dir/unchanged/
+#   Commit A:
+#     (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/
+#      -> folder/, move e into newsubdir, add newfile.rs, remove f, modify
+#      both both Makefiles and jumps)
+#              general/{jump1_A, jump2_A}
+#              basename/subdir/{numbers_A, sequence_A, values_A}
+#              folder/subdir/{a,b,c,d,Makefile_TOP_A}
+#              folder/subdir/newsubdir/{e_A,newfile.rs}
+#              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
+#              folder/unchanged/
+#   Commit B(-single):
+#     (add newfile.pl, tweak Makefile_TOP)
+#              general/{leap1_O, leap2_O,newfile.pl}
+#              basename/{numbers_O, sequence_O, values_O}
+#              dir/{a,b,c,d,e_O,Makefile_TOP_B}
+#              dir/tweaked/{f,g,h,Makefile_SUB_O}
+#              dir/unchanged/
+#   Expected:
+#              general/{jump1_A, jump2_A,newfile.pl}
+#              basename/subdir/{numbers_A, sequence_A, values_A}
+#              folder/subdir/{a,b,c,d,Makefile_TOP_Merged}
+#              folder/subdir/newsubdir/{e_A,newfile.rs}
+#              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
+#              folder/unchanged/
+#
+# Objects that need to be fetched:
+#   Rename detection:
+#     Side1 (O->A):
+#       Basename-matches rename detection only needs to fetch these objects:
+#         Makefile_TOP_O, Makefile_TOP_A
+#         (Despite many renames, all others are content irrelevant.  They
+#          are also location irrelevant because newfile.rs was added on
+#          the side doing the directory rename, and newfile.pl was added to
+#          a directory that was not renamed on either side.)
+#       General rename detection only needs to fetch these objects:
+#         
+#          (Even though newfile.rs, jump[12], basename/subdir/*, and e
+#          could all be used as destinations in rename detection, the
+#          basename detection for Makefile matches up all relevant
+#          sources, so these other files never end up needing to be
+#          used)
+#     Side2 (O->B):
+#       Basename-matches rename detection only needs to fetch these objects:
+#         
+#         (there are no deleted files, so no possible sources)
+#       General rename detection only needs to fetch these objects:
+#         
+#         (there are no deleted files, so no possible sources)
+#   Merge:
+#     3-way content merge needs to grab these objects:
+#       Makefile_TOP_B
+#   Nothing else needs to fetch objects
+#
+#   Summary: 2 fetches (1 for 2 objects, 1 for 1 object)
+#
+test_expect_merge_algorithm failure failure 'Objects downloaded for single relevant rename' '
+	test_setup_repo &&
+	git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-single &&
+	(
+		cd objects-single &&
+
+		git rev-list --objects --all --missing=print |
+			grep "^?" | sort >missing-objects-before &&
+
+		git checkout -q origin/A &&
+
+		GIT_TRACE2_PERF="$(pwd)/trace.output" git \
+			-c merge.directoryRenames=true merge --no-stat \
+			--no-progress origin/B-single &&
+
+		# Check the number of objects we reported we would fetch
+		cat >expect <<-EOF &&
+		fetch_count:2
+		fetch_count:1
+		EOF
+		grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
+		test_cmp expect actual &&
+
+		# Check the number of fetch commands exec-ed
+		grep d0.*fetch.negotiationAlgorithm trace.output >fetches &&
+		test_line_count = 2 fetches &&
+
+		git rev-list --objects --all --missing=print |
+			grep "^?" | sort >missing-objects-after &&
+		comm -2 -3 missing-objects-before missing-objects-after >old &&
+		comm -1 -3 missing-objects-before missing-objects-after >new &&
+		# No new missing objects
+		test_must_be_empty new &&
+		# Fetched 2 + 1 = 3 objects
+		test_line_count = 3 old
+	)
+'
+
+# Testcase: Objects downloaded for directory rename
+#   Commit O:
+#              general/{leap1_O, leap2_O}
+#              basename/{numbers_O, sequence_O, values_O}
+#              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
+#              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
+#              dir/unchanged/
+#   Commit A:
+#     (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/ ->
+#      folder/, move e into newsubdir, add newfile.rs, remove f, modify
+#      both Makefiles and jumps)
+#              general/{jump1_A, jump2_A}
+#              basename/subdir/{numbers_A, sequence_A, values_A}
+#              folder/subdir/{a,b,c,d,Makefile_TOP_A}
+#              folder/subdir/newsubdir/{e_A,newfile.rs}
+#              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
+#              folder/unchanged/
+#   Commit B(-dir):
+#     (add dir/subdir/newfile.{java,scala,groovy}
+#              general/{leap1_O, leap2_O}
+#              basename/{numbers_O, sequence_O, values_O}
+#              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O,
+#                          newfile.java,newfile.scala,newfile.groovy}
+#              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
+#              dir/unchanged/
+#   Expected:
+#              general/{jump1_A, jump2_A}
+#              basename/subdir/{numbers_A, sequence_A, values_A}
+#              folder/subdir/{a,b,c,d,Makefile_TOP_A,
+#                             newfile.java,newfile.scala,newfile.groovy}
+#              folder/subdir/newsubdir/{e_A,newfile.rs}
+#              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
+#              folder/unchanged/
+#
+# Objects that need to be fetched:
+#   Makefile_TOP_O, Makefile_TOP_A
+#   Makefile_SUB_O, Makefile_SUB_A
+#   e_O, e_A
+#   * Despite A's rename of jump->leap, those renames are irrelevant.
+#   * Despite A's rename of basename/ -> basename/subdir/, those renames are
+#     irrelevant.
+#   * Because of A's rename of dir/ -> folder/ and B-dir's addition of
+#     newfile.* into dir/subdir/, we need to determine directory renames.
+#     (Technically, there are enough exact renames to determine directory
+#      rename detection, but the current implementation always does
+#      basename searching before directory rename detection.  Running it
+#      also before basename searching would mean doing directory rename
+#      detection twice, but it's a bit expensive to do that and cases like
+#      this are not all that common.)
+#   Summary: 1 fetches for 6 objects
+#
+test_expect_merge_algorithm failure failure 'Objects downloaded when a directory rename triggered' '
+	test_setup_repo &&
+	git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-dir &&
+	(
+		cd objects-dir &&
+
+		git rev-list --objects --all --missing=print |
+			grep "^?" | sort >missing-objects-before &&
+
+		git checkout -q origin/A &&
+
+		GIT_TRACE2_PERF="$(pwd)/trace.output" git \
+			-c merge.directoryRenames=true merge --no-stat \
+			--no-progress origin/B-dir &&
+
+		# Check the number of objects we reported we would fetch
+		cat >expect <<-EOF &&
+		fetch_count:6
+		EOF
+		grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
+		test_cmp expect actual &&
+
+		# Check the number of fetch commands exec-ed
+		grep d0.*fetch.negotiationAlgorithm trace.output >fetches &&
+		test_line_count = 1 fetches &&
+
+		git rev-list --objects --all --missing=print |
+			grep "^?" | sort >missing-objects-after &&
+		comm -2 -3 missing-objects-before missing-objects-after >old &&
+		comm -1 -3 missing-objects-before missing-objects-after >new &&
+		# No new missing objects
+		test_must_be_empty new &&
+		# Fetched 6 objects
+		test_line_count = 6 old
+	)
+'
+
+# Testcase: Objects downloaded with lots of renames and modifications
+#   Commit O:
+#              general/{leap1_O, leap2_O}
+#              basename/{numbers_O, sequence_O, values_O}
+#              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
+#              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
+#              dir/unchanged/
+#   Commit A:
+#     (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/
+#      -> folder/, move e into newsubdir, add newfile.rs, remove f, modify
+#      both both Makefiles and jumps)
+#              general/{jump1_A, jump2_A}
+#              basename/subdir/{numbers_A, sequence_A, values_A}
+#              folder/subdir/{a,b,c,d,Makefile_TOP_A}
+#              folder/subdir/newsubdir/{e_A,newfile.rs}
+#              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
+#              folder/unchanged/
+#   Commit B(-minimal):
+#     (modify both leaps, rename basename/ -> basename/subdir/, add
+#      newfile.{c,py})
+#              general/{leap1_B, leap2_B}
+#              basename/subdir/{numbers_B, sequence_B, values_B}
+#              dir/{a,b,c,d,e_O,Makefile_TOP_O,newfile.c}
+#              dir/tweaked/{f,g,h,Makefile_SUB_O,newfile.py}
+#              dir/unchanged/
+#   Expected:
+#              general/{jump1_Merged, jump2_Merged}
+#              basename/subdir/{numbers_Merged, sequence_Merged, values_Merged}
+#              folder/subdir/{a,b,c,d,Makefile_TOP_A,newfile.c}
+#              folder/subdir/newsubdir/e_A
+#              folder/subdir/tweaked/{g,h,Makefile_SUB_A,newfile.py}
+#              folder/unchanged/
+#
+# Objects that need to be fetched:
+#   Rename detection:
+#     Side1 (O->A):
+#       Basename-matches rename detection only needs to fetch these objects:
+#         numbers_O, numbers_A
+#         sequence_O, sequence_A
+#         values_O, values_A
+#         Makefile_TOP_O, Makefile_TOP_A
+#         Makefile_SUB_O, Makefile_SUB_A
+#         e_O, e_A
+#       General rename detection only needs to fetch these objects:
+#         leap1_O, leap2_O
+#         jump1_A, jump2_A, newfile.rs
+#         (only need remaining relevant sources, but any relevant sources need
+#          to be matched against all possible unpaired destinations)
+#     Side2 (O->B):
+#       Basename-matches rename detection only needs to fetch these objects:
+#         numbers_B
+#         sequence_B
+#         values_B
+#       (because numbers_O, sequence_O, and values_O already fetched above)
+#       General rename detection only needs to fetch these objects:
+#         
+#   Merge:
+#     3-way content merge needs to grab these objects:
+#       leap1_B
+#       leap2_B
+#   Nothing else needs to fetch objects
+#
+#   Summary: 4 fetches (1 for 6 objects, 1 for 8, 1 for 3, 1 for 2)
+#
+test_expect_merge_algorithm failure failure 'Objects downloaded with lots of renames and modifications' '
+	test_setup_repo &&
+	git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-many &&
+	(
+		cd objects-many &&
+
+		git rev-list --objects --all --missing=print |
+			grep "^?" | sort >missing-objects-before &&
+
+		git checkout -q origin/A &&
+
+		GIT_TRACE2_PERF="$(pwd)/trace.output" git \
+			-c merge.directoryRenames=true merge --no-stat \
+			--no-progress origin/B-many &&
+
+		# Check the number of objects we reported we would fetch
+		cat >expect <<-EOF &&
+		fetch_count:12
+		fetch_count:5
+		fetch_count:3
+		fetch_count:2
+		EOF
+		grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
+		test_cmp expect actual &&
+
+		# Check the number of fetch commands exec-ed
+		grep d0.*fetch.negotiationAlgorithm trace.output >fetches &&
+		test_line_count = 4 fetches &&
+
+		git rev-list --objects --all --missing=print |
+			grep "^?" | sort >missing-objects-after &&
+		comm -2 -3 missing-objects-before missing-objects-after >old &&
+		comm -1 -3 missing-objects-before missing-objects-after >new &&
+		# No new missing objects
+		test_must_be_empty new &&
+		# Fetched 12 + 5 + 3 + 2 = 22 objects
+		test_line_count = 22 old
+	)
+'
+
+test_done

From d331dd3b0c829fe9019f0113a095ed95bc06f227 Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Tue, 22 Jun 2021 08:04:39 +0000
Subject: [PATCH 257/397] diffcore-rename: allow different missing_object_cb
 functions

estimate_similarity() was setting up a diff_populate_filespec_options
every time it was called, requiring the caller of estimate_similarity()
to pass in some data needed to set up this option.  Currently the needed
data consisted of a single variable (skip_unmodified), but we want to
also have the different estimate_similarity() callsites start using
different missing_object_cb functions as well.  Rather than also passing
that data in, just have the caller pass in the whole
diff_populate_filespec_options, and reduce the number of times we need to
set it up.

As a side note, this also drops the number of calls to
has_promisor_remote() dramatically.  If L is the number of basename
paths to compare, M is the number of inexact sources, and N is the
number of inexact destinations, then the number of calls to
has_promisor_remote() drops from L+M*N down to at most 2 -- one for each
of the sites that calls estimate_similarity().  has_promisor_remote() is
a very fast function so this almost certainly has no measurable
performance impact, but it seems cleaner to avoid calling that function
so many times.

Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 diffcore-rename.c | 58 +++++++++++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/diffcore-rename.c b/diffcore-rename.c
index 3375e24659..8affa6130e 100644
--- a/diffcore-rename.c
+++ b/diffcore-rename.c
@@ -126,7 +126,7 @@ static int estimate_similarity(struct repository *r,
 			       struct diff_filespec *src,
 			       struct diff_filespec *dst,
 			       int minimum_score,
-			       int skip_unmodified)
+			       struct diff_populate_filespec_options *dpf_opt)
 {
 	/* src points at a file that existed in the original tree (or
 	 * optionally a file in the destination tree) and dst points
@@ -143,15 +143,6 @@ static int estimate_similarity(struct repository *r,
 	 */
 	unsigned long max_size, delta_size, base_size, src_copied, literal_added;
 	int score;
-	struct diff_populate_filespec_options dpf_options = {
-		.check_size_only = 1
-	};
-	struct prefetch_options prefetch_options = {r, skip_unmodified};
-
-	if (r == the_repository && has_promisor_remote()) {
-		dpf_options.missing_object_cb = prefetch;
-		dpf_options.missing_object_data = &prefetch_options;
-	}
 
 	/* We deal only with regular files.  Symlink renames are handled
 	 * only when they are exact matches --- in other words, no edits
@@ -169,11 +160,13 @@ static int estimate_similarity(struct repository *r,
 	 * is a possible size - we really should have a flag to
 	 * say whether the size is valid or not!)
 	 */
+	dpf_opt->check_size_only = 1;
+
 	if (!src->cnt_data &&
-	    diff_populate_filespec(r, src, &dpf_options))
+	    diff_populate_filespec(r, src, dpf_opt))
 		return 0;
 	if (!dst->cnt_data &&
-	    diff_populate_filespec(r, dst, &dpf_options))
+	    diff_populate_filespec(r, dst, dpf_opt))
 		return 0;
 
 	max_size = ((src->size > dst->size) ? src->size : dst->size);
@@ -191,11 +184,11 @@ static int estimate_similarity(struct repository *r,
 	if (max_size * (MAX_SCORE-minimum_score) < delta_size * MAX_SCORE)
 		return 0;
 
-	dpf_options.check_size_only = 0;
+	dpf_opt->check_size_only = 0;
 
-	if (!src->cnt_data && diff_populate_filespec(r, src, &dpf_options))
+	if (!src->cnt_data && diff_populate_filespec(r, src, dpf_opt))
 		return 0;
-	if (!dst->cnt_data && diff_populate_filespec(r, dst, &dpf_options))
+	if (!dst->cnt_data && diff_populate_filespec(r, dst, dpf_opt))
 		return 0;
 
 	if (diffcore_count_changes(r, src, dst,
@@ -862,7 +855,11 @@ static int find_basename_matches(struct diff_options *options,
 	int i, renames = 0;
 	struct strintmap sources;
 	struct strintmap dests;
-
+	struct diff_populate_filespec_options dpf_options = {
+		.check_binary = 0,
+		.missing_object_cb = NULL,
+		.missing_object_data = NULL
+	};
 	/*
 	 * The prefeteching stuff wants to know if it can skip prefetching
 	 * blobs that are unmodified...and will then do a little extra work
@@ -873,7 +870,10 @@ static int find_basename_matches(struct diff_options *options,
 	 * the extra work necessary to check if rename_src entries are
 	 * unmodified would be a small waste.
 	 */
-	int skip_unmodified = 0;
+	struct prefetch_options prefetch_options = {
+		.repo = options->repo,
+		.skip_unmodified = 0
+	};
 
 	/*
 	 * Create maps of basename -> fullname(s) for remaining sources and
@@ -910,6 +910,11 @@ static int find_basename_matches(struct diff_options *options,
 			strintmap_set(&dests, base, i);
 	}
 
+	if (options->repo == the_repository && has_promisor_remote()) {
+		dpf_options.missing_object_cb = prefetch;
+		dpf_options.missing_object_data = &prefetch_options;
+	}
+
 	/* Now look for basename matchups and do similarity estimation */
 	for (i = 0; i < rename_src_nr; ++i) {
 		char *filename = rename_src[i].p->one->path;
@@ -953,7 +958,7 @@ static int find_basename_matches(struct diff_options *options,
 			one = rename_src[src_index].p->one;
 			two = rename_dst[dst_index].p->two;
 			score = estimate_similarity(options->repo, one, two,
-						    minimum_score, skip_unmodified);
+						    minimum_score, &dpf_options);
 
 			/* If sufficiently similar, record as rename pair */
 			if (score < minimum_score)
@@ -1272,6 +1277,14 @@ void diffcore_rename_extended(struct diff_options *options,
 	int num_sources, want_copies;
 	struct progress *progress = NULL;
 	struct dir_rename_info info;
+	struct diff_populate_filespec_options dpf_options = {
+		.check_binary = 0,
+		.missing_object_cb = NULL,
+		.missing_object_data = NULL
+	};
+	struct prefetch_options prefetch_options = {
+		.repo = options->repo
+	};
 
 	trace2_region_enter("diff", "setup", options->repo);
 	info.setup = 0;
@@ -1433,6 +1446,13 @@ void diffcore_rename_extended(struct diff_options *options,
 				(uint64_t)num_destinations * (uint64_t)num_sources);
 	}
 
+	/* Finish setting up dpf_options */
+	prefetch_options.skip_unmodified = skip_unmodified;
+	if (options->repo == the_repository && has_promisor_remote()) {
+		dpf_options.missing_object_cb = prefetch;
+		dpf_options.missing_object_data = &prefetch_options;
+	}
+
 	CALLOC_ARRAY(mx, st_mult(NUM_CANDIDATE_PER_DST, num_destinations));
 	for (dst_cnt = i = 0; i < rename_dst_nr; i++) {
 		struct diff_filespec *two = rename_dst[i].p->two;
@@ -1458,7 +1478,7 @@ void diffcore_rename_extended(struct diff_options *options,
 			this_src.score = estimate_similarity(options->repo,
 							     one, two,
 							     minimum_score,
-							     skip_unmodified);
+							     &dpf_options);
 			this_src.name_score = basename_same(one, two);
 			this_src.dst = i;
 			this_src.src = j;

From 1aedd03afb1592be9e4fbacdc614f45a99a865ea Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Tue, 22 Jun 2021 08:04:40 +0000
Subject: [PATCH 258/397] diffcore-rename: use a different prefetch for
 basename comparisons

merge-ort was designed to minimize the amount of data needed and used,
and several changes were made to diffcore-rename to take advantage of
extra metadata to enable this data minimization (particularly the
relevant_sources variable for skipping "irrelevant" renames).  This
effort obviously succeeded in drastically reducing computation times,
but should also theoretically allow partial clones to download much less
information.  Previously, though, the "prefetch" command used in
diffcore-rename had never been modified and downloaded many blobs that
were unnecessary for merge-ort.  This commit corrects that.

When doing basename comparisons, we want to fetch only the objects that
will be used for basename comparisons.  If after basename fetching this
leaves us with no more relevant sources (or no more destinations), then
we won't need to do the full inexact rename detection and can skip
downloading additional source and destination files.  Even if we have to
do that later full inexact rename detection, irrelevant sources are
culled after basename matching and before the full inexact rename
detection, so we can still avoid downloading the blobs for irrelevant
sources.  Rename prefetch() to inexact_prefetch(), and introduce a
new basename_prefetch() to take advantage of this.

If we modify the testcase from commit 557ac0350d ("merge-ort: begin
performance work; instrument with trace2_region_* calls", 2021-01-23)
to pass
    --sparse --filter=blob:none
to the clone command, and use the new trace2 "fetch_count" output from
a few commits ago to track both the number of fetch subcommands invoked
and the number of objects fetched across all those fetches, then for
the mega-renames testcase we observe the following:

BEFORE this commit, rebasing 35 patches:
    strategy     # of fetches    total # of objects fetched
    ---------    ------------    --------------------------
    recursive    62              11423
    ort          30              11391

AFTER this commit, rebasing the same 35 patches:
    ort          32                 63

This means that the new code only needs to download less than 2 blobs
per patch being rebased.  That is especially interesting given that the
repository at the start only had approximately half a dozen TOTAL blobs
downloaded to start with (because the default sparse-checkout of just
the toplevel directory was in use).

So, for this particular linux kernel testcase that involved ~26,000
renames on the upstream side (drivers/ -> pilots/) across which 35
patches were being rebased, this change reduces the number of blobs that
need to be downloaded by a factor of ~180.

Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 diffcore-rename.c              | 101 +++++++++++++++++++++++++++------
 t/t6421-merge-partial-clone.sh |   4 +-
 2 files changed, 85 insertions(+), 20 deletions(-)

diff --git a/diffcore-rename.c b/diffcore-rename.c
index 8affa6130e..5ab513e78c 100644
--- a/diffcore-rename.c
+++ b/diffcore-rename.c
@@ -87,13 +87,13 @@ struct diff_score {
 	short name_score;
 };
 
-struct prefetch_options {
+struct inexact_prefetch_options {
 	struct repository *repo;
 	int skip_unmodified;
 };
-static void prefetch(void *prefetch_options)
+static void inexact_prefetch(void *prefetch_options)
 {
-	struct prefetch_options *options = prefetch_options;
+	struct inexact_prefetch_options *options = prefetch_options;
 	int i;
 	struct oid_array to_fetch = OID_ARRAY_INIT;
 
@@ -816,6 +816,78 @@ static int idx_possible_rename(char *filename, struct dir_rename_info *info)
 	return idx;
 }
 
+struct basename_prefetch_options {
+	struct repository *repo;
+	struct strintmap *relevant_sources;
+	struct strintmap *sources;
+	struct strintmap *dests;
+	struct dir_rename_info *info;
+};
+static void basename_prefetch(void *prefetch_options)
+{
+	struct basename_prefetch_options *options = prefetch_options;
+	struct strintmap *relevant_sources = options->relevant_sources;
+	struct strintmap *sources = options->sources;
+	struct strintmap *dests = options->dests;
+	struct dir_rename_info *info = options->info;
+	int i;
+	struct oid_array to_fetch = OID_ARRAY_INIT;
+
+	/*
+	 * TODO: The following loops mirror the code/logic from
+	 * find_basename_matches(), though not quite exactly.  Maybe
+	 * abstract the iteration logic out somehow?
+	 */
+	for (i = 0; i < rename_src_nr; ++i) {
+		char *filename = rename_src[i].p->one->path;
+		const char *base = NULL;
+		intptr_t src_index;
+		intptr_t dst_index;
+
+		/* Skip irrelevant sources */
+		if (relevant_sources &&
+		    !strintmap_contains(relevant_sources, filename))
+			continue;
+
+		/*
+		 * If the basename is unique among remaining sources, then
+		 * src_index will equal 'i' and we can attempt to match it
+		 * to a unique basename in the destinations.  Otherwise,
+		 * use directory rename heuristics, if possible.
+		 */
+		base = get_basename(filename);
+		src_index = strintmap_get(sources, base);
+		assert(src_index == -1 || src_index == i);
+
+		if (strintmap_contains(dests, base)) {
+			struct diff_filespec *one, *two;
+
+			/* Find a matching destination, if possible */
+			dst_index = strintmap_get(dests, base);
+			if (src_index == -1 || dst_index == -1) {
+				src_index = i;
+				dst_index = idx_possible_rename(filename, info);
+			}
+			if (dst_index == -1)
+				continue;
+
+			/* Ignore this dest if already used in a rename */
+			if (rename_dst[dst_index].is_rename)
+				continue; /* already used previously */
+
+			one = rename_src[src_index].p->one;
+			two = rename_dst[dst_index].p->two;
+
+			/* Add the pairs */
+			diff_add_if_missing(options->repo, &to_fetch, two);
+			diff_add_if_missing(options->repo, &to_fetch, one);
+		}
+	}
+
+	promisor_remote_get_direct(options->repo, to_fetch.oid, to_fetch.nr);
+	oid_array_clear(&to_fetch);
+}
+
 static int find_basename_matches(struct diff_options *options,
 				 int minimum_score,
 				 struct dir_rename_info *info,
@@ -860,19 +932,12 @@ static int find_basename_matches(struct diff_options *options,
 		.missing_object_cb = NULL,
 		.missing_object_data = NULL
 	};
-	/*
-	 * The prefeteching stuff wants to know if it can skip prefetching
-	 * blobs that are unmodified...and will then do a little extra work
-	 * to verify that the oids are indeed different before prefetching.
-	 * Unmodified blobs are only relevant when doing copy detection;
-	 * when limiting to rename detection, diffcore_rename[_extended]()
-	 * will never be called with unmodified source paths fed to us, so
-	 * the extra work necessary to check if rename_src entries are
-	 * unmodified would be a small waste.
-	 */
-	struct prefetch_options prefetch_options = {
+	struct basename_prefetch_options prefetch_options = {
 		.repo = options->repo,
-		.skip_unmodified = 0
+		.relevant_sources = relevant_sources,
+		.sources = &sources,
+		.dests = &dests,
+		.info = info
 	};
 
 	/*
@@ -911,7 +976,7 @@ static int find_basename_matches(struct diff_options *options,
 	}
 
 	if (options->repo == the_repository && has_promisor_remote()) {
-		dpf_options.missing_object_cb = prefetch;
+		dpf_options.missing_object_cb = basename_prefetch;
 		dpf_options.missing_object_data = &prefetch_options;
 	}
 
@@ -1282,7 +1347,7 @@ void diffcore_rename_extended(struct diff_options *options,
 		.missing_object_cb = NULL,
 		.missing_object_data = NULL
 	};
-	struct prefetch_options prefetch_options = {
+	struct inexact_prefetch_options prefetch_options = {
 		.repo = options->repo
 	};
 
@@ -1449,7 +1514,7 @@ void diffcore_rename_extended(struct diff_options *options,
 	/* Finish setting up dpf_options */
 	prefetch_options.skip_unmodified = skip_unmodified;
 	if (options->repo == the_repository && has_promisor_remote()) {
-		dpf_options.missing_object_cb = prefetch;
+		dpf_options.missing_object_cb = inexact_prefetch;
 		dpf_options.missing_object_data = &prefetch_options;
 	}
 
diff --git a/t/t6421-merge-partial-clone.sh b/t/t6421-merge-partial-clone.sh
index 3dcffc15f8..aad975d24d 100755
--- a/t/t6421-merge-partial-clone.sh
+++ b/t/t6421-merge-partial-clone.sh
@@ -207,7 +207,7 @@ test_setup_repo () {
 #
 #   Summary: 2 fetches (1 for 2 objects, 1 for 1 object)
 #
-test_expect_merge_algorithm failure failure 'Objects downloaded for single relevant rename' '
+test_expect_merge_algorithm failure success 'Objects downloaded for single relevant rename' '
 	test_setup_repo &&
 	git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-single &&
 	(
@@ -296,7 +296,7 @@ test_expect_merge_algorithm failure failure 'Objects downloaded for single relev
 #      this are not all that common.)
 #   Summary: 1 fetches for 6 objects
 #
-test_expect_merge_algorithm failure failure 'Objects downloaded when a directory rename triggered' '
+test_expect_merge_algorithm failure success 'Objects downloaded when a directory rename triggered' '
 	test_setup_repo &&
 	git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-dir &&
 	(

From 2bff554b23e8d5c17f4e6827358bb51811fc7137 Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Tue, 22 Jun 2021 08:04:41 +0000
Subject: [PATCH 259/397] merge-ort: add prefetching for content merges

Commit 7fbbcb21b1 ("diff: batch fetching of missing blobs", 2019-04-05)
introduced batching of fetching missing blobs, so that the diff
machinery would have one fetch subprocess grab N blobs instead of N
processes each grabbing 1.

However, the diff machinery is not the only thing in a merge that needs
to work on blobs.  The 3-way content merges need them as well.  Rather
than download all the blobs 1 at a time, prefetch all the blobs needed
for regular content merges.

This does not cover all possible paths in merge-ort that might need to
download blobs.  Others include:
  - The blob_unchanged() calls to avoid modify/delete conflicts (when
    blob renormalization results in an "unchanged" file)
  - Preliminary content merges needed for rename/add and
    rename/rename(2to1) style conflicts.  (Both of these types of
    conflicts can result in nested conflict markers from the need to do
    two levels of content merging; the first happens before our new
    prefetch_for_content_merges() function.)

The first of these wouldn't be an extreme amount of work to support, and
even the second could be theoretically supported in batching, but all of
these cases seem unusual to me, and this is a minor performance
optimization anyway; in the worst case we only get some of the fetches
batched and have a few additional one-off fetches.  So for now, just
handle the regular 3-way content merges in our prefetching.

For the testcase from the previous commit, the number of downloaded
objects remains at 63, but this drops the number of fetches needed from
32 down to 20, a sizeable reduction.

Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 merge-ort.c                    | 50 ++++++++++++++++++++++++++++++++++
 t/t6421-merge-partial-clone.sh |  2 +-
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/merge-ort.c b/merge-ort.c
index b954f7184a..ccf85b738c 100644
--- a/merge-ort.c
+++ b/merge-ort.c
@@ -29,6 +29,7 @@
 #include "entry.h"
 #include "ll-merge.h"
 #include "object-store.h"
+#include "promisor-remote.h"
 #include "revision.h"
 #include "strmap.h"
 #include "submodule.h"
@@ -3460,6 +3461,54 @@ static void process_entry(struct merge_options *opt,
 	record_entry_for_tree(dir_metadata, path, &ci->merged);
 }
 
+static void prefetch_for_content_merges(struct merge_options *opt,
+					struct string_list *plist)
+{
+	struct string_list_item *e;
+	struct oid_array to_fetch = OID_ARRAY_INIT;
+
+	if (opt->repo != the_repository || !has_promisor_remote())
+		return;
+
+	for (e = &plist->items[plist->nr-1]; e >= plist->items; --e) {
+		/* char *path = e->string; */
+		struct conflict_info *ci = e->util;
+		int i;
+
+		/* Ignore clean entries */
+		if (ci->merged.clean)
+			continue;
+
+		/* Ignore entries that don't need a content merge */
+		if (ci->match_mask || ci->filemask < 6 ||
+		    !S_ISREG(ci->stages[1].mode) ||
+		    !S_ISREG(ci->stages[2].mode) ||
+		    oideq(&ci->stages[1].oid, &ci->stages[2].oid))
+			continue;
+
+		/* Also don't need content merge if base matches either side */
+		if (ci->filemask == 7 &&
+		    S_ISREG(ci->stages[0].mode) &&
+		    (oideq(&ci->stages[0].oid, &ci->stages[1].oid) ||
+		     oideq(&ci->stages[0].oid, &ci->stages[2].oid)))
+			continue;
+
+		for (i = 0; i < 3; i++) {
+			unsigned side_mask = (1 << i);
+			struct version_info *vi = &ci->stages[i];
+
+			if ((ci->filemask & side_mask) &&
+			    S_ISREG(vi->mode) &&
+			    oid_object_info_extended(opt->repo, &vi->oid, NULL,
+						     OBJECT_INFO_FOR_PREFETCH))
+				oid_array_append(&to_fetch, &vi->oid);
+		}
+	}
+
+	promisor_remote_get_direct(opt->repo, to_fetch.oid, to_fetch.nr);
+	oid_array_clear(&to_fetch);
+}
+
 static void process_entries(struct merge_options *opt,
 			    struct object_id *result_oid)
 {
@@ -3506,6 +3555,7 @@ static void process_entries(struct merge_options *opt,
 	 * the way when it is time to process the file at the same path).
 	 */
 	trace2_region_enter("merge", "processing", opt->repo);
+	prefetch_for_content_merges(opt, &plist);
 	for (entry = &plist.items[plist.nr-1]; entry >= plist.items; --entry) {
 		char *path = entry->string;
 		/*
diff --git a/t/t6421-merge-partial-clone.sh b/t/t6421-merge-partial-clone.sh
index aad975d24d..36bcd7c328 100755
--- a/t/t6421-merge-partial-clone.sh
+++ b/t/t6421-merge-partial-clone.sh
@@ -397,7 +397,7 @@ test_expect_merge_algorithm failure success 'Objects downloaded when a directory
 #
 #   Summary: 4 fetches (1 for 6 objects, 1 for 8, 1 for 3, 1 for 2)
 #
-test_expect_merge_algorithm failure failure 'Objects downloaded with lots of renames and modifications' '
+test_expect_merge_algorithm failure success 'Objects downloaded with lots of renames and modifications' '
 	test_setup_repo &&
 	git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-many &&
 	(

From 47c0cb1a5db27dcf1c6117158b8152d1beacc9de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 17 Jun 2021 12:41:59 +0200
Subject: [PATCH 260/397] mktag tests: test hash-object --literally and
 unreachable fsck
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Extend the mktag tests to pass the tag we've created through both
hash-object --literally and fsck.

This checks that fsck itself will not complain about certain invalid
content if a reachable tip isn't involved. Due to how fsck works and
walks the graph the failure will be different if the object is
reachable, so we might succeed before we've created the ref.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3800-mktag.sh | 46 ++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/t/t3800-mktag.sh b/t/t3800-mktag.sh
index 951e6d39c2..78c6f64e36 100755
--- a/t/t3800-mktag.sh
+++ b/t/t3800-mktag.sh
@@ -16,6 +16,8 @@ check_verify_failure () {
 	message=$2 &&
 	shift 2 &&
 
+	no_strict= &&
+	fsck_obj_ok= &&
 	no_strict= &&
 	while test $# != 0
 	do
@@ -23,6 +25,9 @@ check_verify_failure () {
 		--no-strict)
 			no_strict=yes
 			;;
+		--fsck-obj-ok)
+			fsck_obj_ok=yes
+			;;
 		esac &&
 		shift
 	done &&
@@ -37,6 +42,23 @@ check_verify_failure () {
 			git mktag --no-strict  0 +0000
 EOF
 
 check_verify_failure 'verify object (hash/type) check -- correct type, nonexisting object' \
-	'^fatal: could not read tagged object'
+	'^fatal: could not read tagged object' \
+	--fsck-obj-ok
 
 cat >tag.sig < 0 +0000
 EOF
 
 check_verify_failure 'verify object (hash/type) check -- mismatched type, valid object' \
-	'^fatal: object.*tagged as.*tree.*but is.*commit'
+	'^fatal: object.*tagged as.*tree.*but is.*commit' \
+	--fsck-obj-ok
 
 ############################################################
 #  9.5. verify object (hash/type) check -- replacement
@@ -245,7 +269,8 @@ tagger . <> 0 +0000
 EOF
 
 check_verify_failure 'verify object (hash/type) check -- mismatched type, valid object' \
-	'^fatal: object.*tagged as.*tree.*but is.*blob'
+	'^fatal: object.*tagged as.*tree.*but is.*blob' \
+	--fsck-obj-ok
 
 ############################################################
 # 10. verify tag-name check
@@ -260,7 +285,8 @@ EOF
 
 check_verify_failure 'verify tag-name check' \
 	'^error:.* badTagName:' \
-	--no-strict
+	--no-strict \
+	--fsck-obj-ok
 
 ############################################################
 # 11. tagger line label check #1
@@ -275,7 +301,8 @@ EOF
 
 check_verify_failure '"tagger" line label check #1' \
 	'^error:.* missingTaggerEntry:' \
-	--no-strict
+	--no-strict \
+	--fsck-obj-ok
 
 ############################################################
 # 12. tagger line label check #2
@@ -291,7 +318,8 @@ EOF
 
 check_verify_failure '"tagger" line label check #2' \
 	'^error:.* missingTaggerEntry:' \
-	--no-strict
+	--no-strict \
+	--fsck-obj-ok
 
 ############################################################
 # 13. allow missing tag author name like fsck
@@ -321,7 +349,8 @@ EOF
 
 check_verify_failure 'disallow malformed tagger' \
 	'^error:.* badEmail:' \
-	--no-strict
+	--no-strict \
+	--fsck-obj-ok
 
 ############################################################
 # 15. allow empty tag email
@@ -446,7 +475,8 @@ EOF
 
 check_verify_failure 'detect invalid header entry' \
 	'^error:.* extraHeaderEntry:' \
-	--no-strict
+	--no-strict \
+	--fsck-obj-ok
 
 test_expect_success 'invalid header entry config & fsck' '
 	test_must_fail git mktag 
Date: Thu, 17 Jun 2021 12:42:00 +0200
Subject: [PATCH 261/397] mktag tests: test update-ref and reachable fsck
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Extend the mktag tests to pass the created bad tag through update-ref
and fsck.

The reason for passing it through update-ref is to guard against it
having a segfault as for-each-ref did before c6854508808 (ref-filter:
fix NULL check for parse object failure, 2021-04-01).

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3800-mktag.sh | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/t/t3800-mktag.sh b/t/t3800-mktag.sh
index 78c6f64e36..67f6ecbe88 100755
--- a/t/t3800-mktag.sh
+++ b/t/t3800-mktag.sh
@@ -44,11 +44,13 @@ check_verify_failure () {
 	'
 
 	test_expect_success "setup: $subject" '
+		tag_ref=refs/tags/bad_tag &&
+
 		# Reset any leftover state from the last $subject
 		rm -rf bad-tag &&
 
 		git init --bare bad-tag &&
-		git -C bad-tag hash-object -t tag -w --stdin --literally "bad-tag/$tag_ref" &&
+
+		# Unlike fsck-ing unreachable content above, this
+		# will always fail.
+		test_must_fail git -C bad-tag fsck
+	'
 }
 
 test_expect_mktag_success() {

From b48015b3404cb19bfd3b6df1a372164d8cd3f7d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 17 Jun 2021 12:42:01 +0200
Subject: [PATCH 262/397] mktag tests: test for-each-ref
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add a "for-each-ref" for all the mktag tests. This test would have
caught the segfault which was fixed in c6854508808 (ref-filter: fix
NULL check for parse object failure, 2021-04-01). Let's make sure we
test that code more exhaustively.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3800-mktag.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/t/t3800-mktag.sh b/t/t3800-mktag.sh
index 67f6ecbe88..bb71303399 100755
--- a/t/t3800-mktag.sh
+++ b/t/t3800-mktag.sh
@@ -78,6 +78,19 @@ check_verify_failure () {
 		# will always fail.
 		test_must_fail git -C bad-tag fsck
 	'
+
+	test_expect_success "for-each-ref: $subject" '
+		# Make sure the earlier test created it for us
+		git rev-parse "$bad_tag" &&
+
+		echo "$bad_tag" >"bad-tag/$tag_ref" &&
+
+		printf "%s tag\t%s\n" "$bad_tag" "$tag_ref" >expected &&
+		git -C bad-tag for-each-ref "$tag_ref" >actual &&
+		test_cmp expected actual &&
+
+		test_must_fail git -C bad-tag for-each-ref --format="%(*objectname)"
+	'
 }
 
 test_expect_mktag_success() {

From 2f61b3eef386e6e6cb81ab61dc310dd7f03e4b13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 17 Jun 2021 12:42:02 +0200
Subject: [PATCH 263/397] mktag tests: test fast-export
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pass the bad tags we've created in the mktag tests through
fast-export, it will die on the bad object or ref, let's make sure
that happens.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3800-mktag.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/t/t3800-mktag.sh b/t/t3800-mktag.sh
index bb71303399..0544d58a6e 100755
--- a/t/t3800-mktag.sh
+++ b/t/t3800-mktag.sh
@@ -91,6 +91,14 @@ check_verify_failure () {
 
 		test_must_fail git -C bad-tag for-each-ref --format="%(*objectname)"
 	'
+
+	test_expect_success "fast-export & fast-import: $subject" '
+		# Make sure the earlier test created it for us
+		git rev-parse "$bad_tag" &&
+
+		test_must_fail git -C bad-tag fast-export --all &&
+		test_must_fail git -C bad-tag fast-export "$bad_tag"
+	'
 }
 
 test_expect_mktag_success() {

From 4465690cd838017208d297a7f77d00ec51ccf7fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 17 Jun 2021 12:53:37 +0200
Subject: [PATCH 264/397] show-branch: don't  for space
 characters
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change the colored output introduced in ab07ba2a24 (show-branch: color
the commit status signs, 2009-04-22) to not color and reset each
individual space character we use for padding. The intent is to color
just the "!", "+" etc. characters.

This makes the output easier to test, so let's do that now. The test
would be much more verbose without a color/reset for each space
character. Since the coloring cycles through colors we previously had
a "rainbow of space characters".

In theory this breaks things for anyone who's relying on the exact
colored output of show-branch, in practice I'd think anyone parsing it
isn't actively turning on the colored output.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 builtin/show-branch.c  |  9 ++++++---
 t/t3202-show-branch.sh | 30 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/builtin/show-branch.c b/builtin/show-branch.c
index d6d2dabeca..d77ce7aeb3 100644
--- a/builtin/show-branch.c
+++ b/builtin/show-branch.c
@@ -939,9 +939,12 @@ int cmd_show_branch(int ac, const char **av, const char *prefix)
 					mark = '*';
 				else
 					mark = '+';
-				printf("%s%c%s",
-				       get_color_code(i),
-				       mark, get_color_reset_code());
+				if (mark == ' ')
+					putchar(mark);
+				else
+					printf("%s%c%s",
+					       get_color_code(i),
+					       mark, get_color_reset_code());
 			}
 			putchar(' ');
 		}
diff --git a/t/t3202-show-branch.sh b/t/t3202-show-branch.sh
index 7b06048905..54025f0337 100755
--- a/t/t3202-show-branch.sh
+++ b/t/t3202-show-branch.sh
@@ -55,4 +55,34 @@ test_expect_success 'show-branch with showbranch.default' '
 	test_cmp expect actual
 '
 
+test_expect_success 'show-branch --color output' '
+	sed "s/^> //" >expect <<-\EOF &&
+	> ! [branch1] branch1
+	>  ! [branch2] branch2
+	>   ! [branch3] branch3
+	>    ! [branch4] branch4
+	>     ! [branch5] branch5
+	>      ! [branch6] branch6
+	>       ! [branch7] branch7
+	>        ! [branch8] branch8
+	>         ! [branch9] branch9
+	>          * [branch10] branch10
+	> ----------
+	>          * [branch10] branch10
+	>         +  [branch9] branch9
+	>        +   [branch8] branch8
+	>       +    [branch7] branch7
+	>      +     [branch6] branch6
+	>     +      [branch5] branch5
+	>    +       [branch4] branch4
+	>   +        [branch3] branch3
+	>  +         [branch2] branch2
+	> +          [branch1] branch1
+	> +++++++++* [branch10^] initial
+	EOF
+	git show-branch --color=always $(cat branches.sorted) >actual.raw &&
+	test_decode_color actual &&
+	test_cmp expect actual
+'
+
 test_done

From d65aea37d9b3dd64e29aeb0727c6141f264ce24e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 17 Jun 2021 12:53:38 +0200
Subject: [PATCH 265/397] show-branch tests: add missing tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add missing tests for --remotes, --list and --merge-base. These are
not exhaustive, but better than the nothing we have now.

There were some tests for this command added in f76412ed6db ([PATCH]
Add 'git show-branch'., 2005-08-21) has never been properly tested,
namely for the --all option in t6432-merge-recursive-space-options.sh,
and some of --merge-base and --independent in t6010-merge-base.sh.

This fixes a few more blind spots, but there's still a lot of behavior
that's not tested for.

These new tests show the odd (and possibly unintentional) behavior of
--merge-base with one argument, and how its output is the same as "git
merge-base" with N bases in this particular case. See the test added
in f621a8454d1 (git-merge-base/git-show-branch --merge-base:
Documentation and test, 2009-08-05) for a case where the two aren't
the same.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/t3202-show-branch.sh | 61 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/t/t3202-show-branch.sh b/t/t3202-show-branch.sh
index 54025f0337..ad9902a06b 100755
--- a/t/t3202-show-branch.sh
+++ b/t/t3202-show-branch.sh
@@ -85,4 +85,65 @@ test_expect_success 'show-branch --color output' '
 	test_cmp expect actual
 '
 
+test_expect_success 'show branch --remotes' '
+	cat >expect.err <<-\EOF &&
+	No revs to be shown.
+	EOF
+	git show-branch -r 2>actual.err >actual.out &&
+	test_cmp expect.err actual.err &&
+	test_must_be_empty actual.out
+'
+
+test_expect_success 'setup show branch --list' '
+	sed "s/^> //" >expect <<-\EOF
+	>   [branch1] branch1
+	>   [branch2] branch2
+	>   [branch3] branch3
+	>   [branch4] branch4
+	>   [branch5] branch5
+	>   [branch6] branch6
+	>   [branch7] branch7
+	>   [branch8] branch8
+	>   [branch9] branch9
+	> * [branch10] branch10
+	EOF
+'
+
+test_expect_success 'show branch --list' '
+	git show-branch --list $(cat branches.sorted) >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'show branch --list has no --color output' '
+	git show-branch --color=always --list $(cat branches.sorted) >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'show branch --merge-base with one argument' '
+	for branch in $(cat branches.sorted)
+	do
+		git rev-parse $branch >expect &&
+		git show-branch --merge-base $branch >actual &&
+		test_cmp expect actual
+	done
+'
+
+test_expect_success 'show branch --merge-base with two arguments' '
+	for branch in $(cat branches.sorted)
+	do
+		git rev-parse initial >expect &&
+		git show-branch --merge-base initial $branch >actual &&
+		test_cmp expect actual
+	done
+'
+
+test_expect_success 'show branch --merge-base with N arguments' '
+	git rev-parse initial >expect &&
+	git show-branch --merge-base $(cat branches.sorted) >actual &&
+	test_cmp expect actual &&
+
+	git merge-base $(cat branches.sorted) >actual &&
+	test_cmp expect actual
+'
+
 test_done

From 54662d595846dae8884c6b5f9be084e10ada9e4e Mon Sep 17 00:00:00 2001
From: "dorgon.chang" 
Date: Mon, 21 Jun 2021 05:16:13 +0000
Subject: [PATCH 266/397] git-p4: fix failed submit by skip non-text data files

If the submit contain binary files, it will throw exception and stop submit when try to append diff line description.

This commit will skip non-text data files when exception UnicodeDecodeError thrown.

The skip will not affect actual submit files in the resulting cl,
the diff line description will only appear in submit template,
so you can review what changed before actully submit to p4.

I don't know if add any message here will be helpful for users,
so I choose to just skip binary content, since it already append filename previously.

Signed-off-by: dorgon.chang 
Signed-off-by: Junio C Hamano 
---
 git-p4.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/git-p4.py b/git-p4.py
index d34a1946b7..2b4500226a 100755
--- a/git-p4.py
+++ b/git-p4.py
@@ -1977,8 +1977,11 @@ class P4Submit(Command, P4UserMap):
                 newdiff += "+%s\n" % os.readlink(newFile)
             else:
                 f = open(newFile, "r")
-                for line in f.readlines():
-                    newdiff += "+" + line
+                try:
+                    for line in f.readlines():
+                        newdiff += "+" + line
+                except UnicodeDecodeError:
+                    pass # Found non-text data and skip, since diff description should only include text
                 f.close()
 
         return (diff + newdiff).replace('\r\n', '\n')

From ebaf3bcf1aecdc31062ede80fca3a7c98202d8bb Mon Sep 17 00:00:00 2001
From: Jonathan Tan 
Date: Thu, 17 Jun 2021 10:13:22 -0700
Subject: [PATCH 267/397] repository: move global r_f_p_c to repo struct

Move repository_format_partial_clone, which is currently a global
variable, into struct repository. (Full support for per-repository
partial clone config will be done in a subsequent commit - this is split
into its own commit because of the extent of the changes needed.)

The new repo-specific variable cannot be set in
check_repository_format_gently() (as is currently), because that
function does not know which repo it is operating on (or even whether
the value is important); therefore this responsibility is delegated to
the outermost caller that knows. Of all the outermost callers that know
(found by looking at all functions that call clear_repository_format()),
I looked at those that either read from the main Git directory or write
into a struct repository. These callers have been modified accordingly
(write to the_repository in the former case and write to the given
struct repository in the latter case).

Signed-off-by: Jonathan Tan 
Signed-off-by: Junio C Hamano 
Reviewed-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 promisor-remote.c | 13 +++----------
 promisor-remote.h |  6 ------
 repository.c      |  4 ++++
 repository.h      |  3 +++
 setup.c           | 17 +++++++++++++----
 5 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/promisor-remote.c b/promisor-remote.c
index da3f2ca261..d24081dc21 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -5,13 +5,6 @@
 #include "transport.h"
 #include "strvec.h"
 
-static char *repository_format_partial_clone;
-
-void set_repository_format_partial_clone(char *partial_clone)
-{
-	repository_format_partial_clone = xstrdup_or_null(partial_clone);
-}
-
 static int fetch_objects(const char *remote_name,
 			 const struct object_id *oids,
 			 int oid_nr)
@@ -145,15 +138,15 @@ static void promisor_remote_init(void)
 
 	git_config(promisor_remote_config, NULL);
 
-	if (repository_format_partial_clone) {
+	if (the_repository->repository_format_partial_clone) {
 		struct promisor_remote *o, *previous;
 
-		o = promisor_remote_lookup(repository_format_partial_clone,
+		o = promisor_remote_lookup(the_repository->repository_format_partial_clone,
 					   &previous);
 		if (o)
 			promisor_remote_move_to_tail(o, previous);
 		else
-			promisor_remote_new(repository_format_partial_clone);
+			promisor_remote_new(the_repository->repository_format_partial_clone);
 	}
 }
 
diff --git a/promisor-remote.h b/promisor-remote.h
index c7a14063c5..687210ab87 100644
--- a/promisor-remote.h
+++ b/promisor-remote.h
@@ -32,10 +32,4 @@ int promisor_remote_get_direct(struct repository *repo,
 			       const struct object_id *oids,
 			       int oid_nr);
 
-/*
- * This should be used only once from setup.c to set the value we got
- * from the extensions.partialclone config option.
- */
-void set_repository_format_partial_clone(char *partial_clone);
-
 #endif /* PROMISOR_REMOTE_H */
diff --git a/repository.c b/repository.c
index 448cd557d4..057a4748a0 100644
--- a/repository.c
+++ b/repository.c
@@ -172,6 +172,10 @@ int repo_init(struct repository *repo,
 
 	repo_set_hash_algo(repo, format.hash_algo);
 
+	/* take ownership of format.partial_clone */
+	repo->repository_format_partial_clone = format.partial_clone;
+	format.partial_clone = NULL;
+
 	if (worktree)
 		repo_set_worktree(repo, worktree);
 
diff --git a/repository.h b/repository.h
index a45f7520fd..6fb16ed336 100644
--- a/repository.h
+++ b/repository.h
@@ -139,6 +139,9 @@ struct repository {
 	/* True if commit-graph has been disabled within this process. */
 	int commit_graph_disabled;
 
+	/* Configurations related to promisor remotes. */
+	char *repository_format_partial_clone;
+
 	/* Configurations */
 
 	/* Indicate if a repository has a different 'commondir' from 'gitdir' */
diff --git a/setup.c b/setup.c
index 59e2facd9d..5a2dd95f00 100644
--- a/setup.c
+++ b/setup.c
@@ -468,8 +468,6 @@ static enum extension_result handle_extension_v0(const char *var,
 			data->precious_objects = git_config_bool(var, value);
 			return EXTENSION_OK;
 		} else if (!strcmp(ext, "partialclone")) {
-			if (!value)
-				return config_error_nonbool(var);
 			data->partial_clone = xstrdup(value);
 			return EXTENSION_OK;
 		} else if (!strcmp(ext, "worktreeconfig")) {
@@ -566,7 +564,6 @@ static int check_repository_format_gently(const char *gitdir, struct repository_
 	}
 
 	repository_format_precious_objects = candidate->precious_objects;
-	set_repository_format_partial_clone(candidate->partial_clone);
 	repository_format_worktree_config = candidate->worktree_config;
 	string_list_clear(&candidate->unknown_extensions, 0);
 	string_list_clear(&candidate->v1_only_extensions, 0);
@@ -1193,6 +1190,11 @@ int discover_git_directory(struct strbuf *commondir,
 		return -1;
 	}
 
+	/* take ownership of candidate.partial_clone */
+	the_repository->repository_format_partial_clone =
+		candidate.partial_clone;
+	candidate.partial_clone = NULL;
+
 	clear_repository_format(&candidate);
 	return 0;
 }
@@ -1300,8 +1302,13 @@ const char *setup_git_directory_gently(int *nongit_ok)
 				gitdir = DEFAULT_GIT_DIR_ENVIRONMENT;
 			setup_git_env(gitdir);
 		}
-		if (startup_info->have_repository)
+		if (startup_info->have_repository) {
 			repo_set_hash_algo(the_repository, repo_fmt.hash_algo);
+			/* take ownership of repo_fmt.partial_clone */
+			the_repository->repository_format_partial_clone =
+				repo_fmt.partial_clone;
+			repo_fmt.partial_clone = NULL;
+		}
 	}
 	/*
 	 * Since precompose_string_if_needed() needs to look at
@@ -1386,6 +1393,8 @@ void check_repository_format(struct repository_format *fmt)
 	check_repository_format_gently(get_git_dir(), fmt, NULL);
 	startup_info->have_repository = 1;
 	repo_set_hash_algo(the_repository, fmt->hash_algo);
+	the_repository->repository_format_partial_clone =
+		xstrdup_or_null(fmt->partial_clone);
 	clear_repository_format(&repo_fmt);
 }
 

From ef7dc2e9ccb832c1dc29d5102c09f6c4a51d4dca Mon Sep 17 00:00:00 2001
From: Jonathan Tan 
Date: Thu, 17 Jun 2021 10:13:23 -0700
Subject: [PATCH 268/397] promisor-remote: support per-repository config

Instead of using global variables to store promisor remote information,
store this config in struct repository instead, and add
repository-agnostic non-static functions corresponding to the existing
non-static functions that only work on the_repository.

The actual lazy-fetching of missing objects currently does not work on
repositories other than the_repository, and will still not work after
this commit, so add a BUG message explaining this. A subsequent commit
will remove this limitation.

Signed-off-by: Jonathan Tan 
Signed-off-by: Junio C Hamano 
Reviewed-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 promisor-remote.c | 98 ++++++++++++++++++++++++++---------------------
 promisor-remote.h | 22 +++++++++--
 repository.c      |  6 +++
 repository.h      |  2 +
 4 files changed, 82 insertions(+), 46 deletions(-)

diff --git a/promisor-remote.c b/promisor-remote.c
index d24081dc21..1e00e16b0f 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -5,6 +5,11 @@
 #include "transport.h"
 #include "strvec.h"
 
+struct promisor_remote_config {
+	struct promisor_remote *promisors;
+	struct promisor_remote **promisors_tail;
+};
+
 static int fetch_objects(const char *remote_name,
 			 const struct object_id *oids,
 			 int oid_nr)
@@ -35,10 +40,8 @@ static int fetch_objects(const char *remote_name,
 	return finish_command(&child) ? -1 : 0;
 }
 
-static struct promisor_remote *promisors;
-static struct promisor_remote **promisors_tail = &promisors;
-
-static struct promisor_remote *promisor_remote_new(const char *remote_name)
+static struct promisor_remote *promisor_remote_new(struct promisor_remote_config *config,
+						   const char *remote_name)
 {
 	struct promisor_remote *r;
 
@@ -50,18 +53,19 @@ static struct promisor_remote *promisor_remote_new(const char *remote_name)
 
 	FLEX_ALLOC_STR(r, name, remote_name);
 
-	*promisors_tail = r;
-	promisors_tail = &r->next;
+	*config->promisors_tail = r;
+	config->promisors_tail = &r->next;
 
 	return r;
 }
 
-static struct promisor_remote *promisor_remote_lookup(const char *remote_name,
+static struct promisor_remote *promisor_remote_lookup(struct promisor_remote_config *config,
+						      const char *remote_name,
 						      struct promisor_remote **previous)
 {
 	struct promisor_remote *r, *p;
 
-	for (p = NULL, r = promisors; r; p = r, r = r->next)
+	for (p = NULL, r = config->promisors; r; p = r, r = r->next)
 		if (!strcmp(r->name, remote_name)) {
 			if (previous)
 				*previous = p;
@@ -71,7 +75,8 @@ static struct promisor_remote *promisor_remote_lookup(const char *remote_name,
 	return NULL;
 }
 
-static void promisor_remote_move_to_tail(struct promisor_remote *r,
+static void promisor_remote_move_to_tail(struct promisor_remote_config *config,
+					 struct promisor_remote *r,
 					 struct promisor_remote *previous)
 {
 	if (r->next == NULL)
@@ -80,14 +85,15 @@ static void promisor_remote_move_to_tail(struct promisor_remote *r,
 	if (previous)
 		previous->next = r->next;
 	else
-		promisors = r->next ? r->next : r;
+		config->promisors = r->next ? r->next : r;
 	r->next = NULL;
-	*promisors_tail = r;
-	promisors_tail = &r->next;
+	*config->promisors_tail = r;
+	config->promisors_tail = &r->next;
 }
 
 static int promisor_remote_config(const char *var, const char *value, void *data)
 {
+	struct promisor_remote_config *config = data;
 	const char *name;
 	size_t namelen;
 	const char *subkey;
@@ -103,8 +109,8 @@ static int promisor_remote_config(const char *var, const char *value, void *data
 
 		remote_name = xmemdupz(name, namelen);
 
-		if (!promisor_remote_lookup(remote_name, NULL))
-			promisor_remote_new(remote_name);
+		if (!promisor_remote_lookup(config, remote_name, NULL))
+			promisor_remote_new(config, remote_name);
 
 		free(remote_name);
 		return 0;
@@ -113,9 +119,9 @@ static int promisor_remote_config(const char *var, const char *value, void *data
 		struct promisor_remote *r;
 		char *remote_name = xmemdupz(name, namelen);
 
-		r = promisor_remote_lookup(remote_name, NULL);
+		r = promisor_remote_lookup(config, remote_name, NULL);
 		if (!r)
-			r = promisor_remote_new(remote_name);
+			r = promisor_remote_new(config, remote_name);
 
 		free(remote_name);
 
@@ -128,59 +134,63 @@ static int promisor_remote_config(const char *var, const char *value, void *data
 	return 0;
 }
 
-static int initialized;
-
-static void promisor_remote_init(void)
+static void promisor_remote_init(struct repository *r)
 {
-	if (initialized)
+	struct promisor_remote_config *config;
+
+	if (r->promisor_remote_config)
 		return;
-	initialized = 1;
+	config = r->promisor_remote_config =
+		xcalloc(sizeof(*r->promisor_remote_config), 1);
+	config->promisors_tail = &config->promisors;
 
-	git_config(promisor_remote_config, NULL);
+	repo_config(r, promisor_remote_config, config);
 
-	if (the_repository->repository_format_partial_clone) {
+	if (r->repository_format_partial_clone) {
 		struct promisor_remote *o, *previous;
 
-		o = promisor_remote_lookup(the_repository->repository_format_partial_clone,
+		o = promisor_remote_lookup(config,
+					   r->repository_format_partial_clone,
 					   &previous);
 		if (o)
-			promisor_remote_move_to_tail(o, previous);
+			promisor_remote_move_to_tail(config, o, previous);
 		else
-			promisor_remote_new(the_repository->repository_format_partial_clone);
+			promisor_remote_new(config, r->repository_format_partial_clone);
 	}
 }
 
-static void promisor_remote_clear(void)
+void promisor_remote_clear(struct promisor_remote_config *config)
 {
-	while (promisors) {
-		struct promisor_remote *r = promisors;
-		promisors = promisors->next;
+	while (config->promisors) {
+		struct promisor_remote *r = config->promisors;
+		config->promisors = config->promisors->next;
 		free(r);
 	}
 
-	promisors_tail = &promisors;
+	config->promisors_tail = &config->promisors;
 }
 
-void promisor_remote_reinit(void)
+void repo_promisor_remote_reinit(struct repository *r)
 {
-	initialized = 0;
-	promisor_remote_clear();
-	promisor_remote_init();
+	promisor_remote_clear(r->promisor_remote_config);
+	FREE_AND_NULL(r->promisor_remote_config);
+	promisor_remote_init(r);
 }
 
-struct promisor_remote *promisor_remote_find(const char *remote_name)
+struct promisor_remote *repo_promisor_remote_find(struct repository *r,
+						  const char *remote_name)
 {
-	promisor_remote_init();
+	promisor_remote_init(r);
 
 	if (!remote_name)
-		return promisors;
+		return r->promisor_remote_config->promisors;
 
-	return promisor_remote_lookup(remote_name, NULL);
+	return promisor_remote_lookup(r->promisor_remote_config, remote_name, NULL);
 }
 
-int has_promisor_remote(void)
+int repo_has_promisor_remote(struct repository *r)
 {
-	return !!promisor_remote_find(NULL);
+	return !!repo_promisor_remote_find(r, NULL);
 }
 
 static int remove_fetched_oids(struct repository *repo,
@@ -228,9 +238,11 @@ int promisor_remote_get_direct(struct repository *repo,
 	if (oid_nr == 0)
 		return 0;
 
-	promisor_remote_init();
+	promisor_remote_init(repo);
 
-	for (r = promisors; r; r = r->next) {
+	if (repo != the_repository)
+		BUG("only the_repository is supported for now");
+	for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
 		if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) {
 			if (remaining_nr == 1)
 				continue;
diff --git a/promisor-remote.h b/promisor-remote.h
index 687210ab87..edc45ab0f5 100644
--- a/promisor-remote.h
+++ b/promisor-remote.h
@@ -17,9 +17,25 @@ struct promisor_remote {
 	const char name[FLEX_ARRAY];
 };
 
-void promisor_remote_reinit(void);
-struct promisor_remote *promisor_remote_find(const char *remote_name);
-int has_promisor_remote(void);
+void repo_promisor_remote_reinit(struct repository *r);
+static inline void promisor_remote_reinit(void)
+{
+	repo_promisor_remote_reinit(the_repository);
+}
+
+void promisor_remote_clear(struct promisor_remote_config *config);
+
+struct promisor_remote *repo_promisor_remote_find(struct repository *r, const char *remote_name);
+static inline struct promisor_remote *promisor_remote_find(const char *remote_name)
+{
+	return repo_promisor_remote_find(the_repository, remote_name);
+}
+
+int repo_has_promisor_remote(struct repository *r);
+static inline int has_promisor_remote(void)
+{
+	return repo_has_promisor_remote(the_repository);
+}
 
 /*
  * Fetches all requested objects from all promisor remotes, trying them one at
diff --git a/repository.c b/repository.c
index 057a4748a0..b2bf44c6fa 100644
--- a/repository.c
+++ b/repository.c
@@ -11,6 +11,7 @@
 #include "lockfile.h"
 #include "submodule-config.h"
 #include "sparse-index.h"
+#include "promisor-remote.h"
 
 /* The main repository */
 static struct repository the_repo;
@@ -262,6 +263,11 @@ void repo_clear(struct repository *repo)
 		if (repo->index != &the_index)
 			FREE_AND_NULL(repo->index);
 	}
+
+	if (repo->promisor_remote_config) {
+		promisor_remote_clear(repo->promisor_remote_config);
+		FREE_AND_NULL(repo->promisor_remote_config);
+	}
 }
 
 int repo_read_index(struct repository *repo)
diff --git a/repository.h b/repository.h
index 6fb16ed336..3740c93bc0 100644
--- a/repository.h
+++ b/repository.h
@@ -10,6 +10,7 @@ struct lock_file;
 struct pathspec;
 struct raw_object_store;
 struct submodule_cache;
+struct promisor_remote_config;
 
 enum untracked_cache_setting {
 	UNTRACKED_CACHE_UNSET = -1,
@@ -141,6 +142,7 @@ struct repository {
 
 	/* Configurations related to promisor remotes. */
 	char *repository_format_partial_clone;
+	struct promisor_remote_config *promisor_remote_config;
 
 	/* Configurations */
 

From 69bb2e1804d2f3e5ba7ef365fdfd1937b4d7ca0a Mon Sep 17 00:00:00 2001
From: Jonathan Tan 
Date: Thu, 17 Jun 2021 10:13:24 -0700
Subject: [PATCH 269/397] submodule: refrain from filtering GIT_CONFIG_COUNT

14111fc492 ("git: submodule honor -c credential.* from command line",
2016-03-01) taught Git to pass through the GIT_CONFIG_PARAMETERS
environment variable when invoking a subprocess on behalf of a
submodule. But when d8d77153ea ("config: allow specifying config entries
via envvar pairs", 2021-01-15) introduced support for GIT_CONFIG_COUNT
(and its associated GIT_CONFIG_KEY_? and GIT_CONFIG_VALUE_?), the
subprocess mechanism wasn't updated to also pass through these
variables.

Since they are conceptually the same (d8d77153ea was written to address
a shortcoming of GIT_CONFIG_PARAMETERS), update the submodule subprocess
mechanism to also pass through GIT_CONFIG_COUNT.

Signed-off-by: Jonathan Tan 
Signed-off-by: Junio C Hamano 
---
 submodule.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/submodule.c b/submodule.c
index 0b1d9c1dde..f09031e397 100644
--- a/submodule.c
+++ b/submodule.c
@@ -489,7 +489,8 @@ static void prepare_submodule_repo_env_no_git_dir(struct strvec *out)
 	const char * const *var;
 
 	for (var = local_repo_env; *var; var++) {
-		if (strcmp(*var, CONFIG_DATA_ENVIRONMENT))
+		if (strcmp(*var, CONFIG_DATA_ENVIRONMENT) &&
+		    strcmp(*var, CONFIG_COUNT_ENVIRONMENT))
 			strvec_push(out, *var);
 	}
 }

From d1fa94356ddd2a81348532d49030cd08d0df6a4d Mon Sep 17 00:00:00 2001
From: Jonathan Tan 
Date: Thu, 17 Jun 2021 10:13:25 -0700
Subject: [PATCH 270/397] run-command: refactor subprocess env preparation

submodule.c has functionality that prepares the environment for running
a subprocess in a new repo. The lazy-fetching code (used in partial
clones) will need this in a subsequent commit, so move it to a more
central location.

Signed-off-by: Jonathan Tan 
Signed-off-by: Junio C Hamano 
Reviewed-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 run-command.c | 12 ++++++++++++
 run-command.h | 10 ++++++++++
 submodule.c   | 18 ++----------------
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/run-command.c b/run-command.c
index be6bc128cd..549a94a6a4 100644
--- a/run-command.c
+++ b/run-command.c
@@ -1892,3 +1892,15 @@ int run_auto_maintenance(int quiet)
 
 	return run_command(&maint);
 }
+
+void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir)
+{
+	const char * const *var;
+
+	for (var = local_repo_env; *var; var++) {
+		if (strcmp(*var, CONFIG_DATA_ENVIRONMENT) &&
+		    strcmp(*var, CONFIG_COUNT_ENVIRONMENT))
+			strvec_push(env_array, *var);
+	}
+	strvec_pushf(env_array, "%s=%s", GIT_DIR_ENVIRONMENT, new_git_dir);
+}
diff --git a/run-command.h b/run-command.h
index d08414a92e..22132536a9 100644
--- a/run-command.h
+++ b/run-command.h
@@ -483,4 +483,14 @@ int run_processes_parallel_tr2(int n, get_next_task_fn, start_failure_fn,
 			       task_finished_fn, void *pp_cb,
 			       const char *tr2_category, const char *tr2_label);
 
+/**
+ * Convenience function which prepares env_array for a command to be run in a
+ * new repo. This adds all GIT_* environment variables to env_array with the
+ * exception of GIT_CONFIG_PARAMETERS and GIT_CONFIG_COUNT (which cause the
+ * corresponding environment variables to be unset in the subprocess) and adds
+ * an environment variable pointing to new_git_dir. See local_repo_env in
+ * cache.h for more information.
+ */
+void prepare_other_repo_env(struct strvec *env_array, const char *new_git_dir);
+
 #endif
diff --git a/submodule.c b/submodule.c
index f09031e397..8e611fe1db 100644
--- a/submodule.c
+++ b/submodule.c
@@ -484,28 +484,14 @@ static void print_submodule_diff_summary(struct repository *r, struct rev_info *
 	strbuf_release(&sb);
 }
 
-static void prepare_submodule_repo_env_no_git_dir(struct strvec *out)
-{
-	const char * const *var;
-
-	for (var = local_repo_env; *var; var++) {
-		if (strcmp(*var, CONFIG_DATA_ENVIRONMENT) &&
-		    strcmp(*var, CONFIG_COUNT_ENVIRONMENT))
-			strvec_push(out, *var);
-	}
-}
-
 void prepare_submodule_repo_env(struct strvec *out)
 {
-	prepare_submodule_repo_env_no_git_dir(out);
-	strvec_pushf(out, "%s=%s", GIT_DIR_ENVIRONMENT,
-		     DEFAULT_GIT_DIR_ENVIRONMENT);
+	prepare_other_repo_env(out, DEFAULT_GIT_DIR_ENVIRONMENT);
 }
 
 static void prepare_submodule_repo_env_in_gitdir(struct strvec *out)
 {
-	prepare_submodule_repo_env_no_git_dir(out);
-	strvec_pushf(out, "%s=.", GIT_DIR_ENVIRONMENT);
+	prepare_other_repo_env(out, ".");
 }
 
 /*

From ef830cc4341260ef45ffe6c7164e23505d45a5a2 Mon Sep 17 00:00:00 2001
From: Jonathan Tan 
Date: Thu, 17 Jun 2021 10:13:26 -0700
Subject: [PATCH 271/397] promisor-remote: teach lazy-fetch in any repo

This is one step towards supporting partial clone submodules.

Even after this patch, we will still lack partial clone submodules
support, primarily because a lot of Git code that accesses submodule
objects does so by adding their object stores as alternates, meaning
that any lazy fetches that would occur in the submodule would be done
based on the config of the superproject, not of the submodule. This also
prevents testing of the functionality in this patch by user-facing
commands. So for now, test this mechanism using a test helper.

Besides that, there is some code that uses the wrapper functions
like has_promisor_remote(). Those will need to be checked to see if they
could support the non-wrapper functions instead (and thus support any
repository, not just the_repository).

Signed-off-by: Jonathan Tan 
Signed-off-by: Junio C Hamano 
Reviewed-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 Makefile                      |  1 +
 object-file.c                 |  7 ++----
 promisor-remote.c             |  9 ++++----
 t/helper/test-partial-clone.c | 43 +++++++++++++++++++++++++++++++++++
 t/helper/test-tool.c          |  1 +
 t/helper/test-tool.h          |  1 +
 t/t0410-partial-clone.sh      | 23 +++++++++++++++++++
 7 files changed, 76 insertions(+), 9 deletions(-)
 create mode 100644 t/helper/test-partial-clone.c

diff --git a/Makefile b/Makefile
index c3565fc0f8..f6653bcd5e 100644
--- a/Makefile
+++ b/Makefile
@@ -725,6 +725,7 @@ TEST_BUILTINS_OBJS += test-oidmap.o
 TEST_BUILTINS_OBJS += test-online-cpus.o
 TEST_BUILTINS_OBJS += test-parse-options.o
 TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
+TEST_BUILTINS_OBJS += test-partial-clone.o
 TEST_BUILTINS_OBJS += test-path-utils.o
 TEST_BUILTINS_OBJS += test-pcre2-config.o
 TEST_BUILTINS_OBJS += test-pkt-line.o
diff --git a/object-file.c b/object-file.c
index f233b440b2..ebf273e9e7 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1570,15 +1570,12 @@ static int do_oid_object_info_extended(struct repository *r,
 		}
 
 		/* Check if it is a missing object */
-		if (fetch_if_missing && has_promisor_remote() &&
-		    !already_retried && r == the_repository &&
+		if (fetch_if_missing && repo_has_promisor_remote(r) &&
+		    !already_retried &&
 		    !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
 			/*
 			 * TODO Investigate checking promisor_remote_get_direct()
 			 * TODO return value and stopping on error here.
-			 * TODO Pass a repository struct through
-			 * promisor_remote_get_direct(), such that arbitrary
-			 * repositories work.
 			 */
 			promisor_remote_get_direct(r, real, 1);
 			already_retried = 1;
diff --git a/promisor-remote.c b/promisor-remote.c
index 1e00e16b0f..c088dcbff3 100644
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -10,7 +10,8 @@ struct promisor_remote_config {
 	struct promisor_remote **promisors_tail;
 };
 
-static int fetch_objects(const char *remote_name,
+static int fetch_objects(struct repository *repo,
+			 const char *remote_name,
 			 const struct object_id *oids,
 			 int oid_nr)
 {
@@ -20,6 +21,8 @@ static int fetch_objects(const char *remote_name,
 
 	child.git_cmd = 1;
 	child.in = -1;
+	if (repo != the_repository)
+		prepare_other_repo_env(&child.env_array, repo->gitdir);
 	strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
 		     "fetch", remote_name, "--no-tags",
 		     "--no-write-fetch-head", "--recurse-submodules=no",
@@ -240,10 +243,8 @@ int promisor_remote_get_direct(struct repository *repo,
 
 	promisor_remote_init(repo);
 
-	if (repo != the_repository)
-		BUG("only the_repository is supported for now");
 	for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
-		if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) {
+		if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
 			if (remaining_nr == 1)
 				continue;
 			remaining_nr = remove_fetched_oids(repo, &remaining_oids,
diff --git a/t/helper/test-partial-clone.c b/t/helper/test-partial-clone.c
new file mode 100644
index 0000000000..3f102cfddd
--- /dev/null
+++ b/t/helper/test-partial-clone.c
@@ -0,0 +1,43 @@
+#include "cache.h"
+#include "test-tool.h"
+#include "repository.h"
+#include "object-store.h"
+
+/*
+ * Prints the size of the object corresponding to the given hash in a specific
+ * gitdir. This is similar to "git -C gitdir cat-file -s", except that this
+ * exercises the code that accesses the object of an arbitrary repository that
+ * is not the_repository. ("git -C gitdir" makes it so that the_repository is
+ * the one in gitdir.)
+ */
+static void object_info(const char *gitdir, const char *oid_hex)
+{
+	struct repository r;
+	struct object_id oid;
+	unsigned long size;
+	struct object_info oi = {.sizep = &size};
+	const char *p;
+
+	if (repo_init(&r, gitdir, NULL))
+		die("could not init repo");
+	if (parse_oid_hex(oid_hex, &oid, &p))
+		die("could not parse oid");
+	if (oid_object_info_extended(&r, &oid, &oi, 0))
+		die("could not obtain object info");
+	printf("%d\n", (int) size);
+}
+
+int cmd__partial_clone(int argc, const char **argv)
+{
+	setup_git_directory();
+
+	if (argc < 4)
+		die("too few arguments");
+
+	if (!strcmp(argv[1], "object-info"))
+		object_info(argv[2], argv[3]);
+	else
+		die("invalid argument '%s'", argv[1]);
+
+	return 0;
+}
diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c
index c5bd0c6d4c..b21e8f1519 100644
--- a/t/helper/test-tool.c
+++ b/t/helper/test-tool.c
@@ -46,6 +46,7 @@ static struct test_cmd cmds[] = {
 	{ "online-cpus", cmd__online_cpus },
 	{ "parse-options", cmd__parse_options },
 	{ "parse-pathspec-file", cmd__parse_pathspec_file },
+	{ "partial-clone", cmd__partial_clone },
 	{ "path-utils", cmd__path_utils },
 	{ "pcre2-config", cmd__pcre2_config },
 	{ "pkt-line", cmd__pkt_line },
diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h
index e8069a3b22..f845ced4b3 100644
--- a/t/helper/test-tool.h
+++ b/t/helper/test-tool.h
@@ -35,6 +35,7 @@ int cmd__oidmap(int argc, const char **argv);
 int cmd__online_cpus(int argc, const char **argv);
 int cmd__parse_options(int argc, const char **argv);
 int cmd__parse_pathspec_file(int argc, const char** argv);
+int cmd__partial_clone(int argc, const char **argv);
 int cmd__path_utils(int argc, const char **argv);
 int cmd__pcre2_config(int argc, const char **argv);
 int cmd__pkt_line(int argc, const char **argv);
diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index 584a039b85..a211a66c67 100755
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -604,6 +604,29 @@ test_expect_success 'do not fetch when checking existence of tree we construct o
 	git -C repo cherry-pick side1
 '
 
+test_expect_success 'lazy-fetch when accessing object not in the_repository' '
+	rm -rf full partial.git &&
+	test_create_repo full &&
+	test_commit -C full create-a-file file.txt &&
+
+	test_config -C full uploadpack.allowfilter 1 &&
+	test_config -C full uploadpack.allowanysha1inwant 1 &&
+	git clone --filter=blob:none --bare "file://$(pwd)/full" partial.git &&
+	FILE_HASH=$(git -C full rev-parse HEAD:file.txt) &&
+
+	# Sanity check that the file is missing
+	git -C partial.git rev-list --objects --missing=print HEAD >out &&
+	grep "[?]$FILE_HASH" out &&
+
+	git -C full cat-file -s "$FILE_HASH" >expect &&
+	test-tool partial-clone object-info partial.git "$FILE_HASH" >actual &&
+	test_cmp expect actual &&
+
+	# Sanity check that the file is now present
+	git -C partial.git rev-list --objects --missing=print HEAD >out &&
+	! grep "[?]$FILE_HASH" out
+'
+
 . "$TEST_DIRECTORY"/lib-httpd.sh
 start_httpd
 

From 3fca954172274668e79df188924865265ed29d03 Mon Sep 17 00:00:00 2001
From: Andrei Rybak 
Date: Fri, 25 Jun 2021 21:38:49 +0200
Subject: [PATCH 272/397] blame: correct name of config option in docs

As can be seen in files "Documentation/blame-options.txt" and
"builtin/blame.c", the name of this configuration option is
"blame.markUnblamableLines".

Signed-off-by: Andrei Rybak 
Reviewed-by: Bagas Sanjaya 
Signed-off-by: Junio C Hamano 
---
 Documentation/config/blame.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Documentation/config/blame.txt b/Documentation/config/blame.txt
index 9468e8599c..4d047c1790 100644
--- a/Documentation/config/blame.txt
+++ b/Documentation/config/blame.txt
@@ -27,7 +27,7 @@ blame.ignoreRevsFile::
 	file names will reset the list of ignored revisions.  This option will
 	be handled before the command line option `--ignore-revs-file`.
 
-blame.markUnblamables::
+blame.markUnblamableLines::
 	Mark lines that were changed by an ignored revision that we could not
 	attribute to another commit with a '*' in the output of
 	linkgit:git-blame[1].

From 6fc53692636c75748975a84a6d7f3d83daf8a50a Mon Sep 17 00:00:00 2001
From: Andrei Rybak 
Date: Fri, 25 Jun 2021 21:38:50 +0200
Subject: [PATCH 273/397] t: fix typos in test messages

Both in t4258 and in t9001, the code of the tests following shows the
proper name for the configuration variables.  So use the correct names
in the test messages as well.

Signed-off-by: Andrei Rybak 
Signed-off-by: Junio C Hamano 
---
 t/t4258-am-quoted-cr.sh | 2 +-
 t/t9001-send-email.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/t4258-am-quoted-cr.sh b/t/t4258-am-quoted-cr.sh
index fb5071f914..201915b45a 100755
--- a/t/t4258-am-quoted-cr.sh
+++ b/t/t4258-am-quoted-cr.sh
@@ -26,7 +26,7 @@ test_expect_success 'am --quoted-cr=strip' '
 	git diff --exit-code HEAD two
 '
 
-test_expect_success 'am with config mailinfo.quotecr=strip' '
+test_expect_success 'am with config mailinfo.quotedCr=strip' '
 	test_might_fail git am --abort &&
 	git reset --hard one &&
 	test_config mailinfo.quotedCr strip &&
diff --git a/t/t9001-send-email.sh b/t/t9001-send-email.sh
index 3b7540050c..09360ca683 100755
--- a/t/t9001-send-email.sh
+++ b/t/t9001-send-email.sh
@@ -1829,7 +1829,7 @@ test_expect_success $PREREQ 'sendemail.aliasfiletype=mailrc' '
 	grep "^!somebody@example\.org!$" commandline1
 '
 
-test_expect_success $PREREQ 'sendemail.aliasfile=~/.mailrc' '
+test_expect_success $PREREQ 'sendemail.aliasesfile=~/.mailrc' '
 	clean_fake_sendmail &&
 	echo "alias sbd  someone@example.org" >"$HOME/.mailrc" &&
 	git config --replace-all sendemail.aliasesfile "~/.mailrc" &&

From 98c7656a18b032b92180c945484a51a15502821d Mon Sep 17 00:00:00 2001
From: Andrei Rybak 
Date: Fri, 25 Jun 2021 21:38:51 +0200
Subject: [PATCH 274/397] git-worktree.txt: fix typo in example path

Signed-off-by: Andrei Rybak 
Signed-off-by: Junio C Hamano 
---
 Documentation/git-worktree.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Documentation/git-worktree.txt b/Documentation/git-worktree.txt
index f1bb1fa5f5..66e67e6cbf 100644
--- a/Documentation/git-worktree.txt
+++ b/Documentation/git-worktree.txt
@@ -387,7 +387,7 @@ These annotations are:
 ------------
 $ git worktree list
 /path/to/linked-worktree    abcd1234 [master]
-/path/to/locked-worktreee   acbd5678 (brancha) locked
+/path/to/locked-worktree    acbd5678 (brancha) locked
 /path/to/prunable-worktree  5678abc  (detached HEAD) prunable
 ------------
 

From 9b6e2c8b98f6a382a6ee8d9110bfe93b1b48d6a7 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Mon, 21 Jun 2021 16:57:58 +0000
Subject: [PATCH 275/397] pager: avoid setting COLUMNS when we're guessing its
 value

We query `TIOCGWINSZ` in Git to determine the correct value for
`COLUMNS`, and then set that environment variable.

If `TIOCGWINSZ` is not available, we fall back to the hard-coded value
80 _and still_ set the environment variable.

On Windows this is a problem. The reason is that Git for
Windows uses a version of `less` that relies on the MSYS2 runtime to
interact with the pseudo terminal (typically inside a MinTTY window,
which is also aware of the MSYS2 runtime). Both MinTTY and `less.exe`
interact with that pseudo terminal via `ioctl()` calls (which the MSYS2
runtime emulates even if there is no such thing on Windows).
Since https://github.com/gwsw/less/commit/bb0ee4e76c2, `less` prefers
the `COLUMNS` variable over asking ncurses itself.

But `git.exe` itself is _not_ aware of the MSYS2 runtime, or for that
matter of that pseudo terminal, and has no way to call `ioctl()` or
`TIOCGWINSZ`.

Therefore, `git.exe` will fall back to hard-coding 80 columns, no matter
what the actual terminal size is.

But `less.exe` is totally able to interact with the MSYS2 runtime and
would not actually require Git's help (which actually makes things
worse here). So let's not override `COLUMNS` on Windows.

Let's just not set `COLUMNS` unless we managed to query the actual value
from the terminal.

This fixes https://github.com/git-for-windows/git/issues/3235

Co-authored-by: Junio C Hamano 
Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 pager.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/pager.c b/pager.c
index 3d37dd7ada..52f27a6765 100644
--- a/pager.c
+++ b/pager.c
@@ -11,6 +11,10 @@
 static struct child_process pager_process = CHILD_PROCESS_INIT;
 static const char *pager_program;
 
+/* Is the value coming back from term_columns() just a guess? */
+static int term_columns_guessed;
+
+
 static void close_pager_fds(void)
 {
 	/* signal EOF to pager */
@@ -114,7 +118,8 @@ void setup_pager(void)
 	{
 		char buf[64];
 		xsnprintf(buf, sizeof(buf), "%d", term_columns());
-		setenv("COLUMNS", buf, 0);
+		if (!term_columns_guessed)
+			setenv("COLUMNS", buf, 0);
 	}
 
 	setenv("GIT_PAGER_IN_USE", "true", 1);
@@ -158,15 +163,20 @@ int term_columns(void)
 		return term_columns_at_startup;
 
 	term_columns_at_startup = 80;
+	term_columns_guessed = 1;
 
 	col_string = getenv("COLUMNS");
-	if (col_string && (n_cols = atoi(col_string)) > 0)
+	if (col_string && (n_cols = atoi(col_string)) > 0) {
 		term_columns_at_startup = n_cols;
+		term_columns_guessed = 0;
+	}
 #ifdef TIOCGWINSZ
 	else {
 		struct winsize ws;
-		if (!ioctl(1, TIOCGWINSZ, &ws) && ws.ws_col)
+		if (!ioctl(1, TIOCGWINSZ, &ws) && ws.ws_col) {
 			term_columns_at_startup = ws.ws_col;
+			term_columns_guessed = 0;
+		}
 	}
 #endif
 

From 9fffc385830a3abc54e76473e9332166942e116e Mon Sep 17 00:00:00 2001
From: Beshr Kayali 
Date: Mon, 28 Jun 2021 21:37:18 +0200
Subject: [PATCH 276/397] Documentation: fix typo in the --patch option of the
 commit command

Typofix (chose -> choose) in the documentation of the patch option
under the commit command.

Signed-off-by: Beshr Kayali 
Signed-off-by: Junio C Hamano 
---
 Documentation/git-commit.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Documentation/git-commit.txt b/Documentation/git-commit.txt
index 340c5fbb48..95fec5f069 100644
--- a/Documentation/git-commit.txt
+++ b/Documentation/git-commit.txt
@@ -72,7 +72,7 @@ OPTIONS
 
 -p::
 --patch::
-	Use the interactive patch selection interface to chose
+	Use the interactive patch selection interface to choose
 	which changes to commit. See linkgit:git-add[1] for
 	details.
 

From ebbf5d2b70d72a9a4407c12567fbbd9cde892e26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlo=20Marcelo=20Arenas=20Bel=C3=B3n?= 
Date: Tue, 22 Jun 2021 10:42:41 +0000
Subject: [PATCH 277/397] config.mak.uname: PCRE1 cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Style issue: a space was missing.

Signed-off-by: Carlo Marcelo Arenas Belón 
Signed-off-by: Junio C Hamano 
---
 config.mak.uname | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.mak.uname b/config.mak.uname
index cb443b4e02..e8ceaed3d3 100644
--- a/config.mak.uname
+++ b/config.mak.uname
@@ -668,7 +668,7 @@ else
 		HAVE_LIBCHARSET_H = YesPlease
 		NO_GETTEXT =
 		USE_GETTEXT_SCHEME = fallthrough
-		USE_LIBPCRE= YesPlease
+		USE_LIBPCRE = YesPlease
 		NO_CURL =
 		USE_NED_ALLOCATOR = YesPlease
 	else

From fb5e3378f8427ef970544848974db2dc90828a7a Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Tue, 22 Jun 2021 10:46:46 +0000
Subject: [PATCH 278/397] mingw: move Git for Windows' system config where
 users expect it

Git for Windows' prefix is `/mingw64/` (or `/mingw32/` for 32-bit
versions), therefore the system config is located at the clunky location
`C:\Program Files\Git\mingw64\etc\gitconfig`.

This moves the system config into a more logical location: the `mingw64`
part of `C:\Program Files\Git\mingw64\etc\gitconfig` never made sense,
as it is a mere implementation detail. Let's skip the `mingw64` part and
move this to `C:\Program Files\Git\etc\gitconfig`.

Side note: in the rare (and not recommended) case a user chooses to
install 32-bit Git for Windows on a 64-bit system, the path will of
course be `C:\Program Files (x86)\Git\etc\gitconfig`.

Background: During the Git for Windows v1.x days, the system config was
located at `C:\Program Files (x86)\Git\etc\gitconfig`. With Git for
Windows v2.x, it moved to `C:\Program Files\Git\mingw64\gitconfig` (or
`C:\Program Files (x86)\Git\mingw32\gitconfig`). Rather than fixing it
back then, we tried to introduce a "Windows-wide" config, but that never
caught on.

Likewise, we move the system `gitattributes` into the same directory.

Obviously, we are cautious to do this only for the known install
locations `/mingw64` and `/mingw32`; If anybody wants to override that
while building their version of Git (e.g. via `make prefix=$HOME`), we
leave the default location of the system config and gitattributes alone.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 config.mak.uname | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/config.mak.uname b/config.mak.uname
index cb443b4e02..0587a23c1c 100644
--- a/config.mak.uname
+++ b/config.mak.uname
@@ -437,6 +437,11 @@ ifeq ($(uname_S),Windows)
 	NO_POSIX_GOODIES = UnfortunatelyYes
 	NATIVE_CRLF = YesPlease
 	DEFAULT_HELP_FORMAT = html
+ifeq (/mingw64,$(subst 32,64,$(prefix)))
+	# Move system config into top-level /etc/
+	ETC_GITCONFIG = ../etc/gitconfig
+	ETC_GITATTRIBUTES = ../etc/gitattributes
+endif
 
 	CC = compat/vcbuild/scripts/clink.pl
 	AR = compat/vcbuild/scripts/lib.pl
@@ -671,6 +676,11 @@ else
 		USE_LIBPCRE= YesPlease
 		NO_CURL =
 		USE_NED_ALLOCATOR = YesPlease
+		ifeq (/mingw64,$(subst 32,64,$(prefix)))
+			# Move system config into top-level /etc/
+			ETC_GITCONFIG = ../etc/gitconfig
+			ETC_GITATTRIBUTES = ../etc/gitattributes
+		endif
 	else
 		COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO
 		NO_CURL = YesPlease

From 50101b93ca69babb389e6bdefd733ef014338d29 Mon Sep 17 00:00:00 2001
From: Dennis Ameling 
Date: Tue, 22 Jun 2021 10:46:47 +0000
Subject: [PATCH 279/397] cmake(windows): set correct path to the system Git
 config

Currently, when Git for Windows is built with CMake, the system Git config is
expected in a different location than when building via `make`: the former
expects it to be in `/mingw64/etc/gitconfig`, the latter in
`/etc/gitconfig`.

Because of this, things like `git clone` do not work correctly (because cURL is
no longer able to find its certificate bundle that it needs to validate HTTPS
certificates). See the full bug report and discussion here:
https://github.com/git-for-windows/git/issues/3071#issuecomment-789261386.

This commit aligns the CMake-based build by mimicking what is already done in
`config.mak.uname`.

This closes https://github.com/git-for-windows/git/issues/3071.

Signed-off-by: Dennis Ameling 
Signed-off-by: Junio C Hamano 
---
 contrib/buildsystems/CMakeLists.txt | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt
index a87841340e..bdc5ab58d0 100644
--- a/contrib/buildsystems/CMakeLists.txt
+++ b/contrib/buildsystems/CMakeLists.txt
@@ -204,8 +204,6 @@ list(APPEND compat_SOURCES sha1dc_git.c sha1dc/sha1.c sha1dc/ubc_check.c block-s
 
 
 add_compile_definitions(PAGER_ENV="LESS=FRX LV=-c"
-			ETC_GITATTRIBUTES="etc/gitattributes"
-			ETC_GITCONFIG="etc/gitconfig"
 			GIT_EXEC_PATH="libexec/git-core"
 			GIT_LOCALE_PATH="share/locale"
 			GIT_MAN_PATH="share/man"
@@ -220,10 +218,15 @@ add_compile_definitions(PAGER_ENV="LESS=FRX LV=-c"
 
 if(WIN32)
 	set(FALLBACK_RUNTIME_PREFIX /mingw64)
-	add_compile_definitions(FALLBACK_RUNTIME_PREFIX="${FALLBACK_RUNTIME_PREFIX}")
+	# Move system config into top-level /etc/
+	add_compile_definitions(FALLBACK_RUNTIME_PREFIX="${FALLBACK_RUNTIME_PREFIX}"
+		ETC_GITATTRIBUTES="../etc/gitattributes"
+		ETC_GITCONFIG="../etc/gitconfig")
 else()
 	set(FALLBACK_RUNTIME_PREFIX /home/$ENV{USER})
-	add_compile_definitions(FALLBACK_RUNTIME_PREFIX="${FALLBACK_RUNTIME_PREFIX}")
+	add_compile_definitions(FALLBACK_RUNTIME_PREFIX="${FALLBACK_RUNTIME_PREFIX}"
+		ETC_GITATTRIBUTES="etc/gitattributes"
+		ETC_GITCONFIG="etc/gitconfig")
 endif()
 
 

From e355307692f49f2f02275b083e91532efed9f7c0 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Tue, 22 Jun 2021 10:46:48 +0000
Subject: [PATCH 280/397] config: normalize the path of the system gitconfig

Git for Windows is compiled with a runtime prefix, and that runtime
prefix is typically `C:/Program Files/Git/mingw64`. As we want the
system gitconfig to live in the sibling directory `etc`, we define the
relative path as `../etc/gitconfig`.

However, as reported by Philip Oakley, the output of `git config
--show-origin --system -l` looks rather ugly, as it shows the path as
`file:C:/Program Files/Git/mingw64/../etc/gitconfig`, i.e. with the
`mingw64/../` part.

By normalizing the path, we get a prettier path.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 config.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/config.c b/config.c
index f9c400ad30..3cd10aeb90 100644
--- a/config.c
+++ b/config.c
@@ -1833,9 +1833,10 @@ static int git_config_from_blob_ref(config_fn_t fn,
 char *git_system_config(void)
 {
 	char *system_config = xstrdup_or_null(getenv("GIT_CONFIG_SYSTEM"));
-	if (system_config)
-		return system_config;
-	return system_path(ETC_GITCONFIG);
+	if (!system_config)
+		system_config = system_path(ETC_GITCONFIG);
+	normalize_path_copy(system_config, system_config);
+	return system_config;
 }
 
 void git_global_config(char **user_out, char **xdg_out)

From 1cf823d8f000e2c85a5cae9674cf38c1503f214d Mon Sep 17 00:00:00 2001
From: Kaartic Sivaraam 
Date: Tue, 22 Jun 2021 23:44:52 +0530
Subject: [PATCH 281/397] submodule: remove unnecessary `prefix` based option
 logic

Over time when parts of submodule have been ported from shell to
builtin, many instances of the submodule helper have been added.
Also added with them are some unnecessary option passing
logic that are based on the `prefix` shell variable which never
gets set in their code flows.

On analysis, the only shell functions which have a valid usage
for the `prefix` shell variable are:

    - cmd_update: which is the only function which sets the variable
      and thus uses it properly

    - cmd_init: which uses the variable via a call from cmd_update

So, remove the unnecessary option parsing logic based on the `prefix`
shell variable.

Signed-off-by: Kaartic Sivaraam 
Signed-off-by: Junio C Hamano 
---
 git-submodule.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/git-submodule.sh b/git-submodule.sh
index 4678378424..cb06aa02c8 100755
--- a/git-submodule.sh
+++ b/git-submodule.sh
@@ -335,7 +335,7 @@ cmd_foreach()
 		shift
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} ${prefix:+--super-prefix "$prefix"} submodule--helper foreach ${GIT_QUIET:+--quiet} ${recursive:+--recursive} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper foreach ${GIT_QUIET:+--quiet} ${recursive:+--recursive} -- "$@"
 }
 
 #
@@ -402,7 +402,7 @@ cmd_deinit()
 		shift
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper deinit ${GIT_QUIET:+--quiet} ${prefix:+--prefix "$prefix"} ${force:+--force} ${deinit_all:+--all} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper deinit ${GIT_QUIET:+--quiet} ${force:+--force} ${deinit_all:+--all} -- "$@"
 }
 
 is_tip_reachable () (
@@ -726,7 +726,7 @@ cmd_set_branch() {
 		shift
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} ${prefix:+--super-prefix "$prefix"} submodule--helper set-branch ${GIT_QUIET:+--quiet} ${branch:+--branch "$branch"} ${default:+--default} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper set-branch ${GIT_QUIET:+--quiet} ${branch:+--branch "$branch"} ${default:+--default} -- "$@"
 }
 
 #
@@ -755,7 +755,7 @@ cmd_set_url() {
 		shift
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} ${prefix:+--super-prefix "$prefix"} submodule--helper set-url ${GIT_QUIET:+--quiet} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper set-url ${GIT_QUIET:+--quiet} -- "$@"
 }
 
 #
@@ -807,7 +807,7 @@ cmd_summary() {
 		shift
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper summary ${prefix:+--prefix "$prefix"} ${files:+--files} ${cached:+--cached} ${for_status:+--for-status} ${summary_limit:+-n $summary_limit} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper summary ${files:+--files} ${cached:+--cached} ${for_status:+--for-status} ${summary_limit:+-n $summary_limit} -- "$@"
 }
 #
 # List all submodules, prefixed with:
@@ -848,7 +848,7 @@ cmd_status()
 		shift
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} ${prefix:+--super-prefix "$prefix"} submodule--helper status ${GIT_QUIET:+--quiet} ${cached:+--cached} ${recursive:+--recursive} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper status ${GIT_QUIET:+--quiet} ${cached:+--cached} ${recursive:+--recursive} -- "$@"
 }
 #
 # Sync remote urls for submodules
@@ -881,7 +881,7 @@ cmd_sync()
 		esac
 	done
 
-	git ${wt_prefix:+-C "$wt_prefix"} ${prefix:+--super-prefix "$prefix"} submodule--helper sync ${GIT_QUIET:+--quiet} ${recursive:+--recursive} -- "$@"
+	git ${wt_prefix:+-C "$wt_prefix"} submodule--helper sync ${GIT_QUIET:+--quiet} ${recursive:+--recursive} -- "$@"
 }
 
 cmd_absorbgitdirs()

From 3c7e2e8f0acfe94220cdd3bbd3a35a955ae80294 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 22 Jun 2021 12:03:54 -0400
Subject: [PATCH 282/397] pretty.h: update and expand docstring for
 userformat_find_requirements()

The comment only mentions "notes", but there are more fields now (and
I'm about to add another). Let's make it more general, and stick the
struct next to the function to make the list of possibilities obvious.

While we're touching this comment, let's also mention the behavior of
NULL, which some callers rely on (though in the long run, this global is
pretty nasty and probably should get moved into rev_info).

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 pretty.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pretty.h b/pretty.h
index f034609e4d..c81cf40d38 100644
--- a/pretty.h
+++ b/pretty.h
@@ -65,12 +65,15 @@ static inline int cmit_fmt_is_mail(enum cmit_fmt fmt)
 	return (fmt == CMIT_FMT_EMAIL || fmt == CMIT_FMT_MBOXRD);
 }
 
+/*
+ * Examine the user-specified format given by "fmt" (or if NULL, the global one
+ * previously saved by get_commit_format()), and set flags based on which items
+ * the format will need when it is expanded.
+ */
 struct userformat_want {
 	unsigned notes:1;
 	unsigned source:1;
 };
-
-/* Set the flag "w->notes" if there is placeholder %N in "fmt". */
 void userformat_find_requirements(const char *fmt, struct userformat_want *w);
 
 /*

From b2086b518366ed71caac498857b9c5765dd73ed1 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 22 Jun 2021 12:04:50 -0400
Subject: [PATCH 283/397] log: avoid loading decorations for userformats that
 don't need it

If no --decorate option is given, we default to auto-decoration. And
when that kicks in, cmd_log_init_finish() will unconditionally load the
decoration refs.

However, if we are using a user-format that does not include "%d" or
"%D", we won't show the decorations at all, so we don't need to load
them. We can detect this case and auto-disable them by adding a new
field to our userformat_want helper. We can do this even when the user
explicitly asked for --decorate, because it can't affect the output at
all.

This patch consistently reduces the time to run "git log -1 --format=%H"
on my git.git clone (with ~2k refs) from 34ms to 7ms. On a much more
extreme real-world repository (with ~220k refs), it goes from 2.5s to
4ms.

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 builtin/log.c | 3 +++
 pretty.c      | 4 ++++
 pretty.h      | 1 +
 3 files changed, 8 insertions(+)

diff --git a/builtin/log.c b/builtin/log.c
index 6102893fcc..6ba7f20726 100644
--- a/builtin/log.c
+++ b/builtin/log.c
@@ -245,6 +245,9 @@ static void cmd_log_init_finish(int argc, const char **argv, const char *prefix,
 			rev->abbrev_commit = 0;
 	}
 
+	if (rev->commit_format == CMIT_FMT_USERFORMAT && !w.decorate)
+		decoration_style = 0;
+
 	if (decoration_style) {
 		const struct string_list *config_exclude =
 			repo_config_get_value_multi(the_repository,
diff --git a/pretty.c b/pretty.c
index b1ecd039ce..9631529c10 100644
--- a/pretty.c
+++ b/pretty.c
@@ -1735,6 +1735,10 @@ static size_t userformat_want_item(struct strbuf *sb, const char *placeholder,
 	case 'S':
 		w->source = 1;
 		break;
+	case 'd':
+	case 'D':
+		w->decorate = 1;
+		break;
 	}
 	return 0;
 }
diff --git a/pretty.h b/pretty.h
index c81cf40d38..2f16acd213 100644
--- a/pretty.h
+++ b/pretty.h
@@ -73,6 +73,7 @@ static inline int cmit_fmt_is_mail(enum cmit_fmt fmt)
 struct userformat_want {
 	unsigned notes:1;
 	unsigned source:1;
+	unsigned decorate:1;
 };
 void userformat_find_requirements(const char *fmt, struct userformat_want *w);
 

From 542d6abbb4e838cf1c47612a2b4b33f2a31e6277 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 22 Jun 2021 12:05:31 -0400
Subject: [PATCH 284/397] object.h: expand docstring for
 lookup_unknown_object()

The lookup_unknown_object() system is not often used and is somewhat
confusing. Let's try to explain it a bit more (which is especially
important as I'm adding a related but slightly different function in the
next commit).

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 object.h | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/object.h b/object.h
index 8bca310713..eb7e481c39 100644
--- a/object.h
+++ b/object.h
@@ -144,7 +144,18 @@ struct object *parse_object_or_die(const struct object_id *oid, const char *name
  */
 struct object *parse_object_buffer(struct repository *r, const struct object_id *oid, enum object_type type, unsigned long size, void *buffer, int *eaten_p);
 
-/** Returns the object, with potentially excess memory allocated. **/
+/*
+ * Allocate and return an object struct, even if you do not know the type of
+ * the object. The returned object may have its "type" field set to a real type
+ * (if somebody previously called lookup_blob(), etc), or it may be set to
+ * OBJ_NONE. In the latter case, subsequent calls to lookup_blob(), etc, will
+ * set the type field as appropriate.
+ *
+ * Use this when you do not know the expected type of an object and want to
+ * avoid parsing it for efficiency reasons. Try to avoid it otherwise; it
+ * may allocate excess memory, since the returned object must be as large as
+ * the maximum struct of any type.
+ */
 struct object *lookup_unknown_object(struct repository *r, const struct object_id *oid);
 
 struct object_list *object_list_insert(struct object *item,

From 7463064b28086c0a765e247bc8336f8e32356494 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 22 Jun 2021 12:06:41 -0400
Subject: [PATCH 285/397] object.h: add lookup_object_by_type() function

In some cases it's useful for efficiency reasons to get the type of an
object before deciding whether to parse it, but we still want an object
struct. E.g., in reachable.c, bitmaps give us the type, but we just want
to mark flags on each object. Likewise, we may loop over every object
and only parse tags in order to peel them; checking the type first lets
us avoid parsing the non-tags.

But our lookup_blob(), etc, functions make getting an object struct
annoying: we have to call the right function for every type. And we
cannot just use the generic lookup_object(), because it only returns an
already-seen object; it won't allocate a new object struct.

Let's provide a function that dispatches to the correct lookup_*
function based on a run-time type. In fact, reachable.c already has such
a helper, so we'll just make that public.

I did change the return type from "void *" to "struct object *". While
the former is a clever way to avoid casting inside the function, it's
less safe and less informative to people reading the function
declaration.

The next commit will add a new caller.

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 object.c    | 18 ++++++++++++++++++
 object.h    |  7 +++++++
 reachable.c | 18 ------------------
 3 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/object.c b/object.c
index 14188453c5..07fcf23d7b 100644
--- a/object.c
+++ b/object.c
@@ -185,6 +185,24 @@ struct object *lookup_unknown_object(struct repository *r, const struct object_i
 	return obj;
 }
 
+struct object *lookup_object_by_type(struct repository *r,
+			    const struct object_id *oid,
+			    enum object_type type)
+{
+	switch (type) {
+	case OBJ_COMMIT:
+		return (struct object *)lookup_commit(r, oid);
+	case OBJ_TREE:
+		return (struct object *)lookup_tree(r, oid);
+	case OBJ_TAG:
+		return (struct object *)lookup_tag(r, oid);
+	case OBJ_BLOB:
+		return (struct object *)lookup_blob(r, oid);
+	default:
+		die("BUG: unknown object type %d", type);
+	}
+}
+
 struct object *parse_object_buffer(struct repository *r, const struct object_id *oid, enum object_type type, unsigned long size, void *buffer, int *eaten_p)
 {
 	struct object *obj;
diff --git a/object.h b/object.h
index eb7e481c39..3b38c9cc98 100644
--- a/object.h
+++ b/object.h
@@ -158,6 +158,13 @@ struct object *parse_object_buffer(struct repository *r, const struct object_id
  */
 struct object *lookup_unknown_object(struct repository *r, const struct object_id *oid);
 
+/*
+ * Dispatch to the appropriate lookup_blob(), lookup_commit(), etc, based on
+ * "type".
+ */
+struct object *lookup_object_by_type(struct repository *r, const struct object_id *oid,
+				     enum object_type type);
+
 struct object_list *object_list_insert(struct object *item,
 				       struct object_list **list_p);
 
diff --git a/reachable.c b/reachable.c
index c59847257a..84e3d0d75e 100644
--- a/reachable.c
+++ b/reachable.c
@@ -159,24 +159,6 @@ int add_unseen_recent_objects_to_traversal(struct rev_info *revs,
 				      FOR_EACH_OBJECT_LOCAL_ONLY);
 }
 
-static void *lookup_object_by_type(struct repository *r,
-				   const struct object_id *oid,
-				   enum object_type type)
-{
-	switch (type) {
-	case OBJ_COMMIT:
-		return lookup_commit(r, oid);
-	case OBJ_TREE:
-		return lookup_tree(r, oid);
-	case OBJ_TAG:
-		return lookup_tag(r, oid);
-	case OBJ_BLOB:
-		return lookup_blob(r, oid);
-	default:
-		die("BUG: unknown object type %d", type);
-	}
-}
-
 static int mark_object_seen(const struct object_id *oid,
 			     enum object_type type,
 			     int exclude,

From 88473c8baeefafe6b95ab0f62eb2f12e2b098ac7 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 22 Jun 2021 13:06:48 -0400
Subject: [PATCH 286/397] load_ref_decorations(): avoid parsing non-tag objects

When we load the ref decorations, we parse the object pointed to by each
ref in order to get a "struct object". This is unnecessarily expensive;
we really only need the object struct, and don't even look at the parsed
contents. The exception is tags, which we do need to peel.

We can improve this by looking up the object type first (which is much
cheaper), and skipping the parse entirely for non-tags. This increases
the work slightly for annotated tags (which now do a type lookup _and_ a
parse), but decreases it a lot for other types. On balance, this seems
to be a good tradeoff.

In my git.git clone, with ~2k refs, most of which are branches, the time
to run "git log -1 --decorate" drops from 34ms to 11ms. Even on my
linux.git clone, which contains mostly tags and only a handful of
branches, the time drops from 30ms to 19ms. And on a more extreme
real-world case with ~220k refs, mostly non-tags, the time drops from
2.6s to 650ms.

That command is a lop-sided example, of course, because it does as
little non-loading work as possible. But it does show the absolute time
improvement. Even in something like a full "git log --decorate" on that
extreme repo, we'd still be saving 2s of CPU time.

Ideally we could push this even further, and avoid parsing even tags, by
relying on the packed-refs "peel" optimization (which we could do by
calling peel_iterated_oid() instead of peeling manually). But we can't
do that here. The packed-refs file only stores the bottom-layer of the
peel (so in a "tag->tag->commit" chain, it stores only the commit as the
peel result).  But the decoration code wants to peel the layers
individually, annotating the middle layers of the chain.

If the packed-refs file ever learns to store all of the peeled layers,
then we could switch to it. Or even if it stored a flag to indicate the
peel was not multi-layer (because most of them aren't), then we could
use it most of the time and fall back to a manual peel for the rare
cases.

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 log-tree.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/log-tree.c b/log-tree.c
index 7b823786c2..c3c8e7e1df 100644
--- a/log-tree.c
+++ b/log-tree.c
@@ -134,6 +134,7 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid,
 			      int flags, void *cb_data)
 {
 	struct object *obj;
+	enum object_type objtype;
 	enum decoration_type type = DECORATION_NONE;
 	struct decoration_filter *filter = (struct decoration_filter *)cb_data;
 
@@ -155,9 +156,10 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid,
 		return 0;
 	}
 
-	obj = parse_object(the_repository, oid);
-	if (!obj)
+	objtype = oid_object_info(the_repository, oid, NULL);
+	if (objtype < 0)
 		return 0;
+	obj = lookup_object_by_type(the_repository, oid, objtype);
 
 	if (starts_with(refname, "refs/heads/"))
 		type = DECORATION_REF_LOCAL;

From 6afb265b9675b1e6061f53a323825db239f6e0fa Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 22 Jun 2021 13:09:26 -0400
Subject: [PATCH 287/397] add_ref_decoration(): rename s/type/deco_type/

Now that we have two types (a decoration type and an object type) in the
function, let's give them both unique names to avoid accidentally using
one instead of the other.

Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 log-tree.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/log-tree.c b/log-tree.c
index c3c8e7e1df..4f69ed176d 100644
--- a/log-tree.c
+++ b/log-tree.c
@@ -135,7 +135,7 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid,
 {
 	struct object *obj;
 	enum object_type objtype;
-	enum decoration_type type = DECORATION_NONE;
+	enum decoration_type deco_type = DECORATION_NONE;
 	struct decoration_filter *filter = (struct decoration_filter *)cb_data;
 
 	if (filter && !ref_filter_match(refname, filter))
@@ -162,17 +162,17 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid,
 	obj = lookup_object_by_type(the_repository, oid, objtype);
 
 	if (starts_with(refname, "refs/heads/"))
-		type = DECORATION_REF_LOCAL;
+		deco_type = DECORATION_REF_LOCAL;
 	else if (starts_with(refname, "refs/remotes/"))
-		type = DECORATION_REF_REMOTE;
+		deco_type = DECORATION_REF_REMOTE;
 	else if (starts_with(refname, "refs/tags/"))
-		type = DECORATION_REF_TAG;
+		deco_type = DECORATION_REF_TAG;
 	else if (!strcmp(refname, "refs/stash"))
-		type = DECORATION_REF_STASH;
+		deco_type = DECORATION_REF_STASH;
 	else if (!strcmp(refname, "HEAD"))
-		type = DECORATION_REF_HEAD;
+		deco_type = DECORATION_REF_HEAD;
 
-	add_name_decoration(type, refname, obj);
+	add_name_decoration(deco_type, refname, obj);
 	while (obj->type == OBJ_TAG) {
 		obj = ((struct tag *)obj)->tagged;
 		if (!obj)

From 0eb6c189a3576aa8745f471c1ad23b3ca123cc75 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Wed, 23 Jun 2021 15:24:11 +0000
Subject: [PATCH 288/397] ci: use the new GitHub Action to download
 git-sdk-64-minimal

In our continuous builds, Windows is the odd cookie that requires a
complete development environment to be downloaded because there is no
suitable one installed by default on Windows.

Side note: technically, there _is_ a development environment present in
GitHub Actions' build agents: MSYS2. But it differs from Git for
Windows' SDK in subtle points, unfortunately enough so to prevent Git's
test suite from running without failures.

Traditionally, we support downloading this environment (which we
nicknamed `git-sdk-64-minimal`) via a PowerShell scriptlet that accesses
the build artifacts of a dedicated Azure Pipeline (which packages a tiny
subset of the full Git for Windows SDK, containing just enough to build
Git and run its test suite).

This PowerShell script is unfortunately not very robust and sometimes
fails due to network issues.

Of course, we could add code to detect that situation, wait a little,
try again, if it fails again wait a little longer, lather, rinse and
repeat.

Instead of doing all of this in Git's own `.github/workflows/`, though,
let's offload this logic to the new GitHub Action at
https://github.com/marketplace/actions/setup-git-for-windows-sdk

This Action not only downloads and extracts git-sdk-64-minimal _outside_
the worktree (making it no longer necessary to meddle with
`.gitignore` or `.git/info/exclude`), it also adds the `bash.exe` to the
`PATH` and sets the environment variable `MSYSTEM` (an implementation
detail that Git's workflow should never have needed to know about).

This allows us to convert all those funny PowerShell tasks that wanted
to call git-sdk-64-minimal's `bash.exe`: they all are now regular `bash`
scriptlets.

This finally lets us get rid of the funny quoting and escaping where we
had to pay attention not only to quote and escape the Bash scriptlets
properly, but also to add a second level of escaping (with backslashes
for double quotes and backticks for dollar signs) to stop PowerShell
from doing unintended things.

Further, this Action uses a fast caching strategy native to GitHub
Actions that should accelerate the download across CI runs:
git-sdk-64-minimal is usually updated once per 24h, and needs to be
cached only once within that period. Caching it (unfortunately only on
a per-branch basis) speeds up the download step, and makes it much more
robust at the same time by virtue of accessing a cache location that is
closer in the network topology.

With this we can drop the home-rolled caching where we try to accelerate
the test phase by uploading git-sdk-64-minimal as a workflow artifact
after using it to build Git, and then download it as workflow artifact
in the test phase.

Even better: the `vs-test` job no longer needs to depend on the
`windows-build` job. The only reason it depended on it was to ensure
that the `git-sdk-64-minimal` workflow artifact was available.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 103 +++++++------------------------------
 1 file changed, 19 insertions(+), 84 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 73856bafc9..c62766e7b1 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -82,43 +82,18 @@ jobs:
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v1
-    - name: download git-sdk-64-minimal
-      shell: bash
-      run: |
-        ## Get artifact
-        urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
-        id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" |
-          jq -r ".value[] | .id")
-        download_url="$(curl "$urlbase/$id/artifacts" |
-          jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')"
-        curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \
-          -o artifacts.zip "$download_url"
-
-        ## Unzip and remove the artifact
-        unzip artifacts.zip
-        rm artifacts.zip
+    - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: build
-      shell: powershell
+      shell: bash
       env:
         HOME: ${{runner.workspace}}
-        MSYSTEM: MINGW64
         NO_PERL: 1
-      run: |
-        & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @"
-        printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude
-
-          ci/make-test-artifacts.sh artifacts
-        "@
+      run: ci/make-test-artifacts.sh artifacts
     - name: upload build artifacts
       uses: actions/upload-artifact@v1
       with:
         name: windows-artifacts
         path: artifacts
-    - name: upload git-sdk-64-minimal
-      uses: actions/upload-artifact@v1
-      with:
-        name: git-sdk-64-minimal
-        path: git-sdk-64-minimal
   windows-test:
     runs-on: windows-latest
     needs: [windows-build]
@@ -136,25 +111,14 @@ jobs:
     - name: extract build artifacts
       shell: bash
       run: tar xf artifacts.tar.gz
-    - name: download git-sdk-64-minimal
-      uses: actions/download-artifact@v1
-      with:
-        name: git-sdk-64-minimal
-        path: ${{github.workspace}}/git-sdk-64-minimal/
+    - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: test
-      shell: powershell
-      run: |
-        & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @"
-          # Let Git ignore the SDK
-          printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude
-
-          ci/run-test-slice.sh ${{matrix.nr}} 10
-        "@
+      shell: bash
+      run: ci/run-test-slice.sh ${{matrix.nr}} 10
     - name: ci/print-test-failures.sh
       if: failure()
-      shell: powershell
-      run: |
-        & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh
+      shell: bash
+      run: ci/print-test-failures.sh
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       uses: actions/upload-artifact@v1
@@ -165,27 +129,12 @@ jobs:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'
     env:
-      MSYSTEM: MINGW64
       NO_PERL: 1
       GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v1
-    - name: download git-sdk-64-minimal
-      shell: bash
-      run: |
-        ## Get artifact
-        urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds
-        id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" |
-          jq -r ".value[] | .id")
-        download_url="$(curl "$urlbase/$id/artifacts" |
-          jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')"
-        curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \
-          -o artifacts.zip "$download_url"
-
-        ## Unzip and remove the artifact
-        unzip artifacts.zip
-        rm artifacts.zip
+    - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: initialize vcpkg
       uses: actions/checkout@v2
       with:
@@ -211,19 +160,17 @@ jobs:
       shell: bash
       run: |
         cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \
-        -DMSGFMT_EXE=`pwd`/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON
+        -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON
     - name: MSBuild
       run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142
     - name: bundle artifact tar
-      shell: powershell
+      shell: bash
       env:
         MSVC: 1
         VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg
       run: |
-        & git-sdk-64-minimal\usr\bin\bash.exe -lc @"
-          mkdir -p artifacts &&
-          eval \"`$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)\"
-        "@
+        mkdir -p artifacts &&
+        eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)"
     - name: upload build artifacts
       uses: actions/upload-artifact@v1
       with:
@@ -231,18 +178,14 @@ jobs:
         path: artifacts
   vs-test:
     runs-on: windows-latest
-    needs: [vs-build, windows-build]
+    needs: vs-build
     strategy:
       fail-fast: false
       matrix:
         nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     steps:
     - uses: actions/checkout@v1
-    - name: download git-sdk-64-minimal
-      uses: actions/download-artifact@v1
-      with:
-        name: git-sdk-64-minimal
-        path: ${{github.workspace}}/git-sdk-64-minimal/
+    - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: download build artifacts
       uses: actions/download-artifact@v1
       with:
@@ -252,23 +195,15 @@ jobs:
       shell: bash
       run: tar xf artifacts.tar.gz
     - name: test
-      shell: powershell
+      shell: bash
       env:
-        MSYSTEM: MINGW64
         NO_SVN_TESTS: 1
         GIT_TEST_SKIP_REBASE_P: 1
-      run: |
-        & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @"
-          # Let Git ignore the SDK and the test-cache
-          printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude
-
-          ci/run-test-slice.sh ${{matrix.nr}} 10
-        "@
+      run: ci/run-test-slice.sh ${{matrix.nr}} 10
     - name: ci/print-test-failures.sh
       if: failure()
-      shell: powershell
-      run: |
-        & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh
+      shell: bash
+      run: ci/print-test-failures.sh
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
       uses: actions/upload-artifact@v1

From abb2b389f7604704d3e78ae24e52e1a9429b699d Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Wed, 23 Jun 2021 15:24:12 +0000
Subject: [PATCH 289/397] ci (vs-build): use `cmd` to copy the DLLs, not
 `powershell`

We use a `.bat` script to copy the DLLs in the `vs-build` job, and those
type of scripts are native to CMD, not to PowerShell.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c62766e7b1..d430c4e0d2 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -152,10 +152,8 @@ jobs:
     - name: add msbuild to PATH
       uses: microsoft/setup-msbuild@v1
     - name: copy dlls to root
-      shell: powershell
-      run: |
-        & compat\vcbuild\vcpkg_copy_dlls.bat release
-        if (!$?) { exit(1) }
+      shell: cmd
+      run: compat\vcbuild\vcpkg_copy_dlls.bat release
     - name: generate Visual Studio solution
       shell: bash
       run: |

From e9f79acb28cca67e4c8c13148501e612b7f1dd56 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Wed, 23 Jun 2021 15:24:13 +0000
Subject: [PATCH 290/397] ci: upgrade to using actions/{up,down}load-artifacts
 v2

The GitHub Actions to upload/download workflow artifacts saw a major
upgrade since Git's GitHub workflow was established. Let's use it.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d430c4e0d2..a399114c0f 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -90,7 +90,7 @@ jobs:
         NO_PERL: 1
       run: ci/make-test-artifacts.sh artifacts
     - name: upload build artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
         name: windows-artifacts
         path: artifacts
@@ -104,7 +104,7 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: download build artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v2
       with:
         name: windows-artifacts
         path: ${{github.workspace}}
@@ -121,7 +121,7 @@ jobs:
       run: ci/print-test-failures.sh
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
         name: failed-tests-windows
         path: ${{env.FAILED_TEST_ARTIFACTS}}
@@ -170,7 +170,7 @@ jobs:
         mkdir -p artifacts &&
         eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)"
     - name: upload build artifacts
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
         name: vs-artifacts
         path: artifacts
@@ -185,7 +185,7 @@ jobs:
     - uses: actions/checkout@v1
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: download build artifacts
-      uses: actions/download-artifact@v1
+      uses: actions/download-artifact@v2
       with:
         name: vs-artifacts
         path: ${{github.workspace}}
@@ -204,7 +204,7 @@ jobs:
       run: ci/print-test-failures.sh
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
         name: failed-tests-windows
         path: ${{env.FAILED_TEST_ARTIFACTS}}
@@ -242,7 +242,7 @@ jobs:
       if: failure()
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
         name: failed-tests-${{matrix.vector.jobname}}
         path: ${{env.FAILED_TEST_ARTIFACTS}}
@@ -269,7 +269,7 @@ jobs:
       if: failure()
     - name: Upload failed tests' directories
       if: failure() && env.FAILED_TEST_ARTIFACTS != ''
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v2
       with:
         name: failed-tests-${{matrix.vector.jobname}}
         path: ${{env.FAILED_TEST_ARTIFACTS}}

From f9221e2cf5049805d9151b3db6a5eef07b1cc92e Mon Sep 17 00:00:00 2001
From: Taylor Blau 
Date: Wed, 23 Jun 2021 14:39:07 -0400
Subject: [PATCH 291/397] csum-file: introduce checksum_valid()

Introduce a new function which checks the validity of a file's trailing
checksum. This is similar to hashfd_check(), but different since it is
intended to be used by callers who aren't writing the same data (like
`git index-pack --verify`), but who instead want to validate the
integrity of data that they are reading.

Rewrite the first of two callers which could benefit from this new
function in pack-check.c. Subsequent callers will be added in the
following patches.

Helped-by: Jeff King 
Signed-off-by: Jeff King 
Signed-off-by: Taylor Blau 
Signed-off-by: Junio C Hamano 
---
 csum-file.c  | 16 ++++++++++++++++
 csum-file.h  |  3 +++
 pack-check.c | 11 +----------
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/csum-file.c b/csum-file.c
index 7510950fa3..60f58f662a 100644
--- a/csum-file.c
+++ b/csum-file.c
@@ -187,3 +187,19 @@ uint32_t crc32_end(struct hashfile *f)
 	f->do_crc = 0;
 	return f->crc32;
 }
+
+int hashfile_checksum_valid(const unsigned char *data, size_t total_len)
+{
+	unsigned char got[GIT_MAX_RAWSZ];
+	git_hash_ctx ctx;
+	size_t data_len = total_len - the_hash_algo->rawsz;
+
+	if (total_len < the_hash_algo->rawsz)
+		return 0; /* say "too short"? */
+
+	the_hash_algo->init_fn(&ctx);
+	the_hash_algo->update_fn(&ctx, data, data_len);
+	the_hash_algo->final_fn(got, &ctx);
+
+	return hasheq(got, data + data_len);
+}
diff --git a/csum-file.h b/csum-file.h
index e54d53d1d0..87e3879f1c 100644
--- a/csum-file.h
+++ b/csum-file.h
@@ -42,6 +42,9 @@ void hashflush(struct hashfile *f);
 void crc32_begin(struct hashfile *);
 uint32_t crc32_end(struct hashfile *);
 
+/* Verify checksum validity while reading. Returns non-zero on success. */
+int hashfile_checksum_valid(const unsigned char *data, size_t len);
+
 /*
  * Returns the total number of bytes fed to the hashfile so far (including ones
  * that have not been written out to the descriptor yet).
diff --git a/pack-check.c b/pack-check.c
index 4b089fe8ec..c8e560d71a 100644
--- a/pack-check.c
+++ b/pack-check.c
@@ -164,22 +164,13 @@ static int verify_packfile(struct repository *r,
 
 int verify_pack_index(struct packed_git *p)
 {
-	size_t len;
-	const unsigned char *index_base;
-	git_hash_ctx ctx;
-	unsigned char hash[GIT_MAX_RAWSZ];
 	int err = 0;
 
 	if (open_pack_index(p))
 		return error("packfile %s index not opened", p->pack_name);
-	index_base = p->index_data;
-	len = p->index_size - the_hash_algo->rawsz;
 
 	/* Verify SHA1 sum of the index file */
-	the_hash_algo->init_fn(&ctx);
-	the_hash_algo->update_fn(&ctx, index_base, len);
-	the_hash_algo->final_fn(hash, &ctx);
-	if (!hasheq(hash, index_base + len))
+	if (!hashfile_checksum_valid(p->index_data, p->index_size))
 		err = error("Packfile index for %s hash mismatch",
 			    p->pack_name);
 	return err;

From 15316a4732eeb0dab27ba406cb80e8704cb9b46d Mon Sep 17 00:00:00 2001
From: Taylor Blau 
Date: Wed, 23 Jun 2021 14:39:09 -0400
Subject: [PATCH 292/397] commit-graph: rewrite to use checksum_valid()

Rewrite an existing caller in `git commit-graph verify` to take
advantage of checksum_valid().

Note that the replacement isn't a verbatim cut-and-paste, since the new
function avoids using hashfile at all and instead talks to the_hash_algo
directly, but it is functionally equivalent.

Signed-off-by: Taylor Blau 
Signed-off-by: Junio C Hamano 
---
 commit-graph.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/commit-graph.c b/commit-graph.c
index 2bcb4e0f89..1a2602da61 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -2422,14 +2422,16 @@ static void graph_report(const char *fmt, ...)
 #define GENERATION_ZERO_EXISTS 1
 #define GENERATION_NUMBER_EXISTS 2
 
+static int commit_graph_checksum_valid(struct commit_graph *g)
+{
+	return hashfile_checksum_valid(g->data, g->data_len);
+}
+
 int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
 {
 	uint32_t i, cur_fanout_pos = 0;
 	struct object_id prev_oid, cur_oid;
-	unsigned char checksum[GIT_MAX_HEXSZ];
 	int generation_zero = 0;
-	struct hashfile *f;
-	int devnull;
 	struct progress *progress = NULL;
 	int local_error = 0;
 
@@ -2442,11 +2444,7 @@ int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
 	if (verify_commit_graph_error)
 		return verify_commit_graph_error;
 
-	devnull = open("/dev/null", O_WRONLY);
-	f = hashfd(devnull, NULL);
-	hashwrite(f, g->data, g->data_len - g->hash_len);
-	finalize_hashfile(f, checksum, CSUM_CLOSE);
-	if (!hasheq(checksum, g->data + g->data_len - g->hash_len)) {
+	if (!commit_graph_checksum_valid(g)) {
 		graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
 		verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
 	}

From ec1e28ef9c30468d2e76e41c88a1611e63047f61 Mon Sep 17 00:00:00 2001
From: Taylor Blau 
Date: Wed, 23 Jun 2021 14:39:12 -0400
Subject: [PATCH 293/397] midx: don't reuse corrupt MIDXs when writing

When writing a new multi-pack index, Git tries to reuse as much of the
data from an existing MIDX as possible, like object offsets. This is
done to avoid re-opening a bunch of *.idx files unnecessarily, but can
lead to problems if the data we are reusing is corrupt.

That's because we'll blindly reuse data from an existing MIDX without
checking its trailing checksum for validity. So if there is memory
corruption while writing a MIDX, or disk corruption in the intervening
period between writing and reuse, we'll blindly propagate those bad
values forward.

Suppose we experience a memory corruption while writing a MIDX such that
we write an incorrect object offset (or alternatively, the disk corrupts
the data after being written, but before being reused). Then when we go
to write a new MIDX, we'll reuse the bad object offset without checking
its validity. This means that the MIDX we just wrote is broken, but its
trailing checksum is in-tact, since we never bothered to look at the
values before writing.

In the above, a "git multi-pack-index verify" would have caught the
problem before writing, but writing a new MIDX wouldn't have noticed
anything wrong, blindly carrying forward the corrupt offset.

Individual pack indexes check their validity by verifying the crc32
attached to each entry when carrying data forward during a repack.
We could solve this problem for MIDXs in the same way, but individual
crc32's don't make much sense, since their entries are so small.
Likewise, checking the whole file on every read may be prohibitively
expensive if a repository has a lot of objects, packs, or both.

But we can check the trailing checksum when reusing an existing MIDX
when writing a new one. And a corrupt MIDX need not stop us from writing
a new one, since we can just avoid reusing the existing one at all and
pretend as if we are writing a new MIDX from scratch.

Suggested-by: Derrick Stolee 
Signed-off-by: Taylor Blau 
Signed-off-by: Junio C Hamano 
---
 midx.c                      | 10 ++++++++++
 t/t5319-multi-pack-index.sh |  8 ++++++++
 2 files changed, 18 insertions(+)

diff --git a/midx.c b/midx.c
index 21d6a05e88..a12cbbf928 100644
--- a/midx.c
+++ b/midx.c
@@ -885,6 +885,11 @@ static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
 static void clear_midx_files_ext(struct repository *r, const char *ext,
 				 unsigned char *keep_hash);
 
+static int midx_checksum_valid(struct multi_pack_index *m)
+{
+	return hashfile_checksum_valid(m->data, m->data_len);
+}
+
 static int write_midx_internal(const char *object_dir, struct multi_pack_index *m,
 			       struct string_list *packs_to_drop,
 			       const char *preferred_pack_name,
@@ -911,6 +916,11 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
 	else
 		ctx.m = load_multi_pack_index(object_dir, 1);
 
+	if (ctx.m && !midx_checksum_valid(ctx.m)) {
+		warning(_("ignoring existing multi-pack-index; checksum mismatch"));
+		ctx.m = NULL;
+	}
+
 	ctx.nr = 0;
 	ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
 	ctx.info = NULL;
diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh
index 5641d158df..d582f370c4 100755
--- a/t/t5319-multi-pack-index.sh
+++ b/t/t5319-multi-pack-index.sh
@@ -410,6 +410,14 @@ test_expect_success 'git-fsck incorrect offset' '
 		"git -c core.multipackindex=true fsck"
 '
 
+test_expect_success 'corrupt MIDX is not reused' '
+	corrupt_midx_and_verify $MIDX_BYTE_OFFSET "\377" $objdir \
+		"incorrect object offset" &&
+	git multi-pack-index write 2>err &&
+	test_i18ngrep checksum.mismatch err &&
+	git multi-pack-index verify
+'
+
 test_expect_success 'repack progress off for redirected stderr' '
 	GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir repack 2>err &&
 	test_line_count = 0 err

From f89ecf79888a48e0adf14d0e05c69ee09e853fd5 Mon Sep 17 00:00:00 2001
From: Taylor Blau 
Date: Wed, 23 Jun 2021 14:39:15 -0400
Subject: [PATCH 294/397] midx: report checksum mismatches during 'verify'

'git multi-pack-index verify' inspects the data in an existing MIDX for
correctness by checking that the recorded object offsets are correct,
and so on.

But it does not check that the file's trailing checksum matches the data
that it records. So, if an on-disk corruption happened to occur in the
final few bytes (and all other data was recorded correctly), we would:

  - get a clean result from 'git multi-pack-index verify', but
  - be unable to reuse the existing MIDX when writing a new one (since
    we now check for checksum mismatches before reusing a MIDX)

Teach the 'verify' sub-command to recognize corruption in the checksum
by calling midx_checksum_valid().

Suggested-by: Derrick Stolee 
Signed-off-by: Taylor Blau 
Signed-off-by: Junio C Hamano 
---
 midx.c                      | 3 +++
 t/t5319-multi-pack-index.sh | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/midx.c b/midx.c
index a12cbbf928..9a35b0255d 100644
--- a/midx.c
+++ b/midx.c
@@ -1228,6 +1228,9 @@ int verify_midx_file(struct repository *r, const char *object_dir, unsigned flag
 		return result;
 	}
 
+	if (!midx_checksum_valid(m))
+		midx_report(_("incorrect checksum"));
+
 	if (flags & MIDX_PROGRESS)
 		progress = start_delayed_progress(_("Looking for referenced packfiles"),
 					  m->num_packs);
diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh
index d582f370c4..7609f1ea64 100755
--- a/t/t5319-multi-pack-index.sh
+++ b/t/t5319-multi-pack-index.sh
@@ -418,6 +418,11 @@ test_expect_success 'corrupt MIDX is not reused' '
 	git multi-pack-index verify
 '
 
+test_expect_success 'verify incorrect checksum' '
+	pos=$(($(wc -c <$objdir/pack/multi-pack-index) - 1)) &&
+	corrupt_midx_and_verify $pos "\377" $objdir "incorrect checksum"
+'
+
 test_expect_success 'repack progress off for redirected stderr' '
 	GIT_PROGRESS_DELAY=0 git multi-pack-index --object-dir=$objdir repack 2>err &&
 	test_line_count = 0 err

From 7b76d6bf221071d733a63cb65a616ec9f8fc4b92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Tue, 29 Jun 2021 10:44:50 +0200
Subject: [PATCH 295/397] Makefile: add and use the ".DELETE_ON_ERROR" flag
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Use the GNU make ".DELETE_ON_ERROR" flag in our main Makefile, as we
already do in the Documentation/Makefile since db10fc6c09f (doc:
simplify Makefile using .DELETE_ON_ERROR, 2021-05-21).

Now if a command to make X fails X will be removed, the default
behavior of GNU make is to only do so if "make" itself is interrupted
with a signal.

E.g. if we now intentionally break one of the rules with:

    -       mv $@+ $@
    +       mv $@+ $@ && \
    +       false

We'll get output like:

    $ make git
        CC git.o
        LINK git
    make: *** [Makefile:2179: git] Error 1
    make: *** Deleting file 'git'
    $ file git
    git: cannot open `git' (No such file or directory)

Before this change we'd leave the file in place in under this
scenario.

As in db10fc6c09f this allows us to remove patterns of removing
leftover $@ files at the start of rules, since previous failing runs
of the Makefile won't have left those littered around anymore.

I'm not as confident that we should be replacing the "mv $@+ $@"
pattern entirely, since that means that external programs or one of
our other Makefiles might race and get partial content.

I'm not changing $(REMOTE_CURL_ALIASES) since that uses a ln/ln -s/cp
dance, and would require the addition of "-f" flags if the "rm" at the
start was removed. I've also got plans to fix that ln/ln -s/cp pattern
in another series.

For $(LIB_FILE) and $(XDIFF_LIB) we can rely on the "c" (create) being
present in ARFLAGS.

I'm not changing "$(ETAGS_TARGET)", "tags" and "cscope" because
they've got a messy combination of removing "$@+" not "$@" at the
beginning, or "$@*". I'm also addressing those in another series.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 Makefile | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index c3565fc0f8..157293b555 100644
--- a/Makefile
+++ b/Makefile
@@ -2160,6 +2160,16 @@ shell_compatibility_test: please_set_SHELL_PATH_to_a_more_modern_shell
 strip: $(PROGRAMS) git$X
 	$(STRIP) $(STRIP_OPTS) $^
 
+### Flags affecting all rules
+
+# A GNU make extension since gmake 3.72 (released in late 1994) to
+# remove the target of rules if commands in those rules fail. The
+# default is to only do that if make itself receives a signal. Affects
+# all targets, see:
+#
+#    info make --index-search=.DELETE_ON_ERROR
+.DELETE_ON_ERROR:
+
 ### Target-specific flags and dependencies
 
 # The generic compilation pattern rule and automatically
@@ -2243,7 +2253,6 @@ SCRIPT_DEFINES = $(SHELL_PATH_SQ):$(DIFF_SQ):$(GIT_VERSION):\
 	$(gitwebdir_SQ):$(PERL_PATH_SQ):$(SANE_TEXT_GREP):$(PAGER_ENV):\
 	$(perllibdir_SQ)
 define cmd_munge_script
-$(RM) $@ $@+ && \
 sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \
     -e 's|@SHELL_PATH@|$(SHELL_PATH_SQ)|' \
     -e 's|@@DIFF@@|$(DIFF_SQ)|' \
@@ -2313,7 +2322,7 @@ endif
 PERL_DEFINES += $(gitexecdir) $(perllibdir) $(localedir)
 
 $(SCRIPT_PERL_GEN): % : %.perl GIT-PERL-DEFINES GIT-PERL-HEADER GIT-VERSION-FILE
-	$(QUIET_GEN)$(RM) $@ $@+ && \
+	$(QUIET_GEN) \
 	sed -e '1{' \
 	    -e '	s|#!.*perl|#!$(PERL_PATH_SQ)|' \
 	    -e '	r GIT-PERL-HEADER' \
@@ -2333,7 +2342,7 @@ GIT-PERL-DEFINES: FORCE
 	    fi
 
 GIT-PERL-HEADER: $(PERL_HEADER_TEMPLATE) GIT-PERL-DEFINES Makefile
-	$(QUIET_GEN)$(RM) $@ && \
+	$(QUIET_GEN) \
 	INSTLIBDIR='$(perllibdir_SQ)' && \
 	INSTLIBDIR_EXTRA='$(PERLLIB_EXTRA_SQ)' && \
 	INSTLIBDIR="$$INSTLIBDIR$${INSTLIBDIR_EXTRA:+:$$INSTLIBDIR_EXTRA}" && \
@@ -2359,7 +2368,7 @@ git-instaweb: git-instaweb.sh GIT-SCRIPT-DEFINES
 	mv $@+ $@
 else # NO_PERL
 $(SCRIPT_PERL_GEN) git-instaweb: % : unimplemented.sh
-	$(QUIET_GEN)$(RM) $@ $@+ && \
+	$(QUIET_GEN) \
 	sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \
 	    -e 's|@@REASON@@|NO_PERL=$(NO_PERL)|g' \
 	    unimplemented.sh >$@+ && \
@@ -2373,14 +2382,14 @@ $(SCRIPT_PYTHON_GEN): GIT-BUILD-OPTIONS
 ifndef NO_PYTHON
 $(SCRIPT_PYTHON_GEN): GIT-CFLAGS GIT-PREFIX GIT-PYTHON-VARS
 $(SCRIPT_PYTHON_GEN): % : %.py
-	$(QUIET_GEN)$(RM) $@ $@+ && \
+	$(QUIET_GEN) \
 	sed -e '1s|#!.*python|#!$(PYTHON_PATH_SQ)|' \
 	    $< >$@+ && \
 	chmod +x $@+ && \
 	mv $@+ $@
 else # NO_PYTHON
 $(SCRIPT_PYTHON_GEN): % : unimplemented.sh
-	$(QUIET_GEN)$(RM) $@ $@+ && \
+	$(QUIET_GEN) \
 	sed -e '1s|#!.*/sh|#!$(SHELL_PATH_SQ)|' \
 	    -e 's|@@REASON@@|NO_PYTHON=$(NO_PYTHON)|g' \
 	    unimplemented.sh >$@+ && \
@@ -2388,8 +2397,7 @@ $(SCRIPT_PYTHON_GEN): % : unimplemented.sh
 	mv $@+ $@
 endif # NO_PYTHON
 
-CONFIGURE_RECIPE = $(RM) configure configure.ac+ && \
-		   sed -e 's/@@GIT_VERSION@@/$(GIT_VERSION)/g' \
+CONFIGURE_RECIPE = sed -e 's/@@GIT_VERSION@@/$(GIT_VERSION)/g' \
 			configure.ac >configure.ac+ && \
 		   autoconf -o configure configure.ac+ && \
 		   $(RM) configure.ac+
@@ -2514,7 +2522,6 @@ endif
 ifeq ($(GENERATE_COMPILATION_DATABASE),yes)
 all:: compile_commands.json
 compile_commands.json:
-	@$(RM) $@
 	$(QUIET_GEN)sed -e '1s/^/[/' -e '$$s/,$$/]/' $(compdb_dir)/*.o.json > $@+
 	@if test -s $@+; then mv $@+ $@; else $(RM) $@+; fi
 endif
@@ -2587,10 +2594,10 @@ $(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS
 		$(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS)
 
 $(LIB_FILE): $(LIB_OBJS)
-	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
+	$(QUIET_AR)$(AR) $(ARFLAGS) $@ $^
 
 $(XDIFF_LIB): $(XDIFF_OBJS)
-	$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
+	$(QUIET_AR)$(AR) $(ARFLAGS) $@ $^
 
 export DEFAULT_EDITOR DEFAULT_PAGER
 

From c49a177beca2f03f205437a3fa8c52e2011bd581 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Tue, 29 Jun 2021 13:29:36 +0200
Subject: [PATCH 296/397] test-lib.sh: set COLUMNS=80 for --verbose
 repeatability
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Some tests will fail under --verbose because while we've unset COLUMNS
since b1d645b58ac (tests: unset COLUMNS inherited from environment,
2012-03-27), we also look for the columns with an ioctl(..,
TIOCGWINSZ, ...) on some platforms. By setting COLUMNS again we
preempt the TIOCGWINSZ lookup in pager.c's term_columns(), it'll take
COLUMNS over TIOCGWINSZ,

This fixes t0500-progress-display.sh., which broke because of a
combination of the this issue and the progress output reacting to the
column width since 545dc345ebd (progress: break too long progress bar
lines, 2019-04-12). The t5324-split-commit-graph.sh fails in a similar
manner due to progress output, see [1] for details.

The issue is not specific to progress.c, the diff code also checks
COLUMNS and some of its tests can be made to fail in a similar
manner[2], anything that invokes a pager is potentially affected.

See ea77e675e56 (Make "git help" react to window size correctly,
2005-12-18) and ad6c3739a33 (pager: find out the terminal width before
spawning the pager, 2012-02-12) for how the TIOCGWINSZ code ended up
in pager.c

1. http://lore.kernel.org/git/20210624051253.GG6312@szeder.dev
2. https://lore.kernel.org/git/20210627074419.GH6312@szeder.dev/

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 t/test-lib.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/t/test-lib.sh b/t/test-lib.sh
index adaf03543e..4d4439a917 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -401,14 +401,15 @@ LANG=C
 LC_ALL=C
 PAGER=cat
 TZ=UTC
-export LANG LC_ALL PAGER TZ
+COLUMNS=80
+export LANG LC_ALL PAGER TZ COLUMNS
 EDITOR=:
 
 # A call to "unset" with no arguments causes at least Solaris 10
 # /usr/xpg4/bin/sh and /bin/ksh to bail out.  So keep the unsets
 # deriving from the command substitution clustered with the other
 # ones.
-unset VISUAL EMAIL LANGUAGE COLUMNS $("$PERL_PATH" -e '
+unset VISUAL EMAIL LANGUAGE $("$PERL_PATH" -e '
 	my @env = keys %ENV;
 	my $ok = join("|", qw(
 		TRACE

From dc0592941138df684770bfe800ccad6b810214c3 Mon Sep 17 00:00:00 2001
From: Eric Wong 
Date: Wed, 30 Jun 2021 00:01:32 +0000
Subject: [PATCH 297/397] xmmap: inform Linux users of tuning knobs on ENOMEM

Linux users may benefit from additional information on how to
avoid ENOMEM from mmap despite the system having enough RAM to
accomodate them.  We can't reliably unmap pack windows to work
around the issue since malloc and other library routines may
mmap without our knowledge.

Signed-off-by: Eric Wong 
Signed-off-by: Junio C Hamano 
---
 config.c          |  3 ++-
 git-compat-util.h |  1 +
 object-file.c     | 16 +++++++++++++++-
 packfile.c        |  4 ++--
 read-cache.c      |  3 ++-
 5 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/config.c b/config.c
index f9c400ad30..79ae9f2dea 100644
--- a/config.c
+++ b/config.c
@@ -3051,7 +3051,8 @@ int git_config_set_multivar_in_file_gently(const char *config_filename,
 		if (contents == MAP_FAILED) {
 			if (errno == ENODEV && S_ISDIR(st.st_mode))
 				errno = EISDIR;
-			error_errno(_("unable to mmap '%s'"), config_filename);
+			error_errno(_("unable to mmap '%s'%s"),
+					config_filename, mmap_os_err());
 			ret = CONFIG_INVALID_FILE;
 			contents = NULL;
 			goto out_free;
diff --git a/git-compat-util.h b/git-compat-util.h
index fb6e9af76b..fa6dd92219 100644
--- a/git-compat-util.h
+++ b/git-compat-util.h
@@ -876,6 +876,7 @@ char *xstrndup(const char *str, size_t len);
 void *xrealloc(void *ptr, size_t size);
 void *xcalloc(size_t nmemb, size_t size);
 void *xmmap(void *start, size_t length, int prot, int flags, int fd, off_t offset);
+const char *mmap_os_err(void);
 void *xmmap_gently(void *start, size_t length, int prot, int flags, int fd, off_t offset);
 int xopen(const char *path, int flags, ...);
 ssize_t xread(int fd, void *buf, size_t len);
diff --git a/object-file.c b/object-file.c
index f233b440b2..b9c3219793 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1023,12 +1023,26 @@ void *xmmap_gently(void *start, size_t length,
 	return ret;
 }
 
+const char *mmap_os_err(void)
+{
+	static const char blank[] = "";
+#if defined(__linux__)
+	if (errno == ENOMEM) {
+		/* this continues an existing error message: */
+		static const char enomem[] =
+", check sys.vm.max_map_count and/or RLIMIT_DATA";
+		return enomem;
+	}
+#endif /* OS-specific bits */
+	return blank;
+}
+
 void *xmmap(void *start, size_t length,
 	int prot, int flags, int fd, off_t offset)
 {
 	void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
 	if (ret == MAP_FAILED)
-		die_errno(_("mmap failed"));
+		die_errno(_("mmap failed%s"), mmap_os_err());
 	return ret;
 }
 
diff --git a/packfile.c b/packfile.c
index 755aa7aec5..9ef6d98292 100644
--- a/packfile.c
+++ b/packfile.c
@@ -652,8 +652,8 @@ unsigned char *use_pack(struct packed_git *p,
 				PROT_READ, MAP_PRIVATE,
 				p->pack_fd, win->offset);
 			if (win->base == MAP_FAILED)
-				die_errno("packfile %s cannot be mapped",
-					  p->pack_name);
+				die_errno(_("packfile %s cannot be mapped%s"),
+					  p->pack_name, mmap_os_err());
 			if (!win->offset && win->len == p->pack_size
 				&& !p->do_not_close)
 				close_pack_fd(p);
diff --git a/read-cache.c b/read-cache.c
index 77961a3885..a80902155c 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -2236,7 +2236,8 @@ int do_read_index(struct index_state *istate, const char *path, int must_exist)
 
 	mmap = xmmap_gently(NULL, mmap_size, PROT_READ, MAP_PRIVATE, fd, 0);
 	if (mmap == MAP_FAILED)
-		die_errno(_("%s: unable to map index file"), path);
+		die_errno(_("%s: unable to map index file%s"), path,
+			mmap_os_err());
 	close(fd);
 
 	hdr = (const struct cache_header *)mmap;

From fe7fe62d8da0949d9b2bca34467b349bd294e91b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= 
Date: Wed, 30 Jun 2021 18:12:43 +0200
Subject: [PATCH 298/397] grep: report missing left operand of --and
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Git grep allows combining two patterns with --and.  It checks and
reports if the second pattern is missing when compiling the expression.
A missing first pattern, however, is only reported later at match time.
Thus no error is returned if no matching is done, e.g. because no file
matches the also given pathspec.

When that happens we get an expression tree with an GREP_NODE_AND node
and a NULL pointer to the missing left child.  free_pattern_expr()
tries to dereference it during the cleanup at the end, which results
in a segmentation fault.

Fix this by verifying the presence of the left operand at expression
compilation time.

Reported-by: Matthew Hughes 
Helped-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: René Scharfe 
Signed-off-by: Junio C Hamano 
---
 grep.c          | 2 ++
 t/t7810-grep.sh | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/grep.c b/grep.c
index 4db1510d16..71e44402dc 100644
--- a/grep.c
+++ b/grep.c
@@ -774,6 +774,8 @@ static struct grep_expr *compile_pattern_and(struct grep_pat **list)
 	x = compile_pattern_not(list);
 	p = *list;
 	if (p && p->token == GREP_AND) {
+		if (!x)
+			die("--and not preceded by pattern expression");
 		if (!p->next)
 			die("--and not followed by pattern expression");
 		*list = p->next;
diff --git a/t/t7810-grep.sh b/t/t7810-grep.sh
index 43aa4161cf..47434e6687 100755
--- a/t/t7810-grep.sh
+++ b/t/t7810-grep.sh
@@ -8,6 +8,13 @@ test_description='git grep various.
 
 . ./test-lib.sh
 
+test_invalid_grep_expression() {
+	params="$@" &&
+	test_expect_success "invalid expression: grep $params" '
+		test_must_fail git grep $params -- nonexisting
+	'
+}
+
 cat >hello.c <
 #include 
@@ -81,6 +88,8 @@ test_expect_success 'grep should not segfault with a bad input' '
 	test_must_fail git grep "("
 '
 
+test_invalid_grep_expression --and -e A
+
 for H in HEAD ''
 do
 	case "$H" in

From 806f83287f8d6c0568beed7ca5d603f936d53c40 Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Wed, 30 Jun 2021 17:29:58 +0000
Subject: [PATCH 299/397] t6423: test directory renames causing rename-to-self

Directory rename detection can cause transitive renames, e.g. if the two
different sides of history each do one half of:
    A/file -> B/file
    B/     -> C/
then directory rename detection transitively renames to give us C/file.
Since the default for merge.directoryRenames is conflict, this results
in an error message saying it is unclear whether the file should be
placed at B/file or C/file.

What if C/ is A/, though?  In such a case, the transitive rename would
give us A/file, the original name we started with.  Logically, having
an error message with B/file vs. A/file should be fine, as should
leaving the file where it started.  But the logic in both
merge-recursive and merge-ort did not handle a case of a filename being
renamed to itself correctly; merge-recursive had two bugs, and merge-ort
had one.  Add some testcases covering such a scenario.

Based-on-testcase-by: Anders Kaseorg 
Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 t/t6423-merge-rename-directories.sh | 175 ++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)

diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh
index 7134769149..afb88c347f 100755
--- a/t/t6423-merge-rename-directories.sh
+++ b/t/t6423-merge-rename-directories.sh
@@ -4966,6 +4966,181 @@ test_expect_success '12g: Testcase with two kinds of "relevant" renames' '
 	)
 '
 
+# Testcase 12i, Directory rename causes rename-to-self
+#   Commit O: source/{subdir/foo, bar, baz_1}
+#   Commit A: source/{foo, bar, baz_1}
+#   Commit B: source/{subdir/{foo, bar}, baz_2}
+#   Expected: source/{foo, bar, baz_2}, with conflicts on
+#                source/bar vs. source/subdir/bar
+
+test_setup_12i () {
+	test_create_repo 12i &&
+	(
+		cd 12i &&
+
+		mkdir -p source/subdir &&
+		echo foo >source/subdir/foo &&
+		echo bar >source/bar &&
+		echo baz >source/baz &&
+		git add source &&
+		git commit -m orig &&
+
+		git branch O &&
+		git branch A &&
+		git branch B &&
+
+		git switch A &&
+		git mv source/subdir/foo source/foo &&
+		git commit -m A &&
+
+		git switch B &&
+		git mv source/bar source/subdir/bar &&
+		echo more baz >>source/baz &&
+		git commit -m B
+	)
+}
+
+test_expect_merge_algorithm failure failure '12i: Directory rename causes rename-to-self' '
+	test_setup_12i &&
+	(
+		cd 12i &&
+
+		git checkout A^0 &&
+
+		test_must_fail git -c merge.directoryRenames=conflict merge -s recursive B^0 &&
+
+		test_path_is_missing source/subdir &&
+		test_path_is_file source/bar &&
+		test_path_is_file source/baz &&
+
+		git ls-files | uniq >tracked &&
+		test_line_count = 3 tracked &&
+
+		git status --porcelain -uno >actual &&
+		cat >expect <<-\EOF &&
+		UU source/bar
+		 M source/baz
+		EOF
+		test_cmp expect actual
+	)
+'
+
+# Testcase 12j, Directory rename to root causes rename-to-self
+#   Commit O: {subdir/foo, bar, baz_1}
+#   Commit A: {foo, bar, baz_1}
+#   Commit B: {subdir/{foo, bar}, baz_2}
+#   Expected: {foo, bar, baz_2}, with conflicts on bar vs. subdir/bar
+
+test_setup_12j () {
+	test_create_repo 12j &&
+	(
+		cd 12j &&
+
+		mkdir -p subdir &&
+		echo foo >subdir/foo &&
+		echo bar >bar &&
+		echo baz >baz &&
+		git add . &&
+		git commit -m orig &&
+
+		git branch O &&
+		git branch A &&
+		git branch B &&
+
+		git switch A &&
+		git mv subdir/foo foo &&
+		git commit -m A &&
+
+		git switch B &&
+		git mv bar subdir/bar &&
+		echo more baz >>baz &&
+		git commit -m B
+	)
+}
+
+test_expect_merge_algorithm failure failure '12j: Directory rename to root causes rename-to-self' '
+	test_setup_12j &&
+	(
+		cd 12j &&
+
+		git checkout A^0 &&
+
+		test_must_fail git -c merge.directoryRenames=conflict merge -s recursive B^0 &&
+
+		test_path_is_missing subdir &&
+		test_path_is_file bar &&
+		test_path_is_file baz &&
+
+		git ls-files | uniq >tracked &&
+		test_line_count = 3 tracked &&
+
+		git status --porcelain -uno >actual &&
+		cat >expect <<-\EOF &&
+		UU bar
+		 M baz
+		EOF
+		test_cmp expect actual
+	)
+'
+
+# Testcase 12k, Directory rename with sibling causes rename-to-self
+#   Commit O: dirB/foo, dirA/{bar, baz_1}
+#   Commit A: dirA/{foo, bar, baz_1}
+#   Commit B: dirB/{foo, bar}, dirA/baz_2
+#   Expected: dirA/{foo, bar, baz_2}, with conflicts on dirA/bar vs. dirB/bar
+
+test_setup_12k () {
+	test_create_repo 12k &&
+	(
+		cd 12k &&
+
+		mkdir dirA dirB &&
+		echo foo >dirB/foo &&
+		echo bar >dirA/bar &&
+		echo baz >dirA/baz &&
+		git add . &&
+		git commit -m orig &&
+
+		git branch O &&
+		git branch A &&
+		git branch B &&
+
+		git switch A &&
+		git mv dirB/* dirA/ &&
+		git commit -m A &&
+
+		git switch B &&
+		git mv dirA/bar dirB/bar &&
+		echo more baz >>dirA/baz &&
+		git commit -m B
+	)
+}
+
+test_expect_merge_algorithm failure failure '12k: Directory rename with sibling causes rename-to-self' '
+	test_setup_12k &&
+	(
+		cd 12k &&
+
+		git checkout A^0 &&
+
+		test_must_fail git -c merge.directoryRenames=conflict merge -s recursive B^0 &&
+
+		test_path_is_missing dirB &&
+		test_path_is_file dirA/bar &&
+		test_path_is_file dirA/baz &&
+
+		git ls-files | uniq >tracked &&
+		test_line_count = 3 tracked &&
+
+		git status --porcelain -uno >actual &&
+		cat >expect <<-\EOF &&
+		UU dirA/bar
+		 M dirA/baz
+		EOF
+		test_cmp expect actual
+	)
+'
+
 ###########################################################################
 # SECTION 13: Checking informational and conflict messages
 #

From a492d5331cb93d8293e72741b4fb9e1ec4ff294b Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Wed, 30 Jun 2021 17:29:59 +0000
Subject: [PATCH 300/397] merge-ort: ensure we consult df_conflict and
 path_conflicts

Path conflicts (typically rename path conflicts, e.g.
rename/rename(1to2) or rename/add/delete), and directory/file conflicts
should obviously result in files not being marked as clean in the merge.
We had a codepath where we missed consulting the path_conflict and
df_conflict flags, based on match_mask.  Granted, it requires an unusual
setup to trigger this codepath (directory rename causing rename-to-self
is the only case I can think of), but we still need to handle it.  To
make it clear that we have audited the other codepaths that do not
explicitly mention these flags, add some assertions that the flags are
not set.

Reported-by: Anders Kaseorg 
Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 merge-ort.c                         | 6 +++++-
 t/t6423-merge-rename-directories.sh | 6 +++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/merge-ort.c b/merge-ort.c
index 4a9ce2a822..8b0f2448f0 100644
--- a/merge-ort.c
+++ b/merge-ort.c
@@ -3002,7 +3002,7 @@ static void process_entry(struct merge_options *opt,
 	 *       above.
 	 */
 	if (ci->match_mask) {
-		ci->merged.clean = 1;
+		ci->merged.clean = !ci->df_conflict && !ci->path_conflict;
 		if (ci->match_mask == 6) {
 			/* stages[1] == stages[2] */
 			ci->merged.result.mode = ci->stages[1].mode;
@@ -3014,6 +3014,8 @@ static void process_entry(struct merge_options *opt,
 
 			ci->merged.result.mode = ci->stages[side].mode;
 			ci->merged.is_null = !ci->merged.result.mode;
+			if (ci->merged.is_null)
+				ci->merged.clean = 1;
 			oidcpy(&ci->merged.result.oid, &ci->stages[side].oid);
 
 			assert(othermask == 2 || othermask == 4);
@@ -3186,6 +3188,7 @@ static void process_entry(struct merge_options *opt,
 				   path)) {
 			ci->merged.is_null = 1;
 			ci->merged.clean = 1;
+			assert(!ci->df_conflict && !ci->path_conflict);
 		} else if (ci->path_conflict &&
 			   oideq(&ci->stages[0].oid, &ci->stages[side].oid)) {
 			/*
@@ -3212,6 +3215,7 @@ static void process_entry(struct merge_options *opt,
 		ci->merged.is_null = 1;
 		ci->merged.result.mode = 0;
 		oidcpy(&ci->merged.result.oid, null_oid());
+		assert(!ci->df_conflict);
 		ci->merged.clean = !ci->path_conflict;
 	}
 
diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh
index afb88c347f..316339cb6c 100755
--- a/t/t6423-merge-rename-directories.sh
+++ b/t/t6423-merge-rename-directories.sh
@@ -5000,7 +5000,7 @@ test_setup_12i () {
 	)
 }
 
-test_expect_merge_algorithm failure failure '12i: Directory rename causes rename-to-self' '
+test_expect_merge_algorithm failure success '12i: Directory rename causes rename-to-self' '
 	test_setup_12i &&
 	(
 		cd 12i &&
@@ -5058,7 +5058,7 @@ test_setup_12j () {
 	)
 }
 
-test_expect_merge_algorithm failure failure '12j: Directory rename to root causes rename-to-self' '
+test_expect_merge_algorithm failure success '12j: Directory rename to root causes rename-to-self' '
 	test_setup_12j &&
 	(
 		cd 12j &&
@@ -5116,7 +5116,7 @@ test_setup_12k () {
 	)
 }
 
-test_expect_merge_algorithm failure failure '12k: Directory rename with sibling causes rename-to-self' '
+test_expect_merge_algorithm failure success '12k: Directory rename with sibling causes rename-to-self' '
 	test_setup_12k &&
 	(
 		cd 12k &&

From 3585d0ea232b1a9c5498ab5785b11f61e93967c8 Mon Sep 17 00:00:00 2001
From: Elijah Newren 
Date: Wed, 30 Jun 2021 17:30:00 +0000
Subject: [PATCH 301/397] merge-recursive: handle rename-to-self case

Directory rename detection can cause transitive renames, e.g. if the two
different sides of history each do one half of:
    A/file -> B/file
    B/     -> C/
then directory rename detection transitively renames to give us
    A/file -> C/file

However, when C/ == A/, note that this gives us
    A/file -> A/file.

merge-recursive assumed that any rename D -> E would have D != E.  While
that is almost always true, the above is a special case where it is not.
So we cannot do things like delete the rename source, we cannot assume
that a file existing at path E implies a rename/add conflict and we have
to be careful about what stages end up in the output.

This change feels a bit hackish.  It took me surprisingly many hours to
find, and given merge-recursive's design causing it to attempt to
enumerate all combinations of edge and corner cases with special code
for each combination, I'm worried there are other similar fixes needed
elsewhere if we can just come up with the right special testcase.
Perhaps an audit would rule it out, but I have not the energy.
merge-recursive deserves to die, and since it is on its way out anyway,
fixing this particular bug narrowly will have to be good enough.

Reported-by: Anders Kaseorg 
Signed-off-by: Elijah Newren 
Signed-off-by: Junio C Hamano 
---
 merge-recursive.c                   | 19 +++++++++++++------
 t/t6423-merge-rename-directories.sh |  6 +++---
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/merge-recursive.c b/merge-recursive.c
index d146bb116f..c895145a8f 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -2804,12 +2804,19 @@ static int process_renames(struct merge_options *opt,
 			int renamed_stage = a_renames == renames1 ? 2 : 3;
 			int other_stage =   a_renames == renames1 ? 3 : 2;
 
+			/*
+			 * Directory renames have a funny corner case...
+			 */
+			int renamed_to_self = !strcmp(ren1_src, ren1_dst);
+
 			/* BUG: We should only remove ren1_src in the base
 			 * stage and in other_stage (think of rename +
 			 * add-source case).
 			 */
-			remove_file(opt, 1, ren1_src,
-				    renamed_stage == 2 || !was_tracked(opt, ren1_src));
+			if (!renamed_to_self)
+				remove_file(opt, 1, ren1_src,
+					    renamed_stage == 2 ||
+					    !was_tracked(opt, ren1_src));
 
 			oidcpy(&src_other.oid,
 			       &ren1->src_entry->stages[other_stage].oid);
@@ -2823,6 +2830,9 @@ static int process_renames(struct merge_options *opt,
 			    ren1->dir_rename_original_type == 'A') {
 				setup_rename_conflict_info(RENAME_VIA_DIR,
 							   opt, ren1, NULL);
+			} else if (renamed_to_self) {
+				setup_rename_conflict_info(RENAME_NORMAL,
+							   opt, ren1, NULL);
 			} else if (oideq(&src_other.oid, null_oid())) {
 				setup_rename_conflict_info(RENAME_DELETE,
 							   opt, ren1, NULL);
@@ -3180,7 +3190,6 @@ static int handle_rename_normal(struct merge_options *opt,
 	struct rename *ren = ci->ren1;
 	struct merge_file_info mfi;
 	int clean;
-	int side = (ren->branch == opt->branch1 ? 2 : 3);
 
 	/* Merge the content and write it out */
 	clean = handle_content_merge(&mfi, opt, path, was_dirty(opt, path),
@@ -3190,9 +3199,7 @@ static int handle_rename_normal(struct merge_options *opt,
 	    opt->detect_directory_renames == MERGE_DIRECTORY_RENAMES_CONFLICT &&
 	    ren->dir_rename_original_dest) {
 		if (update_stages(opt, path,
-				  NULL,
-				  side == 2 ? &mfi.blob : NULL,
-				  side == 2 ? NULL : &mfi.blob))
+				  &mfi.blob, &mfi.blob, &mfi.blob))
 			return -1;
 		clean = 0; /* not clean, but conflicted */
 	}
diff --git a/t/t6423-merge-rename-directories.sh b/t/t6423-merge-rename-directories.sh
index 316339cb6c..4d056f0465 100755
--- a/t/t6423-merge-rename-directories.sh
+++ b/t/t6423-merge-rename-directories.sh
@@ -5000,7 +5000,7 @@ test_setup_12i () {
 	)
 }
 
-test_expect_merge_algorithm failure success '12i: Directory rename causes rename-to-self' '
+test_expect_success '12i: Directory rename causes rename-to-self' '
 	test_setup_12i &&
 	(
 		cd 12i &&
@@ -5058,7 +5058,7 @@ test_setup_12j () {
 	)
 }
 
-test_expect_merge_algorithm failure success '12j: Directory rename to root causes rename-to-self' '
+test_expect_success '12j: Directory rename to root causes rename-to-self' '
 	test_setup_12j &&
 	(
 		cd 12j &&
@@ -5116,7 +5116,7 @@ test_setup_12k () {
 	)
 }
 
-test_expect_merge_algorithm failure success '12k: Directory rename with sibling causes rename-to-self' '
+test_expect_success '12k: Directory rename with sibling causes rename-to-self' '
 	test_setup_12k &&
 	(
 		cd 12k &&

From 1e5b5ea5386121fd80c7fe1a05c4e3419584f3c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Wed, 30 Jun 2021 18:38:10 +0200
Subject: [PATCH 302/397] send-pack.c: move "no refs in common" abort earlier
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move the early return if we have no remote refs in send_pack()
earlier.

When this was added in 4c353e890c0 (Warn when send-pack does nothing,
2005-12-04) one of the first things we'd do was to abort, but as of
cfee10a773b (send-pack/receive-pack: allow errors to be reported back
to pusher., 2005-12-25) we've added numerous server_supports()
conditions that are acted on later in the function, that won't be used
if we don't have remote refs.

Then as of 477673d6f39 (send-pack: support push negotiation,
2021-05-04) we started doing even more work on the assumption that we
had some remote refs to feed to --negotiation-tip=* options.

We only hit this condition if we have nothing to push, so we don't
need to consider "push.negotiate" etc. only to do nothing with that
information.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 send-pack.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/send-pack.c b/send-pack.c
index 9cb9f71650..5a79e0e711 100644
--- a/send-pack.c
+++ b/send-pack.c
@@ -486,6 +486,12 @@ int send_pack(struct send_pack_args *args,
 	const char *push_cert_nonce = NULL;
 	struct packet_reader reader;
 
+	if (!remote_refs) {
+		fprintf(stderr, "No refs in common and none specified; doing nothing.\n"
+			"Perhaps you should specify a branch.\n");
+		return 0;
+	}
+
 	git_config_get_bool("push.negotiate", &push_negotiate);
 	if (push_negotiate)
 		get_commons_through_negotiation(args->url, remote_refs, &commons);
@@ -534,11 +540,6 @@ int send_pack(struct send_pack_args *args,
 		}
 	}
 
-	if (!remote_refs) {
-		fprintf(stderr, "No refs in common and none specified; doing nothing.\n"
-			"Perhaps you should specify a branch.\n");
-		return 0;
-	}
 	if (args->atomic && !atomic_supported)
 		die(_("the receiving end does not support --atomic push"));
 

From 60fadf8bd2abe6bede48bcf42377f6c8b7c1d0bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Wed, 30 Jun 2021 18:38:11 +0200
Subject: [PATCH 303/397] fetch: document the --negotiate-only option
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There was no documentation for the --negotiate-only option added in
9c1e657a8fd (fetch: teach independent negotiation (no packfile),
2021-05-04), only documentation for the related push.negotiation
option added in the following commit in 477673d6f39 (send-pack:
support push negotiation, 2021-05-04).

Let's document it, and update the cross-linking I'd added between
--negotiation-tip=* and 'fetch.negotiationAlgorithm' in
526608284a7 (fetch doc: cross-link two new negotiation options,
2018-08-01).

I think it would be better to say "in common with the remote" here
than "...the server", but the documentation for --negotiation-tip=*
above this talks about "the server", so let's continue doing that in
this related option. See 3390e42adb3 (fetch-pack: support negotiation
tip whitelist, 2018-07-02) for that documentation.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 Documentation/config/fetch.txt  |  3 ++-
 Documentation/fetch-options.txt | 13 +++++++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/Documentation/config/fetch.txt b/Documentation/config/fetch.txt
index 6af6f5edb2..63748c02b7 100644
--- a/Documentation/config/fetch.txt
+++ b/Documentation/config/fetch.txt
@@ -69,7 +69,8 @@ fetch.negotiationAlgorithm::
 	setting defaults to "skipping".
 	Unknown values will cause 'git fetch' to error out.
 +
-See also the `--negotiation-tip` option for linkgit:git-fetch[1].
+See also the `--negotiate-only` and `--negotiation-tip` options to
+linkgit:git-fetch[1].
 
 fetch.showForcedUpdates::
 	Set to false to enable `--no-show-forced-updates` in
diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 9e7b4e189c..e967ff1874 100644
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -62,8 +62,17 @@ The argument to this option may be a glob on ref names, a ref, or the (possibly
 abbreviated) SHA-1 of a commit. Specifying a glob is equivalent to specifying
 this option multiple times, one for each matching ref name.
 +
-See also the `fetch.negotiationAlgorithm` configuration variable
-documented in linkgit:git-config[1].
+See also the `fetch.negotiationAlgorithm` and `push.negotiate`
+configuration variables documented in linkgit:git-config[1], and the
+`--negotiate-only` option below.
+
+--negotiate-only::
+	Do not fetch anything from the server, and instead print the
+	ancestors of the provided `--negotiation-tip=*` arguments,
+	which we have in common with the server.
++
+Internally this is used to implement the `push.negotiate` option, see
+linkgit:git-config[1].
 
 --dry-run::
 	Show what would be done, without making any changes.

From 560bf518923e0bf40b43d71bb6238cbe13b016b1 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Wed, 16 Jun 2021 06:23:07 -0400
Subject: [PATCH 304/397] test-lib: avoid accidental globbing in
 match_pattern_list()

We have a custom match_pattern_list() function which we use for matching
test names (like "t1234") against glob-like patterns (like "t1???") for
$GIT_SKIP_TESTS, --verbose-only, etc.

Those patterns may have multiple whitespace-separated elements (e.g.,
"t0* t1234 t5?78"). The callers of match_pattern_list thus pass the
strings unquoted, so that the shell does the usual field-splitting into
separate arguments.

But this also means the shell will do the usual globbing for each
argument, which can result in us seeing an expansion based on what's in
the filesystem, rather than the real pattern. For example, if I have the
path "t5000" in the filesystem, and you feed the pattern "t?000", that
_should_ match the string "t0000", but it won't after the shell has
expanded it to "t5000".

This has been a bug ever since that function was introduced. But it
didn't usually trigger since we typically use the function inside the
trash directory, which has a very limited set of files that are unlikely
to match. It became a lot easier to trigger after edc23840b0 (test-lib:
bring $remove_trash out of retirement, 2021-05-10), because now we match
$GIT_SKIP_TESTS before even entering the trash directory. So the t5000
example above can be seen with:

  GIT_SKIP_TESTS=t?000 ./t0000-basic.sh

which should skip all tests but doesn't.

We can fix this by using "set -f" to ask the shell not to glob (which is
in POSIX, so should hopefully be portable enough). We only want to do
this in a subshell (to avoid polluting the rest of the script), which
means we need to get the whole string intact into the match_pattern_list
function by quoting it. Arguably this is a good idea anyway, since it
makes it much more obvious that we intend to split, and it's not simply
sloppy scripting.

Diagnosed-by: Junio C Hamano 
Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 t/test-lib.sh | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/t/test-lib.sh b/t/test-lib.sh
index 54938c6427..a7badbf1dd 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -732,14 +732,24 @@ match_pattern_list () {
 	arg="$1"
 	shift
 	test -z "$*" && return 1
-	for pattern_
-	do
-		case "$arg" in
-		$pattern_)
-			return 0
-		esac
-	done
-	return 1
+	# We need to use "$*" to get field-splitting, but we want to
+	# disable globbing, since we are matching against an arbitrary
+	# $arg, not what's in the filesystem. Using "set -f" accomplishes
+	# that, but we must do it in a subshell to avoid impacting the
+	# rest of the script. The exit value of the subshell becomes
+	# the function's return value.
+	(
+		set -f
+		for pattern_ in $*
+		do
+			case "$arg" in
+			$pattern_)
+				exit 0
+				;;
+			esac
+		done
+		exit 1
+	)
 }
 
 match_test_selector_list () {
@@ -848,7 +858,7 @@ maybe_teardown_verbose () {
 last_verbose=t
 maybe_setup_verbose () {
 	test -z "$verbose_only" && return
-	if match_pattern_list $test_count $verbose_only
+	if match_pattern_list $test_count "$verbose_only"
 	then
 		exec 4>&2 3>&1
 		# Emit a delimiting blank line when going from
@@ -878,7 +888,7 @@ maybe_setup_valgrind () {
 		return
 	fi
 	GIT_VALGRIND_ENABLED=
-	if match_pattern_list $test_count $valgrind_only
+	if match_pattern_list $test_count "$valgrind_only"
 	then
 		GIT_VALGRIND_ENABLED=t
 	fi
@@ -1006,7 +1016,7 @@ test_finish_ () {
 test_skip () {
 	to_skip=
 	skipped_reason=
-	if match_pattern_list $this_test.$test_count $GIT_SKIP_TESTS
+	if match_pattern_list $this_test.$test_count "$GIT_SKIP_TESTS"
 	then
 		to_skip=t
 		skipped_reason="GIT_SKIP_TESTS"
@@ -1346,7 +1356,7 @@ fi
 remove_trash=
 this_test=${0##*/}
 this_test=${this_test%%-*}
-if match_pattern_list "$this_test" $GIT_SKIP_TESTS
+if match_pattern_list "$this_test" "$GIT_SKIP_TESTS"
 then
 	say_color info >&3 "skipping test $this_test altogether"
 	skip_all="skip all tests in $this_test"

From 3d97ea479fdcb88671105e2f2d04064bab110bd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 1 Jul 2021 12:51:25 +0200
Subject: [PATCH 305/397] *.h: move some *_INIT to designated initializers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move *_INIT macros I'll use in a subsequent commits to designated
initializers. This isn't required for those follow-up changes, but
since next commits will change things in this area, let's use the
modern pattern over the old one while we're at it.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 credential.h  | 4 +++-
 json-writer.h | 5 ++++-
 run-command.h | 5 ++++-
 string-list.h | 4 ++--
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/credential.h b/credential.h
index c0e17e3554..f430e77fea 100644
--- a/credential.h
+++ b/credential.h
@@ -128,7 +128,9 @@ struct credential {
 	char *path;
 };
 
-#define CREDENTIAL_INIT { STRING_LIST_INIT_DUP }
+#define CREDENTIAL_INIT { \
+	.helpers = STRING_LIST_INIT_DUP, \
+}
 
 /* Initialize a credential structure, setting all fields to empty. */
 void credential_init(struct credential *);
diff --git a/json-writer.h b/json-writer.h
index 83906b09c1..209355e0f1 100644
--- a/json-writer.h
+++ b/json-writer.h
@@ -64,7 +64,10 @@ struct json_writer
 	unsigned int pretty:1;
 };
 
-#define JSON_WRITER_INIT { STRBUF_INIT, STRBUF_INIT, 0, 0 }
+#define JSON_WRITER_INIT { \
+	.json = STRBUF_INIT, \
+	.open_stack = STRBUF_INIT, \
+}
 
 void jw_init(struct json_writer *jw);
 void jw_release(struct json_writer *jw);
diff --git a/run-command.h b/run-command.h
index d08414a92e..62a922d23f 100644
--- a/run-command.h
+++ b/run-command.h
@@ -141,7 +141,10 @@ struct child_process {
 	void *clean_on_exit_handler_cbdata;
 };
 
-#define CHILD_PROCESS_INIT { NULL, STRVEC_INIT, STRVEC_INIT }
+#define CHILD_PROCESS_INIT { \
+	.args = STRVEC_INIT, \
+	.env_array = STRVEC_INIT, \
+}
 
 /**
  * The functions: child_process_init, start_command, finish_command,
diff --git a/string-list.h b/string-list.h
index 6c5d274126..521b9c0748 100644
--- a/string-list.h
+++ b/string-list.h
@@ -91,8 +91,8 @@ struct string_list {
 	compare_strings_fn cmp; /* NULL uses strcmp() */
 };
 
-#define STRING_LIST_INIT_NODUP { NULL, 0, 0, 0, NULL }
-#define STRING_LIST_INIT_DUP   { NULL, 0, 0, 1, NULL }
+#define STRING_LIST_INIT_NODUP { 0 }
+#define STRING_LIST_INIT_DUP   { .strdup_strings = 1 }
 
 /* General functions which work with both sorted and unsorted lists. */
 

From 5726a6b4012cd41701927a6637b9f2070e7760ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 1 Jul 2021 12:51:26 +0200
Subject: [PATCH 306/397] *.c *_init(): define in terms of corresponding *_INIT
 macro
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change the common patter in the codebase of duplicating the
initialization logic between an *_INIT macro and a
corresponding *_init() function to use the macro as the canonical
source of truth.

Now we no longer need to keep the function up-to-date with the macro
version. This implements a suggestion by Jeff King who found that
under -O2 [1] modern compilers will init new version in place without
the extra copy[1]. The performance of a single *_init() won't matter
in most cases, but even if it does we're going to be producing
efficient machine code to perform these operations.

1. https://lore.kernel.org/git/YNyrDxUO1PlGJvCn@coredump.intra.peff.net/

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 credential.c  | 4 ++--
 json-writer.c | 6 ++----
 run-command.c | 5 ++---
 strbuf.c      | 4 ++--
 strmap.c      | 3 ++-
 strvec.c      | 5 ++---
 6 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/credential.c b/credential.c
index e5202fbef2..3c05c7c669 100644
--- a/credential.c
+++ b/credential.c
@@ -10,8 +10,8 @@
 
 void credential_init(struct credential *c)
 {
-	memset(c, 0, sizeof(*c));
-	c->helpers.strdup_strings = 1;
+	struct credential blank = CREDENTIAL_INIT;
+	memcpy(c, &blank, sizeof(*c));
 }
 
 void credential_clear(struct credential *c)
diff --git a/json-writer.c b/json-writer.c
index aadb9dbddc..f1cfd8fa8c 100644
--- a/json-writer.c
+++ b/json-writer.c
@@ -3,10 +3,8 @@
 
 void jw_init(struct json_writer *jw)
 {
-	strbuf_init(&jw->json, 0);
-	strbuf_init(&jw->open_stack, 0);
-	jw->need_comma = 0;
-	jw->pretty = 0;
+	struct json_writer blank = JSON_WRITER_INIT;
+	memcpy(jw, &blank, sizeof(*jw));;
 }
 
 void jw_release(struct json_writer *jw)
diff --git a/run-command.c b/run-command.c
index be6bc128cd..8750df16d8 100644
--- a/run-command.c
+++ b/run-command.c
@@ -11,9 +11,8 @@
 
 void child_process_init(struct child_process *child)
 {
-	memset(child, 0, sizeof(*child));
-	strvec_init(&child->args);
-	strvec_init(&child->env_array);
+	struct child_process blank = CHILD_PROCESS_INIT;
+	memcpy(child, &blank, sizeof(*child));
 }
 
 void child_process_clear(struct child_process *child)
diff --git a/strbuf.c b/strbuf.c
index 4df30b4549..c8a5789694 100644
--- a/strbuf.c
+++ b/strbuf.c
@@ -52,8 +52,8 @@ char strbuf_slopbuf[1];
 
 void strbuf_init(struct strbuf *sb, size_t hint)
 {
-	sb->alloc = sb->len = 0;
-	sb->buf = strbuf_slopbuf;
+	struct strbuf blank = STRBUF_INIT;
+	memcpy(sb, &blank, sizeof(*sb));
 	if (hint)
 		strbuf_grow(sb, hint);
 }
diff --git a/strmap.c b/strmap.c
index 4fb9f6100e..ee48635708 100644
--- a/strmap.c
+++ b/strmap.c
@@ -25,7 +25,8 @@ static struct strmap_entry *find_strmap_entry(struct strmap *map,
 
 void strmap_init(struct strmap *map)
 {
-	strmap_init_with_options(map, NULL, 1);
+	struct strmap blank = STRMAP_INIT;
+	memcpy(map, &blank, sizeof(*map));
 }
 
 void strmap_init_with_options(struct strmap *map,
diff --git a/strvec.c b/strvec.c
index 21dce0a7a4..61a76ce6cb 100644
--- a/strvec.c
+++ b/strvec.c
@@ -6,9 +6,8 @@ const char *empty_strvec[] = { NULL };
 
 void strvec_init(struct strvec *array)
 {
-	array->v = empty_strvec;
-	array->nr = 0;
-	array->alloc = 0;
+	struct strvec blank = STRVEC_INIT;
+	memcpy(array, &blank, sizeof(*array));
 }
 
 static void strvec_push_nodup(struct strvec *array, const char *value)

From ce93a4c6127abdf1ad9eacd537edd1c571a18e41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 1 Jul 2021 12:51:27 +0200
Subject: [PATCH 307/397] dir.[ch]: replace dir_init() with DIR_INIT
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove the dir_init() function and replace it with a DIR_INIT
macro. In many cases in the codebase we need to initialize things with
a function for good reasons, e.g. needing to call another function on
initialization. The "dir_init()" function was not one such case, and
could trivially be replaced with a more idiomatic macro initialization
pattern.

The only place where we made use of its use of memset() was in
dir_clear() itself, which resets the contents of an an existing struct
pointer. Let's use the new "memcpy() a 'blank' struct on the stack"
idiom to do that reset.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 builtin/add.c          | 3 +--
 builtin/check-ignore.c | 3 +--
 builtin/clean.c        | 6 ++----
 builtin/grep.c         | 3 +--
 builtin/ls-files.c     | 3 +--
 builtin/stash.c        | 3 +--
 dir.c                  | 9 ++-------
 dir.h                  | 4 ++--
 merge.c                | 3 +--
 wt-status.c            | 3 +--
 10 files changed, 13 insertions(+), 27 deletions(-)

diff --git a/builtin/add.c b/builtin/add.c
index b773b5a499..09e684585d 100644
--- a/builtin/add.c
+++ b/builtin/add.c
@@ -470,7 +470,7 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 {
 	int exit_status = 0;
 	struct pathspec pathspec;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	int flags;
 	int add_new_files;
 	int require_pathspec;
@@ -577,7 +577,6 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 	die_in_unpopulated_submodule(&the_index, prefix);
 	die_path_inside_submodule(&the_index, &pathspec);
 
-	dir_init(&dir);
 	if (add_new_files) {
 		int baselen;
 
diff --git a/builtin/check-ignore.c b/builtin/check-ignore.c
index 81234552b7..2191256965 100644
--- a/builtin/check-ignore.c
+++ b/builtin/check-ignore.c
@@ -153,7 +153,7 @@ static int check_ignore_stdin_paths(struct dir_struct *dir, const char *prefix)
 int cmd_check_ignore(int argc, const char **argv, const char *prefix)
 {
 	int num_ignored;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 
 	git_config(git_default_config, NULL);
 
@@ -182,7 +182,6 @@ int cmd_check_ignore(int argc, const char **argv, const char *prefix)
 	if (!no_index && read_cache() < 0)
 		die(_("index file corrupt"));
 
-	dir_init(&dir);
 	setup_standard_excludes(&dir);
 
 	if (stdin_paths) {
diff --git a/builtin/clean.c b/builtin/clean.c
index 4944cf440b..98a2860409 100644
--- a/builtin/clean.c
+++ b/builtin/clean.c
@@ -641,7 +641,7 @@ static int clean_cmd(void)
 
 static int filter_by_patterns_cmd(void)
 {
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	struct strbuf confirm = STRBUF_INIT;
 	struct strbuf **ignore_list;
 	struct string_list_item *item;
@@ -665,7 +665,6 @@ static int filter_by_patterns_cmd(void)
 		if (!confirm.len)
 			break;
 
-		dir_init(&dir);
 		pl = add_pattern_list(&dir, EXC_CMDL, "manual exclude");
 		ignore_list = strbuf_split_max(&confirm, ' ', 0);
 
@@ -890,7 +889,7 @@ int cmd_clean(int argc, const char **argv, const char *prefix)
 	int ignored_only = 0, config_set = 0, errors = 0, gone = 1;
 	int rm_flags = REMOVE_DIR_KEEP_NESTED_GIT;
 	struct strbuf abs_path = STRBUF_INIT;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	struct pathspec pathspec;
 	struct strbuf buf = STRBUF_INIT;
 	struct string_list exclude_list = STRING_LIST_INIT_NODUP;
@@ -921,7 +920,6 @@ int cmd_clean(int argc, const char **argv, const char *prefix)
 	argc = parse_options(argc, argv, prefix, options, builtin_clean_usage,
 			     0);
 
-	dir_init(&dir);
 	if (!interactive && !dry_run && !force) {
 		if (config_set)
 			die(_("clean.requireForce set to true and neither -i, -n, nor -f given; "
diff --git a/builtin/grep.c b/builtin/grep.c
index ab8822e68f..7d2f8e5adb 100644
--- a/builtin/grep.c
+++ b/builtin/grep.c
@@ -704,10 +704,9 @@ static int grep_objects(struct grep_opt *opt, const struct pathspec *pathspec,
 static int grep_directory(struct grep_opt *opt, const struct pathspec *pathspec,
 			  int exc_std, int use_index)
 {
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	int i, hit = 0;
 
-	dir_init(&dir);
 	if (!use_index)
 		dir.flags |= DIR_NO_GITLINKS;
 	if (exc_std)
diff --git a/builtin/ls-files.c b/builtin/ls-files.c
index 45cc3b23dd..29a26ad8ae 100644
--- a/builtin/ls-files.c
+++ b/builtin/ls-files.c
@@ -608,7 +608,7 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 {
 	int require_work_tree = 0, show_tag = 0, i;
 	char *max_prefix;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	struct pattern_list *pl;
 	struct string_list exclude_list = STRING_LIST_INIT_NODUP;
 	struct option builtin_ls_files_options[] = {
@@ -678,7 +678,6 @@ int cmd_ls_files(int argc, const char **argv, const char *cmd_prefix)
 	if (argc == 2 && !strcmp(argv[1], "-h"))
 		usage_with_options(ls_files_usage, builtin_ls_files_options);
 
-	dir_init(&dir);
 	prefix = cmd_prefix;
 	if (prefix)
 		prefix_len = strlen(prefix);
diff --git a/builtin/stash.c b/builtin/stash.c
index 01066d7085..dc50a4b292 100644
--- a/builtin/stash.c
+++ b/builtin/stash.c
@@ -991,9 +991,8 @@ static int get_untracked_files(const struct pathspec *ps, int include_untracked,
 {
 	int i;
 	int found = 0;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 
-	dir_init(&dir);
 	if (include_untracked != INCLUDE_ALL_FILES)
 		setup_standard_excludes(&dir);
 
diff --git a/dir.c b/dir.c
index ebe5ec046e..313e932459 100644
--- a/dir.c
+++ b/dir.c
@@ -53,12 +53,6 @@ static enum path_treatment read_directory_recursive(struct dir_struct *dir,
 	int check_only, int stop_at_first_file, const struct pathspec *pathspec);
 static int resolve_dtype(int dtype, struct index_state *istate,
 			 const char *path, int len);
-
-void dir_init(struct dir_struct *dir)
-{
-	memset(dir, 0, sizeof(*dir));
-}
-
 struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp)
 {
 	struct dirent *e;
@@ -3105,6 +3099,7 @@ void dir_clear(struct dir_struct *dir)
 	struct exclude_list_group *group;
 	struct pattern_list *pl;
 	struct exclude_stack *stk;
+	struct dir_struct new = DIR_INIT;
 
 	for (i = EXC_CMDL; i <= EXC_FILE; i++) {
 		group = &dir->exclude_list_group[i];
@@ -3132,7 +3127,7 @@ void dir_clear(struct dir_struct *dir)
 	}
 	strbuf_release(&dir->basebuf);
 
-	dir_init(dir);
+	memcpy(dir, &new, sizeof(*dir));
 }
 
 struct ondisk_untracked_cache {
diff --git a/dir.h b/dir.h
index e3db9b9ec6..8d0ddd8f18 100644
--- a/dir.h
+++ b/dir.h
@@ -342,6 +342,8 @@ struct dir_struct {
 	unsigned visited_directories;
 };
 
+#define DIR_INIT { 0 }
+
 struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp);
 
 /*Count the number of slashes for string s*/
@@ -367,8 +369,6 @@ int match_pathspec(struct index_state *istate,
 int report_path_error(const char *ps_matched, const struct pathspec *pathspec);
 int within_depth(const char *name, int namelen, int depth, int max_depth);
 
-void dir_init(struct dir_struct *dir);
-
 int fill_directory(struct dir_struct *dir,
 		   struct index_state *istate,
 		   const struct pathspec *pathspec);
diff --git a/merge.c b/merge.c
index 5fb88af102..6e736881d9 100644
--- a/merge.c
+++ b/merge.c
@@ -53,7 +53,7 @@ int checkout_fast_forward(struct repository *r,
 	struct unpack_trees_options opts;
 	struct tree_desc t[MAX_UNPACK_TREES];
 	int i, nr_trees = 0;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	struct lock_file lock_file = LOCK_INIT;
 
 	refresh_index(r->index, REFRESH_QUIET, NULL, NULL, NULL);
@@ -80,7 +80,6 @@ int checkout_fast_forward(struct repository *r,
 	}
 
 	memset(&opts, 0, sizeof(opts));
-	dir_init(&dir);
 	if (overwrite_ignore) {
 		dir.flags |= DIR_SHOW_IGNORED;
 		setup_standard_excludes(&dir);
diff --git a/wt-status.c b/wt-status.c
index 42b6735716..b5a3e1cc25 100644
--- a/wt-status.c
+++ b/wt-status.c
@@ -699,14 +699,13 @@ static void wt_status_collect_changes_initial(struct wt_status *s)
 static void wt_status_collect_untracked(struct wt_status *s)
 {
 	int i;
-	struct dir_struct dir;
+	struct dir_struct dir = DIR_INIT;
 	uint64_t t_begin = getnanotime();
 	struct index_state *istate = s->repo->index;
 
 	if (!s->show_untracked_files)
 		return;
 
-	dir_init(&dir);
 	if (s->show_untracked_files != SHOW_ALL_UNTRACKED_FILES)
 		dir.flags |=
 			DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES;

From 770fedaf9fb156bd8c18da41770eac0cb63fba63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 1 Jul 2021 12:51:28 +0200
Subject: [PATCH 308/397] string-list.[ch]: add a
 string_list_init_{nodup,dup}()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In order to use the new "memcpy() a 'blank' struct on the stack"
pattern for string_list_init(), and to make the macro initialization
consistent with the function initialization introduce two new
string_list_init_{nodup,dup}() functions. These are like the old
string_list_init() when called with a false and true second argument,
respectively.

I think this not only makes things more consistent, but also easier to
read. I often had to lookup what the ", 0)" or ", 1)" in these
invocations meant, now it's right there in the function name, and
corresponds to the macros.

A subsequent commit will convert existing API users to this pattern,
but as this is a very common API let's leave a compatibility function
in place for later removal. This intermediate state also proves that
the compatibility function works.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 string-list.c | 18 ++++++++++++++++--
 string-list.h | 11 +++++++++--
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/string-list.c b/string-list.c
index a917955fbd..43576ad126 100644
--- a/string-list.c
+++ b/string-list.c
@@ -1,10 +1,24 @@
 #include "cache.h"
 #include "string-list.h"
 
+void string_list_init_nodup(struct string_list *list)
+{
+	struct string_list blank = STRING_LIST_INIT_NODUP;
+	memcpy(list, &blank, sizeof(*list));
+}
+
+void string_list_init_dup(struct string_list *list)
+{
+	struct string_list blank = STRING_LIST_INIT_DUP;
+	memcpy(list, &blank, sizeof(*list));
+}
+
 void string_list_init(struct string_list *list, int strdup_strings)
 {
-	memset(list, 0, sizeof(*list));
-	list->strdup_strings = strdup_strings;
+	if (strdup_strings)
+		string_list_init_dup(list);
+	else
+		string_list_init_nodup(list);
 }
 
 /* if there is no exact match, point to the index where the entry could be
diff --git a/string-list.h b/string-list.h
index 521b9c0748..0d6b469239 100644
--- a/string-list.h
+++ b/string-list.h
@@ -97,8 +97,15 @@ struct string_list {
 /* General functions which work with both sorted and unsorted lists. */
 
 /**
- * Initialize the members of the string_list, set `strdup_strings`
- * member according to the value of the second parameter.
+ * Initialize the members of a string_list pointer in the same way as
+ * the corresponding `STRING_LIST_INIT_NODUP` and
+ * `STRING_LIST_INIT_DUP` macros.
+ */
+void string_list_init_nodup(struct string_list *list);
+void string_list_init_dup(struct string_list *list);
+
+/**
+ * TODO remove: For compatibility with any in-flight older API users
  */
 void string_list_init(struct string_list *list, int strdup_strings);
 

From bc40dfb10a06612813a3f9b733d65af0732208b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 1 Jul 2021 12:51:29 +0200
Subject: [PATCH 309/397] string-list.h users: change to use *_{nodup,dup}()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change all in-tree users of the string_list_init(LIST, BOOL) API to
use string_list_init_{nodup,dup}(LIST) instead.

As noted in the preceding commit let's leave the now-unused
string_list_init() wrapper in-place for any in-flight users, it can be
removed at some later date.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 apply.c               | 6 +++---
 archive.c             | 2 +-
 config.c              | 2 +-
 entry.c               | 4 ++--
 merge-ort.c           | 4 ++--
 merge-recursive.c     | 4 ++--
 refs/packed-backend.c | 2 +-
 transport.c           | 2 +-
 8 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/apply.c b/apply.c
index 853d3ed385..44bc31d6eb 100644
--- a/apply.c
+++ b/apply.c
@@ -101,9 +101,9 @@ int init_apply_state(struct apply_state *state,
 	state->ws_error_action = warn_on_ws_error;
 	state->ws_ignore_action = ignore_ws_none;
 	state->linenr = 1;
-	string_list_init(&state->fn_table, 0);
-	string_list_init(&state->limit_by_name, 0);
-	string_list_init(&state->symlink_changes, 0);
+	string_list_init_nodup(&state->fn_table);
+	string_list_init_nodup(&state->limit_by_name);
+	string_list_init_nodup(&state->symlink_changes);
 	strbuf_init(&state->root, 0);
 
 	git_apply_config();
diff --git a/archive.c b/archive.c
index ff2bb54f62..3c266d1d7b 100644
--- a/archive.c
+++ b/archive.c
@@ -645,7 +645,7 @@ int write_archive(int argc, const char **argv, const char *prefix,
 	args.pretty_ctx = &ctx;
 	args.repo = repo;
 	args.prefix = prefix;
-	string_list_init(&args.extra_files, 1);
+	string_list_init_dup(&args.extra_files);
 	argc = parse_archive_args(argc, argv, &ar, &args, name_hint, remote);
 	if (!startup_info->have_repository) {
 		/*
diff --git a/config.c b/config.c
index f9c400ad30..2edb282b43 100644
--- a/config.c
+++ b/config.c
@@ -2072,7 +2072,7 @@ static int configset_add_value(struct config_set *cs, const char *key, const cha
 		e = xmalloc(sizeof(*e));
 		hashmap_entry_init(&e->ent, strhash(key));
 		e->key = xstrdup(key);
-		string_list_init(&e->value_list, 1);
+		string_list_init_dup(&e->value_list);
 		hashmap_add(&cs->config_hash, &e->ent);
 	}
 	si = string_list_append_nodup(&e->value_list, xstrdup_or_null(value));
diff --git a/entry.c b/entry.c
index 711ee0693c..125fabdbd5 100644
--- a/entry.c
+++ b/entry.c
@@ -143,8 +143,8 @@ void enable_delayed_checkout(struct checkout *state)
 	if (!state->delayed_checkout) {
 		state->delayed_checkout = xmalloc(sizeof(*state->delayed_checkout));
 		state->delayed_checkout->state = CE_CAN_DELAY;
-		string_list_init(&state->delayed_checkout->filters, 0);
-		string_list_init(&state->delayed_checkout->paths, 0);
+		string_list_init_nodup(&state->delayed_checkout->filters);
+		string_list_init_nodup(&state->delayed_checkout->paths);
 	}
 }
 
diff --git a/merge-ort.c b/merge-ort.c
index 4a9ce2a822..ef49a93513 100644
--- a/merge-ort.c
+++ b/merge-ort.c
@@ -1729,7 +1729,7 @@ static void compute_collisions(struct strmap *collisions,
 			free(new_path);
 		} else {
 			CALLOC_ARRAY(collision_info, 1);
-			string_list_init(&collision_info->source_files, 0);
+			string_list_init_nodup(&collision_info->source_files);
 			strmap_put(collisions, new_path, collision_info);
 		}
 		string_list_insert(&collision_info->source_files,
@@ -3689,7 +3689,7 @@ static void merge_start(struct merge_options *opt, struct merge_result *result)
 	 */
 	strmap_init_with_options(&opt->priv->paths, NULL, 0);
 	strmap_init_with_options(&opt->priv->conflicted, NULL, 0);
-	string_list_init(&opt->priv->paths_to_free, 0);
+	string_list_init_nodup(&opt->priv->paths_to_free);
 
 	/*
 	 * keys & strbufs in output will sometimes need to outlive "paths",
diff --git a/merge-recursive.c b/merge-recursive.c
index d146bb116f..be473f854b 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -121,7 +121,7 @@ static void dir_rename_entry_init(struct dir_rename_entry *entry,
 	entry->dir = directory;
 	entry->non_unique_new_dir = 0;
 	strbuf_init(&entry->new_dir, 0);
-	string_list_init(&entry->possible_new_dirs, 0);
+	string_list_init_nodup(&entry->possible_new_dirs);
 }
 
 struct collision_entry {
@@ -3703,7 +3703,7 @@ static int merge_start(struct merge_options *opt, struct tree *head)
 	}
 
 	CALLOC_ARRAY(opt->priv, 1);
-	string_list_init(&opt->priv->df_conflict_file_set, 1);
+	string_list_init_dup(&opt->priv->df_conflict_file_set);
 	return 0;
 }
 
diff --git a/refs/packed-backend.c b/refs/packed-backend.c
index dfecdbc1db..5f50def076 100644
--- a/refs/packed-backend.c
+++ b/refs/packed-backend.c
@@ -1425,7 +1425,7 @@ static int packed_transaction_prepare(struct ref_store *ref_store,
 	 */
 
 	CALLOC_ARRAY(data, 1);
-	string_list_init(&data->updates, 0);
+	string_list_init_nodup(&data->updates);
 
 	transaction->backend_data = data;
 
diff --git a/transport.c b/transport.c
index 6cf3da19eb..7ee11e93e7 100644
--- a/transport.c
+++ b/transport.c
@@ -1050,7 +1050,7 @@ struct transport *transport_get(struct remote *remote, const char *url)
 	struct transport *ret = xcalloc(1, sizeof(*ret));
 
 	ret->progress = isatty(2);
-	string_list_init(&ret->pack_lockfiles, 1);
+	string_list_init_dup(&ret->pack_lockfiles);
 
 	if (!remote)
 		BUG("No remote provided to transport_get()");

From 3663e5904d7c0060f3b51ffe7c7469caeefb51e5 Mon Sep 17 00:00:00 2001
From: Patrick Steinhardt 
Date: Fri, 18 Jun 2021 15:56:08 +0200
Subject: [PATCH 310/397] perf: fix when running with TEST_OUTPUT_DIRECTORY

When the TEST_OUTPUT_DIRECTORY is defined, then all test data will be
written in that directory instead of the default directory located in
"t/". While this works as expected for our normal tests, performance
tests fail to locate and aggregate performance data because they don't
know to handle TEST_OUTPUT_DIRECTORY correctly and always look at the
default location.

Fix the issue by adding a `--results-dir` parameter to "aggregate.perl"
which identifies the directory where results are and by making the "run"
script awake of the TEST_OUTPUT_DIRECTORY variable.

Signed-off-by: Patrick Steinhardt 
Signed-off-by: Junio C Hamano 
---
 t/perf/aggregate.perl |  5 +++--
 t/perf/perf-lib.sh    |  7 +++++--
 t/perf/run            | 25 ++++++++++++++++---------
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/t/perf/aggregate.perl b/t/perf/aggregate.perl
index 14e4cda287..82c0df4553 100755
--- a/t/perf/aggregate.perl
+++ b/t/perf/aggregate.perl
@@ -58,6 +58,7 @@ sub usage {
   Options:
     --codespeed          * Format output for Codespeed
     --reponame      * Send given reponame to codespeed
+    --results-dir   * Directory where test results are located
     --sort-by       * Sort output (only "regression" criteria is supported)
     --subsection    * Use results from given subsection
 
@@ -91,11 +92,13 @@ sub sane_backticks {
 
 my (@dirs, %dirnames, %dirabbrevs, %prefixes, @tests,
     $codespeed, $sortby, $subsection, $reponame);
+my $resultsdir = "test-results";
 
 Getopt::Long::Configure qw/ require_order /;
 
 my $rc = GetOptions("codespeed"     => \$codespeed,
 		    "reponame=s"    => \$reponame,
+		    "results-dir=s" => \$resultsdir,
 		    "sort-by=s"     => \$sortby,
 		    "subsection=s"  => \$subsection);
 usage() unless $rc;
@@ -137,8 +140,6 @@ if (not @tests) {
 	@tests = glob "p????-*.sh";
 }
 
-my $resultsdir = "test-results";
-
 if (! $subsection and
     exists $ENV{GIT_PERF_SUBSECTION} and
     $ENV{GIT_PERF_SUBSECTION} ne "") {
diff --git a/t/perf/perf-lib.sh b/t/perf/perf-lib.sh
index 601d9f67dd..f5ed092ee5 100644
--- a/t/perf/perf-lib.sh
+++ b/t/perf/perf-lib.sh
@@ -45,7 +45,7 @@ export TEST_DIRECTORY TRASH_DIRECTORY GIT_BUILD_DIR GIT_TEST_CMP
 MODERN_GIT=$GIT_BUILD_DIR/bin-wrappers/git
 export MODERN_GIT
 
-perf_results_dir=$TEST_OUTPUT_DIRECTORY/test-results
+perf_results_dir=$TEST_RESULTS_DIR
 test -n "$GIT_PERF_SUBSECTION" && perf_results_dir="$perf_results_dir/$GIT_PERF_SUBSECTION"
 mkdir -p "$perf_results_dir"
 rm -f "$perf_results_dir"/$(basename "$0" .sh).subtests
@@ -253,7 +253,10 @@ test_size () {
 # and does it after running everything)
 test_at_end_hook_ () {
 	if test -z "$GIT_PERF_AGGREGATING_LATER"; then
-		( cd "$TEST_DIRECTORY"/perf && ./aggregate.perl $(basename "$0") )
+		(
+			cd "$TEST_DIRECTORY"/perf &&
+			./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $(basename "$0")
+		)
 	fi
 }
 
diff --git a/t/perf/run b/t/perf/run
index c7b86104e1..d19dec258a 100755
--- a/t/perf/run
+++ b/t/perf/run
@@ -188,10 +188,10 @@ run_subsection () {
 
 	if test -z "$GIT_PERF_SEND_TO_CODESPEED"
 	then
-		./aggregate.perl $codespeed_opt "$@"
+		./aggregate.perl --results-dir="$TEST_RESULTS_DIR" $codespeed_opt "$@"
 	else
-		json_res_file="test-results/$GIT_PERF_SUBSECTION/aggregate.json"
-		./aggregate.perl --codespeed "$@" | tee "$json_res_file"
+		json_res_file=""$TEST_RESULTS_DIR"/$GIT_PERF_SUBSECTION/aggregate.json"
+		./aggregate.perl --results-dir="$TEST_RESULTS_DIR" --codespeed "$@" | tee "$json_res_file"
 		send_data_url="$GIT_PERF_SEND_TO_CODESPEED/result/add/json/"
 		curl -v --request POST --data-urlencode "json=$(cat "$json_res_file")" "$send_data_url"
 	fi
@@ -203,10 +203,17 @@ get_var_from_env_or_config "GIT_PERF_SEND_TO_CODESPEED" "perf" "sendToCodespeed"
 cd "$(dirname $0)"
 . ../../GIT-BUILD-OPTIONS
 
-mkdir -p test-results
-get_subsections "perf" >test-results/run_subsections.names
+if test -n "$TEST_OUTPUT_DIRECTORY"
+then
+	TEST_RESULTS_DIR="$TEST_OUTPUT_DIRECTORY/test-results"
+else
+	TEST_RESULTS_DIR=test-results
+fi
 
-if test $(wc -l "$TEST_RESULTS_DIR"/run_subsections.names
+
+if test $(wc -l <"$TEST_RESULTS_DIR"/run_subsections.names) -eq 0
 then
 	if test -n "$GIT_PERF_SUBSECTION"
 	then
@@ -222,10 +229,10 @@ then
 	)
 elif test -n "$GIT_PERF_SUBSECTION"
 then
-	egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names >/dev/null ||
+	egrep "^$GIT_PERF_SUBSECTION\$" "$TEST_RESULTS_DIR"/run_subsections.names >/dev/null ||
 		die "subsection '$GIT_PERF_SUBSECTION' not found in '$GIT_PERF_CONFIG_FILE'"
 
-	egrep "^$GIT_PERF_SUBSECTION\$" test-results/run_subsections.names | while read -r subsec
+	egrep "^$GIT_PERF_SUBSECTION\$" "$TEST_RESULTS_DIR"/run_subsections.names | while read -r subsec
 	do
 		(
 			GIT_PERF_SUBSECTION="$subsec"
@@ -243,5 +250,5 @@ else
 			echo "======== Run for subsection '$GIT_PERF_SUBSECTION' ========"
 			run_subsection "$@"
 		)
-	done 
Date: Fri, 2 Jul 2021 11:57:30 +0200
Subject: [PATCH 311/397] bundle cmd: stop leaking memory from
 parse_options_cmd_bundle()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fix a memory leak from the prefix_filename() function introduced with
its use in 3b754eedd5 (bundle: use prefix_filename with bundle path,
2017-03-20).

As noted in that commit the leak was intentional as a part of being
sloppy about freeing resources just before we exit, I'm changing this
because I'll be fixing other memory leaks in the bundle API (including
the library version) in subsequent commits. It's easier to reason
about those fixes if valgrind runs cleanly at the end without any
leaks whatsoever.

An earlier version of this change[1] went out of its way to not leak
memory on the die() codepaths here, but doing so will only avoid
reports of potential leaks under heap-only leak trackers such as
valgrind, not the SANITIZE=leak mode.

Avoiding those leaks as well might be useful to enable us to run
cleanly under the likes of valgrind in the future. But for now the
relative verbosity of the resulting code, and the fact that we don't
have some valgrind or SANITIZE=leak mode as part of our CI (it's only
run ad-hoc, see [2]), means we're not worrying about that for now.

1. https://lore.kernel.org/git/87v95vdxrc.fsf@evledraar.gmail.com/
2. https://lore.kernel.org/git/87czsv2idy.fsf@evledraar.gmail.com/

Signed-off-by: Ævar Arnfjörð Bjarmason 
Acked-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 builtin/bundle.c | 62 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 21 deletions(-)

diff --git a/builtin/bundle.c b/builtin/bundle.c
index ea6948110b..15e2bd6196 100644
--- a/builtin/bundle.c
+++ b/builtin/bundle.c
@@ -46,7 +46,7 @@ static int parse_options_cmd_bundle(int argc,
 		const char* prefix,
 		const char * const usagestr[],
 		const struct option options[],
-		const char **bundle_file) {
+		char **bundle_file) {
 	int newargc;
 	newargc = parse_options(argc, argv, NULL, options, usagestr,
 			     PARSE_OPT_STOP_AT_NON_OPTION);
@@ -61,7 +61,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
 	int progress = isatty(STDERR_FILENO);
 	struct strvec pack_opts;
 	int version = -1;
-
+	int ret;
 	struct option options[] = {
 		OPT_SET_INT('q', "quiet", &progress,
 			    N_("do not show progress meter"), 0),
@@ -76,7 +76,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
 			    N_("specify bundle format version")),
 		OPT_END()
 	};
-	const char* bundle_file;
+	char *bundle_file;
 
 	argc = parse_options_cmd_bundle(argc, argv, prefix,
 			builtin_bundle_create_usage, options, &bundle_file);
@@ -94,75 +94,95 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
 
 	if (!startup_info->have_repository)
 		die(_("Need a repository to create a bundle."));
-	return !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts, version);
+	ret = !!create_bundle(the_repository, bundle_file, argc, argv, &pack_opts, version);
+	free(bundle_file);
+	return ret;
 }
 
 static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
 	struct bundle_header header;
 	int bundle_fd = -1;
 	int quiet = 0;
-
+	int ret;
 	struct option options[] = {
 		OPT_BOOL('q', "quiet", &quiet,
 			    N_("do not show bundle details")),
 		OPT_END()
 	};
-	const char* bundle_file;
+	char *bundle_file;
 
 	argc = parse_options_cmd_bundle(argc, argv, prefix,
 			builtin_bundle_verify_usage, options, &bundle_file);
 	/* bundle internals use argv[1] as further parameters */
 
 	memset(&header, 0, sizeof(header));
-	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0)
-		return 1;
+	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) {
+		ret = 1;
+		goto cleanup;
+	}
 	close(bundle_fd);
-	if (verify_bundle(the_repository, &header, !quiet))
-		return 1;
+	if (verify_bundle(the_repository, &header, !quiet)) {
+		ret = 1;
+		goto cleanup;
+	}
+
 	fprintf(stderr, _("%s is okay\n"), bundle_file);
-	return 0;
+	ret = 0;
+cleanup:
+	free(bundle_file);
+	return ret;
 }
 
 static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix) {
 	struct bundle_header header;
 	int bundle_fd = -1;
-
+	int ret;
 	struct option options[] = {
 		OPT_END()
 	};
-	const char* bundle_file;
+	char *bundle_file;
 
 	argc = parse_options_cmd_bundle(argc, argv, prefix,
 			builtin_bundle_list_heads_usage, options, &bundle_file);
 	/* bundle internals use argv[1] as further parameters */
 
 	memset(&header, 0, sizeof(header));
-	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0)
-		return 1;
+	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) {
+		ret = 1;
+		goto cleanup;
+	}
 	close(bundle_fd);
-	return !!list_bundle_refs(&header, argc, argv);
+	ret = !!list_bundle_refs(&header, argc, argv);
+cleanup:
+	free(bundle_file);
+	return ret;
 }
 
 static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix) {
 	struct bundle_header header;
 	int bundle_fd = -1;
-
+	int ret;
 	struct option options[] = {
 		OPT_END()
 	};
-	const char* bundle_file;
+	char *bundle_file;
 
 	argc = parse_options_cmd_bundle(argc, argv, prefix,
 			builtin_bundle_unbundle_usage, options, &bundle_file);
 	/* bundle internals use argv[1] as further parameters */
 
 	memset(&header, 0, sizeof(header));
-	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0)
-		return 1;
+	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) {
+		ret = 1;
+		goto cleanup;
+	}
 	if (!startup_info->have_repository)
 		die(_("Need a repository to unbundle."));
-	return !!unbundle(the_repository, &header, bundle_fd, 0) ||
+	ret = !!unbundle(the_repository, &header, bundle_fd, 0) ||
 		list_bundle_refs(&header, argc, argv);
+cleanup:
+	free(bundle_file);
+	return ret;
 }
 
 int cmd_bundle(int argc, const char **argv, const char *prefix)

From 15e7c7dca66ab7b020316696f54433f76e5e1084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Fri, 2 Jul 2021 11:57:31 +0200
Subject: [PATCH 312/397] bundle.c: use a temporary variable for OIDs and names
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In preparation for moving away from accessing the OID and name via the
"oid" and "name" slots in a subsequent commit, change the code that
accesses it to use named variables. This makes the subsequent change
smaller.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Acked-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 bundle.c    | 26 ++++++++++++++++++--------
 transport.c |  6 ++++--
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/bundle.c b/bundle.c
index 693d619551..7210e5e710 100644
--- a/bundle.c
+++ b/bundle.c
@@ -156,6 +156,9 @@ static int list_refs(struct ref_list *r, int argc, const char **argv)
 	int i;
 
 	for (i = 0; i < r->nr; i++) {
+		struct object_id *oid;
+		const char *name;
+
 		if (argc > 1) {
 			int j;
 			for (j = 1; j < argc; j++)
@@ -164,8 +167,10 @@ static int list_refs(struct ref_list *r, int argc, const char **argv)
 			if (j == argc)
 				continue;
 		}
-		printf("%s %s\n", oid_to_hex(&r->list[i].oid),
-				r->list[i].name);
+
+		oid = &r->list[i].oid;
+		name = r->list[i].name;
+		printf("%s %s\n", oid_to_hex(oid), name);
 	}
 	return 0;
 }
@@ -194,15 +199,17 @@ int verify_bundle(struct repository *r,
 	repo_init_revisions(r, &revs, NULL);
 	for (i = 0; i < p->nr; i++) {
 		struct ref_list_entry *e = p->list + i;
-		struct object *o = parse_object(r, &e->oid);
+		const char *name = e->name;
+		struct object_id *oid = &e->oid;
+		struct object *o = parse_object(r, oid);
 		if (o) {
 			o->flags |= PREREQ_MARK;
-			add_pending_object(&revs, o, e->name);
+			add_pending_object(&revs, o, name);
 			continue;
 		}
 		if (++ret == 1)
 			error("%s", message);
-		error("%s %s", oid_to_hex(&e->oid), e->name);
+		error("%s %s", oid_to_hex(oid), name);
 	}
 	if (revs.pending.nr != p->nr)
 		return ret;
@@ -219,19 +226,22 @@ int verify_bundle(struct repository *r,
 
 	for (i = 0; i < p->nr; i++) {
 		struct ref_list_entry *e = p->list + i;
-		struct object *o = parse_object(r, &e->oid);
+		const char *name = e->name;
+		struct object_id *oid = &e->oid;
+		struct object *o = parse_object(r, oid);
 		assert(o); /* otherwise we'd have returned early */
 		if (o->flags & SHOWN)
 			continue;
 		if (++ret == 1)
 			error("%s", message);
-		error("%s %s", oid_to_hex(&e->oid), e->name);
+		error("%s %s", oid_to_hex(oid), name);
 	}
 
 	/* Clean up objects used, as they will be reused. */
 	for (i = 0; i < p->nr; i++) {
 		struct ref_list_entry *e = p->list + i;
-		commit = lookup_commit_reference_gently(r, &e->oid, 1);
+		struct object_id *oid = &e->oid;
+		commit = lookup_commit_reference_gently(r, oid, 1);
 		if (commit)
 			clear_commit_marks(commit, ALL_REV_FLAGS);
 	}
diff --git a/transport.c b/transport.c
index 50f5830eb6..95c1138e9a 100644
--- a/transport.c
+++ b/transport.c
@@ -148,8 +148,10 @@ static struct ref *get_refs_from_bundle(struct transport *transport,
 
 	for (i = 0; i < data->header.references.nr; i++) {
 		struct ref_list_entry *e = data->header.references.list + i;
-		struct ref *ref = alloc_ref(e->name);
-		oidcpy(&ref->old_oid, &e->oid);
+		const char *name = e->name;
+		struct ref *ref = alloc_ref(name);
+		struct object_id *oid = &e->oid;
+		oidcpy(&ref->old_oid, oid);
 		ref->next = result;
 		result = ref;
 	}

From 10b635b77311badcbb5045b7421e6826c4536613 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Fri, 2 Jul 2021 11:57:32 +0200
Subject: [PATCH 313/397] bundle: remove "ref_list" in favor of string-list.c
 API
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move away from the "struct ref_list" in bundle.c in favor of the
almost identical string-list.c API.

That API fits this use-case perfectly, but did not exist in its
current form when this code was added in 2e0afafebd (Add git-bundle:
move objects and references by archive, 2007-02-22), with hindsight we
could have used the path-list API, which later got renamed to
string-list. See 8fd2cb4069 (Extract helper bits from
c-merge-recursive work, 2006-07-25)

We need to change "name" to "string" and "oid" to "util" to make this
conversion, but other than that the APIs are pretty much identical for
what bundle.c made use of.

Let's also replace the memset(..,0,...) pattern with a more idiomatic
"INIT" macro, and finally add a *_release() function so to free the
allocated memory.

Before this the add_to_ref_list() would leak memory, now e.g. "bundle
list-heads" reports no memory leaks at all under valgrind.

In the bundle_header_init() function we're using a clever trick to
memcpy() what we'd get from the corresponding
BUNDLE_HEADER_INIT. There is a concurrent series to make use of that
pattern more generally, see [1].

1. https://lore.kernel.org/git/cover-0.5-00000000000-20210701T104855Z-avarab@gmail.com/

Signed-off-by: Ævar Arnfjörð Bjarmason 
Acked-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 builtin/bundle.c | 12 +++++------
 bundle.c         | 52 ++++++++++++++++++++++++++----------------------
 bundle.h         | 21 +++++++++----------
 transport.c      |  8 +++++---
 4 files changed, 50 insertions(+), 43 deletions(-)

diff --git a/builtin/bundle.c b/builtin/bundle.c
index 15e2bd6196..053a51bea1 100644
--- a/builtin/bundle.c
+++ b/builtin/bundle.c
@@ -100,7 +100,7 @@ static int cmd_bundle_create(int argc, const char **argv, const char *prefix) {
 }
 
 static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
-	struct bundle_header header;
+	struct bundle_header header = BUNDLE_HEADER_INIT;
 	int bundle_fd = -1;
 	int quiet = 0;
 	int ret;
@@ -115,7 +115,6 @@ static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
 			builtin_bundle_verify_usage, options, &bundle_file);
 	/* bundle internals use argv[1] as further parameters */
 
-	memset(&header, 0, sizeof(header));
 	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) {
 		ret = 1;
 		goto cleanup;
@@ -130,11 +129,12 @@ static int cmd_bundle_verify(int argc, const char **argv, const char *prefix) {
 	ret = 0;
 cleanup:
 	free(bundle_file);
+	bundle_header_release(&header);
 	return ret;
 }
 
 static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix) {
-	struct bundle_header header;
+	struct bundle_header header = BUNDLE_HEADER_INIT;
 	int bundle_fd = -1;
 	int ret;
 	struct option options[] = {
@@ -146,7 +146,6 @@ static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix
 			builtin_bundle_list_heads_usage, options, &bundle_file);
 	/* bundle internals use argv[1] as further parameters */
 
-	memset(&header, 0, sizeof(header));
 	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) {
 		ret = 1;
 		goto cleanup;
@@ -155,11 +154,12 @@ static int cmd_bundle_list_heads(int argc, const char **argv, const char *prefix
 	ret = !!list_bundle_refs(&header, argc, argv);
 cleanup:
 	free(bundle_file);
+	bundle_header_release(&header);
 	return ret;
 }
 
 static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix) {
-	struct bundle_header header;
+	struct bundle_header header = BUNDLE_HEADER_INIT;
 	int bundle_fd = -1;
 	int ret;
 	struct option options[] = {
@@ -171,7 +171,6 @@ static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix)
 			builtin_bundle_unbundle_usage, options, &bundle_file);
 	/* bundle internals use argv[1] as further parameters */
 
-	memset(&header, 0, sizeof(header));
 	if ((bundle_fd = read_bundle_header(bundle_file, &header)) < 0) {
 		ret = 1;
 		goto cleanup;
@@ -180,6 +179,7 @@ static int cmd_bundle_unbundle(int argc, const char **argv, const char *prefix)
 		die(_("Need a repository to unbundle."));
 	ret = !!unbundle(the_repository, &header, bundle_fd, 0) ||
 		list_bundle_refs(&header, argc, argv);
+	bundle_header_release(&header);
 cleanup:
 	free(bundle_file);
 	return ret;
diff --git a/bundle.c b/bundle.c
index 7210e5e710..ab63f40226 100644
--- a/bundle.c
+++ b/bundle.c
@@ -23,13 +23,16 @@ static struct {
 	{ 3, v3_bundle_signature },
 };
 
-static void add_to_ref_list(const struct object_id *oid, const char *name,
-		struct ref_list *list)
+void bundle_header_init(struct bundle_header *header)
 {
-	ALLOC_GROW(list->list, list->nr + 1, list->alloc);
-	oidcpy(&list->list[list->nr].oid, oid);
-	list->list[list->nr].name = xstrdup(name);
-	list->nr++;
+	struct bundle_header blank = BUNDLE_HEADER_INIT;
+	memcpy(header, &blank, sizeof(*header));
+}
+
+void bundle_header_release(struct bundle_header *header)
+{
+	string_list_clear(&header->prerequisites, 1);
+	string_list_clear(&header->references, 1);
 }
 
 static int parse_capability(struct bundle_header *header, const char *capability)
@@ -112,10 +115,11 @@ static int parse_bundle_header(int fd, struct bundle_header *header,
 			status = -1;
 			break;
 		} else {
+			struct object_id *dup = oiddup(&oid);
 			if (is_prereq)
-				add_to_ref_list(&oid, "", &header->prerequisites);
+				string_list_append(&header->prerequisites, "")->util = dup;
 			else
-				add_to_ref_list(&oid, p + 1, &header->references);
+				string_list_append(&header->references, p + 1)->util = dup;
 		}
 	}
 
@@ -139,19 +143,19 @@ int read_bundle_header(const char *path, struct bundle_header *header)
 
 int is_bundle(const char *path, int quiet)
 {
-	struct bundle_header header;
+	struct bundle_header header = BUNDLE_HEADER_INIT;
 	int fd = open(path, O_RDONLY);
 
 	if (fd < 0)
 		return 0;
-	memset(&header, 0, sizeof(header));
 	fd = parse_bundle_header(fd, &header, quiet ? NULL : path);
 	if (fd >= 0)
 		close(fd);
+	bundle_header_release(&header);
 	return (fd >= 0);
 }
 
-static int list_refs(struct ref_list *r, int argc, const char **argv)
+static int list_refs(struct string_list *r, int argc, const char **argv)
 {
 	int i;
 
@@ -162,14 +166,14 @@ static int list_refs(struct ref_list *r, int argc, const char **argv)
 		if (argc > 1) {
 			int j;
 			for (j = 1; j < argc; j++)
-				if (!strcmp(r->list[i].name, argv[j]))
+				if (!strcmp(r->items[i].string, argv[j]))
 					break;
 			if (j == argc)
 				continue;
 		}
 
-		oid = &r->list[i].oid;
-		name = r->list[i].name;
+		oid = r->items[i].util;
+		name = r->items[i].string;
 		printf("%s %s\n", oid_to_hex(oid), name);
 	}
 	return 0;
@@ -186,7 +190,7 @@ int verify_bundle(struct repository *r,
 	 * Do fast check, then if any prereqs are missing then go line by line
 	 * to be verbose about the errors
 	 */
-	struct ref_list *p = &header->prerequisites;
+	struct string_list *p = &header->prerequisites;
 	struct rev_info revs;
 	const char *argv[] = {NULL, "--all", NULL};
 	struct commit *commit;
@@ -198,9 +202,9 @@ int verify_bundle(struct repository *r,
 
 	repo_init_revisions(r, &revs, NULL);
 	for (i = 0; i < p->nr; i++) {
-		struct ref_list_entry *e = p->list + i;
-		const char *name = e->name;
-		struct object_id *oid = &e->oid;
+		struct string_list_item *e = p->items + i;
+		const char *name = e->string;
+		struct object_id *oid = e->util;
 		struct object *o = parse_object(r, oid);
 		if (o) {
 			o->flags |= PREREQ_MARK;
@@ -225,9 +229,9 @@ int verify_bundle(struct repository *r,
 			i--;
 
 	for (i = 0; i < p->nr; i++) {
-		struct ref_list_entry *e = p->list + i;
-		const char *name = e->name;
-		struct object_id *oid = &e->oid;
+		struct string_list_item *e = p->items + i;
+		const char *name = e->string;
+		const struct object_id *oid = e->util;
 		struct object *o = parse_object(r, oid);
 		assert(o); /* otherwise we'd have returned early */
 		if (o->flags & SHOWN)
@@ -239,15 +243,15 @@ int verify_bundle(struct repository *r,
 
 	/* Clean up objects used, as they will be reused. */
 	for (i = 0; i < p->nr; i++) {
-		struct ref_list_entry *e = p->list + i;
-		struct object_id *oid = &e->oid;
+		struct string_list_item *e = p->items + i;
+		struct object_id *oid = e->util;
 		commit = lookup_commit_reference_gently(r, oid, 1);
 		if (commit)
 			clear_commit_marks(commit, ALL_REV_FLAGS);
 	}
 
 	if (verbose) {
-		struct ref_list *r;
+		struct string_list *r;
 
 		r = &header->references;
 		printf_ln(Q_("The bundle contains this ref:",
diff --git a/bundle.h b/bundle.h
index f9e2d1c8ef..1927d8cd6a 100644
--- a/bundle.h
+++ b/bundle.h
@@ -3,22 +3,23 @@
 
 #include "strvec.h"
 #include "cache.h"
-
-struct ref_list {
-	unsigned int nr, alloc;
-	struct ref_list_entry {
-		struct object_id oid;
-		char *name;
-	} *list;
-};
+#include "string-list.h"
 
 struct bundle_header {
 	unsigned version;
-	struct ref_list prerequisites;
-	struct ref_list references;
+	struct string_list prerequisites;
+	struct string_list references;
 	const struct git_hash_algo *hash_algo;
 };
 
+#define BUNDLE_HEADER_INIT \
+{ \
+	.prerequisites = STRING_LIST_INIT_DUP, \
+	.references = STRING_LIST_INIT_DUP, \
+}
+void bundle_header_init(struct bundle_header *header);
+void bundle_header_release(struct bundle_header *header);
+
 int is_bundle(const char *path, int quiet);
 int read_bundle_header(const char *path, struct bundle_header *header);
 int create_bundle(struct repository *r, const char *path,
diff --git a/transport.c b/transport.c
index 95c1138e9a..745ffa2247 100644
--- a/transport.c
+++ b/transport.c
@@ -147,10 +147,10 @@ static struct ref *get_refs_from_bundle(struct transport *transport,
 	transport->hash_algo = data->header.hash_algo;
 
 	for (i = 0; i < data->header.references.nr; i++) {
-		struct ref_list_entry *e = data->header.references.list + i;
-		const char *name = e->name;
+		struct string_list_item *e = data->header.references.items + i;
+		const char *name = e->string;
 		struct ref *ref = alloc_ref(name);
-		struct object_id *oid = &e->oid;
+		struct object_id *oid = e->util;
 		oidcpy(&ref->old_oid, oid);
 		ref->next = result;
 		result = ref;
@@ -177,6 +177,7 @@ static int close_bundle(struct transport *transport)
 	struct bundle_transport_data *data = transport->data;
 	if (data->fd > 0)
 		close(data->fd);
+	bundle_header_release(&data->header);
 	free(data);
 	return 0;
 }
@@ -1083,6 +1084,7 @@ struct transport *transport_get(struct remote *remote, const char *url)
 		die(_("git-over-rsync is no longer supported"));
 	} else if (url_is_local_not_ssh(url) && is_file(url) && is_bundle(url, 1)) {
 		struct bundle_transport_data *data = xcalloc(1, sizeof(*data));
+		bundle_header_init(&data->header);
 		transport_check_allowed("file");
 		ret->data = data;
 		ret->vtable = &bundle_vtable;

From d681d0dc3a77016caa7e26abfe734afbdab44de5 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Sun, 4 Jul 2021 22:55:11 +0000
Subject: [PATCH 314/397] ci (windows): transfer also the Git-tracked files to
 the test jobs

Git's test suite is excruciatingly slow on Windows, mainly due to the
fact that it executes a lot of shell script code, and that's simply not
native to Windows.

To help with that, we established the pattern where the artifacts are
first built in one job, and then multiple test jobs run in parallel
using the artifacts built in the first job.

We take pains in transferring only the build outputs, and letting
`actions/checkout` fill in the rest of the files.

One major downside of that strategy is that the test jobs might fail to
check out the intended revision (e.g. because the branch has been
updated while the build was running, as is frequently the case with the
`seen` branch).

Let's transfer also the files tracked by Git, and skip the checkout step
in the test jobs.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a399114c0f..0f7516c9ef 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -89,7 +89,9 @@ jobs:
         HOME: ${{runner.workspace}}
         NO_PERL: 1
       run: ci/make-test-artifacts.sh artifacts
-    - name: upload build artifacts
+    - name: zip up tracked files
+      run: git archive -o artifacts/tracked.tar.gz HEAD
+    - name: upload tracked files and build artifacts
       uses: actions/upload-artifact@v2
       with:
         name: windows-artifacts
@@ -102,15 +104,14 @@ jobs:
       matrix:
         nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     steps:
-    - uses: actions/checkout@v1
-    - name: download build artifacts
+    - name: download tracked files and build artifacts
       uses: actions/download-artifact@v2
       with:
         name: windows-artifacts
         path: ${{github.workspace}}
-    - name: extract build artifacts
+    - name: extract tracked files and build artifacts
       shell: bash
-      run: tar xf artifacts.tar.gz
+      run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: test
       shell: bash
@@ -169,7 +170,9 @@ jobs:
       run: |
         mkdir -p artifacts &&
         eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)"
-    - name: upload build artifacts
+    - name: zip up tracked files
+      run: git archive -o artifacts/tracked.tar.gz HEAD
+    - name: upload tracked files and build artifacts
       uses: actions/upload-artifact@v2
       with:
         name: vs-artifacts
@@ -182,16 +185,15 @@ jobs:
       matrix:
         nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
     steps:
-    - uses: actions/checkout@v1
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
-    - name: download build artifacts
+    - name: download tracked files and build artifacts
       uses: actions/download-artifact@v2
       with:
         name: vs-artifacts
         path: ${{github.workspace}}
-    - name: extract build artifacts
+    - name: extract tracked files and build artifacts
       shell: bash
-      run: tar xf artifacts.tar.gz
+      run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz
     - name: test
       shell: bash
       env:

From 434882405931bf240d9ad0c08549d651fbf7daf0 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Sun, 4 Jul 2021 22:55:12 +0000
Subject: [PATCH 315/397] artifacts-tar: respect NO_GETTEXT
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We obviously do not want to bundle `.mo` files during `make
artifacts-tar NO_GETTEXT=Yep`, but that was the case.

To fix that, go a step beyond just fixing the symptom, and simply
define the lists of `.po` and `.mo` files as empty if `NO_GETTEXT` is
set.

Helped-by: Matthias Aßhauer 
Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c3565fc0f8..04e852be01 100644
--- a/Makefile
+++ b/Makefile
@@ -2675,10 +2675,13 @@ po/git.pot: $(GENERATED_H) FORCE
 .PHONY: pot
 pot: po/git.pot
 
+ifdef NO_GETTEXT
+POFILES :=
+MOFILES :=
+else
 POFILES := $(wildcard po/*.po)
 MOFILES := $(patsubst po/%.po,po/build/locale/%/LC_MESSAGES/git.mo,$(POFILES))
 
-ifndef NO_GETTEXT
 all:: $(MOFILES)
 endif
 

From 9ab0b6612918b833bdec775f7c9bf22c4c4ddd07 Mon Sep 17 00:00:00 2001
From: Dennis Ameling 
Date: Sun, 4 Jul 2021 22:55:13 +0000
Subject: [PATCH 316/397] ci (vs-build): build with NO_GETTEXT
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We already build Git for Windows with `NO_GETTEXT` when compiling with
GCC. Let's do the same with Visual C, too.

Note that we do not technically _need_ to pass `NO_GETTEXT` explicitly
in that `make artifacts-tar` invocation because we do this while `MSVC`
is set (which will set `uname_S := Windows`, which in turn will set
`NO_GETTEXT = YesPlease`). But it is definitely nicer to be explicit
here.

Signed-off-by: Dennis Ameling 
Helped-by: Matthias Aßhauer 
Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0f7516c9ef..c99628681e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -159,7 +159,7 @@ jobs:
       shell: bash
       run: |
         cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \
-        -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON
+        -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON
     - name: MSBuild
       run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142
     - name: bundle artifact tar
@@ -169,7 +169,7 @@ jobs:
         VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg
       run: |
         mkdir -p artifacts &&
-        eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)"
+        eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)"
     - name: zip up tracked files
       run: git archive -o artifacts/tracked.tar.gz HEAD
     - name: upload tracked files and build artifacts

From 0dc787a9f23f78902b8792e44ff8187398e49de8 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Sun, 4 Jul 2021 22:55:14 +0000
Subject: [PATCH 317/397] ci: accelerate the checkout

By upgrading from v1 to v2 of `actions/checkout`, we avoid fetching all
the tags and the complete history: v2 only fetches one revision by
default. This should make things a lot faster.

Note that `actions/checkout@v2` seems to be incompatible with running in
containers: https://github.com/actions/checkout/issues/151. Therefore,
we stick with v1 there.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c99628681e..e6f99e29a3 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -81,7 +81,7 @@ jobs:
     if: needs.ci-config.outputs.enabled == 'yes'
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: build
       shell: bash
@@ -134,7 +134,7 @@ jobs:
       GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'"
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - uses: git-for-windows/setup-git-for-windows-sdk@v1
     - name: initialize vcpkg
       uses: actions/checkout@v2
@@ -237,7 +237,7 @@ jobs:
       jobname: ${{matrix.vector.jobname}}
     runs-on: ${{matrix.vector.pool}}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
     - run: ci/run-build-and-tests.sh
     - run: ci/print-test-failures.sh
@@ -282,7 +282,7 @@ jobs:
       jobname: StaticAnalysis
     runs-on: ubuntu-18.04
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
     - run: ci/run-static-analysis.sh
   documentation:
@@ -292,6 +292,6 @@ jobs:
       jobname: Documentation
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - run: ci/install-dependencies.sh
     - run: ci/test-documentation.sh

From cdff1bb5a3d6f0e6bf26d8ff088a0e10b40bc4f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?=
 
Date: Sun, 4 Jul 2021 12:46:10 +0700
Subject: [PATCH 318/397] test-lib-functions: introduce test_stdout_line_count
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In some tests, we're checking the number of lines in output of some
commands, including but not limited to Git's command.

We're doing the check by running those commands in the left side of
a pipe, thus losing the exit status code of those commands. Meanwhile,
we really want to check the exit status code of Git's command.

Let's write the output of those commands to a temporary file, and use
test_line_count separately in order to check exit status code of
those commands properly.

Signed-off-by: Đoàn Trần Công Danh 
Signed-off-by: Junio C Hamano 
---
 t/test-lib-functions.sh | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh
index f0448daa74..4345027c76 100644
--- a/t/test-lib-functions.sh
+++ b/t/test-lib-functions.sh
@@ -845,6 +845,32 @@ test_line_count () {
 	fi
 }
 
+# SYNOPSIS:
+# 	test_stdout_line_count    [...]
+#
+# test_stdout_line_count checks that the output of a command has the number
+# of lines it ought to. For example:
+#
+# test_stdout_line_count = 3 git ls-files -u
+# test_stdout_line_count -gt 10 ls
+test_stdout_line_count () {
+	local ops val trashdir &&
+	if test "$#" -le 3
+	then
+		BUG "expect 3 or more arguments"
+	fi &&
+	ops="$1" &&
+	val="$2" &&
+	shift 2 &&
+	if ! trashdir="$(git rev-parse --git-dir)/trash"; then
+		BUG "expect to be run inside a worktree"
+	fi &&
+	mkdir -p "$trashdir" &&
+	"$@" >"$trashdir/output" &&
+	test_line_count "$ops" "$val" "$trashdir/output"
+}
+
+
 test_file_size () {
 	test "$#" -ne 1 && BUG "1 param"
 	test-tool path-utils file-size "$1"

From 66c9562013de1d8fc04fb46a75fb88122df303c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?=
 
Date: Sun, 4 Jul 2021 12:46:11 +0700
Subject: [PATCH 319/397] t6400: preserve git ls-files exit status code
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In t6400, we're checking number of files in the index and the working
tree by piping the output of "git ls-files" to "wc -l", thus losing the
exit status code of git.

Let's use the newly introduced test_stdout_line_count in order to check
the exit status code of Git's command.

Signed-off-by: Đoàn Trần Công Danh 
Signed-off-by: Junio C Hamano 
---
 t/t6400-merge-df.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/t/t6400-merge-df.sh b/t/t6400-merge-df.sh
index 38700d29b5..57a67cf362 100755
--- a/t/t6400-merge-df.sh
+++ b/t/t6400-merge-df.sh
@@ -82,13 +82,13 @@ test_expect_success 'modify/delete + directory/file conflict' '
 	git checkout delete^0 &&
 	test_must_fail git merge modify &&
 
-	test 5 -eq $(git ls-files -s | wc -l) &&
-	test 4 -eq $(git ls-files -u | wc -l) &&
+	test_stdout_line_count = 5 git ls-files -s &&
+	test_stdout_line_count = 4 git ls-files -u &&
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 0 -eq $(git ls-files -o | wc -l)
+		test_stdout_line_count = 0 git ls-files -o
 	else
-		test 1 -eq $(git ls-files -o | wc -l)
+		test_stdout_line_count = 1 git ls-files -o
 	fi &&
 
 	test_path_is_file letters/file &&
@@ -103,13 +103,13 @@ test_expect_success 'modify/delete + directory/file conflict; other way' '
 
 	test_must_fail git merge delete &&
 
-	test 5 -eq $(git ls-files -s | wc -l) &&
-	test 4 -eq $(git ls-files -u | wc -l) &&
+	test_stdout_line_count = 5 git ls-files -s &&
+	test_stdout_line_count = 4 git ls-files -u &&
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 0 -eq $(git ls-files -o | wc -l)
+		test_stdout_line_count = 0 git ls-files -o
 	else
-		test 1 -eq $(git ls-files -o | wc -l)
+		test_stdout_line_count = 1 git ls-files -o
 	fi &&
 
 	test_path_is_file letters/file &&

From ba8c2680ec996160ae16750b48e4cc10c537bdd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?=
 
Date: Sun, 4 Jul 2021 12:46:12 +0700
Subject: [PATCH 320/397] t6402: preserve git exit status code
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In t6402, we're checking number of files in the index and the working
tree by piping the output of Git's command to "wc -l", thus losing the
exit status code of git.

Let's use the new helper test_stdout_line_count in order to preserve
Git's exit status code.

Signed-off-by: Đoàn Trần Công Danh 
Signed-off-by: Junio C Hamano 
---
 t/t6402-merge-rename.sh | 132 +++++++++++++++++++---------------------
 1 file changed, 64 insertions(+), 68 deletions(-)

diff --git a/t/t6402-merge-rename.sh b/t/t6402-merge-rename.sh
index 425dad97d5..3da2896e3b 100755
--- a/t/t6402-merge-rename.sh
+++ b/t/t6402-merge-rename.sh
@@ -105,10 +105,8 @@ test_expect_success 'pull renaming branch into unrenaming one' \
 	git show-branch &&
 	test_expect_code 1 git pull . white &&
 	git ls-files -s &&
-	git ls-files -u B >b.stages &&
-	test_line_count = 3 b.stages &&
-	git ls-files -s N >n.stages &&
-	test_line_count = 1 n.stages &&
+	test_stdout_line_count = 3 git ls-files -u B &&
+	test_stdout_line_count = 1 git ls-files -s N &&
 	sed -ne "/^g/{
 	p
 	q
@@ -122,10 +120,8 @@ test_expect_success 'pull renaming branch into another renaming one' \
 	git reset --hard &&
 	git checkout red &&
 	test_expect_code 1 git pull . white &&
-	git ls-files -u B >b.stages &&
-	test_line_count = 3 b.stages &&
-	git ls-files -s N >n.stages &&
-	test_line_count = 1 n.stages &&
+	test_stdout_line_count = 3 git ls-files -u B &&
+	test_stdout_line_count = 1 git ls-files -s N &&
 	sed -ne "/^g/{
 	p
 	q
@@ -138,10 +134,8 @@ test_expect_success 'pull unrenaming branch into renaming one' \
 	git reset --hard &&
 	git show-branch &&
 	test_expect_code 1 git pull . main &&
-	git ls-files -u B >b.stages &&
-	test_line_count = 3 b.stages &&
-	git ls-files -s N >n.stages &&
-	test_line_count = 1 n.stages &&
+	test_stdout_line_count = 3 git ls-files -u B &&
+	test_stdout_line_count = 1 git ls-files -s N &&
 	sed -ne "/^g/{
 	p
 	q
@@ -154,14 +148,10 @@ test_expect_success 'pull conflicting renames' \
 	git reset --hard &&
 	git show-branch &&
 	test_expect_code 1 git pull . blue &&
-	git ls-files -u A >a.stages &&
-	test_line_count = 1 a.stages &&
-	git ls-files -u B >b.stages &&
-	test_line_count = 1 b.stages &&
-	git ls-files -u C >c.stages &&
-	test_line_count = 1 c.stages &&
-	git ls-files -s N >n.stages &&
-	test_line_count = 1 n.stages &&
+	test_stdout_line_count = 1 git ls-files -u A &&
+	test_stdout_line_count = 1 git ls-files -u B &&
+	test_stdout_line_count = 1 git ls-files -u C &&
+	test_stdout_line_count = 1 git ls-files -s N &&
 	sed -ne "/^g/{
 	p
 	q
@@ -330,8 +320,8 @@ test_expect_success 'Rename+D/F conflict; renamed file merges but dir in way' '
 		test_i18ngrep "Adding as dir~HEAD instead" output
 	fi &&
 
-	test 3 -eq "$(git ls-files -u | wc -l)" &&
-	test 2 -eq "$(git ls-files -u dir/file-in-the-way | wc -l)" &&
+	test_stdout_line_count = 3 git ls-files -u &&
+	test_stdout_line_count = 2 git ls-files -u dir/file-in-the-way &&
 
 	test_must_fail git diff --quiet &&
 	test_must_fail git diff --cached --quiet &&
@@ -357,8 +347,8 @@ test_expect_success 'Same as previous, but merged other way' '
 		test_i18ngrep "Adding as dir~renamed-file-has-no-conflicts instead" output
 	fi &&
 
-	test 3 -eq "$(git ls-files -u | wc -l)" &&
-	test 2 -eq "$(git ls-files -u dir/file-in-the-way | wc -l)" &&
+	test_stdout_line_count = 3 git ls-files -u &&
+	test_stdout_line_count = 2 git ls-files -u dir/file-in-the-way &&
 
 	test_must_fail git diff --quiet &&
 	test_must_fail git diff --cached --quiet &&
@@ -374,8 +364,8 @@ test_expect_success 'Rename+D/F conflict; renamed file cannot merge, dir not in
 	git checkout -q renamed-file-has-conflicts^0 &&
 	test_must_fail git merge --strategy=recursive dir-not-in-way &&
 
-	test 3 -eq "$(git ls-files -u | wc -l)" &&
-	test 3 -eq "$(git ls-files -u dir | wc -l)" &&
+	test_stdout_line_count = 3 git ls-files -u &&
+	test_stdout_line_count = 3 git ls-files -u dir &&
 
 	test_must_fail git diff --quiet &&
 	test_must_fail git diff --cached --quiet &&
@@ -409,14 +399,16 @@ test_expect_success 'Rename+D/F conflict; renamed file cannot merge and dir in t
 	git checkout -q renamed-file-has-conflicts^0 &&
 	test_must_fail git merge --strategy=recursive dir-in-way &&
 
-	test 5 -eq "$(git ls-files -u | wc -l)" &&
+	test_stdout_line_count = 5 git ls-files -u &&
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 3 -eq "$(git ls-files -u dir~HEAD | wc -l)"
+		test_stdout_line_count = 3 git ls-files -u dir~HEAD
 	else
-		test 3 -eq "$(git ls-files -u dir | grep -v file-in-the-way | wc -l)"
+		git ls-files -u dir >out &&
+		test 3 -eq $(grep -v file-in-the-way out | wc -l) &&
+		rm -f out
 	fi &&
-	test 2 -eq "$(git ls-files -u dir/file-in-the-way | wc -l)" &&
+	test_stdout_line_count = 2 git ls-files -u dir/file-in-the-way &&
 
 	test_must_fail git diff --quiet &&
 	test_must_fail git diff --cached --quiet &&
@@ -432,14 +424,16 @@ test_expect_success 'Same as previous, but merged other way' '
 	git checkout -q dir-in-way^0 &&
 	test_must_fail git merge --strategy=recursive renamed-file-has-conflicts &&
 
-	test 5 -eq "$(git ls-files -u | wc -l)" &&
+	test_stdout_line_count = 5 git ls-files -u &&
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 3 -eq "$(git ls-files -u dir~renamed-file-has-conflicts | wc -l)"
+		test_stdout_line_count = 3 git ls-files -u dir~renamed-file-has-conflicts
 	else
-		test 3 -eq "$(git ls-files -u dir | grep -v file-in-the-way | wc -l)"
+		git ls-files -u dir >out &&
+		test 3 -eq $(grep -v file-in-the-way out | wc -l) &&
+		rm -f out
 	fi &&
-	test 2 -eq "$(git ls-files -u dir/file-in-the-way | wc -l)" &&
+	test_stdout_line_count = 2 git ls-files -u dir/file-in-the-way &&
 
 	test_must_fail git diff --quiet &&
 	test_must_fail git diff --cached --quiet &&
@@ -496,9 +490,9 @@ test_expect_success 'both rename source and destination involved in D/F conflict
 
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 2 -eq "$(git ls-files -u | wc -l)"
+		test_stdout_line_count = 2 git ls-files -u
 	else
-		test 1 -eq "$(git ls-files -u | wc -l)"
+		test_stdout_line_count = 1 git ls-files -u
 	fi &&
 
 	test_must_fail git diff --quiet &&
@@ -540,9 +534,9 @@ then
 		mkdir one &&
 		test_must_fail git merge --strategy=recursive rename-two &&
 
-		test 4 -eq "$(git ls-files -u | wc -l)" &&
-		test 2 -eq "$(git ls-files -u one | wc -l)" &&
-		test 2 -eq "$(git ls-files -u two | wc -l)" &&
+		test_stdout_line_count = 4 git ls-files -u &&
+		test_stdout_line_count = 2 git ls-files -u one &&
+		test_stdout_line_count = 2 git ls-files -u two &&
 
 		test_must_fail git diff --quiet &&
 
@@ -559,9 +553,9 @@ else
 		mkdir one &&
 		test_must_fail git merge --strategy=recursive rename-two &&
 
-		test 2 -eq "$(git ls-files -u | wc -l)" &&
-		test 1 -eq "$(git ls-files -u one | wc -l)" &&
-		test 1 -eq "$(git ls-files -u two | wc -l)" &&
+		test_stdout_line_count = 2 git ls-files -u &&
+		test_stdout_line_count = 1 git ls-files -u one &&
+		test_stdout_line_count = 1 git ls-files -u two &&
 
 		test_must_fail git diff --quiet &&
 
@@ -582,13 +576,13 @@ test_expect_success 'pair rename to parent of other (D/F conflicts) w/ clean sta
 
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 4 -eq "$(git ls-files -u | wc -l)" &&
-		test 2 -eq "$(git ls-files -u one | wc -l)" &&
-		test 2 -eq "$(git ls-files -u two | wc -l)"
+		test_stdout_line_count = 4 git ls-files -u &&
+		test_stdout_line_count = 2 git ls-files -u one &&
+		test_stdout_line_count = 2 git ls-files -u two
 	else
-		test 2 -eq "$(git ls-files -u | wc -l)" &&
-		test 1 -eq "$(git ls-files -u one | wc -l)" &&
-		test 1 -eq "$(git ls-files -u two | wc -l)"
+		test_stdout_line_count = 2 git ls-files -u &&
+		test_stdout_line_count = 1 git ls-files -u one &&
+		test_stdout_line_count = 1 git ls-files -u two
 	fi &&
 
 	test_must_fail git diff --quiet &&
@@ -631,19 +625,19 @@ test_expect_success 'check handling of differently renamed file with D/F conflic
 
 	if test "$GIT_TEST_MERGE_ALGORITHM" = ort
 	then
-		test 5 -eq "$(git ls-files -s | wc -l)" &&
-		test 3 -eq "$(git ls-files -u | wc -l)" &&
-		test 1 -eq "$(git ls-files -u one~HEAD | wc -l)" &&
-		test 1 -eq "$(git ls-files -u two~second-rename | wc -l)" &&
-		test 1 -eq "$(git ls-files -u original | wc -l)" &&
-		test 0 -eq "$(git ls-files -o | wc -l)"
+		test_stdout_line_count = 5 git ls-files -s &&
+		test_stdout_line_count = 3 git ls-files -u &&
+		test_stdout_line_count = 1 git ls-files -u one~HEAD &&
+		test_stdout_line_count = 1 git ls-files -u two~second-rename &&
+		test_stdout_line_count = 1 git ls-files -u original &&
+		test_stdout_line_count = 0 git ls-files -o
 	else
-		test 5 -eq "$(git ls-files -s | wc -l)" &&
-		test 3 -eq "$(git ls-files -u | wc -l)" &&
-		test 1 -eq "$(git ls-files -u one | wc -l)" &&
-		test 1 -eq "$(git ls-files -u two | wc -l)" &&
-		test 1 -eq "$(git ls-files -u original | wc -l)" &&
-		test 2 -eq "$(git ls-files -o | wc -l)"
+		test_stdout_line_count = 5 git ls-files -s &&
+		test_stdout_line_count = 3 git ls-files -u &&
+		test_stdout_line_count = 1 git ls-files -u one &&
+		test_stdout_line_count = 1 git ls-files -u two &&
+		test_stdout_line_count = 1 git ls-files -u original &&
+		test_stdout_line_count = 2 git ls-files -o
 	fi &&
 
 	test_path_is_file one/file &&
@@ -679,11 +673,11 @@ test_expect_success 'check handling of differently renamed file with D/F conflic
 	git checkout -q first-rename-redo^0 &&
 	test_must_fail git merge --strategy=recursive second-rename-redo &&
 
-	test 3 -eq "$(git ls-files -u | wc -l)" &&
-	test 1 -eq "$(git ls-files -u one | wc -l)" &&
-	test 1 -eq "$(git ls-files -u two | wc -l)" &&
-	test 1 -eq "$(git ls-files -u original | wc -l)" &&
-	test 0 -eq "$(git ls-files -o | wc -l)" &&
+	test_stdout_line_count = 3 git ls-files -u &&
+	test_stdout_line_count = 1 git ls-files -u one &&
+	test_stdout_line_count = 1 git ls-files -u two &&
+	test_stdout_line_count = 1 git ls-files -u original &&
+	test_stdout_line_count = 0 git ls-files -o &&
 
 	test_path_is_file one &&
 	test_path_is_file two &&
@@ -861,9 +855,11 @@ test_expect_success 'setup merge of rename + small change' '
 test_expect_success 'merge rename + small change' '
 	git merge rename_branch &&
 
-	test 1 -eq $(git ls-files -s | wc -l) &&
-	test 0 -eq $(git ls-files -o | wc -l) &&
-	test $(git rev-parse HEAD:renamed_file) = $(git rev-parse HEAD~1:file)
+	test_stdout_line_count = 1 git ls-files -s &&
+	test_stdout_line_count = 0 git ls-files -o &&
+	newhash=$(git rev-parse HEAD:renamed_file) &&
+	oldhash=$(git rev-parse HEAD~1:file) &&
+	test $newhash = $oldhash
 '
 
 test_expect_success 'setup for use of extended merge markers' '

From ae815940f6efde993083e6e7ae655f8b3d16754c Mon Sep 17 00:00:00 2001
From: Han-Wen Nienhuys 
Date: Tue, 6 Jul 2021 18:47:56 +0000
Subject: [PATCH 321/397] t1415: avoid direct filesystem access for writing
 refs

Signed-off-by: Han-Wen Nienhuys 
Signed-off-by: Junio C Hamano 
---
 t/t1415-worktree-refs.sh | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/t/t1415-worktree-refs.sh b/t/t1415-worktree-refs.sh
index 7ab91241ab..011a1a5e5e 100755
--- a/t/t1415-worktree-refs.sh
+++ b/t/t1415-worktree-refs.sh
@@ -37,9 +37,8 @@ test_expect_success 'resolve main-worktree/HEAD' '
 '
 
 test_expect_success 'ambiguous main-worktree/HEAD' '
-	mkdir -p .git/refs/heads/main-worktree &&
-	test_when_finished rm -f .git/refs/heads/main-worktree/HEAD &&
-	cp .git/HEAD .git/refs/heads/main-worktree/HEAD &&
+	test_when_finished git update-ref -d refs/heads/main-worktree/HEAD &&
+	git update-ref refs/heads/main-worktree/HEAD $(git rev-parse HEAD) &&
 	git rev-parse main-worktree/HEAD 2>warn &&
 	grep "main-worktree/HEAD.*ambiguous" warn
 '
@@ -51,9 +50,8 @@ test_expect_success 'resolve worktrees/xx/HEAD' '
 '
 
 test_expect_success 'ambiguous worktrees/xx/HEAD' '
-	mkdir -p .git/refs/heads/worktrees/wt1 &&
-	test_when_finished rm -f .git/refs/heads/worktrees/wt1/HEAD &&
-	cp .git/HEAD .git/refs/heads/worktrees/wt1/HEAD &&
+	git update-ref refs/heads/worktrees/wt1/HEAD $(git rev-parse HEAD) &&
+	test_when_finished git update-ref -d refs/heads/worktrees/wt1/HEAD &&
 	git rev-parse worktrees/wt1/HEAD 2>warn &&
 	grep "worktrees/wt1/HEAD.*ambiguous" warn
 '

From 52a47aea70875cb64a5a975189d6b5ba35b81574 Mon Sep 17 00:00:00 2001
From: Han-Wen Nienhuys 
Date: Tue, 6 Jul 2021 18:47:57 +0000
Subject: [PATCH 322/397] t7509: avoid direct file access for writing
 CHERRY_PICK_HEAD

Signed-off-by: Han-Wen Nienhuys 
Signed-off-by: Junio C Hamano 
---
 t/t7509-commit-authorship.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/t7509-commit-authorship.sh b/t/t7509-commit-authorship.sh
index ee6c47416e..d568593382 100755
--- a/t/t7509-commit-authorship.sh
+++ b/t/t7509-commit-authorship.sh
@@ -147,7 +147,7 @@ test_expect_success 'commit respects CHERRY_PICK_HEAD and MERGE_MSG' '
 	test_tick &&
 	git commit -am "cherry-pick 1" --author="Cherry " &&
 	git tag cherry-pick-head &&
-	git rev-parse cherry-pick-head >.git/CHERRY_PICK_HEAD &&
+	git update-ref CHERRY_PICK_HEAD $(git rev-parse cherry-pick-head) &&
 	echo "This is a MERGE_MSG" >.git/MERGE_MSG &&
 	echo "cherry-pick 1b" >>foo &&
 	test_tick &&
@@ -162,7 +162,7 @@ test_expect_success 'commit respects CHERRY_PICK_HEAD and MERGE_MSG' '
 '
 
 test_expect_success '--reset-author with CHERRY_PICK_HEAD' '
-	git rev-parse cherry-pick-head >.git/CHERRY_PICK_HEAD &&
+	git update-ref CHERRY_PICK_HEAD $(git rev-parse cherry-pick-head) &&
 	echo "cherry-pick 2" >>foo &&
 	test_tick &&
 	git commit -am "cherry-pick 2" --reset-author &&

From 5632e838f8fa73abfce3f66d2781b8e6d7b14001 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ren=C3=A9=20Scharfe?= 
Date: Sat, 3 Jul 2021 14:57:30 +0200
Subject: [PATCH 323/397] khash: clarify that allocations never fail
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We use our standard allocation functions and macros (xcalloc,
ALLOC_ARRAY, REALLOC_ARRAY) in our version of khash.h.  They terminate
the program on error instead, so code that's using them doesn't have to
handle allocation failures.  Make this behavior explicit by turning
kh_resize_ into a void function and removing the related unreachable
error handling code.

Helped-by: Jeff King 
Signed-off-by: René Scharfe 
Acked-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 khash.h | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/khash.h b/khash.h
index 21c2095216..cb79bf8856 100644
--- a/khash.h
+++ b/khash.h
@@ -74,7 +74,7 @@ static const double __ac_HASH_UPPER = 0.77;
 	void kh_destroy_##name(kh_##name##_t *h);					\
 	void kh_clear_##name(kh_##name##_t *h);						\
 	khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
-	int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+	void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
 	khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
 	void kh_del_##name(kh_##name##_t *h, khint_t x);
 
@@ -116,7 +116,7 @@ static const double __ac_HASH_UPPER = 0.77;
 			return __ac_iseither(h->flags, i)? h->n_buckets : i;		\
 		} else return 0;												\
 	}																	\
-	SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+	SCOPE void kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
 	{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
 		khint32_t *new_flags = NULL;										\
 		khint_t j = 1;													\
@@ -126,7 +126,6 @@ static const double __ac_HASH_UPPER = 0.77;
 			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \
 			else { /* hash table size to be changed (shrink or expand); rehash */ \
 				ALLOC_ARRAY(new_flags, __ac_fsize(new_n_buckets)); \
-				if (!new_flags) return -1;								\
 				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
 				if (h->n_buckets < new_n_buckets) {	/* expand */		\
 					REALLOC_ARRAY(h->keys, new_n_buckets); \
@@ -173,18 +172,15 @@ static const double __ac_HASH_UPPER = 0.77;
 			h->n_occupied = h->size;									\
 			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
 		}																\
-		return 0;														\
 	}																	\
 	SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
 	{																	\
 		khint_t x;														\
 		if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
 			if (h->n_buckets > (h->size<<1)) {							\
-				if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
-					*ret = -1; return h->n_buckets;						\
-				}														\
-			} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
-				*ret = -1; return h->n_buckets;							\
+				kh_resize_##name(h, h->n_buckets - 1); /* clear "deleted" elements */ \
+			} else { \
+				kh_resize_##name(h, h->n_buckets + 1); /* expand the hash table */ \
 			}															\
 		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
 		{																\

From d5659f856f3084d33d2379958dd3eefd8a24379f Mon Sep 17 00:00:00 2001
From: Andrei Rybak 
Date: Sun, 4 Jul 2021 17:39:12 +0200
Subject: [PATCH 324/397] help: convert git_cmd to page in one place
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Depending on the chosen format of help pages, git-help uses function
show_man_page, show_info_page, or show_html_page.  The first thing all
three functions do is to convert given `git_cmd` to a `page` using
function cmd_to_page.

Move the common part of these three functions to function cmd_help to
avoid code duplication.

Signed-off-by: Andrei Rybak 
Reviewed-by: Felipe Contreras 
Acked-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 builtin/help.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/builtin/help.c b/builtin/help.c
index bb339f0fc8..b7eec06c3d 100644
--- a/builtin/help.c
+++ b/builtin/help.c
@@ -436,10 +436,9 @@ static void exec_viewer(const char *name, const char *page)
 		warning(_("'%s': unknown man viewer."), name);
 }
 
-static void show_man_page(const char *git_cmd)
+static void show_man_page(const char *page)
 {
 	struct man_viewer_list *viewer;
-	const char *page = cmd_to_page(git_cmd);
 	const char *fallback = getenv("GIT_MAN_VIEWER");
 
 	setup_man_path();
@@ -453,9 +452,8 @@ static void show_man_page(const char *git_cmd)
 	die(_("no man viewer handled the request"));
 }
 
-static void show_info_page(const char *git_cmd)
+static void show_info_page(const char *page)
 {
-	const char *page = cmd_to_page(git_cmd);
 	setenv("INFOPATH", system_path(GIT_INFO_PATH), 1);
 	execlp("info", "info", "gitman", page, (char *)NULL);
 	die(_("no info viewer handled the request"));
@@ -486,9 +484,8 @@ static void open_html(const char *path)
 	execl_git_cmd("web--browse", "-c", "help.browser", path, (char *)NULL);
 }
 
-static void show_html_page(const char *git_cmd)
+static void show_html_page(const char *page)
 {
-	const char *page = cmd_to_page(git_cmd);
 	struct strbuf page_path; /* it leaks but we exec bellow */
 
 	get_html_page_path(&page_path, page);
@@ -548,6 +545,7 @@ int cmd_help(int argc, const char **argv, const char *prefix)
 {
 	int nongit;
 	enum help_format parsed_help_format;
+	const char *page;
 
 	argc = parse_options(argc, argv, prefix, builtin_help_options,
 			builtin_help_usage, 0);
@@ -606,16 +604,17 @@ int cmd_help(int argc, const char **argv, const char *prefix)
 
 	argv[0] = check_git_cmd(argv[0]);
 
+	page = cmd_to_page(argv[0]);
 	switch (help_format) {
 	case HELP_FORMAT_NONE:
 	case HELP_FORMAT_MAN:
-		show_man_page(argv[0]);
+		show_man_page(page);
 		break;
 	case HELP_FORMAT_INFO:
-		show_info_page(argv[0]);
+		show_info_page(page);
 		break;
 	case HELP_FORMAT_WEB:
-		show_html_page(argv[0]);
+		show_html_page(page);
 		break;
 	}
 

From 6a24cc71ed4e7297e3fa1fc8abbd604eaaef4a16 Mon Sep 17 00:00:00 2001
From: Atharva Raykar 
Date: Tue, 6 Jul 2021 23:54:08 +0530
Subject: [PATCH 325/397] submodule--helper: remove redundant include

"dir.h" should have been included only once.

Signed-off-by: Atharva Raykar 
Mentored-by: Christian Couder 
Mentored-by: Shourya Shukla 
Signed-off-by: Junio C Hamano 
---
 builtin/submodule--helper.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c
index 79156fac45..9f7daf5604 100644
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@@ -13,7 +13,6 @@
 #include "remote.h"
 #include "refs.h"
 #include "connect.h"
-#include "dir.h"
 
 static char *get_default_remote(void)
 {

From e04170697a2848189492d53c7b3aab59c32d044f Mon Sep 17 00:00:00 2001
From: Andrew Berry 
Date: Tue, 6 Jul 2021 16:57:12 -0400
Subject: [PATCH 326/397] docs: .gitignore parsing is to the top of the repo

The current documentation reads as if .gitignore files will be parsed in
every parent directory, and not until they reach a repository boundary.
This clarifies the current behaviour.

As well, this corrects 'toplevel' to 'top-level', matching usage for
'top-level domain'.

Signed-off-by: Andrew Berry 
Signed-off-by: Junio C Hamano 
---
 Documentation/gitignore.txt | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Documentation/gitignore.txt b/Documentation/gitignore.txt
index 53e7d5c914..f8a1fc2014 100644
--- a/Documentation/gitignore.txt
+++ b/Documentation/gitignore.txt
@@ -27,12 +27,11 @@ precedence, the last matching pattern decides the outcome):
    them.
 
  * Patterns read from a `.gitignore` file in the same directory
-   as the path, or in any parent directory, with patterns in the
-   higher level files (up to the toplevel of the work tree) being overridden
-   by those in lower level files down to the directory containing the file.
-   These patterns match relative to the location of the
-   `.gitignore` file.  A project normally includes such
-   `.gitignore` files in its repository, containing patterns for
+   as the path, or in any parent directory (up to the top-level of the working
+   tree), with patterns in the higher level files being overridden by those in
+   lower level files down to the directory containing the file. These patterns
+   match relative to the location of the `.gitignore` file.  A project normally
+   includes such `.gitignore` files in its repository, containing patterns for
    files generated as part of the project build.
 
  * Patterns read from `$GIT_DIR/info/exclude`.

From 351bca2d1f814e69740ac0b023bdfb7978b5c215 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Wed, 7 Jul 2021 11:05:24 +0200
Subject: [PATCH 327/397] imap-send.c: use less verbose strbuf_fread() idiom
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When looking for things that hardcoded a non-zero "hint" parameter to
strbuf_fread() I discovered that since f2561fda364 (Add git-imap-send,
derived from isync 1.0.1., 2006-03-10) we've been passing a hardcoded
4096 in imap-send.c to read stdin.

Since we're not doing anything unusual here let's use a less verbose
pattern used in a lot of other places (the hint of "0" will default to
8192). We don't need to take a FILE * here either, so we can use "0"
instead of "stdin". While we're at it improve the error message if we
can't read the input to use error_errno().

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 imap-send.c | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/imap-send.c b/imap-send.c
index bb085d66d1..9d06ef7cd2 100644
--- a/imap-send.c
+++ b/imap-send.c
@@ -1266,18 +1266,6 @@ static void wrap_in_html(struct strbuf *msg)
 	*msg = buf;
 }
 
-#define CHUNKSIZE 0x1000
-
-static int read_message(FILE *f, struct strbuf *all_msgs)
-{
-	do {
-		if (strbuf_fread(all_msgs, CHUNKSIZE, f) <= 0)
-			break;
-	} while (!feof(f));
-
-	return ferror(f) ? -1 : 0;
-}
-
 static int count_messages(struct strbuf *all_msgs)
 {
 	int count = 0;
@@ -1582,8 +1570,8 @@ int cmd_main(int argc, const char **argv)
 	}
 
 	/* read the messages */
-	if (read_message(stdin, &all_msgs)) {
-		fprintf(stderr, "error reading input\n");
+	if (strbuf_read(&all_msgs, 0, 0) < 0) {
+		error_errno(_("could not read from stdin"));
 		return 1;
 	}
 

From cf2dc1c238c6fd5f93c315a3045ccf95459701cd Mon Sep 17 00:00:00 2001
From: Eric Wong 
Date: Wed, 7 Jul 2021 23:10:15 +0000
Subject: [PATCH 328/397] speed up alt_odb_usable() with many alternates

With many alternates, the duplicate check in alt_odb_usable()
wastes many cycles doing repeated fspathcmp() on every existing
alternate.  Use a khash to speed up lookups by odb->path.

Since the kh_put_* API uses the supplied key without
duplicating it, we also take advantage of it to replace both
xstrdup() and strbuf_release() in link_alt_odb_entry() with
strbuf_detach() to avoid the allocation and copy.

In a test repository with 50K alternates and each of those 50K
alternates having one alternate each (for a total of 100K total
alternates); this speeds up lookup of a non-existent blob from
over 16 minutes to roughly 2.7 seconds on my busy workstation.

Note: all underlying git object directories were small and
unpacked with only loose objects and no packs.  Having to load
packs increases times significantly.

Signed-off-by: Eric Wong 
Signed-off-by: Junio C Hamano 
---
 dir.c          | 10 ++++++++++
 dir.h          |  2 ++
 object-file.c  | 35 ++++++++++++++++++++++-------------
 object-store.h |  7 +++++++
 object.c       |  2 ++
 5 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/dir.c b/dir.c
index ebe5ec046e..20b942d161 100644
--- a/dir.c
+++ b/dir.c
@@ -84,11 +84,21 @@ int fspathcmp(const char *a, const char *b)
 	return ignore_case ? strcasecmp(a, b) : strcmp(a, b);
 }
 
+int fspatheq(const char *a, const char *b)
+{
+	return !fspathcmp(a, b);
+}
+
 int fspathncmp(const char *a, const char *b, size_t count)
 {
 	return ignore_case ? strncasecmp(a, b, count) : strncmp(a, b, count);
 }
 
+unsigned int fspathhash(const char *str)
+{
+	return ignore_case ? strihash(str) : strhash(str);
+}
+
 int git_fnmatch(const struct pathspec_item *item,
 		const char *pattern, const char *string,
 		int prefix)
diff --git a/dir.h b/dir.h
index e3db9b9ec6..2af7bcd7e5 100644
--- a/dir.h
+++ b/dir.h
@@ -489,7 +489,9 @@ int remove_dir_recursively(struct strbuf *path, int flag);
 int remove_path(const char *path);
 
 int fspathcmp(const char *a, const char *b);
+int fspatheq(const char *a, const char *b);
 int fspathncmp(const char *a, const char *b, size_t count);
+unsigned int fspathhash(const char *str);
 
 /*
  * The prefix part of pattern must not contains wildcards.
diff --git a/object-file.c b/object-file.c
index f233b440b2..a13f49b192 100644
--- a/object-file.c
+++ b/object-file.c
@@ -517,9 +517,9 @@ const char *loose_object_path(struct repository *r, struct strbuf *buf,
  */
 static int alt_odb_usable(struct raw_object_store *o,
 			  struct strbuf *path,
-			  const char *normalized_objdir)
+			  const char *normalized_objdir, khiter_t *pos)
 {
-	struct object_directory *odb;
+	int r;
 
 	/* Detect cases where alternate disappeared */
 	if (!is_directory(path->buf)) {
@@ -533,14 +533,20 @@ static int alt_odb_usable(struct raw_object_store *o,
 	 * Prevent the common mistake of listing the same
 	 * thing twice, or object directory itself.
 	 */
-	for (odb = o->odb; odb; odb = odb->next) {
-		if (!fspathcmp(path->buf, odb->path))
-			return 0;
-	}
-	if (!fspathcmp(path->buf, normalized_objdir))
-		return 0;
+	if (!o->odb_by_path) {
+		khiter_t p;
 
-	return 1;
+		o->odb_by_path = kh_init_odb_path_map();
+		assert(!o->odb->next);
+		p = kh_put_odb_path_map(o->odb_by_path, o->odb->path, &r);
+		assert(r == 1); /* never used */
+		kh_value(o->odb_by_path, p) = o->odb;
+	}
+	if (fspatheq(path->buf, normalized_objdir))
+		return 0;
+	*pos = kh_put_odb_path_map(o->odb_by_path, path->buf, &r);
+	/* r: 0 = exists, 1 = never used, 2 = deleted */
+	return r == 0 ? 0 : 1;
 }
 
 /*
@@ -566,6 +572,7 @@ static int link_alt_odb_entry(struct repository *r, const char *entry,
 {
 	struct object_directory *ent;
 	struct strbuf pathbuf = STRBUF_INIT;
+	khiter_t pos;
 
 	if (!is_absolute_path(entry) && relative_base) {
 		strbuf_realpath(&pathbuf, relative_base, 1);
@@ -587,23 +594,25 @@ static int link_alt_odb_entry(struct repository *r, const char *entry,
 	while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
 		strbuf_setlen(&pathbuf, pathbuf.len - 1);
 
-	if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir)) {
+	if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir, &pos)) {
 		strbuf_release(&pathbuf);
 		return -1;
 	}
 
 	CALLOC_ARRAY(ent, 1);
-	ent->path = xstrdup(pathbuf.buf);
+	/* pathbuf.buf is already in r->objects->odb_by_path */
+	ent->path = strbuf_detach(&pathbuf, NULL);
 
 	/* add the alternate entry */
 	*r->objects->odb_tail = ent;
 	r->objects->odb_tail = &(ent->next);
 	ent->next = NULL;
+	assert(r->objects->odb_by_path);
+	kh_value(r->objects->odb_by_path, pos) = ent;
 
 	/* recursively add alternates */
-	read_info_alternates(r, pathbuf.buf, depth + 1);
+	read_info_alternates(r, ent->path, depth + 1);
 
-	strbuf_release(&pathbuf);
 	return 0;
 }
 
diff --git a/object-store.h b/object-store.h
index ec32c23dcb..6077065d90 100644
--- a/object-store.h
+++ b/object-store.h
@@ -7,6 +7,8 @@
 #include "oid-array.h"
 #include "strbuf.h"
 #include "thread-utils.h"
+#include "khash.h"
+#include "dir.h"
 
 struct object_directory {
 	struct object_directory *next;
@@ -30,6 +32,9 @@ struct object_directory {
 	char *path;
 };
 
+KHASH_INIT(odb_path_map, const char * /* key: odb_path */,
+	struct object_directory *, 1, fspathhash, fspatheq);
+
 void prepare_alt_odb(struct repository *r);
 char *compute_alternate_path(const char *path, struct strbuf *err);
 typedef int alt_odb_fn(struct object_directory *, void *);
@@ -116,6 +121,8 @@ struct raw_object_store {
 	 */
 	struct object_directory *odb;
 	struct object_directory **odb_tail;
+	kh_odb_path_map_t *odb_by_path;
+
 	int loaded_alternates;
 
 	/*
diff --git a/object.c b/object.c
index 14188453c5..2b3c075a15 100644
--- a/object.c
+++ b/object.c
@@ -511,6 +511,8 @@ static void free_object_directories(struct raw_object_store *o)
 		free_object_directory(o->odb);
 		o->odb = next;
 	}
+	kh_destroy_odb_path_map(o->odb_by_path);
+	o->odb_by_path = NULL;
 }
 
 void raw_object_store_clear(struct raw_object_store *o)

From 407532f82d3fdfd18d4ec276ddeb359e7c724aa6 Mon Sep 17 00:00:00 2001
From: Eric Wong 
Date: Wed, 7 Jul 2021 23:10:16 +0000
Subject: [PATCH 329/397] avoid strlen via strbuf_addstr in link_alt_odb_entry

We can save a few milliseconds (across 100K odbs) by using
strbuf_addbuf() instead of strbuf_addstr() by passing `entry' as
a strbuf pointer rather than a "const char *".

Signed-off-by: Eric Wong 
Signed-off-by: Junio C Hamano 
---
 object-file.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/object-file.c b/object-file.c
index a13f49b192..2dd70ddf3a 100644
--- a/object-file.c
+++ b/object-file.c
@@ -567,18 +567,18 @@ static int alt_odb_usable(struct raw_object_store *o,
 static void read_info_alternates(struct repository *r,
 				 const char *relative_base,
 				 int depth);
-static int link_alt_odb_entry(struct repository *r, const char *entry,
+static int link_alt_odb_entry(struct repository *r, const struct strbuf *entry,
 	const char *relative_base, int depth, const char *normalized_objdir)
 {
 	struct object_directory *ent;
 	struct strbuf pathbuf = STRBUF_INIT;
 	khiter_t pos;
 
-	if (!is_absolute_path(entry) && relative_base) {
+	if (!is_absolute_path(entry->buf) && relative_base) {
 		strbuf_realpath(&pathbuf, relative_base, 1);
 		strbuf_addch(&pathbuf, '/');
 	}
-	strbuf_addstr(&pathbuf, entry);
+	strbuf_addbuf(&pathbuf, entry);
 
 	if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
 		error(_("unable to normalize alternate object path: %s"),
@@ -669,7 +669,7 @@ static void link_alt_odb_entries(struct repository *r, const char *alt,
 		alt = parse_alt_odb_entry(alt, sep, &entry);
 		if (!entry.len)
 			continue;
-		link_alt_odb_entry(r, entry.buf,
+		link_alt_odb_entry(r, &entry,
 				   relative_base, depth, objdirbuf.buf);
 	}
 	strbuf_release(&entry);

From 33f379eee63a0529f85079857598ac6325d3aec5 Mon Sep 17 00:00:00 2001
From: Eric Wong 
Date: Wed, 7 Jul 2021 23:10:17 +0000
Subject: [PATCH 330/397] make object_directory.loose_objects_subdir_seen a
 bitmap

There's no point in using 8 bits per-directory when 1 bit
will do.  This saves us 224 bytes per object directory, which
ends up being 22MB when dealing with 100K alternates.

Signed-off-by: Eric Wong 
Signed-off-by: Junio C Hamano 
---
 object-file.c  | 11 ++++++++---
 object-store.h |  2 +-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/object-file.c b/object-file.c
index 2dd70ddf3a..91ded8c22a 100644
--- a/object-file.c
+++ b/object-file.c
@@ -2461,12 +2461,17 @@ struct oid_array *odb_loose_cache(struct object_directory *odb,
 {
 	int subdir_nr = oid->hash[0];
 	struct strbuf buf = STRBUF_INIT;
+	size_t word_bits = bitsizeof(odb->loose_objects_subdir_seen[0]);
+	size_t word_index = subdir_nr / word_bits;
+	size_t mask = 1 << (subdir_nr % word_bits);
+	uint32_t *bitmap;
 
 	if (subdir_nr < 0 ||
-	    subdir_nr >= ARRAY_SIZE(odb->loose_objects_subdir_seen))
+	    subdir_nr >= bitsizeof(odb->loose_objects_subdir_seen))
 		BUG("subdir_nr out of range");
 
-	if (odb->loose_objects_subdir_seen[subdir_nr])
+	bitmap = &odb->loose_objects_subdir_seen[word_index];
+	if (*bitmap & mask)
 		return &odb->loose_objects_cache[subdir_nr];
 
 	strbuf_addstr(&buf, odb->path);
@@ -2474,7 +2479,7 @@ struct oid_array *odb_loose_cache(struct object_directory *odb,
 				    append_loose_object,
 				    NULL, NULL,
 				    &odb->loose_objects_cache[subdir_nr]);
-	odb->loose_objects_subdir_seen[subdir_nr] = 1;
+	*bitmap |= mask;
 	strbuf_release(&buf);
 	return &odb->loose_objects_cache[subdir_nr];
 }
diff --git a/object-store.h b/object-store.h
index 6077065d90..ab6d469970 100644
--- a/object-store.h
+++ b/object-store.h
@@ -22,7 +22,7 @@ struct object_directory {
 	 *
 	 * Be sure to call odb_load_loose_cache() before using.
 	 */
-	char loose_objects_subdir_seen[256];
+	uint32_t loose_objects_subdir_seen[8]; /* 256 bits */
 	struct oid_array loose_objects_cache[256];
 
 	/*

From 90e07f0a342df836a33b92e179eb105243dba88d Mon Sep 17 00:00:00 2001
From: Eric Wong 
Date: Wed, 7 Jul 2021 23:10:18 +0000
Subject: [PATCH 331/397] oidcpy_with_padding: constify `src' arg

As with `oidcpy', the source struct will not be modified and
this will allow an upcoming const-correct caller to use it.

Signed-off-by: Eric Wong 
Signed-off-by: Junio C Hamano 
---
 hash.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hash.h b/hash.h
index 9c6df4d952..27a180248f 100644
--- a/hash.h
+++ b/hash.h
@@ -265,7 +265,7 @@ static inline void oidcpy(struct object_id *dst, const struct object_id *src)
 
 /* Like oidcpy() but zero-pads the unused bytes in dst's hash array. */
 static inline void oidcpy_with_padding(struct object_id *dst,
-				       struct object_id *src)
+				       const struct object_id *src)
 {
 	size_t hashsz;
 

From 92d8ed8ac101d62183d51f280b90efb1de1bda5c Mon Sep 17 00:00:00 2001
From: Eric Wong 
Date: Wed, 7 Jul 2021 23:10:19 +0000
Subject: [PATCH 332/397] oidtree: a crit-bit tree for odb_loose_cache

This saves 8K per `struct object_directory', meaning it saves
around 800MB in my case involving 100K alternates (half or more
of those alternates are unlikely to hold loose objects).

This is implemented in two parts: a generic, allocation-free
`cbtree' and the `oidtree' wrapper on top of it.  The latter
provides allocation using alloc_state as a memory pool to
improve locality and reduce free(3) overhead.

Unlike oid-array, the crit-bit tree does not require sorting.
Performance is bound by the key length, for oidtree that is
fixed at sizeof(struct object_id).  There's no need to have
256 oidtrees to mitigate the O(n log n) overhead like we did
with oid-array.

Being a prefix trie, it is natively suited for expanding short
object IDs via prefix-limited iteration in
`find_short_object_filename'.

On my busy workstation, p4205 performance seems to be roughly
unchanged (+/-8%).  Startup with 100K total alternates with no
loose objects seems around 10-20% faster on a hot cache.
(800MB in memory savings means more memory for the kernel FS
cache).

The generic cbtree implementation does impose some extra
overhead for oidtree in that it uses memcmp(3) on
"struct object_id" so it wastes cycles comparing 12 extra bytes
on SHA-1 repositories.  I've not yet explored reducing this
overhead, but I expect there are many places in our code base
where we'd want to investigate this.

More information on crit-bit trees: https://cr.yp.to/critbit.html

Signed-off-by: Eric Wong 
Signed-off-by: Junio C Hamano 
---
 Makefile                |   3 +
 cbtree.c                | 167 ++++++++++++++++++++++++++++++++++++++++
 cbtree.h                |  56 ++++++++++++++
 object-file.c           |  23 +++---
 object-name.c           |  28 +++----
 object-store.h          |   5 +-
 oidtree.c               | 104 +++++++++++++++++++++++++
 oidtree.h               |  22 ++++++
 t/helper/test-oidtree.c |  49 ++++++++++++
 t/helper/test-tool.c    |   1 +
 t/helper/test-tool.h    |   1 +
 t/t0069-oidtree.sh      |  49 ++++++++++++
 12 files changed, 478 insertions(+), 30 deletions(-)
 create mode 100644 cbtree.c
 create mode 100644 cbtree.h
 create mode 100644 oidtree.c
 create mode 100644 oidtree.h
 create mode 100644 t/helper/test-oidtree.c
 create mode 100755 t/t0069-oidtree.sh

diff --git a/Makefile b/Makefile
index c3565fc0f8..a1525978fb 100644
--- a/Makefile
+++ b/Makefile
@@ -722,6 +722,7 @@ TEST_BUILTINS_OBJS += test-mergesort.o
 TEST_BUILTINS_OBJS += test-mktemp.o
 TEST_BUILTINS_OBJS += test-oid-array.o
 TEST_BUILTINS_OBJS += test-oidmap.o
+TEST_BUILTINS_OBJS += test-oidtree.o
 TEST_BUILTINS_OBJS += test-online-cpus.o
 TEST_BUILTINS_OBJS += test-parse-options.o
 TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
@@ -845,6 +846,7 @@ LIB_OBJS += branch.o
 LIB_OBJS += bulk-checkin.o
 LIB_OBJS += bundle.o
 LIB_OBJS += cache-tree.o
+LIB_OBJS += cbtree.o
 LIB_OBJS += chdir-notify.o
 LIB_OBJS += checkout.o
 LIB_OBJS += chunk-format.o
@@ -940,6 +942,7 @@ LIB_OBJS += object.o
 LIB_OBJS += oid-array.o
 LIB_OBJS += oidmap.o
 LIB_OBJS += oidset.o
+LIB_OBJS += oidtree.o
 LIB_OBJS += pack-bitmap-write.o
 LIB_OBJS += pack-bitmap.o
 LIB_OBJS += pack-check.o
diff --git a/cbtree.c b/cbtree.c
new file mode 100644
index 0000000000..b0c65d810f
--- /dev/null
+++ b/cbtree.c
@@ -0,0 +1,167 @@
+/*
+ * crit-bit tree implementation, does no allocations internally
+ * For more information on crit-bit trees: https://cr.yp.to/critbit.html
+ * Based on Adam Langley's adaptation of Dan Bernstein's public domain code
+ * git clone https://github.com/agl/critbit.git
+ */
+#include "cbtree.h"
+
+static struct cb_node *cb_node_of(const void *p)
+{
+	return (struct cb_node *)((uintptr_t)p - 1);
+}
+
+/* locate the best match, does not do a final comparision */
+static struct cb_node *cb_internal_best_match(struct cb_node *p,
+					const uint8_t *k, size_t klen)
+{
+	while (1 & (uintptr_t)p) {
+		struct cb_node *q = cb_node_of(p);
+		uint8_t c = q->byte < klen ? k[q->byte] : 0;
+		size_t direction = (1 + (q->otherbits | c)) >> 8;
+
+		p = q->child[direction];
+	}
+	return p;
+}
+
+/* returns NULL if successful, existing cb_node if duplicate */
+struct cb_node *cb_insert(struct cb_tree *t, struct cb_node *node, size_t klen)
+{
+	size_t newbyte, newotherbits;
+	uint8_t c;
+	int newdirection;
+	struct cb_node **wherep, *p;
+
+	assert(!((uintptr_t)node & 1)); /* allocations must be aligned */
+
+	if (!t->root) {		/* insert into empty tree */
+		t->root = node;
+		return NULL;	/* success */
+	}
+
+	/* see if a node already exists */
+	p = cb_internal_best_match(t->root, node->k, klen);
+
+	/* find first differing byte */
+	for (newbyte = 0; newbyte < klen; newbyte++) {
+		if (p->k[newbyte] != node->k[newbyte])
+			goto different_byte_found;
+	}
+	return p;	/* element exists, let user deal with it */
+
+different_byte_found:
+	newotherbits = p->k[newbyte] ^ node->k[newbyte];
+	newotherbits |= newotherbits >> 1;
+	newotherbits |= newotherbits >> 2;
+	newotherbits |= newotherbits >> 4;
+	newotherbits = (newotherbits & ~(newotherbits >> 1)) ^ 255;
+	c = p->k[newbyte];
+	newdirection = (1 + (newotherbits | c)) >> 8;
+
+	node->byte = newbyte;
+	node->otherbits = newotherbits;
+	node->child[1 - newdirection] = node;
+
+	/* find a place to insert it */
+	wherep = &t->root;
+	for (;;) {
+		struct cb_node *q;
+		size_t direction;
+
+		p = *wherep;
+		if (!(1 & (uintptr_t)p))
+			break;
+		q = cb_node_of(p);
+		if (q->byte > newbyte)
+			break;
+		if (q->byte == newbyte && q->otherbits > newotherbits)
+			break;
+		c = q->byte < klen ? node->k[q->byte] : 0;
+		direction = (1 + (q->otherbits | c)) >> 8;
+		wherep = q->child + direction;
+	}
+
+	node->child[newdirection] = *wherep;
+	*wherep = (struct cb_node *)(1 + (uintptr_t)node);
+
+	return NULL; /* success */
+}
+
+struct cb_node *cb_lookup(struct cb_tree *t, const uint8_t *k, size_t klen)
+{
+	struct cb_node *p = cb_internal_best_match(t->root, k, klen);
+
+	return p && !memcmp(p->k, k, klen) ? p : NULL;
+}
+
+struct cb_node *cb_unlink(struct cb_tree *t, const uint8_t *k, size_t klen)
+{
+	struct cb_node **wherep = &t->root;
+	struct cb_node **whereq = NULL;
+	struct cb_node *q = NULL;
+	size_t direction = 0;
+	uint8_t c;
+	struct cb_node *p = t->root;
+
+	if (!p) return NULL;	/* empty tree, nothing to delete */
+
+	/* traverse to find best match, keeping link to parent */
+	while (1 & (uintptr_t)p) {
+		whereq = wherep;
+		q = cb_node_of(p);
+		c = q->byte < klen ? k[q->byte] : 0;
+		direction = (1 + (q->otherbits | c)) >> 8;
+		wherep = q->child + direction;
+		p = *wherep;
+	}
+
+	if (memcmp(p->k, k, klen))
+		return NULL;		/* no match, nothing unlinked */
+
+	/* found an exact match */
+	if (whereq)	/* update parent */
+		*whereq = q->child[1 - direction];
+	else
+		t->root = NULL;
+	return p;
+}
+
+static enum cb_next cb_descend(struct cb_node *p, cb_iter fn, void *arg)
+{
+	if (1 & (uintptr_t)p) {
+		struct cb_node *q = cb_node_of(p);
+		enum cb_next n = cb_descend(q->child[0], fn, arg);
+
+		return n == CB_BREAK ? n : cb_descend(q->child[1], fn, arg);
+	} else {
+		return fn(p, arg);
+	}
+}
+
+void cb_each(struct cb_tree *t, const uint8_t *kpfx, size_t klen,
+			cb_iter fn, void *arg)
+{
+	struct cb_node *p = t->root;
+	struct cb_node *top = p;
+	size_t i = 0;
+
+	if (!p) return; /* empty tree */
+
+	/* Walk tree, maintaining top pointer */
+	while (1 & (uintptr_t)p) {
+		struct cb_node *q = cb_node_of(p);
+		uint8_t c = q->byte < klen ? kpfx[q->byte] : 0;
+		size_t direction = (1 + (q->otherbits | c)) >> 8;
+
+		p = q->child[direction];
+		if (q->byte < klen)
+			top = p;
+	}
+
+	for (i = 0; i < klen; i++) {
+		if (p->k[i] != kpfx[i])
+			return; /* "best" match failed */
+	}
+	cb_descend(top, fn, arg);
+}
diff --git a/cbtree.h b/cbtree.h
new file mode 100644
index 0000000000..fe4587087e
--- /dev/null
+++ b/cbtree.h
@@ -0,0 +1,56 @@
+/*
+ * crit-bit tree implementation, does no allocations internally
+ * For more information on crit-bit trees: https://cr.yp.to/critbit.html
+ * Based on Adam Langley's adaptation of Dan Bernstein's public domain code
+ * git clone https://github.com/agl/critbit.git
+ *
+ * This is adapted to store arbitrary data (not just NUL-terminated C strings
+ * and allocates no memory internally.  The user needs to allocate
+ * "struct cb_node" and fill cb_node.k[] with arbitrary match data
+ * for memcmp.
+ * If "klen" is variable, then it should be embedded into "c_node.k[]"
+ * Recursion is bound by the maximum value of "klen" used.
+ */
+#ifndef CBTREE_H
+#define CBTREE_H
+
+#include "git-compat-util.h"
+
+struct cb_node;
+struct cb_node {
+	struct cb_node *child[2];
+	/*
+	 * n.b. uint32_t for `byte' is excessive for OIDs,
+	 * we may consider shorter variants if nothing else gets stored.
+	 */
+	uint32_t byte;
+	uint8_t otherbits;
+	uint8_t k[FLEX_ARRAY]; /* arbitrary data */
+};
+
+struct cb_tree {
+	struct cb_node *root;
+};
+
+enum cb_next {
+	CB_CONTINUE = 0,
+	CB_BREAK = 1
+};
+
+#define CBTREE_INIT { .root = NULL }
+
+static inline void cb_init(struct cb_tree *t)
+{
+	t->root = NULL;
+}
+
+struct cb_node *cb_lookup(struct cb_tree *, const uint8_t *k, size_t klen);
+struct cb_node *cb_insert(struct cb_tree *, struct cb_node *, size_t klen);
+struct cb_node *cb_unlink(struct cb_tree *t, const uint8_t *k, size_t klen);
+
+typedef enum cb_next (*cb_iter)(struct cb_node *, void *arg);
+
+void cb_each(struct cb_tree *, const uint8_t *kpfx, size_t klen,
+		cb_iter, void *arg);
+
+#endif /* CBTREE_H */
diff --git a/object-file.c b/object-file.c
index 91ded8c22a..35f3e7e9bb 100644
--- a/object-file.c
+++ b/object-file.c
@@ -1173,7 +1173,7 @@ static int quick_has_loose(struct repository *r,
 
 	prepare_alt_odb(r);
 	for (odb = r->objects->odb; odb; odb = odb->next) {
-		if (oid_array_lookup(odb_loose_cache(odb, oid), oid) >= 0)
+		if (oidtree_contains(odb_loose_cache(odb, oid), oid))
 			return 1;
 	}
 	return 0;
@@ -2452,11 +2452,11 @@ int for_each_loose_object(each_loose_object_fn cb, void *data,
 static int append_loose_object(const struct object_id *oid, const char *path,
 			       void *data)
 {
-	oid_array_append(data, oid);
+	oidtree_insert(data, oid);
 	return 0;
 }
 
-struct oid_array *odb_loose_cache(struct object_directory *odb,
+struct oidtree *odb_loose_cache(struct object_directory *odb,
 				  const struct object_id *oid)
 {
 	int subdir_nr = oid->hash[0];
@@ -2472,24 +2472,25 @@ struct oid_array *odb_loose_cache(struct object_directory *odb,
 
 	bitmap = &odb->loose_objects_subdir_seen[word_index];
 	if (*bitmap & mask)
-		return &odb->loose_objects_cache[subdir_nr];
-
+		return odb->loose_objects_cache;
+	if (!odb->loose_objects_cache) {
+		ALLOC_ARRAY(odb->loose_objects_cache, 1);
+		oidtree_init(odb->loose_objects_cache);
+	}
 	strbuf_addstr(&buf, odb->path);
 	for_each_file_in_obj_subdir(subdir_nr, &buf,
 				    append_loose_object,
 				    NULL, NULL,
-				    &odb->loose_objects_cache[subdir_nr]);
+				    odb->loose_objects_cache);
 	*bitmap |= mask;
 	strbuf_release(&buf);
-	return &odb->loose_objects_cache[subdir_nr];
+	return odb->loose_objects_cache;
 }
 
 void odb_clear_loose_cache(struct object_directory *odb)
 {
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(odb->loose_objects_cache); i++)
-		oid_array_clear(&odb->loose_objects_cache[i]);
+	oidtree_clear(odb->loose_objects_cache);
+	FREE_AND_NULL(odb->loose_objects_cache);
 	memset(&odb->loose_objects_subdir_seen, 0,
 	       sizeof(odb->loose_objects_subdir_seen));
 }
diff --git a/object-name.c b/object-name.c
index 64202de60b..3263c19457 100644
--- a/object-name.c
+++ b/object-name.c
@@ -87,27 +87,21 @@ static void update_candidates(struct disambiguate_state *ds, const struct object
 
 static int match_hash(unsigned, const unsigned char *, const unsigned char *);
 
+static enum cb_next match_prefix(const struct object_id *oid, void *arg)
+{
+	struct disambiguate_state *ds = arg;
+	/* no need to call match_hash, oidtree_each did prefix match */
+	update_candidates(ds, oid);
+	return ds->ambiguous ? CB_BREAK : CB_CONTINUE;
+}
+
 static void find_short_object_filename(struct disambiguate_state *ds)
 {
 	struct object_directory *odb;
 
-	for (odb = ds->repo->objects->odb; odb && !ds->ambiguous; odb = odb->next) {
-		int pos;
-		struct oid_array *loose_objects;
-
-		loose_objects = odb_loose_cache(odb, &ds->bin_pfx);
-		pos = oid_array_lookup(loose_objects, &ds->bin_pfx);
-		if (pos < 0)
-			pos = -1 - pos;
-		while (!ds->ambiguous && pos < loose_objects->nr) {
-			const struct object_id *oid;
-			oid = loose_objects->oid + pos;
-			if (!match_hash(ds->len, ds->bin_pfx.hash, oid->hash))
-				break;
-			update_candidates(ds, oid);
-			pos++;
-		}
-	}
+	for (odb = ds->repo->objects->odb; odb && !ds->ambiguous; odb = odb->next)
+		oidtree_each(odb_loose_cache(odb, &ds->bin_pfx),
+				&ds->bin_pfx, ds->len, match_prefix, ds);
 }
 
 static int match_hash(unsigned len, const unsigned char *a, const unsigned char *b)
diff --git a/object-store.h b/object-store.h
index ab6d469970..e679acc4c3 100644
--- a/object-store.h
+++ b/object-store.h
@@ -9,6 +9,7 @@
 #include "thread-utils.h"
 #include "khash.h"
 #include "dir.h"
+#include "oidtree.h"
 
 struct object_directory {
 	struct object_directory *next;
@@ -23,7 +24,7 @@ struct object_directory {
 	 * Be sure to call odb_load_loose_cache() before using.
 	 */
 	uint32_t loose_objects_subdir_seen[8]; /* 256 bits */
-	struct oid_array loose_objects_cache[256];
+	struct oidtree *loose_objects_cache;
 
 	/*
 	 * Path to the alternative object store. If this is a relative path,
@@ -59,7 +60,7 @@ void add_to_alternates_memory(const char *dir);
  * Populate and return the loose object cache array corresponding to the
  * given object ID.
  */
-struct oid_array *odb_loose_cache(struct object_directory *odb,
+struct oidtree *odb_loose_cache(struct object_directory *odb,
 				  const struct object_id *oid);
 
 /* Empty the loose object cache for the specified object directory. */
diff --git a/oidtree.c b/oidtree.c
new file mode 100644
index 0000000000..7eb0e9ba05
--- /dev/null
+++ b/oidtree.c
@@ -0,0 +1,104 @@
+/*
+ * A wrapper around cbtree which stores oids
+ * May be used to replace oid-array for prefix (abbreviation) matches
+ */
+#include "oidtree.h"
+#include "alloc.h"
+#include "hash.h"
+
+struct oidtree_node {
+	/* n.k[] is used to store "struct object_id" */
+	struct cb_node n;
+};
+
+struct oidtree_iter_data {
+	oidtree_iter fn;
+	void *arg;
+	size_t *last_nibble_at;
+	int algo;
+	uint8_t last_byte;
+};
+
+void oidtree_init(struct oidtree *ot)
+{
+	cb_init(&ot->tree);
+	mem_pool_init(&ot->mem_pool, 0);
+}
+
+void oidtree_clear(struct oidtree *ot)
+{
+	if (ot) {
+		mem_pool_discard(&ot->mem_pool, 0);
+		oidtree_init(ot);
+	}
+}
+
+void oidtree_insert(struct oidtree *ot, const struct object_id *oid)
+{
+	struct oidtree_node *on;
+
+	if (!oid->algo)
+		BUG("oidtree_insert requires oid->algo");
+
+	on = mem_pool_alloc(&ot->mem_pool, sizeof(*on) + sizeof(*oid));
+	oidcpy_with_padding((struct object_id *)on->n.k, oid);
+
+	/*
+	 * n.b. Current callers won't get us duplicates, here.  If a
+	 * future caller causes duplicates, there'll be a a small leak
+	 * that won't be freed until oidtree_clear.  Currently it's not
+	 * worth maintaining a free list
+	 */
+	cb_insert(&ot->tree, &on->n, sizeof(*oid));
+}
+
+
+int oidtree_contains(struct oidtree *ot, const struct object_id *oid)
+{
+	struct object_id k;
+	size_t klen = sizeof(k);
+
+	oidcpy_with_padding(&k, oid);
+
+	if (oid->algo == GIT_HASH_UNKNOWN)
+		klen -= sizeof(oid->algo);
+
+	/* cb_lookup relies on memcmp on the struct, so order matters: */
+	klen += BUILD_ASSERT_OR_ZERO(offsetof(struct object_id, hash) <
+				offsetof(struct object_id, algo));
+
+	return cb_lookup(&ot->tree, (const uint8_t *)&k, klen) ? 1 : 0;
+}
+
+static enum cb_next iter(struct cb_node *n, void *arg)
+{
+	struct oidtree_iter_data *x = arg;
+	const struct object_id *oid = (const struct object_id *)n->k;
+
+	if (x->algo != GIT_HASH_UNKNOWN && x->algo != oid->algo)
+		return CB_CONTINUE;
+
+	if (x->last_nibble_at) {
+		if ((oid->hash[*x->last_nibble_at] ^ x->last_byte) & 0xf0)
+			return CB_CONTINUE;
+	}
+
+	return x->fn(oid, x->arg);
+}
+
+void oidtree_each(struct oidtree *ot, const struct object_id *oid,
+			size_t oidhexsz, oidtree_iter fn, void *arg)
+{
+	size_t klen = oidhexsz / 2;
+	struct oidtree_iter_data x = { 0 };
+	assert(oidhexsz <= GIT_MAX_HEXSZ);
+
+	x.fn = fn;
+	x.arg = arg;
+	x.algo = oid->algo;
+	if (oidhexsz & 1) {
+		x.last_byte = oid->hash[klen];
+		x.last_nibble_at = &klen;
+	}
+	cb_each(&ot->tree, (const uint8_t *)oid, klen, iter, &x);
+}
diff --git a/oidtree.h b/oidtree.h
new file mode 100644
index 0000000000..77898f510a
--- /dev/null
+++ b/oidtree.h
@@ -0,0 +1,22 @@
+#ifndef OIDTREE_H
+#define OIDTREE_H
+
+#include "cbtree.h"
+#include "hash.h"
+#include "mem-pool.h"
+
+struct oidtree {
+	struct cb_tree tree;
+	struct mem_pool mem_pool;
+};
+
+void oidtree_init(struct oidtree *);
+void oidtree_clear(struct oidtree *);
+void oidtree_insert(struct oidtree *, const struct object_id *);
+int oidtree_contains(struct oidtree *, const struct object_id *);
+
+typedef enum cb_next (*oidtree_iter)(const struct object_id *, void *data);
+void oidtree_each(struct oidtree *, const struct object_id *,
+			size_t oidhexsz, oidtree_iter, void *data);
+
+#endif /* OIDTREE_H */
diff --git a/t/helper/test-oidtree.c b/t/helper/test-oidtree.c
new file mode 100644
index 0000000000..180ee28dd9
--- /dev/null
+++ b/t/helper/test-oidtree.c
@@ -0,0 +1,49 @@
+#include "test-tool.h"
+#include "cache.h"
+#include "oidtree.h"
+
+static enum cb_next print_oid(const struct object_id *oid, void *data)
+{
+	puts(oid_to_hex(oid));
+	return CB_CONTINUE;
+}
+
+int cmd__oidtree(int argc, const char **argv)
+{
+	struct oidtree ot;
+	struct strbuf line = STRBUF_INIT;
+	int nongit_ok;
+	int algo = GIT_HASH_UNKNOWN;
+
+	oidtree_init(&ot);
+	setup_git_directory_gently(&nongit_ok);
+
+	while (strbuf_getline(&line, stdin) != EOF) {
+		const char *arg;
+		struct object_id oid;
+
+		if (skip_prefix(line.buf, "insert ", &arg)) {
+			if (get_oid_hex_any(arg, &oid) == GIT_HASH_UNKNOWN)
+				die("insert not a hexadecimal oid: %s", arg);
+			algo = oid.algo;
+			oidtree_insert(&ot, &oid);
+		} else if (skip_prefix(line.buf, "contains ", &arg)) {
+			if (get_oid_hex(arg, &oid))
+				die("contains not a hexadecimal oid: %s", arg);
+			printf("%d\n", oidtree_contains(&ot, &oid));
+		} else if (skip_prefix(line.buf, "each ", &arg)) {
+			char buf[GIT_MAX_HEXSZ + 1] = { '0' };
+			memset(&oid, 0, sizeof(oid));
+			memcpy(buf, arg, strlen(arg));
+			buf[hash_algos[algo].hexsz] = '\0';
+			get_oid_hex_any(buf, &oid);
+			oid.algo = algo;
+			oidtree_each(&ot, &oid, strlen(arg), print_oid, NULL);
+		} else if (!strcmp(line.buf, "clear")) {
+			oidtree_clear(&ot);
+		} else {
+			die("unknown command: %s", line.buf);
+		}
+	}
+	return 0;
+}
diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c
index c5bd0c6d4c..9d37debf28 100644
--- a/t/helper/test-tool.c
+++ b/t/helper/test-tool.c
@@ -43,6 +43,7 @@ static struct test_cmd cmds[] = {
 	{ "mktemp", cmd__mktemp },
 	{ "oid-array", cmd__oid_array },
 	{ "oidmap", cmd__oidmap },
+	{ "oidtree", cmd__oidtree },
 	{ "online-cpus", cmd__online_cpus },
 	{ "parse-options", cmd__parse_options },
 	{ "parse-pathspec-file", cmd__parse_pathspec_file },
diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h
index e8069a3b22..f683a2f59c 100644
--- a/t/helper/test-tool.h
+++ b/t/helper/test-tool.h
@@ -32,6 +32,7 @@ int cmd__match_trees(int argc, const char **argv);
 int cmd__mergesort(int argc, const char **argv);
 int cmd__mktemp(int argc, const char **argv);
 int cmd__oidmap(int argc, const char **argv);
+int cmd__oidtree(int argc, const char **argv);
 int cmd__online_cpus(int argc, const char **argv);
 int cmd__parse_options(int argc, const char **argv);
 int cmd__parse_pathspec_file(int argc, const char** argv);
diff --git a/t/t0069-oidtree.sh b/t/t0069-oidtree.sh
new file mode 100755
index 0000000000..bfb1397d7b
--- /dev/null
+++ b/t/t0069-oidtree.sh
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+test_description='basic tests for the oidtree implementation'
+. ./test-lib.sh
+
+maxhexsz=$(test_oid hexsz)
+echoid () {
+	prefix="${1:+$1 }"
+	shift
+	while test $# -gt 0
+	do
+		shortoid="$1"
+		shift
+		difference=$(($maxhexsz - ${#shortoid}))
+		printf "%s%s%0${difference}d\\n" "$prefix" "$shortoid" "0"
+	done
+}
+
+test_expect_success 'oidtree insert and contains' '
+	cat >expect <<-\EOF &&
+		0
+		0
+		0
+		1
+		1
+		0
+	EOF
+	{
+		echoid insert 444 1 2 3 4 5 a b c d e &&
+		echoid contains 44 441 440 444 4440 4444
+		echo clear
+	} | test-tool oidtree >actual &&
+	test_cmp expect actual
+'
+
+test_expect_success 'oidtree each' '
+	echoid "" 123 321 321 >expect &&
+	{
+		echoid insert f 9 8 123 321 a b c d e
+		echo each 12300
+		echo each 3211
+		echo each 3210
+		echo each 32100
+		echo clear
+	} | test-tool oidtree >actual &&
+	test_cmp expect actual
+'
+
+test_done

From eff40457a4ab4887c677453d11774322a494a98c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Thu, 8 Jul 2021 12:53:15 +0200
Subject: [PATCH 333/397] fetch: fix segfault in --negotiate-only without
 --negotiation-tip=*
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The recent --negotiate-only option would segfault in the call to
oid_array_for_each() in negotiate_using_fetch() unless one or more
--negotiation-tip=* options were provided.

All of the other tests for the feature combine both, but nothing was
checking this assumption, let's do that and add a test for it. Fixes a
bug in 9c1e657a8fd (fetch: teach independent negotiation (no
packfile), 2021-05-04).

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 builtin/fetch.c        |  3 +++
 t/t5702-protocol-v2.sh | 16 ++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/builtin/fetch.c b/builtin/fetch.c
index dfde96a435..3f3fa0859f 100644
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -1990,6 +1990,9 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
 		fetch_config_from_gitmodules(sfjc, rs);
 	}
 
+	if (negotiate_only && !negotiation_tip.nr)
+		die(_("--negotiate-only needs one or more --negotiate-tip=*"));
+
 	if (deepen_relative) {
 		if (deepen_relative < 0)
 			die(_("Negative depth in --deepen is not supported"));
diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh
index 66af411057..78de1ff2ad 100755
--- a/t/t5702-protocol-v2.sh
+++ b/t/t5702-protocol-v2.sh
@@ -599,6 +599,22 @@ setup_negotiate_only () {
 	test_commit -C client three
 }
 
+test_expect_success 'usage: --negotiate-only without --negotiation-tip' '
+	SERVER="server" &&
+	URI="file://$(pwd)/server" &&
+
+	setup_negotiate_only "$SERVER" "$URI" &&
+
+	cat >err.expect <<-\EOF &&
+	fatal: --negotiate-only needs one or more --negotiate-tip=*
+	EOF
+
+	test_must_fail git -c protocol.version=2 -C client fetch \
+		--negotiate-only \
+		origin 2>err.actual &&
+	test_cmp err.expect err.actual
+'
+
 test_expect_success 'file:// --negotiate-only' '
 	SERVER="server" &&
 	URI="file://$(pwd)/server" &&

From d486ca60a51c9cb1fe068803c3f540724e95e83a Mon Sep 17 00:00:00 2001
From: Junio C Hamano 
Date: Thu, 8 Jul 2021 13:14:36 -0700
Subject: [PATCH 334/397] The third batch

Signed-off-by: Junio C Hamano 
---
 Documentation/RelNotes/2.33.0.txt | 77 +++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/Documentation/RelNotes/2.33.0.txt b/Documentation/RelNotes/2.33.0.txt
index 57443c7466..4749bee61e 100644
--- a/Documentation/RelNotes/2.33.0.txt
+++ b/Documentation/RelNotes/2.33.0.txt
@@ -28,6 +28,8 @@ UI, Workflows & Features
    if any, of diff is desired did not have any visible effect; it now
    implies some form of diff (by default "--patch") is produced.
 
+ * The userdiff pattern for C# learned the token "record".
+
 
 Performance, Internal Implementation, Development Support etc.
 
@@ -68,6 +70,64 @@ Fixes since v2.32
    the other side.
    (merge ae1a7eefff jk/fetch-pack-v2-half-close-early later to maint).
 
+ * The command line completion (in contrib/) learned that "git diff"
+   takes the "--anchored" option.
+   (merge d1e7c2cac9 tb/complete-diff-anchored later to maint).
+
+ * "git-svn" tests assumed that "locale -a", which is used to pick an
+   available UTF-8 locale, is available everywhere.  A knob has been
+   introduced to allow testers to specify a suitable locale to use.
+   (merge 482c962de4 dd/svn-test-wo-locale-a later to maint).
+
+ * Update "git subtree" to work better on Windows.
+   (merge 77f37de39f js/subtree-on-windows-fix later to maint).
+
+ * Remove multimail from contrib/
+   (merge f74d11471f js/no-more-multimail later to maint).
+
+ * Make the codebase MSAN clean.
+   (merge 4dbc55e87d ah/uninitialized-reads-fix later to maint).
+
+ * Work around inefficient glob substitution in older versions of bash
+   by rewriting parts of a test.
+   (merge eb87c6f559 jx/t6020-with-older-bash later to maint).
+
+ * Avoid duplicated work while building reachability bitmaps.
+   (merge aa9ad6fee5 jk/bitmap-tree-optim later to maint).
+
+ * We broke "GIT_SKIP_TESTS=t?000" to skip certain tests in recent
+   update, which got fixed.
+
+ * The side-band demultiplexer that is used to display progress output
+   from the remote end did not clear the line properly when the end of
+   line hits at a packet boundary, which has been corrected.
+
+ * Some test scripts assumed that readlink(1) was universally
+   installed and available, which is not the case.
+   (merge 7c0afdf23c jk/test-without-readlink-1 later to maint).
+
+ * Recent update to completion script (in contrib/) broke those who
+   use the __git_complete helper to define completion to their custom
+   command.
+   (merge cea232194d fw/complete-cmd-idx-fix later to maint).
+
+ * Output from some of our tests were affected by the width of the
+   terminal that they were run in, which has been corrected by
+   exporting a fixed value in the COLUMNS environment.
+   (merge c49a177bec ab/fix-columns-to-80-during-tests later to maint).
+
+ * On Windows, mergetool has been taught to find kdiff3.exe just like
+   it finds winmerge.exe.
+   (merge 47eb4c6890 ms/mergetools-kdiff3-on-windows later to maint).
+
+ * When we cannot figure out how wide the terminal is, we use a
+   fallback value of 80 ourselves (which cannot be avoided), but when
+   we run the pager, we export it in COLUMNS, which forces the pager
+   to use the hardcoded value, even when the pager is perfectly
+   capable to figure it out itself.  Stop exporting COLUMNS when we
+   fall back on the hardcoded default value for our own use.
+   (merge 9b6e2c8b98 js/stop-exporting-bogus-columns later to maint).
+
  * Other code cleanup, docfix, build fix, etc.
    (merge bfe35a6165 ah/doc-describe later to maint).
    (merge f302c1e4aa jc/clarify-revision-range later to maint).
@@ -76,3 +136,20 @@ Fixes since v2.32
    (merge 4e0a64a713 ab/trace2-squelch-gcc-warning later to maint).
    (merge 225f7fa847 ps/rev-list-object-type-filter later to maint).
    (merge 5317dfeaed dd/honor-users-tar-in-tests later to maint).
+   (merge ace6d8e3d6 tk/partial-clone-repack-doc later to maint).
+   (merge 7ba68e0cf1 js/trace2-discard-event-docfix later to maint).
+   (merge 8603c419d3 fc/doc-default-to-upstream-config later to maint).
+   (merge 1d72b604ef jk/revision-squelch-gcc-warning later to maint).
+   (merge abcb66c614 ar/typofix later to maint).
+   (merge 9853830787 ah/graph-typofix later to maint).
+   (merge aac578492d ab/config-hooks-path-testfix later to maint).
+   (merge 98c7656a18 ar/more-typofix later to maint).
+   (merge 6fb9195f6c jk/doc-max-pack-size later to maint).
+   (merge 4184cbd635 ar/mailinfo-memcmp-to-skip-prefix later to maint).
+   (merge 91d2347033 ar/doc-libera-chat-in-my-first-contrib later to maint).
+   (merge 338abb0f04 ab/cmd-foo-should-return later to maint).
+   (merge 546096a5cb ab/xdiff-bug-cleanup later to maint).
+   (merge b7b793d1e7 ab/progress-cleanup later to maint).
+   (merge d94f9b8e90 ba/object-info later to maint).
+   (merge 52ff891c03 ar/test-code-cleanup later to maint).
+   (merge a0538e5c8b dd/document-log-decorate-default later to maint).

From 8232a0ff48ce0959e20db9ffa1c4e16d4657dbec Mon Sep 17 00:00:00 2001
From: Denton Liu 
Date: Thu, 8 Jul 2021 19:27:22 -0700
Subject: [PATCH 335/397] pkt-line: replace "stateless separator" with
 "response end"

In 0181b600a6 (pkt-line: define PACKET_READ_RESPONSE_END, 2020-05-19),
the Response End packet was defined for Git's network protocol. When the
patch was sent, it included an oversight where the error messages
referenced "stateless separator", the work-in-progress name, over
"response end", the final name chosen.

Correct these error messages by having them correctly reference
a "response end" packet.

Signed-off-by: Denton Liu 
Signed-off-by: Junio C Hamano 
---
 pkt-line.c    | 2 +-
 remote-curl.c | 2 +-
 serve.c       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkt-line.c b/pkt-line.c
index 8f9bc68ee2..4f0264b479 100644
--- a/pkt-line.c
+++ b/pkt-line.c
@@ -103,7 +103,7 @@ void packet_response_end(int fd)
 {
 	packet_trace("0002", 4, 1);
 	if (write_in_full(fd, "0002", 4) < 0)
-		die_errno(_("unable to write stateless separator packet"));
+		die_errno(_("unable to write response end packet"));
 }
 
 int packet_flush_gently(int fd)
diff --git a/remote-curl.c b/remote-curl.c
index 75532a8bae..3c6eceb7be 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -602,7 +602,7 @@ static int rpc_read_from_out(struct rpc_state *rpc, int options,
 			memcpy(buf - 4, "0000", 4);
 			break;
 		case PACKET_READ_RESPONSE_END:
-			die(_("remote server sent stateless separator"));
+			die(_("remote server sent unexpected response end packet"));
 		}
 	}
 
diff --git a/serve.c b/serve.c
index c046926ba1..4bf5e9b0c1 100644
--- a/serve.c
+++ b/serve.c
@@ -218,7 +218,7 @@ static int process_request(void)
 			state = PROCESS_REQUEST_DONE;
 			break;
 		case PACKET_READ_RESPONSE_END:
-			BUG("unexpected stateless separator packet");
+			BUG("unexpected response end packet");
 		}
 	}
 

From 54ba2f1862df7144eb3a8b192d992b520bdc3743 Mon Sep 17 00:00:00 2001
From: Hu Jialun 
Date: Sat, 10 Jul 2021 02:07:31 +0800
Subject: [PATCH 336/397] commit: reorganise commit hint strings
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Strings of hint messages inserted into editor on interactive commit was
scattered in-line, rendering the code harder to understand at first
glance.

Extract those messages out into separate variables to make the code
outline easier to follow.

Helped-by: Junio C Hamano 
Helped-by: Đoàn Trần Công Danh 
Signed-off-by: Hu Jialun 
Signed-off-by: Junio C Hamano 
---
 builtin/commit.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/builtin/commit.c b/builtin/commit.c
index 190d215d43..e68d139dee 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -889,6 +889,14 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
 		int ident_shown = 0;
 		int saved_color_setting;
 		struct ident_split ci, ai;
+		const char *hint_cleanup_all = _("Please enter the commit message for your changes."
+						 " Lines starting\nwith '%c' will be ignored,"
+						 " and an empty message aborts the commit.\n");
+		const char *hint_cleanup_space = _("Please enter the commit message for your changes."
+						   " Lines starting\n"
+						   "with '%c' will be kept; you may remove them"
+						   " yourself if you want to.\n"
+						   "An empty message aborts the commit.\n");
 
 		if (whence != FROM_COMMIT) {
 			if (cleanup_mode == COMMIT_MSG_CLEANUP_SCISSORS &&
@@ -911,20 +919,12 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
 
 		fprintf(s->fp, "\n");
 		if (cleanup_mode == COMMIT_MSG_CLEANUP_ALL)
-			status_printf(s, GIT_COLOR_NORMAL,
-				_("Please enter the commit message for your changes."
-				  " Lines starting\nwith '%c' will be ignored, and an empty"
-				  " message aborts the commit.\n"), comment_line_char);
+			status_printf(s, GIT_COLOR_NORMAL, hint_cleanup_all, comment_line_char);
 		else if (cleanup_mode == COMMIT_MSG_CLEANUP_SCISSORS) {
 			if (whence == FROM_COMMIT && !merge_contains_scissors)
 				wt_status_add_cut_line(s->fp);
 		} else /* COMMIT_MSG_CLEANUP_SPACE, that is. */
-			status_printf(s, GIT_COLOR_NORMAL,
-				_("Please enter the commit message for your changes."
-				  " Lines starting\n"
-				  "with '%c' will be kept; you may remove them"
-				  " yourself if you want to.\n"
-				  "An empty message aborts the commit.\n"), comment_line_char);
+			status_printf(s, GIT_COLOR_NORMAL, hint_cleanup_space, comment_line_char);
 
 		/*
 		 * These should never fail because they come from our own

From 6f70f00b4f94c036767cbe0f4cd23db573a4e8ba Mon Sep 17 00:00:00 2001
From: Hu Jialun 
Date: Sat, 10 Jul 2021 02:07:32 +0800
Subject: [PATCH 337/397] commit: remove irrelavent prompt on
 `--allow-empty-message`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Even when the `--allow-empty-message` option is given, "git commit"
offers an interactive editor session with prefilled message that says
the commit will be aborted if the buffer is emptied, which is wrong.

Remove the "an empty message aborts" part from the message when the
option is given to fix it.

Helped-by: Junio C Hamano 
Helped-by: Đoàn Trần Công Danh 
Helped-by: Felipe Contreras 
Signed-off-by: Hu Jialun 
Signed-off-by: Junio C Hamano 
---
 builtin/commit.c                          | 25 +++++++++++++++--------
 t/t7500-commit-template-squash-signoff.sh |  2 +-
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/builtin/commit.c b/builtin/commit.c
index e68d139dee..cfbc83751a 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -889,15 +889,22 @@ static int prepare_to_commit(const char *index_file, const char *prefix,
 		int ident_shown = 0;
 		int saved_color_setting;
 		struct ident_split ci, ai;
-		const char *hint_cleanup_all = _("Please enter the commit message for your changes."
-						 " Lines starting\nwith '%c' will be ignored,"
-						 " and an empty message aborts the commit.\n");
-		const char *hint_cleanup_space = _("Please enter the commit message for your changes."
-						   " Lines starting\n"
-						   "with '%c' will be kept; you may remove them"
-						   " yourself if you want to.\n"
-						   "An empty message aborts the commit.\n");
-
+		const char *hint_cleanup_all = allow_empty_message ?
+			_("Please enter the commit message for your changes."
+			  " Lines starting\nwith '%c' will be ignored.\n") :
+			_("Please enter the commit message for your changes."
+			  " Lines starting\nwith '%c' will be ignored, and an empty"
+			  " message aborts the commit.\n");
+		const char *hint_cleanup_space = allow_empty_message ?
+			_("Please enter the commit message for your changes."
+			  " Lines starting\n"
+			  "with '%c' will be kept; you may remove them"
+			  " yourself if you want to.\n") :
+			_("Please enter the commit message for your changes."
+			  " Lines starting\n"
+			  "with '%c' will be kept; you may remove them"
+			  " yourself if you want to.\n"
+			  "An empty message aborts the commit.\n");
 		if (whence != FROM_COMMIT) {
 			if (cleanup_mode == COMMIT_MSG_CLEANUP_SCISSORS &&
 				!merge_contains_scissors)
diff --git a/t/t7500-commit-template-squash-signoff.sh b/t/t7500-commit-template-squash-signoff.sh
index 7d02f79c0d..54c2082acb 100755
--- a/t/t7500-commit-template-squash-signoff.sh
+++ b/t/t7500-commit-template-squash-signoff.sh
@@ -498,7 +498,7 @@ test_expect_success 'invalid message options when using --fixup' '
 cat >expected-template <
 #

From d1c5ae78ce1260c94c9e626b83dc0901e6843178 Mon Sep 17 00:00:00 2001
From: "brian m. carlson" 
Date: Sun, 11 Jul 2021 21:55:10 +0000
Subject: [PATCH 338/397] rev-list: add option for --pretty=format without
 header

In general, we encourage users to use plumbing commands, like git
rev-list, over porcelain commands, like git log, when scripting.
However, git rev-list has one glaring problem that prevents it from
being used in certain cases: when --pretty is used with a custom format,
it always prints out a line containing "commit" and the object ID.  This
makes it unsuitable for many scripting needs, and forces users to use
git log instead.

While we can't change this behavior for backwards compatibility, we can
add an option to suppress this behavior, so let's do so, and call it
"--no-commit-header".  Additionally, add the corresponding positive
option to switch it back on.

Note that this option doesn't affect the built-in formats, only custom
formats.  This is exactly the same behavior as users already have from
git log and is what most users will be used to.

Signed-off-by: brian m. carlson 
Signed-off-by: Junio C Hamano 
---
 Documentation/rev-list-options.txt |  8 +++
 builtin/rev-list.c                 | 33 ++++++++----
 revision.h                         |  3 +-
 t/t6006-rev-list-format.sh         | 80 +++++++++++++++++++++++++++++-
 4 files changed, 112 insertions(+), 12 deletions(-)

diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt
index 5bf2a85f69..23388f36c3 100644
--- a/Documentation/rev-list-options.txt
+++ b/Documentation/rev-list-options.txt
@@ -1064,6 +1064,14 @@ ifdef::git-rev-list[]
 --header::
 	Print the contents of the commit in raw-format; each record is
 	separated with a NUL character.
+
+--no-commit-header::
+	Suppress the header line containing "commit" and the object ID printed before
+	the specified format.  This has no effect on the built-in formats; only custom
+	formats are affected.
+
+--commit-header::
+	Overrides a previous `--no-commit-header`.
 endif::git-rev-list[]
 
 --parents::
diff --git a/builtin/rev-list.c b/builtin/rev-list.c
index 7677b1af5a..36cb909eba 100644
--- a/builtin/rev-list.c
+++ b/builtin/rev-list.c
@@ -127,13 +127,15 @@ static void show_commit(struct commit *commit, void *data)
 	if (info->header_prefix)
 		fputs(info->header_prefix, stdout);
 
-	if (!revs->graph)
-		fputs(get_revision_mark(revs, commit), stdout);
-	if (revs->abbrev_commit && revs->abbrev)
-		fputs(find_unique_abbrev(&commit->object.oid, revs->abbrev),
-		      stdout);
-	else
-		fputs(oid_to_hex(&commit->object.oid), stdout);
+	if (revs->include_header) {
+		if (!revs->graph)
+			fputs(get_revision_mark(revs, commit), stdout);
+		if (revs->abbrev_commit && revs->abbrev)
+			fputs(find_unique_abbrev(&commit->object.oid, revs->abbrev),
+			      stdout);
+		else
+			fputs(oid_to_hex(&commit->object.oid), stdout);
+	}
 	if (revs->print_parents) {
 		struct commit_list *parents = commit->parents;
 		while (parents) {
@@ -153,7 +155,7 @@ static void show_commit(struct commit *commit, void *data)
 	show_decorations(revs, commit);
 	if (revs->commit_format == CMIT_FMT_ONELINE)
 		putchar(' ');
-	else
+	else if (revs->include_header)
 		putchar('\n');
 
 	if (revs->verbose_header) {
@@ -512,6 +514,7 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 	repo_init_revisions(the_repository, &revs, prefix);
 	revs.abbrev = DEFAULT_ABBREV;
 	revs.commit_format = CMIT_FMT_UNSPECIFIED;
+	revs.include_header = 1;
 
 	/*
 	 * Scan the argument list before invoking setup_revisions(), so that we
@@ -627,6 +630,16 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 			continue;
 		}
 
+		if (!strcmp(arg, ("--commit-header"))) {
+			revs.include_header = 1;
+			continue;
+		}
+
+		if (!strcmp(arg, ("--no-commit-header"))) {
+			revs.include_header = 0;
+			continue;
+		}
+
 		if (!strcmp(arg, "--disk-usage")) {
 			show_disk_usage = 1;
 			info.flags |= REV_LIST_QUIET;
@@ -636,10 +649,12 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
 		usage(rev_list_usage);
 
 	}
+	if (revs.commit_format != CMIT_FMT_USERFORMAT)
+		revs.include_header = 1;
 	if (revs.commit_format != CMIT_FMT_UNSPECIFIED) {
 		/* The command line has a --pretty  */
 		info.hdr_termination = '\n';
-		if (revs.commit_format == CMIT_FMT_ONELINE)
+		if (revs.commit_format == CMIT_FMT_ONELINE || !revs.include_header)
 			info.header_prefix = "";
 		else
 			info.header_prefix = "commit ";
diff --git a/revision.h b/revision.h
index 17698cb51a..7464434f60 100644
--- a/revision.h
+++ b/revision.h
@@ -215,7 +215,8 @@ struct rev_info {
 			missing_newline:1,
 			date_mode_explicit:1,
 			preserve_subject:1,
-			encode_email_headers:1;
+			encode_email_headers:1,
+			include_header:1;
 	unsigned int	disable_stdin:1;
 	/* --show-linear-break */
 	unsigned int	track_linear:1,
diff --git a/t/t6006-rev-list-format.sh b/t/t6006-rev-list-format.sh
index 35a2f62392..41d0ca00b1 100755
--- a/t/t6006-rev-list-format.sh
+++ b/t/t6006-rev-list-format.sh
@@ -41,22 +41,59 @@ test_expect_success 'setup' '
 	echo "$added_iso88591" | git commit -F - &&
 	head1=$(git rev-parse --verify HEAD) &&
 	head1_short=$(git rev-parse --verify --short $head1) &&
+	head1_short4=$(git rev-parse --verify --short=4 $head1) &&
 	tree1=$(git rev-parse --verify HEAD:) &&
 	tree1_short=$(git rev-parse --verify --short $tree1) &&
 	echo "$changed" > foo &&
 	echo "$changed_iso88591" | git commit -a -F - &&
 	head2=$(git rev-parse --verify HEAD) &&
 	head2_short=$(git rev-parse --verify --short $head2) &&
+	head2_short4=$(git rev-parse --verify --short=4 $head2) &&
 	tree2=$(git rev-parse --verify HEAD:) &&
 	tree2_short=$(git rev-parse --verify --short $tree2) &&
 	git config --unset i18n.commitEncoding
 '
 
-# usage: test_format name format_string [failure] expect.$1
 	test_expect_${3:-success} "format $1" "
-		git rev-list --pretty=format:'$2' main >output.$1 &&
+		git rev-list $args --pretty=format:'$2' main >output.$1 &&
+		test_cmp expect.$1 output.$1
+	"
+}
+
+# usage: test_pretty [argument...] name format_name [failure] expect.$1
+	test_expect_${3:-success} "pretty $1 (without --no-commit-header)" "
+		git rev-list $args --pretty='$2' main >output.$1 &&
+		test_cmp expect.$1 output.$1
+	"
+	test_expect_${3:-success} "pretty $1 (with --no-commit-header)" "
+		git rev-list $args --no-commit-header --pretty='$2' main >output.$1 &&
 		test_cmp expect.$1 output.$1
 	"
 }
@@ -93,6 +130,20 @@ $head1
 $head1_short
 EOF
 
+test_format --no-commit-header hash-no-header %H%n%h <
+
+    $changed
+
+commit $head1
+Author: $GIT_AUTHOR_NAME <$GIT_AUTHOR_EMAIL>
+
+    $added
+
+EOF
+
 test_expect_success 'basic colors' '
 	cat >expect <<-EOF &&
 	commit $head2

From 103e02c7001f5c4989d56360c8cfc615f29fc9a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Sat, 10 Jul 2021 10:47:27 +0200
Subject: [PATCH 339/397] *.c static functions: don't forward-declare
 __attribute__
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

9cf6d3357aa (Add git-index-pack utility, 2005-10-12) and
466dbc42f58 (receive-pack: Send internal errors over side-band #2,
2010-02-10) we added these static functions and forward-declared their
__attribute__((printf)).

I think this may have been to work around some compiler limitation at
the time, but in any case we have a lot of code that uses the briefer
way of declaring these that I'm using here, so if we had any such
issues with compilers we'd have seen them already.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 builtin/index-pack.c   | 4 +---
 builtin/receive-pack.c | 5 ++---
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/builtin/index-pack.c b/builtin/index-pack.c
index 3fbc5d7077..8336466865 100644
--- a/builtin/index-pack.c
+++ b/builtin/index-pack.c
@@ -369,9 +369,7 @@ static void parse_pack_header(void)
 	use(sizeof(struct pack_header));
 }
 
-static NORETURN void bad_object(off_t offset, const char *format,
-		       ...) __attribute__((format (printf, 2, 3)));
-
+__attribute__((format (printf, 2, 3)))
 static NORETURN void bad_object(off_t offset, const char *format, ...)
 {
 	va_list params;
diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c
index a34742513a..2d1f97e1ca 100644
--- a/builtin/receive-pack.c
+++ b/builtin/receive-pack.c
@@ -425,9 +425,6 @@ static int proc_receive_ref_matches(struct command *cmd)
 	return 0;
 }
 
-static void rp_error(const char *err, ...) __attribute__((format (printf, 1, 2)));
-static void rp_warning(const char *err, ...) __attribute__((format (printf, 1, 2)));
-
 static void report_message(const char *prefix, const char *err, va_list params)
 {
 	int sz;
@@ -445,6 +442,7 @@ static void report_message(const char *prefix, const char *err, va_list params)
 		xwrite(2, msg, sz);
 }
 
+__attribute__((format (printf, 1, 2)))
 static void rp_warning(const char *err, ...)
 {
 	va_list params;
@@ -453,6 +451,7 @@ static void rp_warning(const char *err, ...)
 	va_end(params);
 }
 
+__attribute__((format (printf, 1, 2)))
 static void rp_error(const char *err, ...)
 {
 	va_list params;

From d4ac305073411bd70ae94e79c8fe8130a7d378aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Sat, 10 Jul 2021 10:47:28 +0200
Subject: [PATCH 340/397] sequencer.c: move static function to avoid forward
 decl
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Move the reflog_message() function added in
96e832a5fd6 (sequencer (rebase -i): refactor setting the reflog
message, 2017-01-02), it gained another user in
9055e401dd6 (sequencer: introduce new commands to reset the revision,
2018-04-25). Let's move it around and remove the forward declaration
added in the latter commit.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 sequencer.c | 41 +++++++++++++++++++----------------------
 1 file changed, 19 insertions(+), 22 deletions(-)

diff --git a/sequencer.c b/sequencer.c
index 0bec01cf38..c316d8374a 100644
--- a/sequencer.c
+++ b/sequencer.c
@@ -3599,7 +3599,25 @@ static int do_label(struct repository *r, const char *name, int len)
 }
 
 static const char *reflog_message(struct replay_opts *opts,
-	const char *sub_action, const char *fmt, ...);
+	const char *sub_action, const char *fmt, ...)
+{
+	va_list ap;
+	static struct strbuf buf = STRBUF_INIT;
+	char *reflog_action = getenv(GIT_REFLOG_ACTION);
+
+	va_start(ap, fmt);
+	strbuf_reset(&buf);
+	strbuf_addstr(&buf, reflog_action ? reflog_action : action_name(opts));
+	if (sub_action)
+		strbuf_addf(&buf, " (%s)", sub_action);
+	if (fmt) {
+		strbuf_addstr(&buf, ": ");
+		strbuf_vaddf(&buf, fmt, ap);
+	}
+	va_end(ap);
+
+	return buf.buf;
+}
 
 static int do_reset(struct repository *r,
 		    const char *name, int len,
@@ -4178,27 +4196,6 @@ int apply_autostash_oid(const char *stash_oid)
 	return apply_save_autostash_oid(stash_oid, 1);
 }
 
-static const char *reflog_message(struct replay_opts *opts,
-	const char *sub_action, const char *fmt, ...)
-{
-	va_list ap;
-	static struct strbuf buf = STRBUF_INIT;
-	char *reflog_action = getenv(GIT_REFLOG_ACTION);
-
-	va_start(ap, fmt);
-	strbuf_reset(&buf);
-	strbuf_addstr(&buf, reflog_action ? reflog_action : action_name(opts));
-	if (sub_action)
-		strbuf_addf(&buf, " (%s)", sub_action);
-	if (fmt) {
-		strbuf_addstr(&buf, ": ");
-		strbuf_vaddf(&buf, fmt, ap);
-	}
-	va_end(ap);
-
-	return buf.buf;
-}
-
 static int run_git_checkout(struct repository *r, struct replay_opts *opts,
 			    const char *commit, const char *action)
 {

From eb448631fb541239d91709d812b92bf2b57bc733 Mon Sep 17 00:00:00 2001
From: Denton Liu 
Date: Sat, 10 Jul 2021 02:28:31 -0700
Subject: [PATCH 341/397] git-diff: fix missing --merge-base docs

When `git diff --merge-base` was introduced at around Git 2.30, the
documentation included a few errors.

In the example given for `git diff --cached --merge-base`, the
`--cached` flag was omitted for the `--merge-base` example. Add the
missing flag.

In the `git diff ` case, we failed to mention that
`--merge-base` is an available option. Give the usage of `--merge-base`
as an option there.

Finally, there are two errors in the usage of `git diff`. Firstly, we do
not mention `--merge-base` in the `git diff --cached` case. Mention it
so that it's consistent with the documentation. Secondly, we put the
`[--merge-base]` in between `` and `[...]`. Move the
`[--merge-base]` so that it's beside `[]` which is a more
logical grouping.

Signed-off-by: Denton Liu 
Signed-off-by: Junio C Hamano 
---
 Documentation/git-diff.txt | 10 +++++++---
 builtin/diff.c             |  4 ++--
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/Documentation/git-diff.txt b/Documentation/git-diff.txt
index 7f4c8a8ce7..6236c75c9b 100644
--- a/Documentation/git-diff.txt
+++ b/Documentation/git-diff.txt
@@ -51,16 +51,20 @@ files on disk.
 	--staged is a synonym of --cached.
 +
 If --merge-base is given, instead of using , use the merge base
-of  and HEAD.  `git diff --merge-base A` is equivalent to
-`git diff $(git merge-base A HEAD)`.
+of  and HEAD.  `git diff --cached --merge-base A` is equivalent to
+`git diff --cached $(git merge-base A HEAD)`.
 
-'git diff' []  [--] [...]::
+'git diff' [] [--merge-base]  [--] [...]::
 
 	This form is to view the changes you have in your
 	working tree relative to the named .  You can
 	use HEAD to compare it with the latest commit, or a
 	branch name to compare with the tip of a different
 	branch.
++
+If --merge-base is given, instead of using , use the merge base
+of  and HEAD.  `git diff --merge-base A` is equivalent to
+`git diff $(git merge-base A HEAD)`.
 
 'git diff' [] [--merge-base]   [--] [...]::
 
diff --git a/builtin/diff.c b/builtin/diff.c
index b50fc68c2a..4c62876073 100644
--- a/builtin/diff.c
+++ b/builtin/diff.c
@@ -25,8 +25,8 @@
 
 static const char builtin_diff_usage[] =
 "git diff [] [] [--] [...]\n"
-"   or: git diff [] --cached [] [--] [...]\n"
-"   or: git diff []  [--merge-base] [...]  [--] [...]\n"
+"   or: git diff [] --cached [--merge-base] [] [--] [...]\n"
+"   or: git diff [] [--merge-base]  [...]  [--] [...]\n"
 "   or: git diff [] ...] [--] [...]\n"
 "   or: git diff []  ]\n"
 "   or: git diff [] --no-index [--]  ]\n"

From 48ca53cac4a5bee2dee3a5f3d6550753bf696d28 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Tue, 13 Jul 2021 10:05:18 +0200
Subject: [PATCH 342/397] *.c static functions: add missing
 __attribute__((format))
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add missing __attribute__((format)) function attributes to various
"static" functions that take printf arguments.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 add-patch.c                                                 | 1 +
 builtin/am.c                                                | 1 +
 builtin/bisect--helper.c                                    | 2 ++
 commit-graph.c                                              | 1 +
 contrib/credential/osxkeychain/git-credential-osxkeychain.c | 1 +
 contrib/credential/wincred/git-credential-wincred.c         | 1 +
 gettext.c                                                   | 1 +
 imap-send.c                                                 | 3 +++
 mailmap.c                                                   | 1 +
 merge-ort.c                                                 | 1 +
 merge-recursive.c                                           | 1 +
 midx.c                                                      | 1 +
 ref-filter.c                                                | 1 +
 sequencer.c                                                 | 2 ++
 server-info.c                                               | 1 +
 worktree.c                                                  | 1 +
 16 files changed, 20 insertions(+)

diff --git a/add-patch.c b/add-patch.c
index 2fad92ca37..8c41cdfe39 100644
--- a/add-patch.c
+++ b/add-patch.c
@@ -280,6 +280,7 @@ static void add_p_state_clear(struct add_p_state *s)
 	clear_add_i_state(&s->s);
 }
 
+__attribute__((format (printf, 2, 3)))
 static void err(struct add_p_state *s, const char *fmt, ...)
 {
 	va_list args;
diff --git a/builtin/am.c b/builtin/am.c
index 0b2d886c81..0c2ad96b70 100644
--- a/builtin/am.c
+++ b/builtin/am.c
@@ -210,6 +210,7 @@ static void write_state_bool(const struct am_state *state,
  * If state->quiet is false, calls fprintf(fp, fmt, ...), and appends a newline
  * at the end.
  */
+__attribute__((format (printf, 3, 4)))
 static void say(const struct am_state *state, FILE *fp, const char *fmt, ...)
 {
 	va_list ap;
diff --git a/builtin/bisect--helper.c b/builtin/bisect--helper.c
index 9d9540a0ab..f184eaeac6 100644
--- a/builtin/bisect--helper.c
+++ b/builtin/bisect--helper.c
@@ -117,6 +117,7 @@ static int write_in_file(const char *path, const char *mode, const char *format,
 	return fclose(fp);
 }
 
+__attribute__((format (printf, 2, 3)))
 static int write_to_file(const char *path, const char *format, ...)
 {
 	int res;
@@ -129,6 +130,7 @@ static int write_to_file(const char *path, const char *format, ...)
 	return res;
 }
 
+__attribute__((format (printf, 2, 3)))
 static int append_to_file(const char *path, const char *format, ...)
 {
 	int res;
diff --git a/commit-graph.c b/commit-graph.c
index 2bcb4e0f89..9179a3a647 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -2408,6 +2408,7 @@ cleanup:
 #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
 static int verify_commit_graph_error;
 
+__attribute__((format (printf, 1, 2)))
 static void graph_report(const char *fmt, ...)
 {
 	va_list ap;
diff --git a/contrib/credential/osxkeychain/git-credential-osxkeychain.c b/contrib/credential/osxkeychain/git-credential-osxkeychain.c
index bcd3f575a3..0b44a9b7cc 100644
--- a/contrib/credential/osxkeychain/git-credential-osxkeychain.c
+++ b/contrib/credential/osxkeychain/git-credential-osxkeychain.c
@@ -10,6 +10,7 @@ static char *username;
 static char *password;
 static UInt16 port;
 
+__attribute__((format (printf, 1, 2)))
 static void die(const char *err, ...)
 {
 	char msg[4096];
diff --git a/contrib/credential/wincred/git-credential-wincred.c b/contrib/credential/wincred/git-credential-wincred.c
index 5bdad41de1..5091048f9c 100644
--- a/contrib/credential/wincred/git-credential-wincred.c
+++ b/contrib/credential/wincred/git-credential-wincred.c
@@ -11,6 +11,7 @@
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
 
+__attribute__((format (printf, 1, 2)))
 static void die(const char *err, ...)
 {
 	char msg[4096];
diff --git a/gettext.c b/gettext.c
index af2413b47e..bb5ba1fe7c 100644
--- a/gettext.c
+++ b/gettext.c
@@ -66,6 +66,7 @@ const char *get_preferred_languages(void)
 }
 
 #ifndef NO_GETTEXT
+__attribute__((format (printf, 1, 2)))
 static int test_vsnprintf(const char *fmt, ...)
 {
 	char buf[26];
diff --git a/imap-send.c b/imap-send.c
index bb085d66d1..ce5a0461a4 100644
--- a/imap-send.c
+++ b/imap-send.c
@@ -451,6 +451,7 @@ static int buffer_gets(struct imap_buffer *b, char **s)
 	/* not reached */
 }
 
+__attribute__((format (printf, 1, 2)))
 static void imap_info(const char *msg, ...)
 {
 	va_list va;
@@ -463,6 +464,7 @@ static void imap_info(const char *msg, ...)
 	}
 }
 
+__attribute__((format (printf, 1, 2)))
 static void imap_warn(const char *msg, ...)
 {
 	va_list va;
@@ -504,6 +506,7 @@ static char *next_arg(char **s)
 	return ret;
 }
 
+__attribute__((format (printf, 3, 4)))
 static int nfsnprintf(char *buf, int blen, const char *fmt, ...)
 {
 	int ret;
diff --git a/mailmap.c b/mailmap.c
index d1f7c0d272..462b395634 100644
--- a/mailmap.c
+++ b/mailmap.c
@@ -8,6 +8,7 @@
 #define debug_mm(...) fprintf(stderr, __VA_ARGS__)
 #define debug_str(X) ((X) ? (X) : "(none)")
 #else
+__attribute__((format (printf, 1, 2)))
 static inline void debug_mm(const char *format, ...) {}
 static inline const char *debug_str(const char *s) { return s; }
 #endif
diff --git a/merge-ort.c b/merge-ort.c
index b954f7184a..955d1d0502 100644
--- a/merge-ort.c
+++ b/merge-ort.c
@@ -529,6 +529,7 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti,
 	renames->callback_data_nr = renames->callback_data_alloc = 0;
 }
 
+__attribute__((format (printf, 2, 3)))
 static int err(struct merge_options *opt, const char *err, ...)
 {
 	va_list params;
diff --git a/merge-recursive.c b/merge-recursive.c
index 4327e0cfa3..5d54990af9 100644
--- a/merge-recursive.c
+++ b/merge-recursive.c
@@ -167,6 +167,7 @@ static void flush_output(struct merge_options *opt)
 	}
 }
 
+__attribute__((format (printf, 2, 3)))
 static int err(struct merge_options *opt, const char *err, ...)
 {
 	va_list params;
diff --git a/midx.c b/midx.c
index 21d6a05e88..0956849e2a 100644
--- a/midx.c
+++ b/midx.c
@@ -1162,6 +1162,7 @@ void clear_midx_file(struct repository *r)
 
 static int verify_midx_error;
 
+__attribute__((format (printf, 1, 2)))
 static void midx_report(const char *fmt, ...)
 {
 	va_list ap;
diff --git a/ref-filter.c b/ref-filter.c
index 4db0e40ff4..f45d3a1b26 100644
--- a/ref-filter.c
+++ b/ref-filter.c
@@ -213,6 +213,7 @@ static int used_atom_cnt, need_tagged, need_symref;
  * Expand string, append it to strbuf *sb, then return error code ret.
  * Allow to save few lines of code.
  */
+__attribute__((format (printf, 3, 4)))
 static int strbuf_addf_ret(struct strbuf *sb, int ret, const char *fmt, ...)
 {
 	va_list ap;
diff --git a/sequencer.c b/sequencer.c
index c316d8374a..7f07cd00f3 100644
--- a/sequencer.c
+++ b/sequencer.c
@@ -3521,6 +3521,7 @@ static int do_exec(struct repository *r, const char *command_line)
 	return status;
 }
 
+__attribute__((format (printf, 2, 3)))
 static int safe_append(const char *filename, const char *fmt, ...)
 {
 	va_list ap;
@@ -3598,6 +3599,7 @@ static int do_label(struct repository *r, const char *name, int len)
 	return ret;
 }
 
+__attribute__((format (printf, 3, 4)))
 static const char *reflog_message(struct replay_opts *opts,
 	const char *sub_action, const char *fmt, ...)
 {
diff --git a/server-info.c b/server-info.c
index de0aa4498c..7701d7c20a 100644
--- a/server-info.c
+++ b/server-info.c
@@ -27,6 +27,7 @@ static int uic_is_stale(const struct update_info_ctx *uic)
 	return uic->old_fp == NULL;
 }
 
+__attribute__((format (printf, 2, 3)))
 static int uic_printf(struct update_info_ctx *uic, const char *fmt, ...)
 {
 	va_list ap;
diff --git a/worktree.c b/worktree.c
index 237517baee..092a4f92ad 100644
--- a/worktree.c
+++ b/worktree.c
@@ -265,6 +265,7 @@ const char *worktree_prune_reason(struct worktree *wt, timestamp_t expire)
 }
 
 /* convenient wrapper to deal with NULL strbuf */
+__attribute__((format (printf, 2, 3)))
 static void strbuf_addf_gently(struct strbuf *buf, const char *fmt, ...)
 {
 	va_list params;

From 75d31ceec5870c94c934264b7a9afe029b787410 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Tue, 13 Jul 2021 10:05:19 +0200
Subject: [PATCH 343/397] *.h: add a few missing __attribute__((format))
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add missing format attributes to API functions that take printf
arguments.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 cache.h  | 1 +
 quote.h  | 1 +
 strbuf.h | 1 +
 3 files changed, 3 insertions(+)

diff --git a/cache.h b/cache.h
index ba04ff8bd3..f9aed2d45c 100644
--- a/cache.h
+++ b/cache.h
@@ -1385,6 +1385,7 @@ enum get_oid_result {
 };
 
 int repo_get_oid(struct repository *r, const char *str, struct object_id *oid);
+__attribute__((format (printf, 2, 3)))
 int get_oidf(struct object_id *oid, const char *fmt, ...);
 int repo_get_oid_commit(struct repository *r, const char *str, struct object_id *oid);
 int repo_get_oid_committish(struct repository *r, const char *str, struct object_id *oid);
diff --git a/quote.h b/quote.h
index 768cc6338e..049d8dd0b3 100644
--- a/quote.h
+++ b/quote.h
@@ -31,6 +31,7 @@ struct strbuf;
 
 void sq_quote_buf(struct strbuf *, const char *src);
 void sq_quote_argv(struct strbuf *, const char **argv);
+__attribute__((format (printf, 2, 3)))
 void sq_quotef(struct strbuf *, const char *fmt, ...);
 
 /*
diff --git a/strbuf.h b/strbuf.h
index 223ee2094a..f1e9821a54 100644
--- a/strbuf.h
+++ b/strbuf.h
@@ -263,6 +263,7 @@ static inline void strbuf_insertstr(struct strbuf *sb, size_t pos,
 void strbuf_vinsertf(struct strbuf *sb, size_t pos, const char *fmt,
 		     va_list ap);
 
+__attribute__((format (printf, 3, 4)))
 void strbuf_insertf(struct strbuf *sb, size_t pos, const char *fmt, ...);
 
 /**

From 927dc33070563d6646c903daf919dde80d8878d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=86var=20Arnfj=C3=B6r=C3=B0=20Bjarmason?=
 
Date: Tue, 13 Jul 2021 10:05:20 +0200
Subject: [PATCH 344/397] advice.h: add missing __attribute__((format)) & fix
 usage
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add the missing __attribute__((format)) checking to
advise_if_enabled(). This revealed a trivial issue introduced in
b3b18d16213 (advice: revamp advise API, 2020-03-02). We treated the
argv[1] as a format string, but did not intend to do so. Let's use
"%s" and pass argv[1] as an argument instead.

Signed-off-by: Ævar Arnfjörð Bjarmason 
Signed-off-by: Junio C Hamano 
---
 advice.h               | 1 +
 t/helper/test-advise.c | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/advice.h b/advice.h
index bd26c385d0..9f8ffc7354 100644
--- a/advice.h
+++ b/advice.h
@@ -90,6 +90,7 @@ int advice_enabled(enum advice_type type);
 /**
  * Checks the visibility of the advice before printing.
  */
+__attribute__((format (printf, 2, 3)))
 void advise_if_enabled(enum advice_type type, const char *advice, ...);
 
 int error_resolve_conflict(const char *me);
diff --git a/t/helper/test-advise.c b/t/helper/test-advise.c
index a7043df1d3..cb881139f7 100644
--- a/t/helper/test-advise.c
+++ b/t/helper/test-advise.c
@@ -16,7 +16,7 @@ int cmd__advise_if_enabled(int argc, const char **argv)
 	 * selected here and in t0018 where this command is being
 	 * executed.
 	 */
-	advise_if_enabled(ADVICE_NESTED_TAG, argv[1]);
+	advise_if_enabled(ADVICE_NESTED_TAG, "%s", argv[1]);
 
 	return 0;
 }

From b1d87fbaf1ee7c6c8ba9f4b9729abcb2d19f8a4e Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Tue, 13 Jul 2021 18:31:26 -0400
Subject: [PATCH 345/397] doc/rev-list-options: fix duplicate word typo

Reported-by: Jason Hatton 
Signed-off-by: Jeff King 
Signed-off-by: Junio C Hamano 
---
 Documentation/rev-list-options.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt
index 8d8b7f492a..c8b2bd2d81 100644
--- a/Documentation/rev-list-options.txt
+++ b/Documentation/rev-list-options.txt
@@ -721,7 +721,7 @@ as 'blob:limit=1024'.
 +
 The form '--filter=sparse:oid=' uses a sparse-checkout
 specification contained in the blob (or blob-expression) ''
-to omit blobs that would not be not required for a sparse checkout on
+to omit blobs that would not be required for a sparse checkout on
 the requested refs.
 +
 The form '--filter=sparse:path=' similarly uses a sparse-checkout

From 75ae10bc75336db031ee58d13c5037b929235912 Mon Sep 17 00:00:00 2001
From: Junio C Hamano 
Date: Tue, 13 Jul 2021 16:52:11 -0700
Subject: [PATCH 346/397] The fourth batch

Signed-off-by: Junio C Hamano 
---
 Documentation/RelNotes/2.33.0.txt | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/Documentation/RelNotes/2.33.0.txt b/Documentation/RelNotes/2.33.0.txt
index 4749bee61e..d1e92766b0 100644
--- a/Documentation/RelNotes/2.33.0.txt
+++ b/Documentation/RelNotes/2.33.0.txt
@@ -44,6 +44,12 @@ Performance, Internal Implementation, Development Support etc.
  * Repeated rename detections in a sequence of mergy operations have
    been optimize out.
 
+ * Preliminary clean-up of tests before the main reftable changes
+   hits the codebase.
+
+ * The backend for "diff -G/-S" has been updated to use pcre2 engine
+   when available.
+
 
 Fixes since v2.32
 -----------------
@@ -128,6 +134,23 @@ Fixes since v2.32
    fall back on the hardcoded default value for our own use.
    (merge 9b6e2c8b98 js/stop-exporting-bogus-columns later to maint).
 
+ * "git cat-file --batch-all-objects"" misbehaved when "--batch" is in
+   use and did not ask for certain object traits.
+   (merge ee02ac6164 zh/cat-file-batch-fix later to maint).
+
+ * Some code and doc clarification around "git push".
+
+ * The "union" conflict resultion variant misbehaved when used with
+   binary merge driver.
+   (merge 382b601acd jk/union-merge-binary later to maint).
+
+ * Prevent "git p4" from failing to submit changes to binary file.
+   (merge 54662d5958 dc/p4-binary-submit-fix later to maint).
+
+ * "git grep --and -e foo" ought to have been diagnosed as an error
+   but instead segfaulted, which has been corrected.
+   (merge fe7fe62d8d rs/grep-parser-fix later to maint).
+
  * Other code cleanup, docfix, build fix, etc.
    (merge bfe35a6165 ah/doc-describe later to maint).
    (merge f302c1e4aa jc/clarify-revision-range later to maint).
@@ -153,3 +176,6 @@ Fixes since v2.32
    (merge d94f9b8e90 ba/object-info later to maint).
    (merge 52ff891c03 ar/test-code-cleanup later to maint).
    (merge a0538e5c8b dd/document-log-decorate-default later to maint).
+   (merge ce24797d38 mr/cmake later to maint).
+   (merge 9eb542f2ee ab/pre-auto-gc-hook-test later to maint).
+   (merge 9fffc38583 bk/doc-commit-typofix later to maint).

From f9365c0a24b9d2346a9b2b5a24e1fa02039f23e5 Mon Sep 17 00:00:00 2001
From: Stephen Manz 
Date: Sun, 11 Jul 2021 00:27:18 +0000
Subject: [PATCH 347/397] t2400: clean up '"add" worktree with lock' test

- remove unneeded `git rev-parse` which must have come from a copy-paste
  of another test
- unlock the worktree with test_when_finished

Signed-off-by: Stephen Manz 
Reviewed-by: Eric Sunshine 
Signed-off-by: Junio C Hamano 
---
 t/t2400-worktree-add.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/t2400-worktree-add.sh b/t/t2400-worktree-add.sh
index 96dfca1554..93d3795cab 100755
--- a/t/t2400-worktree-add.sh
+++ b/t/t2400-worktree-add.sh
@@ -67,8 +67,8 @@ test_expect_success '"add" worktree' '
 '
 
 test_expect_success '"add" worktree with lock' '
-	git rev-parse HEAD >expect &&
 	git worktree add --detach --lock here-with-lock main &&
+	test_when_finished "git worktree unlock here-with-lock || :" &&
 	test -f .git/worktrees/here-with-lock/locked
 '
 

From f7c35ea2a1292d581ca79d395fc978f92a3aec5b Mon Sep 17 00:00:00 2001
From: Stephen Manz 
Date: Sun, 11 Jul 2021 00:27:19 +0000
Subject: [PATCH 348/397] worktree: mark lock strings with `_()` for
 translation

- default lock string, "added with --lock"
- temporary lock string, "initializing"

Signed-off-by: Stephen Manz 
Reviewed-by: Eric Sunshine 
Signed-off-by: Junio C Hamano 
---
 builtin/worktree.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/builtin/worktree.c b/builtin/worktree.c
index 976bf8ed06..4829b9507f 100644
--- a/builtin/worktree.c
+++ b/builtin/worktree.c
@@ -303,9 +303,9 @@ static int add_worktree(const char *path, const char *refname,
 	 */
 	strbuf_addf(&sb, "%s/locked", sb_repo.buf);
 	if (!opts->keep_locked)
-		write_file(sb.buf, "initializing");
+		write_file(sb.buf, _("initializing"));
 	else
-		write_file(sb.buf, "added with --lock");
+		write_file(sb.buf, _("added with --lock"));
 
 	strbuf_addf(&sb_git, "%s/.git", path);
 	if (safe_create_leading_directories_const(sb_git.buf))

From d1ed8d6cee57c91ec770a8a183ed40c3ec867ac1 Mon Sep 17 00:00:00 2001
From: Jeff King 
Date: Wed, 14 Jul 2021 12:31:36 -0400
Subject: [PATCH 349/397] load_ref_decorations(): fix decoration with tags
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Commit 88473c8bae ("load_ref_decorations(): avoid parsing non-tag
objects", 2021-06-22) introduced a shortcut to `add_ref_decoration()`:
Rather than calling `parse_object()`, we go for `oid_object_info()` and
then `lookup_object_by_type()` using the type just discovered. As
detailed in the commit message, this provides a significant time saving.

Unfortunately, it also changes the behavior: We lose all annotated tags
from the decoration.

The reason this happens is in the loop where we try to peel the tags, we
won't necessarily have parsed that first object. If we haven't, its
`tagged` field will be NULL, so we won't actually add a decoration for
the pointed-to object.

Make sure to parse the tag object at the top of the peeling loop. This
effectively restores the pre-88473c8bae parsing -- but only of tags,
allowing us to keep most of the possible speedup from 88473c8bae.

On my big ~220k ref test case (where it's mostly non-tags), the
timings [using "git log -1 --decorate"] are:

  - before either patch: 2.945s
  - with my broken patch: 0.707s
  - with [this patch]: 0.788s

The simplest way to do this is to just conditionally parse before the
loop:

  if (obj->type == OBJ_TAG)
          parse_object(&obj->oid);

But we can observe that our tag-peeling loop needs to peel already, to
examine recursive tags-of-tags. So instead of introducing a new call to
parse_object(), we can simply move the parsing higher in the loop:
instead of parsing the new object before we loop, parse each tag object
before we look at its "tagged" field.

This has another beneficial side effect: if a tag points at a commit (or
other non-tag type), we do not bother to parse the commit at all now.
And we know it is a commit without calling oid_object_info(), because
parsing the surrounding tag object will have created the correct in-core
object based on the "type" field of the tag.

Our test coverage for --decorate was obviously not good, since we missed
this quite-basic regression. The new tests covers an annotated tag
(showing the fix), but also that we correctly show annotations for
lightweight tags and double-annotated tag-of-tags.

Reported-by: Martin Ågren 
Helped-by: Martin Ågren 
Signed-off-by: Martin Ågren 
Signed-off-by: Jeff King 
Reviewed-by: Martin Ågren 
Signed-off-by: Junio C Hamano 
---
 log-tree.c     |  4 ++--
 t/t4202-log.sh | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/log-tree.c b/log-tree.c
index 4f69ed176d..6dc4412268 100644
--- a/log-tree.c
+++ b/log-tree.c
@@ -174,11 +174,11 @@ static int add_ref_decoration(const char *refname, const struct object_id *oid,
 
 	add_name_decoration(deco_type, refname, obj);
 	while (obj->type == OBJ_TAG) {
+		if (!obj->parsed)
+			parse_object(the_repository, &obj->oid);
 		obj = ((struct tag *)obj)->tagged;
 		if (!obj)
 			break;
-		if (!obj->parsed)
-			parse_object(the_repository, &obj->oid);
 		add_name_decoration(DECORATION_REF_TAG, refname, obj);
 	}
 	return 0;
diff --git a/t/t4202-log.sh b/t/t4202-log.sh
index 350cfa3593..fe8f5e2067 100755
--- a/t/t4202-log.sh
+++ b/t/t4202-log.sh
@@ -1905,6 +1905,20 @@ test_expect_success '--exclude-promisor-objects does not BUG-crash' '
 	test_must_fail git log --exclude-promisor-objects source-a
 '
 
+test_expect_success 'log --decorate includes all levels of tag annotated tags' '
+	git checkout -b branch &&
+	git commit --allow-empty -m "new commit" &&
+	git tag lightweight HEAD &&
+	git tag -m annotated annotated HEAD &&
+	git tag -m double-0 double-0 HEAD &&
+	git tag -m double-1 double-1 double-0 &&
+	cat >expect <<-\EOF &&
+	HEAD -> branch, tag: lightweight, tag: double-1, tag: double-0, tag: annotated
+	EOF
+	git log -1 --format="%D" >actual &&
+	test_cmp expect actual
+'
+
 test_expect_success 'log --end-of-options' '
        git update-ref refs/heads/--source HEAD &&
        git log --end-of-options --source >actual &&

From e61059660ca00ba69d1721b7d48653c52848bece Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Wed, 14 Jul 2021 11:50:33 +0000
Subject: [PATCH 350/397] ci: run `make sparse` as part of the GitHub workflow

Occasionally we receive reviews after patches were integrated, where
`sparse` (https://sparse.docs.kernel.org/en/latest/ has more information
on that project) identified problems such as file-local variables or
functions being declared as global.

By running `sparse` as part of our Continuous Integration, we can catch
such things much earlier. Even better: developers who activated GitHub
Actions on their forks can catch such issues before even sending their
patches to the Git mailing list.

This addresses https://github.com/gitgitgadget/git/issues/345

Note: Not even Ubuntu 20.04 ships with a new enough version of `sparse`
to accommodate Git's needs. The symptom looks like this:

    add-interactive.c:537:51: error: Using plain integer as NULL pointer

To counter that, we download and install the custom-built `sparse`
package from the Azure Pipeline that we specifically created to address
this issue.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/main.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 73856bafc9..1b5c039207 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -350,6 +350,27 @@ jobs:
     - uses: actions/checkout@v1
     - run: ci/install-dependencies.sh
     - run: ci/run-static-analysis.sh
+  sparse:
+    needs: ci-config
+    if: needs.ci-config.outputs.enabled == 'yes'
+    env:
+      jobname: sparse
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Download a current `sparse` package
+      # Ubuntu's `sparse` version is too old for us
+      uses: git-for-windows/get-azure-pipelines-artifact@v0
+      with:
+        repository: git/git
+        definitionId: 10
+        artifact: sparse-20.04
+    - name: Install the current `sparse` package
+      run: sudo dpkg -i sparse-20.04/sparse_*.deb
+    - name: Install other dependencies
+      run: |
+        sudo apt-get install -q -y libssl-dev libcurl4-openssl-dev libexpat-dev gettext zlib1g-dev
+    - uses: actions/checkout@v2
+    - run: make sparse
   documentation:
     needs: ci-config
     if: needs.ci-config.outputs.enabled == 'yes'

From fc6609d198c47bf511f3388975c7c94e0aa1af2b Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:25 +0000
Subject: [PATCH 351/397] sparse-index: skip indexes with unmerged entries

The sparse-index format is designed to be compatible with merge
conflicts, even those outside the sparse-checkout definition. The reason
is that when converting a full index to a sparse one, a cache entry with
nonzero stage will not be collapsed into a sparse directory entry.

However, this behavior was not tested, and a different behavior within
convert_to_sparse() fails in this scenario. Specifically,
cache_tree_update() will fail when unmerged entries exist.
convert_to_sparse_rec() uses the cache-tree data to recursively walk the
tree structure, but also to compute the OIDs used in the
sparse-directory entries.

Add an index scan to convert_to_sparse() that will detect if these merge
conflict entries exist and skip the conversion before trying to update
the cache-tree. This is marked as NEEDSWORK because this can be removed
with a suitable update to cache_tree_update() or a similar method that
can construct a cache-tree with invalid nodes, but still allow creating
the nodes necessary for creating sparse directory entries.

It is possible that in the future we will not need to make such an
update, since if we do not expand a sparse-index into a full one, this
conversion does not need to happen. Thus, this can be deferred until the
merge machinery is made to integrate with the sparse-index.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 sparse-index.c                           | 18 ++++++++++++++++++
 t/t1092-sparse-checkout-compatibility.sh | 22 ++++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/sparse-index.c b/sparse-index.c
index affc4048f2..2c69593027 100644
--- a/sparse-index.c
+++ b/sparse-index.c
@@ -116,6 +116,17 @@ int set_sparse_index_config(struct repository *repo, int enable)
 	return res;
 }
 
+static int index_has_unmerged_entries(struct index_state *istate)
+{
+	int i;
+	for (i = 0; i < istate->cache_nr; i++) {
+		if (ce_stage(istate->cache[i]))
+			return 1;
+	}
+
+	return 0;
+}
+
 int convert_to_sparse(struct index_state *istate)
 {
 	int test_env;
@@ -152,6 +163,13 @@ int convert_to_sparse(struct index_state *istate)
 		return -1;
 	}
 
+	/*
+	 * NEEDSWORK: If we have unmerged entries, then stay full.
+	 * Unmerged entries prevent the cache-tree extension from working.
+	 */
+	if (index_has_unmerged_entries(istate))
+		return 0;
+
 	if (cache_tree_update(istate, 0)) {
 		warning(_("unable to update cache-tree, staying full"));
 		return -1;
diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index e9a815ca7a..ba2fd94ada 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -352,6 +352,28 @@ test_expect_success 'merge with outside renames' '
 	done
 '
 
+# Sparse-index fails to convert the index in the
+# final 'git cherry-pick' command.
+test_expect_success 'cherry-pick with conflicts' '
+	init_repos &&
+
+	write_script edit-conflict <<-\EOF &&
+	echo $1 >conflict
+	EOF
+
+	test_all_match git checkout -b to-cherry-pick &&
+	run_on_all ../edit-conflict ABC &&
+	test_all_match git add conflict &&
+	test_all_match git commit -m "conflict to pick" &&
+
+	test_all_match git checkout -B base HEAD~1 &&
+	run_on_all ../edit-conflict DEF &&
+	test_all_match git add conflict &&
+	test_all_match git commit -m "conflict in base" &&
+
+	test_all_match test_must_fail git cherry-pick to-cherry-pick
+'
+
 test_expect_success 'clean' '
 	init_repos &&
 

From 47410778fbd2765c9e0a4b4026ece66aa7c83aa4 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:26 +0000
Subject: [PATCH 352/397] sparse-index: include EXTENDED flag when expanding

When creating a full index from a sparse one, we create cache entries
for every blob within a given sparse directory entry. These are
correctly marked with the CE_SKIP_WORKTREE flag, but the CE_EXTENDED
flag is not included. The CE_EXTENDED flag would exist if we loaded a
full index from disk with these entries marked with CE_SKIP_WORKTREE, so
we can add the flag here to be consistent. This allows us to directly
compare the flags present in cache entries when testing the sparse-index
feature, but has no significance to its correctness in the user-facing
functionality.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 sparse-index.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sparse-index.c b/sparse-index.c
index 2c69593027..ef53bd2198 100644
--- a/sparse-index.c
+++ b/sparse-index.c
@@ -213,7 +213,7 @@ static int add_path_to_index(const struct object_id *oid,
 	strbuf_addstr(base, path);
 
 	ce = make_cache_entry(istate, mode, oid, base->buf, 0, 0);
-	ce->ce_flags |= CE_SKIP_WORKTREE;
+	ce->ce_flags |= CE_SKIP_WORKTREE | CE_EXTENDED;
 	set_index_entry(istate, istate->cache_nr++, ce);
 
 	strbuf_setlen(base, len);

From 3d814b5dc0567ebca1c40c6fe10bd9f19ca31f58 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:27 +0000
Subject: [PATCH 353/397] t1092: replace incorrect 'echo' with 'cat'

This fixes the test data shape to be as expected, allowing rename
detection to work properly now that the 'larger-content' file actually
has meaningful lines.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 t/t1092-sparse-checkout-compatibility.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index ba2fd94ada..ebbba044f7 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -40,7 +40,7 @@ test_expect_success 'setup' '
 		done &&
 
 		git checkout -b rename-base base &&
-		echo >folder1/larger-content <<-\EOF &&
+		cat >folder1/larger-content <<-\EOF &&
 		matching
 		lines
 		help

From e669ffb2b8f7effa4f6876f61c35ae5c4a8566b2 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:28 +0000
Subject: [PATCH 354/397] t1092: expand repository data shape

As more features integrate with the sparse-index feature, more and more
special cases arise that require different data shapes within the tree
structure of the repository in order to demonstrate those cases.

Add several interesting special cases all at once instead of sprinkling
them across several commits. The interesting cases being added here are:

* Add sparse-directory entries on both sides of directories within the
  sparse-checkout definition.

* Add directories outside the sparse-checkout definition who have only
  one entry and are the first entry of a directory with multiple
  entries.

* Add filenames adjacent to a sparse directory entry that sort before
  and after the trailing slash.

Later tests will take advantage of these shapes, but they also deepen
the tests that already exist.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 t/t1092-sparse-checkout-compatibility.sh | 44 ++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index ebbba044f7..0e71a62361 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -17,7 +17,7 @@ test_expect_success 'setup' '
 		echo "after folder1" >g &&
 		echo "after x" >z &&
 		mkdir folder1 folder2 deep x &&
-		mkdir deep/deeper1 deep/deeper2 &&
+		mkdir deep/deeper1 deep/deeper2 deep/before deep/later &&
 		mkdir deep/deeper1/deepest &&
 		echo "after deeper1" >deep/e &&
 		echo "after deepest" >deep/deeper1/e &&
@@ -25,10 +25,23 @@ test_expect_success 'setup' '
 		cp a folder2 &&
 		cp a x &&
 		cp a deep &&
+		cp a deep/before &&
 		cp a deep/deeper1 &&
 		cp a deep/deeper2 &&
+		cp a deep/later &&
 		cp a deep/deeper1/deepest &&
 		cp -r deep/deeper1/deepest deep/deeper2 &&
+		mkdir deep/deeper1/0 &&
+		mkdir deep/deeper1/0/0 &&
+		touch deep/deeper1/0/1 &&
+		touch deep/deeper1/0/0/0 &&
+		>folder1- &&
+		>folder1.x &&
+		>folder10 &&
+		cp -r deep/deeper1/0 folder1 &&
+		cp -r deep/deeper1/0 folder2 &&
+		echo >>folder1/0/0/0 &&
+		echo >>folder2/0/1 &&
 		git add . &&
 		git commit -m "initial commit" &&
 		git checkout -b base &&
@@ -56,11 +69,17 @@ test_expect_success 'setup' '
 		mv folder1/a folder2/b &&
 		mv folder1/larger-content folder2/edited-content &&
 		echo >>folder2/edited-content &&
+		echo >>folder2/0/1 &&
+		echo stuff >>deep/deeper1/a &&
 		git add . &&
 		git commit -m "rename folder1/... to folder2/..." &&
 
 		git checkout -b rename-out-to-in rename-base &&
 		mv folder1/a deep/deeper1/b &&
+		echo more stuff >>deep/deeper1/a &&
+		rm folder2/0/1 &&
+		mkdir folder2/0/1 &&
+		echo >>folder2/0/1/1 &&
 		mv folder1/larger-content deep/deeper1/edited-content &&
 		echo >>deep/deeper1/edited-content &&
 		git add . &&
@@ -68,6 +87,9 @@ test_expect_success 'setup' '
 
 		git checkout -b rename-in-to-out rename-base &&
 		mv deep/deeper1/a folder1/b &&
+		echo >>folder2/0/1 &&
+		rm -rf folder1/0/0 &&
+		echo >>folder1/0/0 &&
 		mv deep/deeper1/larger-content folder1/edited-content &&
 		echo >>folder1/edited-content &&
 		git add . &&
@@ -262,13 +284,29 @@ test_expect_success 'diff --staged' '
 	test_all_match git diff --staged
 '
 
-test_expect_success 'diff with renames' '
+test_expect_success 'diff with renames and conflicts' '
 	init_repos &&
 
 	for branch in rename-out-to-out rename-out-to-in rename-in-to-out
 	do
 		test_all_match git checkout rename-base &&
-		test_all_match git checkout $branch -- .&&
+		test_all_match git checkout $branch -- . &&
+		test_all_match git status --porcelain=v2 &&
+		test_all_match git diff --staged --no-renames &&
+		test_all_match git diff --staged --find-renames || return 1
+	done
+'
+
+test_expect_success 'diff with directory/file conflicts' '
+	init_repos &&
+
+	for branch in rename-out-to-out rename-out-to-in rename-in-to-out
+	do
+		git -C full-checkout reset --hard &&
+		test_sparse_match git reset --hard &&
+		test_all_match git checkout $branch &&
+		test_all_match git checkout rename-base -- . &&
+		test_all_match git status --porcelain=v2 &&
 		test_all_match git diff --staged --no-renames &&
 		test_all_match git diff --staged --find-renames || return 1
 	done

From bf26c06f126219498a10ae9f8923cfb0bcbad823 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:29 +0000
Subject: [PATCH 355/397] t1092: add tests for status/add and sparse files

Before moving to update 'git status' and 'git add' to work with sparse
indexes, add an explicit test that ensures the sparse-index works the
same as a normal sparse-checkout when the worktree contains directories
and files outside of the sparse cone.

Specifically, 'folder1/a' is a file in our test repo, but 'folder1' is
not in the sparse cone. When 'folder1/a' is modified, the file is not
shown as modified and adding it will fail. This is new behavior as of
a20f704 (add: warn when asked to update SKIP_WORKTREE entries,
2021-04-08). Before that change, these adds would be silently ignored.

Untracked files are fine: adding new files both with 'git add .' and
'git add folder1/' works just as in a full checkout. This may not be
entirely desirable, but we are not intending to change behavior at the
moment, only document it. A future change could alter the behavior to
be more sensible, and this test could be modified to satisfy the new
expected behavior.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 t/t1092-sparse-checkout-compatibility.sh | 38 ++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index 0e71a62361..2269f44e03 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -254,6 +254,44 @@ test_expect_success 'add, commit, checkout' '
 	test_all_match git checkout -
 '
 
+test_expect_success 'status/add: outside sparse cone' '
+	init_repos &&
+
+	# adding a "missing" file outside the cone should fail
+	test_sparse_match test_must_fail git add folder1/a &&
+
+	# folder1 is at HEAD, but outside the sparse cone
+	run_on_sparse mkdir folder1 &&
+	cp initial-repo/folder1/a sparse-checkout/folder1/a &&
+	cp initial-repo/folder1/a sparse-index/folder1/a &&
+
+	test_sparse_match git status &&
+
+	write_script edit-contents <<-\EOF &&
+	echo text >>$1
+	EOF
+	run_on_sparse ../edit-contents folder1/a &&
+	run_on_all ../edit-contents folder1/new &&
+
+	test_sparse_match git status --porcelain=v2 &&
+
+	# This "git add folder1/a" fails with a warning
+	# in the sparse repos, differing from the full
+	# repo. This is intentional.
+	test_sparse_match test_must_fail git add folder1/a &&
+	test_sparse_match test_must_fail git add --refresh folder1/a &&
+	test_all_match git status --porcelain=v2 &&
+
+	test_all_match git add . &&
+	test_all_match git status --porcelain=v2 &&
+	test_all_match git commit -m folder1/new &&
+
+	run_on_all ../edit-contents folder1/newer &&
+	test_all_match git add folder1/ &&
+	test_all_match git status --porcelain=v2 &&
+	test_all_match git commit -m folder1/newer
+'
+
 test_expect_success 'checkout and reset --hard' '
 	init_repos &&
 

From 17a1bb570bcd165a3dc1c28c441bee45a941bb12 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:30 +0000
Subject: [PATCH 356/397] unpack-trees: preserve cache_bottom

The cache_bottom member of 'struct unpack_trees_options' is used to
track the range of index entries corresponding to a node of the cache
tree. While recursing with traverse_by_cache_tree(), this value is
preserved on the call stack using a local and then restored as that
method returns.

The mark_ce_used() method normally modifies the cache_bottom member when
it refers to the marked cache entry. However, sparse directory entries
are stored as nodes in the cache-tree data structure as of 2de37c53
(cache-tree: integrate with sparse directory entries, 2021-03-30). Thus,
the cache_bottom will be modified as the cache-tree walk advances. Do
not update it as well within mark_ce_used().

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 unpack-trees.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/unpack-trees.c b/unpack-trees.c
index f88a69f8e7..87c1ed204c 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -600,6 +600,13 @@ static void mark_ce_used(struct cache_entry *ce, struct unpack_trees_options *o)
 {
 	ce->ce_flags |= CE_UNPACKED;
 
+	/*
+	 * If this is a sparse directory, don't advance cache_bottom.
+	 * That will be advanced later using the cache-tree data.
+	 */
+	if (S_ISSPARSEDIR(ce->ce_mode))
+		return;
+
 	if (o->cache_bottom < o->src_index->cache_nr &&
 	    o->src_index->cache[o->cache_bottom] == ce) {
 		int bottom = o->cache_bottom;

From cd807a5cdabe5720071d91801dbc76faa8a643ba Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:31 +0000
Subject: [PATCH 357/397] unpack-trees: compare sparse directories correctly

As we further integrate the sparse-index into unpack-trees, we need to
ensure that we compare sparse directory entries correctly with other
entries. This affects searching for an exact path as well as sorting
index entries.

Sparse directory entries contain the trailing directory separator. This
is important for the sorting, in particular. Thus, within
do_compare_entry() we stop using S_IFREG in all cases, since sparse
directories should use S_IFDIR to indicate that the comparison should
treat the entry name as a dirctory.

Within compare_entry(), it first calls do_compare_entry() to check the
leading portion of the name. When the input path is a directory name, we
could match exactly already. Thus, we should return 0 if we have an
exact string match on a sparse directory entry. The final check is a
length comparison between the strings.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 unpack-trees.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index 87c1ed204c..b113cc750f 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -983,6 +983,7 @@ static int do_compare_entry(const struct cache_entry *ce,
 	int pathlen, ce_len;
 	const char *ce_name;
 	int cmp;
+	unsigned ce_mode;
 
 	/*
 	 * If we have not precomputed the traverse path, it is quicker
@@ -1005,7 +1006,8 @@ static int do_compare_entry(const struct cache_entry *ce,
 	ce_len -= pathlen;
 	ce_name = ce->name + pathlen;
 
-	return df_name_compare(ce_name, ce_len, S_IFREG, name, namelen, mode);
+	ce_mode = S_ISSPARSEDIR(ce->ce_mode) ? S_IFDIR : S_IFREG;
+	return df_name_compare(ce_name, ce_len, ce_mode, name, namelen, mode);
 }
 
 static int compare_entry(const struct cache_entry *ce, const struct traverse_info *info, const struct name_entry *n)
@@ -1014,6 +1016,16 @@ static int compare_entry(const struct cache_entry *ce, const struct traverse_inf
 	if (cmp)
 		return cmp;
 
+	/*
+	 * At this point, we know that we have a prefix match. If ce
+	 * is a sparse directory, then allow an exact match. This only
+	 * works when the input name is a directory, since ce->name
+	 * ends in a directory separator.
+	 */
+	if (S_ISSPARSEDIR(ce->ce_mode) &&
+	    ce->ce_namelen == traverse_path_len(info, tree_entry_len(n)) + 1)
+		return 0;
+
 	/*
 	 * Even if the beginning compared identically, the ce should
 	 * compare as bigger than a directory leading up to it!

From bd6a3fd7f1ab698c7ed1222c70a7a0f6a6592bd8 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:32 +0000
Subject: [PATCH 358/397] unpack-trees: rename unpack_nondirectories()

In the next change, we will use this method to unpack a sparse directory
entry, so change the name to unpack_single_entry() so these entries
apply. The new name reflects that we will not recurse into trees in
order to resolve the conflicts.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 unpack-trees.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index b113cc750f..d26386ce8b 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -804,7 +804,7 @@ static int traverse_by_cache_tree(int pos, int nr_entries, int nr_names,
 		BUG("We need cache-tree to do this optimization");
 
 	/*
-	 * Do what unpack_callback() and unpack_nondirectories() normally
+	 * Do what unpack_callback() and unpack_single_entry() normally
 	 * do. But we walk all paths in an iterative loop instead.
 	 *
 	 * D/F conflicts and higher stage entries are not a concern
@@ -1075,11 +1075,11 @@ static struct cache_entry *create_ce_entry(const struct traverse_info *info,
  * without actually calling it. If you change the logic here you may need to
  * check and change there as well.
  */
-static int unpack_nondirectories(int n, unsigned long mask,
-				 unsigned long dirmask,
-				 struct cache_entry **src,
-				 const struct name_entry *names,
-				 const struct traverse_info *info)
+static int unpack_single_entry(int n, unsigned long mask,
+			       unsigned long dirmask,
+			       struct cache_entry **src,
+			       const struct name_entry *names,
+			       const struct traverse_info *info)
 {
 	int i;
 	struct unpack_trees_options *o = info->data;
@@ -1322,7 +1322,7 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str
 		}
 	}
 
-	if (unpack_nondirectories(n, mask, dirmask, src, names, info) < 0)
+	if (unpack_single_entry(n, mask, dirmask, src, names, info) < 0)
 		return -1;
 
 	if (o->merge && src[0]) {

From 523506df51e2e13c8c354dff1eea3378bac8dec8 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:33 +0000
Subject: [PATCH 359/397] unpack-trees: unpack sparse directory entries

During unpack_callback(), index entries are compared against tree
entries. These are matched according to names and types. One goal is to
decide if we should recurse into subtrees or simply operate on one index
entry.

In the case of a sparse-directory entry, we do not want to recurse into
that subtree and instead simply compare the trees. In some cases, we
might want to perform a merge operation on the entry, such as during
'git checkout ' which wants to replace a sparse tree entry with
the tree for that path at the target commit. We extend the logic within
unpack_single_entry() to create a sparse-directory entry in this case,
and then that is sent to call_unpack_fn().

There are some subtleties in this process. For instance, we need to
update find_cache_entry() to allow finding a sparse-directory entry that
exactly matches a given path. Use the new helper method
sparse_dir_matches_path() for this. We also need to ignore conflict
markers in the case that the entries correspond to directories and we
already have a sparse directory entry.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 unpack-trees.c | 107 +++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 99 insertions(+), 8 deletions(-)

diff --git a/unpack-trees.c b/unpack-trees.c
index d26386ce8b..0a5135ab39 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -1052,13 +1052,15 @@ static struct cache_entry *create_ce_entry(const struct traverse_info *info,
 	const struct name_entry *n,
 	int stage,
 	struct index_state *istate,
-	int is_transient)
+	int is_transient,
+	int is_sparse_directory)
 {
 	size_t len = traverse_path_len(info, tree_entry_len(n));
+	size_t alloc_len = is_sparse_directory ? len + 1 : len;
 	struct cache_entry *ce =
 		is_transient ?
-		make_empty_transient_cache_entry(len, NULL) :
-		make_empty_cache_entry(istate, len);
+		make_empty_transient_cache_entry(alloc_len, NULL) :
+		make_empty_cache_entry(istate, alloc_len);
 
 	ce->ce_mode = create_ce_mode(n->mode);
 	ce->ce_flags = create_ce_flags(stage);
@@ -1067,6 +1069,13 @@ static struct cache_entry *create_ce_entry(const struct traverse_info *info,
 	/* len+1 because the cache_entry allocates space for NUL */
 	make_traverse_path(ce->name, len + 1, info, n->path, n->pathlen);
 
+	if (is_sparse_directory) {
+		ce->name[len] = '/';
+		ce->name[len + 1] = '\0';
+		ce->ce_namelen++;
+		ce->ce_flags |= CE_SKIP_WORKTREE;
+	}
+
 	return ce;
 }
 
@@ -1085,10 +1094,17 @@ static int unpack_single_entry(int n, unsigned long mask,
 	struct unpack_trees_options *o = info->data;
 	unsigned long conflicts = info->df_conflicts | dirmask;
 
-	/* Do we have *only* directories? Nothing to do */
 	if (mask == dirmask && !src[0])
 		return 0;
 
+	/*
+	 * When we have a sparse directory entry for src[0],
+	 * then this isn't necessarily a directory-file conflict.
+	 */
+	if (mask == dirmask && src[0] &&
+	    S_ISSPARSEDIR(src[0]->ce_mode))
+		conflicts = 0;
+
 	/*
 	 * Ok, we've filled in up to any potential index entry in src[0],
 	 * now do the rest.
@@ -1118,7 +1134,9 @@ static int unpack_single_entry(int n, unsigned long mask,
 		 * not stored in the index.  otherwise construct the
 		 * cache entry from the index aware logic.
 		 */
-		src[i + o->merge] = create_ce_entry(info, names + i, stage, &o->result, o->merge);
+		src[i + o->merge] = create_ce_entry(info, names + i, stage,
+						    &o->result, o->merge,
+						    bit & dirmask);
 	}
 
 	if (o->merge) {
@@ -1222,16 +1240,71 @@ static int find_cache_pos(struct traverse_info *info,
 	return -1;
 }
 
+/*
+ * Given a sparse directory entry 'ce', compare ce->name to
+ * info->name + '/' + p->path + '/' if info->name is non-empty.
+ * Compare ce->name to p->path + '/' otherwise. Note that
+ * ce->name must end in a trailing '/' because it is a sparse
+ * directory entry.
+ */
+static int sparse_dir_matches_path(const struct cache_entry *ce,
+				   struct traverse_info *info,
+				   const struct name_entry *p)
+{
+	assert(S_ISSPARSEDIR(ce->ce_mode));
+	assert(ce->name[ce->ce_namelen - 1] == '/');
+
+	if (info->namelen)
+		return ce->ce_namelen == info->namelen + p->pathlen + 2 &&
+		       ce->name[info->namelen] == '/' &&
+		       !strncmp(ce->name, info->name, info->namelen) &&
+		       !strncmp(ce->name + info->namelen + 1, p->path, p->pathlen);
+	return ce->ce_namelen == p->pathlen + 1 &&
+	       !strncmp(ce->name, p->path, p->pathlen);
+}
+
 static struct cache_entry *find_cache_entry(struct traverse_info *info,
 					    const struct name_entry *p)
 {
+	struct cache_entry *ce;
 	int pos = find_cache_pos(info, p->path, p->pathlen);
 	struct unpack_trees_options *o = info->data;
 
 	if (0 <= pos)
 		return o->src_index->cache[pos];
-	else
+
+	/*
+	 * Check for a sparse-directory entry named "path/".
+	 * Due to the input p->path not having a trailing
+	 * slash, the negative 'pos' value overshoots the
+	 * expected position, hence "-2" instead of "-1".
+	 */
+	pos = -pos - 2;
+
+	if (pos < 0 || pos >= o->src_index->cache_nr)
 		return NULL;
+
+	/*
+	 * Due to lexicographic sorting and sparse directory
+	 * entries ending with a trailing slash, our path as a
+	 * sparse directory (e.g "subdir/") and	our path as a
+	 * file (e.g. "subdir") might be separated by other
+	 * paths (e.g. "subdir-").
+	 */
+	while (pos >= 0) {
+		ce = o->src_index->cache[pos];
+
+		if (strncmp(ce->name, p->path, p->pathlen))
+			return NULL;
+
+		if (S_ISSPARSEDIR(ce->ce_mode) &&
+		    sparse_dir_matches_path(ce, info, p))
+			return ce;
+
+		pos--;
+	}
+
+	return NULL;
 }
 
 static void debug_path(struct traverse_info *info)
@@ -1266,6 +1339,21 @@ static void debug_unpack_callback(int n,
 		debug_name_entry(i, names + i);
 }
 
+/*
+ * Returns true if and only if the given cache_entry is a
+ * sparse-directory entry that matches the given name_entry
+ * from the tree walk at the given traverse_info.
+ */
+static int is_sparse_directory_entry(struct cache_entry *ce,
+				     struct name_entry *name,
+				     struct traverse_info *info)
+{
+	if (!ce || !name || !S_ISSPARSEDIR(ce->ce_mode))
+		return 0;
+
+	return sparse_dir_matches_path(ce, info, name);
+}
+
 /*
  * Note that traverse_by_cache_tree() duplicates some logic in this function
  * without actually calling it. If you change the logic here you may need to
@@ -1352,9 +1440,12 @@ static int unpack_callback(int n, unsigned long mask, unsigned long dirmask, str
 			}
 		}
 
-		if (traverse_trees_recursive(n, dirmask, mask & ~dirmask,
-					     names, info) < 0)
+		if (!is_sparse_directory_entry(src[0], names, info) &&
+		    traverse_trees_recursive(n, dirmask, mask & ~dirmask,
+						    names, info) < 0) {
 			return -1;
+		}
+
 		return mask;
 	}
 

From 69bdbdb0ee25bd64bd3ad2fe241d434442e96f75 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:34 +0000
Subject: [PATCH 360/397] dir.c: accept a directory as part of cone-mode
 patterns

When we have sparse directory entries in the index, we want to compare
that directory against sparse-checkout patterns. Those pattern matching
algorithms are built expecting a file path, not a directory path. This
is especially important in the "cone mode" patterns which will match
files that exist within the "parent directories" as well as the
recursive directory matches.

If path_matches_pattern_list() is given a directory, we can add a fake
filename ("-") to the directory and get the same results as before,
assuming we are in cone mode. Since sparse index requires cone mode
patterns, this is an acceptable assumption.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 dir.c | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/dir.c b/dir.c
index ebe5ec046e..0c5264b3b2 100644
--- a/dir.c
+++ b/dir.c
@@ -1376,7 +1376,7 @@ enum pattern_match_result path_matches_pattern_list(
 	struct path_pattern *pattern;
 	struct strbuf parent_pathname = STRBUF_INIT;
 	int result = NOT_MATCHED;
-	const char *slash_pos;
+	size_t slash_pos;
 
 	if (!pl->use_cone_patterns) {
 		pattern = last_matching_pattern_from_list(pathname, pathlen, basename,
@@ -1397,21 +1397,35 @@ enum pattern_match_result path_matches_pattern_list(
 	strbuf_addch(&parent_pathname, '/');
 	strbuf_add(&parent_pathname, pathname, pathlen);
 
+	/*
+	 * Directory entries are matched if and only if a file
+	 * contained immediately within them is matched. For the
+	 * case of a directory entry, modify the path to create
+	 * a fake filename within this directory, allowing us to
+	 * use the file-base matching logic in an equivalent way.
+	 */
+	if (parent_pathname.len > 0 &&
+	    parent_pathname.buf[parent_pathname.len - 1] == '/') {
+		slash_pos = parent_pathname.len - 1;
+		strbuf_add(&parent_pathname, "-", 1);
+	} else {
+		const char *slash_ptr = strrchr(parent_pathname.buf, '/');
+		slash_pos = slash_ptr ? slash_ptr - parent_pathname.buf : 0;
+	}
+
 	if (hashmap_contains_path(&pl->recursive_hashmap,
 				  &parent_pathname)) {
 		result = MATCHED_RECURSIVE;
 		goto done;
 	}
 
-	slash_pos = strrchr(parent_pathname.buf, '/');
-
-	if (slash_pos == parent_pathname.buf) {
+	if (!slash_pos) {
 		/* include every file in root */
 		result = MATCHED;
 		goto done;
 	}
 
-	strbuf_setlen(&parent_pathname, slash_pos - parent_pathname.buf);
+	strbuf_setlen(&parent_pathname, slash_pos);
 
 	if (hashmap_contains_path(&pl->parent_hashmap, &parent_pathname)) {
 		result = MATCHED;

From 9eb00af5627f1a04ca083128085ca6774fce0524 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:35 +0000
Subject: [PATCH 361/397] diff-lib: handle index diffs with sparse dirs

While comparing an index to a tree, we may see a sparse directory entry.
In this case, we should compare that portion of the tree to the tree
represented by that entry. This could include a new tree which needs to
be expanded to a full list of added files. It could also include an
existing tree, in which case all of the changes inside are important to
describe, including the modifications, additions, and deletions. Note
that the case where the tree has a path and the index does not remains
identical to before: the lack of a cache entry is the same with a sparse
index.

Use diff_tree_oid() appropriately to compute the diff.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 diff-lib.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/diff-lib.c b/diff-lib.c
index c2ac9250fe..f9eadc4fc1 100644
--- a/diff-lib.c
+++ b/diff-lib.c
@@ -325,6 +325,11 @@ static void show_new_file(struct rev_info *revs,
 	unsigned dirty_submodule = 0;
 	struct index_state *istate = revs->diffopt.repo->index;
 
+	if (new_file && S_ISSPARSEDIR(new_file->ce_mode)) {
+		diff_tree_oid(NULL, &new_file->oid, new_file->name, &revs->diffopt);
+		return;
+	}
+
 	/*
 	 * New file in the index: it might actually be different in
 	 * the working tree.
@@ -347,6 +352,20 @@ static int show_modified(struct rev_info *revs,
 	unsigned dirty_submodule = 0;
 	struct index_state *istate = revs->diffopt.repo->index;
 
+	assert(S_ISSPARSEDIR(old_entry->ce_mode) ==
+	       S_ISSPARSEDIR(new_entry->ce_mode));
+
+	/*
+	 * If both are sparse directory entries, then expand the
+	 * modifications to the file level. If only one was a sparse
+	 * directory, then they appear as an add and delete instead of
+	 * a modification.
+	 */
+	if (S_ISSPARSEDIR(new_entry->ce_mode)) {
+		diff_tree_oid(&old_entry->oid, &new_entry->oid, new_entry->name, &revs->diffopt);
+		return 0;
+	}
+
 	if (get_stat_data(istate, new_entry, &oid, &mode, cached, match_missing,
 			  &dirty_submodule, &revs->diffopt) < 0) {
 		if (report_missing)

From bf48e5acdbf25a98fd142d9c90157c10b427119a Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:36 +0000
Subject: [PATCH 362/397] status: skip sparse-checkout percentage with
 sparse-index

'git status' began reporting a percentage of populated paths when
sparse-checkout is enabled in 051df3cf (wt-status: show sparse
checkout status as well, 2020-07-18). This percentage is incorrect when
the index has sparse directories. It would also be expensive to
calculate as we would need to parse trees to count the total number of
possible paths.

Avoid the expensive computation by simplifying the output to only report
that a sparse checkout exists, without the percentage.

This change is the reason we use 'git status --porcelain=v2' in
t1092-sparse-checkout-compatibility.sh. We don't want to ensure that
this message is equal across both modes, but instead just the important
information about staged, modified, and untracked files are compared.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 t/t1092-sparse-checkout-compatibility.sh |  8 ++++++++
 wt-status.c                              | 14 +++++++++++---
 wt-status.h                              |  1 +
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index 2269f44e03..375b0d3556 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -218,6 +218,14 @@ test_expect_success 'status with options' '
 	test_all_match git status --porcelain=v2 -uno
 '
 
+test_expect_success 'status reports sparse-checkout' '
+	init_repos &&
+	git -C sparse-checkout status >full &&
+	git -C sparse-index status >sparse &&
+	test_i18ngrep "You are in a sparse checkout with " full &&
+	test_i18ngrep "You are in a sparse checkout." sparse
+'
+
 test_expect_success 'add, commit, checkout' '
 	init_repos &&
 
diff --git a/wt-status.c b/wt-status.c
index 42b6735716..96db3e7496 100644
--- a/wt-status.c
+++ b/wt-status.c
@@ -1493,9 +1493,12 @@ static void show_sparse_checkout_in_use(struct wt_status *s,
 	if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_DISABLED)
 		return;
 
-	status_printf_ln(s, color,
-			 _("You are in a sparse checkout with %d%% of tracked files present."),
-			 s->state.sparse_checkout_percentage);
+	if (s->state.sparse_checkout_percentage == SPARSE_CHECKOUT_SPARSE_INDEX)
+		status_printf_ln(s, color, _("You are in a sparse checkout."));
+	else
+		status_printf_ln(s, color,
+				_("You are in a sparse checkout with %d%% of tracked files present."),
+				s->state.sparse_checkout_percentage);
 	wt_longstatus_print_trailer(s);
 }
 
@@ -1653,6 +1656,11 @@ static void wt_status_check_sparse_checkout(struct repository *r,
 		return;
 	}
 
+	if (r->index->sparse_index) {
+		state->sparse_checkout_percentage = SPARSE_CHECKOUT_SPARSE_INDEX;
+		return;
+	}
+
 	for (i = 0; i < r->index->cache_nr; i++) {
 		struct cache_entry *ce = r->index->cache[i];
 		if (ce_skip_worktree(ce))
diff --git a/wt-status.h b/wt-status.h
index 0d32799b28..ab9cc9d8f0 100644
--- a/wt-status.h
+++ b/wt-status.h
@@ -78,6 +78,7 @@ enum wt_status_format {
 };
 
 #define SPARSE_CHECKOUT_DISABLED -1
+#define SPARSE_CHECKOUT_SPARSE_INDEX -2
 
 struct wt_status_state {
 	int merge_in_progress;

From d76723ee5313d0176e8777af4aa104d7562543c1 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:37 +0000
Subject: [PATCH 363/397] status: use sparse-index throughout

By testing 'git -c core.fsmonitor= status -uno', we can check for the
simplest index operations that can be made sparse-aware. The necessary
implementation details are already integrated with sparse-checkout, so
modify command_requires_full_index to be zero for cmd_status().

In refresh_index(), we loop through the index entries to refresh their
stat() information. However, sparse directories have no stat()
information to populate. Ignore these entries.

This allows 'git status' to no longer expand a sparse index to a full
one. This is further tested by dropping the "-uno" option and adding an
untracked file into the worktree.

The performance test p2000-sparse-checkout-operations.sh demonstrates
these improvements:

Test                                  HEAD~1           HEAD
-----------------------------------------------------------------------------
2000.2: git status (full-index-v3)    0.31(0.30+0.05)  0.31(0.29+0.06) +0.0%
2000.3: git status (full-index-v4)    0.31(0.29+0.07)  0.34(0.30+0.08) +9.7%
2000.4: git status (sparse-index-v3)  2.35(2.28+0.10)  0.04(0.04+0.05) -98.3%
2000.5: git status (sparse-index-v4)  2.35(2.24+0.15)  0.05(0.04+0.06) -97.9%

Note that since HEAD~1 was expanding the sparse index by parsing trees,
it was artificially slower than the full index case. Thus, the 98%
improvement is misleading, and instead we should celebrate the 0.34s to
0.05s improvement of 85%. This is more indicative of the peformance
gains we are expecting by using a sparse index.

Note: we are dropping the assignment of core.fsmonitor here. This is not
necessary for the test script as we are not altering the config any
other way. Correct integration with FS Monitor will be validated in
later changes.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 builtin/commit.c                         |  3 +++
 read-cache.c                             | 10 ++++++++--
 t/t1092-sparse-checkout-compatibility.sh | 15 ++++++++++-----
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/builtin/commit.c b/builtin/commit.c
index 190d215d43..12f51db158 100644
--- a/builtin/commit.c
+++ b/builtin/commit.c
@@ -1510,6 +1510,9 @@ int cmd_status(int argc, const char **argv, const char *prefix)
 	if (argc == 2 && !strcmp(argv[1], "-h"))
 		usage_with_options(builtin_status_usage, builtin_status_options);
 
+	prepare_repo_settings(the_repository);
+	the_repository->settings.command_requires_full_index = 0;
+
 	status_init_config(&s, git_status_config);
 	argc = parse_options(argc, argv, prefix,
 			     builtin_status_options,
diff --git a/read-cache.c b/read-cache.c
index 77961a3885..f67e04949d 100644
--- a/read-cache.c
+++ b/read-cache.c
@@ -1585,8 +1585,7 @@ int refresh_index(struct index_state *istate, unsigned int flags,
 	 */
 	preload_index(istate, pathspec, 0);
 	trace2_region_enter("index", "refresh", NULL);
-	/* TODO: audit for interaction with sparse-index. */
-	ensure_full_index(istate);
+
 	for (i = 0; i < istate->cache_nr; i++) {
 		struct cache_entry *ce, *new_entry;
 		int cache_errno = 0;
@@ -1601,6 +1600,13 @@ int refresh_index(struct index_state *istate, unsigned int flags,
 		if (ignore_skip_worktree && ce_skip_worktree(ce))
 			continue;
 
+		/*
+		 * If this entry is a sparse directory, then there isn't
+		 * any stat() information to update. Ignore the entry.
+		 */
+		if (S_ISSPARSEDIR(ce->ce_mode))
+			continue;
+
 		if (pathspec && !ce_path_match(istate, ce, pathspec, seen))
 			filtered = 1;
 
diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index 375b0d3556..751f397cc7 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -511,12 +511,17 @@ test_expect_success 'sparse-index is expanded and converted back' '
 	GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
 		git -C sparse-index -c core.fsmonitor="" reset --hard &&
 	test_region index convert_to_sparse trace2.txt &&
-	test_region index ensure_full_index trace2.txt &&
-
-	rm trace2.txt &&
-	GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
-		git -C sparse-index -c core.fsmonitor="" status -uno &&
 	test_region index ensure_full_index trace2.txt
 '
 
+test_expect_success 'sparse-index is not expanded' '
+	init_repos &&
+
+	rm -f trace2.txt &&
+	echo >>sparse-index/untracked.txt &&
+	GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
+		git -C sparse-index status &&
+	test_region ! index ensure_full_index trace2.txt
+'
+
 test_done

From fe0d576153117953189e03d6c2c09445ccad4977 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:38 +0000
Subject: [PATCH 364/397] wt-status: expand added sparse directory entries

It is difficult, but possible, to get into a state where we intend to
add a directory that is outside of the sparse-checkout definition. Add a
test to t1092-sparse-checkout-compatibility.sh that demonstrates this
using a combination of 'git reset --mixed' and 'git checkout --orphan'.

This test failed before because the output of 'git status
--porcelain=v2' would not match on the lines for folder1/:

* The sparse-checkout repo (with a full index) would output each path
  name that is intended to be added.

* The sparse-index repo would only output that "folder1/" is staged for
  addition.

The status should report the full list of files to be added, and so this
sparse-directory entry should be expanded to a full list when reaching
it inside the wt_status_collect_changes_initial() method. Use
read_tree_at() to assist.

Somehow, this loop over the cache entries was not guarded by
ensure_full_index() as intended.

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 t/t1092-sparse-checkout-compatibility.sh | 33 +++++++++++++++
 wt-status.c                              | 51 ++++++++++++++++++++++++
 2 files changed, 84 insertions(+)

diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index 751f397cc7..2394c36d88 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -524,4 +524,37 @@ test_expect_success 'sparse-index is not expanded' '
 	test_region ! index ensure_full_index trace2.txt
 '
 
+test_expect_success 'reset mixed and checkout orphan' '
+	init_repos &&
+
+	test_all_match git checkout rename-out-to-in &&
+
+	# Sparse checkouts do not agree with full checkouts about
+	# how to report a directory/file conflict during a reset.
+	# This command would fail with test_all_match because the
+	# full checkout reports "T folder1/0/1" while a sparse
+	# checkout reports "D folder1/0/1". This matches because
+	# the sparse checkouts skip "adding" the other side of
+	# the conflict.
+	test_sparse_match git reset --mixed HEAD~1 &&
+	test_sparse_match test-tool read-cache --table --expand &&
+	test_sparse_match git status --porcelain=v2 &&
+
+	# At this point, sparse-checkouts behave differently
+	# from the full-checkout.
+	test_sparse_match git checkout --orphan new-branch &&
+	test_sparse_match test-tool read-cache --table --expand &&
+	test_sparse_match git status --porcelain=v2
+'
+
+test_expect_success 'add everything with deep new file' '
+	init_repos &&
+
+	run_on_sparse git sparse-checkout set deep/deeper1/deepest &&
+
+	run_on_all touch deep/deeper1/x &&
+	test_all_match git add . &&
+	test_all_match git status --porcelain=v2
+'
+
 test_done
diff --git a/wt-status.c b/wt-status.c
index 96db3e7496..0317baef87 100644
--- a/wt-status.c
+++ b/wt-status.c
@@ -657,6 +657,36 @@ static void wt_status_collect_changes_index(struct wt_status *s)
 	clear_pathspec(&rev.prune_data);
 }
 
+static int add_file_to_list(const struct object_id *oid,
+			    struct strbuf *base, const char *path,
+			    unsigned int mode, void *context)
+{
+	struct string_list_item *it;
+	struct wt_status_change_data *d;
+	struct wt_status *s = context;
+	struct strbuf full_name = STRBUF_INIT;
+
+	if (S_ISDIR(mode))
+		return READ_TREE_RECURSIVE;
+
+	strbuf_add(&full_name, base->buf, base->len);
+	strbuf_addstr(&full_name, path);
+	it = string_list_insert(&s->change, full_name.buf);
+	d = it->util;
+	if (!d) {
+		CALLOC_ARRAY(d, 1);
+		it->util = d;
+	}
+
+	d->index_status = DIFF_STATUS_ADDED;
+	/* Leave {mode,oid}_head zero for adds. */
+	d->mode_index = mode;
+	oidcpy(&d->oid_index, oid);
+	s->committable = 1;
+	strbuf_release(&full_name);
+	return 0;
+}
+
 static void wt_status_collect_changes_initial(struct wt_status *s)
 {
 	struct index_state *istate = s->repo->index;
@@ -671,6 +701,27 @@ static void wt_status_collect_changes_initial(struct wt_status *s)
 			continue;
 		if (ce_intent_to_add(ce))
 			continue;
+		if (S_ISSPARSEDIR(ce->ce_mode)) {
+			/*
+			 * This is a sparse directory entry, so we want to collect all
+			 * of the added files within the tree. This requires recursively
+			 * expanding the trees to find the elements that are new in this
+			 * tree and marking them with DIFF_STATUS_ADDED.
+			 */
+			struct strbuf base = STRBUF_INIT;
+			struct pathspec ps = { 0 };
+			struct tree *tree = lookup_tree(istate->repo, &ce->oid);
+
+			ps.recursive = 1;
+			ps.has_wildcard = 1;
+			ps.max_depth = -1;
+
+			strbuf_add(&base, ce->name, ce->ce_namelen);
+			read_tree_at(istate->repo, tree, &base, &ps,
+				     add_file_to_list, s);
+			continue;
+		}
+
 		it = string_list_insert(&s->change, ce->name);
 		d = it->util;
 		if (!d) {

From f8fe49e53958f19b0e62e9549a80fcaa56d2f3cf Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:39 +0000
Subject: [PATCH 365/397] fsmonitor: integrate with sparse index

If we need to expand a sparse-index into a full one, then the FS Monitor
bitmap is going to be incorrect. Ensure that we start fresh at such an
event.

While this is currently a performance drawback, the eventual hope of the
sparse-index feature is that these expansions will be rare and hence we
will be able to keep the FS Monitor data accurate across multiple Git
commands.

These tests are added to demonstrate that the behavior is the same
across a full index and a sparse index, but also that file modifications
to a tracked directory outside of the sparse cone will trigger
ensure_full_index().

Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 sparse-index.c              |  7 ++++++
 t/t7519-status-fsmonitor.sh | 49 +++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/sparse-index.c b/sparse-index.c
index ef53bd2198..53c8f711cc 100644
--- a/sparse-index.c
+++ b/sparse-index.c
@@ -186,6 +186,10 @@ int convert_to_sparse(struct index_state *istate)
 	cache_tree_free(&istate->cache_tree);
 	cache_tree_update(istate, 0);
 
+	istate->fsmonitor_has_run_once = 0;
+	FREE_AND_NULL(istate->fsmonitor_dirty);
+	FREE_AND_NULL(istate->fsmonitor_last_update);
+
 	istate->sparse_index = 1;
 	trace2_region_leave("index", "convert_to_sparse", istate->repo);
 	return 0;
@@ -282,6 +286,9 @@ void ensure_full_index(struct index_state *istate)
 	istate->cache = full->cache;
 	istate->cache_nr = full->cache_nr;
 	istate->cache_alloc = full->cache_alloc;
+	istate->fsmonitor_has_run_once = 0;
+	FREE_AND_NULL(istate->fsmonitor_dirty);
+	FREE_AND_NULL(istate->fsmonitor_last_update);
 
 	strbuf_release(&base);
 	free(full);
diff --git a/t/t7519-status-fsmonitor.sh b/t/t7519-status-fsmonitor.sh
index 637391c6ce..deea88d443 100755
--- a/t/t7519-status-fsmonitor.sh
+++ b/t/t7519-status-fsmonitor.sh
@@ -73,6 +73,7 @@ test_expect_success 'setup' '
 	expect*
 	actual*
 	marker*
+	trace2*
 	EOF
 '
 
@@ -383,4 +384,52 @@ test_expect_success 'status succeeds after staging/unstaging' '
 	)
 '
 
+# Usage:
+# check_sparse_index_behavior [!]
+# If "!" is supplied, then we verify that we do not call ensure_full_index
+# during a call to 'git status'. Otherwise, we verify that we _do_ call it.
+check_sparse_index_behavior () {
+	git status --porcelain=v2 >expect &&
+	git sparse-checkout init --cone --sparse-index &&
+	git sparse-checkout set dir1 dir2 &&
+	GIT_TRACE2_EVENT="$(pwd)/trace2.txt" GIT_TRACE2_EVENT_NESTING=10 \
+		git status --porcelain=v2 >actual &&
+	test_region $1 index ensure_full_index trace2.txt &&
+	test_region fsm_hook query trace2.txt &&
+	test_cmp expect actual &&
+	rm trace2.txt &&
+	git sparse-checkout disable
+}
+
+test_expect_success 'status succeeds with sparse index' '
+	git reset --hard &&
+
+	test_config core.fsmonitor "$TEST_DIRECTORY/t7519/fsmonitor-all" &&
+	check_sparse_index_behavior ! &&
+
+	write_script .git/hooks/fsmonitor-test<<-\EOF &&
+		printf "last_update_token\0"
+	EOF
+	git config core.fsmonitor .git/hooks/fsmonitor-test &&
+	check_sparse_index_behavior ! &&
+
+	write_script .git/hooks/fsmonitor-test<<-\EOF &&
+		printf "last_update_token\0"
+		printf "dir1/modified\0"
+	EOF
+	check_sparse_index_behavior ! &&
+
+	cp -r dir1 dir1a &&
+	git add dir1a &&
+	git commit -m "add dir1a" &&
+
+	# This one modifies outside the sparse-checkout definition
+	# and hence we expect to expand the sparse-index.
+	write_script .git/hooks/fsmonitor-test<<-\EOF &&
+		printf "last_update_token\0"
+		printf "dir1a/modified\0"
+	EOF
+	check_sparse_index_behavior
+'
+
 test_done

From e5ca291076a8a936283bb2c57433c4393d3f80c2 Mon Sep 17 00:00:00 2001
From: Derrick Stolee 
Date: Wed, 14 Jul 2021 13:12:40 +0000
Subject: [PATCH 366/397] t1092: document bad sparse-checkout behavior

There are several situations where a repository with sparse-checkout
enabled will act differently than a normal repository, and in ways that
are not intentional. The test t1092-sparse-checkout-compatibility.sh
documents some of these deviations, but a casual reader might think
these are intentional behavior changes.

Add comments on these tests that make it clear that these behaviors
should be updated. Using 'NEEDSWORK' helps contributors find that these
are potential areas for improvement.

Helped-by: Elijah Newren 
Reviewed-by: Elijah Newren 
Signed-off-by: Derrick Stolee 
Signed-off-by: Junio C Hamano 
---
 t/t1092-sparse-checkout-compatibility.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/t/t1092-sparse-checkout-compatibility.sh b/t/t1092-sparse-checkout-compatibility.sh
index 2394c36d88..cabbd42e33 100755
--- a/t/t1092-sparse-checkout-compatibility.sh
+++ b/t/t1092-sparse-checkout-compatibility.sh
@@ -392,8 +392,8 @@ test_expect_failure 'blame with pathspec outside sparse definition' '
 	test_all_match git blame deep/deeper2/deepest/a
 '
 
-# TODO: reset currently does not behave as expected when in a
-# sparse-checkout.
+# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
+# in this scenario, but it shouldn't.
 test_expect_failure 'checkout and reset (mixed)' '
 	init_repos &&
 
@@ -403,8 +403,8 @@ test_expect_failure 'checkout and reset (mixed)' '
 	test_all_match git reset update-folder2
 '
 
-# Ensure that sparse-index behaves identically to
-# sparse-checkout with a full index.
+# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
+# in this scenario, but it shouldn't.
 test_expect_success 'checkout and reset (mixed) [sparse]' '
 	init_repos &&
 
@@ -524,6 +524,8 @@ test_expect_success 'sparse-index is not expanded' '
 	test_region ! index ensure_full_index trace2.txt
 '
 
+# NEEDSWORK: a sparse-checkout behaves differently from a full checkout
+# in this scenario, but it shouldn't.
 test_expect_success 'reset mixed and checkout orphan' '
 	init_repos &&
 

From cc00362125c7726551d2b6bda85e1a4b17d0bc81 Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Wed, 14 Jul 2021 22:09:36 +0000
Subject: [PATCH 367/397] ci(check-whitespace): stop requiring a read/write
 token

As part of some recent security tightening, GitHub introduced the
ability to configure GitHub workflows to be run with a read-only token.
This is much more secure, in particular when working in a public
repository: While the regular read/write token might be restricted to
writing to the current branch, it is not necessarily restricted to
access only the current Pull Request.

However, the `check-whitespace` workflow threw a wrench into this plan:
it _requires_ write access (because it wants to add a PR comment in case
of a whitespace issue).

Let's just skip that PR comment. The user can always click through to
the actual error, even if it is slightly less convenient.

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/check-whitespace.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml
index f1483059c7..c53614d603 100644
--- a/.github/workflows/check-whitespace.yml
+++ b/.github/workflows/check-whitespace.yml
@@ -51,21 +51,5 @@ jobs:
 
         if test -n "${log}"
         then
-          echo "::set-output name=checkout::"${log}""
           exit 2
         fi
-
-    - name: Add Check Output as Comment
-      uses: actions/github-script@v3
-      id: add-comment
-      env:
-        log: ${{ steps.check_out.outputs.checkout }}
-      with:
-        script: |
-            await github.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: `Whitespace errors found in workflow ${{ github.workflow }}:\n\n\`\`\`\n${process.env.log.replace(/\\n/g, "\n")}\n\`\`\``
-            })
-      if: ${{ failure() }}

From a066a90db68da5262e81e74a50d18eaeddc6783f Mon Sep 17 00:00:00 2001
From: Johannes Schindelin 
Date: Wed, 14 Jul 2021 22:09:37 +0000
Subject: [PATCH 368/397] ci(check-whitespace): restrict to the intended
 commits

During a run of the `check-whitespace` we want to verify that the
commits introduced in the Pull Request have no whitespace issues. We
only want to look at those commits, not the upstream commits (because
the contributor cannot do anything about the latter).

However, by using the `-` form in `git log --check`, we run the
risk of looking at the wrong commits. The reason is that the
`actions/checkout` step does _not_ check out the tip commit of the Pull
Request's branch: Instead, it checks out a merge commit that merges that
branch into the target branch. For that reason, we already adjust the
commit count by incrementing it, but that is not enough: if the upstream
branch has newer commits, they are traversed _first_. And obviously we
will then miss some of the commits that we _actually_ wanted to look at.

Therefore, let's be careful to stop assuming a linear, up to date commit
topology in the contributed commits, and instead specify the correct
commit range.

Unfortunately, this means that we no longer can rely on a shallow clone:
There is no way of knowing just how many commits the upstream branch
advanced after the commit from which the PR branch branched off. So
let's just go with a full clone instead, and be safe rather than sorry
(if we have "too shallow" a situation, a commit range `@{u}..` may very
well include a shallow commit itself, and the output of `git show
--check ` is _not_ pretty).

Signed-off-by: Johannes Schindelin 
Signed-off-by: Junio C Hamano 
---
 .github/workflows/check-whitespace.yml | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml
index c53614d603..8c4358d805 100644
--- a/.github/workflows/check-whitespace.yml
+++ b/.github/workflows/check-whitespace.yml
@@ -12,15 +12,9 @@ jobs:
   check-whitespace:
     runs-on: ubuntu-latest
     steps:
-    - name: Set commit count
-      shell: bash
-      run: echo "COMMIT_DEPTH=$((1+$COMMITS))" >>$GITHUB_ENV
-      env:
-        COMMITS: ${{ github.event.pull_request.commits }}
-
     - uses: actions/checkout@v2
       with:
-        fetch-depth: ${{ env.COMMIT_DEPTH }}
+        fetch-depth: 0
 
     - name: git log --check
       id: check_out
@@ -47,7 +41,7 @@ jobs:
             echo "${dash} ${etc}"
             ;;
           esac
-        done <<< $(git log --check --pretty=format:"---% h% s" -${{github.event.pull_request.commits}})
+        done <<< $(git log --check --pretty=format:"---% h% s" ${{github.event.pull_request.base.sha}}..)
 
         if test -n "${log}"
         then

From 0db4961c49bb70cef89ed3d7c90f8f5d9dfc822d Mon Sep 17 00:00:00 2001
From: Stephen Manz 
Date: Thu, 15 Jul 2021 02:32:30 +0000
Subject: [PATCH 369/397] worktree: teach `add` to accept --reason 
 with --lock

The default reason stored in the lock file, "added with --lock",
is unlikely to be what the user would have given in a separate
`git worktree lock` command. Allowing `--reason` to be specified
along with `--lock` when adding a working tree gives the user control
over the reason for locking without needing a second command.

Signed-off-by: Stephen Manz 
Reviewed-by: Eric Sunshine 
Signed-off-by: Junio C Hamano 
---
 Documentation/git-worktree.txt |  4 ++--
 builtin/worktree.c             | 21 ++++++++++++++++-----
 t/t2400-worktree-add.sh        | 14 ++++++++++++++
 3 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/Documentation/git-worktree.txt b/Documentation/git-worktree.txt
index f1bb1fa5f5..720663746b 100644
--- a/Documentation/git-worktree.txt
+++ b/Documentation/git-worktree.txt
@@ -9,7 +9,7 @@ git-worktree - Manage multiple working trees
 SYNOPSIS
 --------
 [verse]
-'git worktree add' [-f] [--detach] [--checkout] [--lock] [-b ]  []
+'git worktree add' [-f] [--detach] [--checkout] [--lock [--reason ]] [-b ]  []
 'git worktree list' [--porcelain]
 'git worktree lock' [--reason ] 
 'git worktree move'  
@@ -242,7 +242,7 @@ With `list`, annotate missing working trees as prunable if they are
 older than `