Merge branch 'jx/l10n-workflow-change'

A workflow change for translators are being proposed. * jx/l10n-workflow-change: l10n: Document the new l10n workflow Makefile: add "po-init" rule to initialize po/XX.po Makefile: add "po-update" rule to update po/XX.po po/git.pot: don't check in result of "make pot" po/git.pot: this is now a generated file Makefile: remove duplicate and unwanted files in FOUND_SOURCE_FILES i18n CI: stop allowing non-ASCII source messages in po/git.pot Makefile: have "make pot" not "reset --hard" Makefile: generate "po/git.pot" from stable LOCALIZED_C Makefile: sort source files before feeding to xgettext
2022-06-03 14:30:36 -07:00 · 2022-06-03 14:30:36 -07:00 · 28db3b7b71
commit 28db3b7b71
parent 16a0e92ddc e2f4045fc4
8 changed files with 255 additions and 25298 deletions
--- a/.gitignore
+++ b/.gitignore
@ -200,6 +200,7 @@
 *.[aos]
 *.o.json
 *.py[co]
+.build/
 .depend/
 *.gcda
 *.gcno
--- a/153
+++ b/153
@ -569,7 +569,9 @@ INSTALL = install
 TCL_PATH = tclsh
 TCLTK_PATH = wish
 XGETTEXT = xgettext
+MSGCAT = msgcat
 MSGFMT = msgfmt
+MSGMERGE = msgmerge
 CURL_CONFIG = curl-config
 GCOV = gcov
 STRIP = strip
@ -844,7 +846,7 @@ generated-hdrs: $(GENERATED_H)
 ## Exhaustive lists of our source files, either dynamically generated,
 ## or hardcoded.
 SOURCES_CMD = ( \
-	git ls-files \
+	git ls-files --deduplicate \
 		'*.[hcS]' \
 		'*.sh' \
 		':!*[tp][0-9][0-9][0-9][0-9]*' \
@ -855,12 +857,13 @@ SOURCES_CMD = ( \
 		-o \( -name '[tp][0-9][0-9][0-9][0-9]*' -prune \) \
 		-o \( -name contrib -type d -prune \) \
 		-o \( -name build -type d -prune \) \
+		-o \( -name .build -type d -prune \) \
 		-o \( -name 'trash*' -type d -prune \) \
 		-o \( -name '*.[hcS]' -type f -print \) \
 		-o \( -name '*.sh' -type f -print \) \
 		| sed -e 's|^\./||' \
 	)
-FOUND_SOURCE_FILES := $(shell $(SOURCES_CMD))
+FOUND_SOURCE_FILES := $(filter-out $(GENERATED_H),$(shell $(SOURCES_CMD)))

 FOUND_C_SOURCES = $(filter %.c,$(FOUND_SOURCE_FILES))
 FOUND_H_SOURCES = $(filter %.h,$(FOUND_SOURCE_FILES))
@ -2708,17 +2711,18 @@ XGETTEXT_FLAGS = \
 	--force-po \
 	--add-comments=TRANSLATORS: \
 	--msgid-bugs-address="Git Mailing List <git@vger.kernel.org>" \
-	--from-code=UTF-8
+	--package-name=Git
 XGETTEXT_FLAGS_C = $(XGETTEXT_FLAGS) --language=C \
 	--keyword=_ --keyword=N_ --keyword="Q_:1,2"
 XGETTEXT_FLAGS_SH = $(XGETTEXT_FLAGS) --language=Shell \
 	--keyword=gettextln --keyword=eval_gettextln
 XGETTEXT_FLAGS_PERL = $(XGETTEXT_FLAGS) --language=Perl \
 	--keyword=__ --keyword=N__ --keyword="__n:1,2"
-LOCALIZED_C = $(C_OBJ:o=c) $(LIB_H) $(GENERATED_H)
-LOCALIZED_SH = $(SCRIPT_SH)
-LOCALIZED_SH += git-sh-setup.sh
-LOCALIZED_PERL = $(SCRIPT_PERL)
+MSGMERGE_FLAGS = --add-location --backup=off --update
+LOCALIZED_C = $(sort $(FOUND_C_SOURCES) $(FOUND_H_SOURCES) $(SCALAR_SOURCES) \
+	        $(GENERATED_H))
+LOCALIZED_SH = $(sort $(SCRIPT_SH) git-sh-setup.sh)
+LOCALIZED_PERL = $(sort $(SCRIPT_PERL))

 ifdef XGETTEXT_INCLUDE_TESTS
 LOCALIZED_C += t/t0200/test.c
@ -2726,38 +2730,129 @@ LOCALIZED_SH += t/t0200/test.sh
 LOCALIZED_PERL += t/t0200/test.perl
 endif

-## Note that this is meant to be run only by the localization coordinator
-## under a very controlled condition, i.e. (1) it is to be run in a
-## Git repository (not a tarball extract), (2) any local modifications
-## will be lost.
+## We generate intermediate .build/pot/po/%.po files containing a
+## extract of the translations we find in each file in the source
+## tree. We will assemble them using msgcat to create the final
+## "po/git.pot" file.
+LOCALIZED_ALL_GEN_PO =
+
+LOCALIZED_C_GEN_PO = $(LOCALIZED_C:%=.build/pot/po/%.po)
+LOCALIZED_ALL_GEN_PO += $(LOCALIZED_C_GEN_PO)
+
+LOCALIZED_SH_GEN_PO = $(LOCALIZED_SH:%=.build/pot/po/%.po)
+LOCALIZED_ALL_GEN_PO += $(LOCALIZED_SH_GEN_PO)
+
+LOCALIZED_PERL_GEN_PO = $(LOCALIZED_PERL:%=.build/pot/po/%.po)
+LOCALIZED_ALL_GEN_PO += $(LOCALIZED_PERL_GEN_PO)
+
 ## Gettext tools cannot work with our own custom PRItime type, so
 ## we replace PRItime with PRIuMAX.  We need to update this to
 ## PRIdMAX if we switch to a signed type later.
+$(LOCALIZED_C_GEN_PO): .build/pot/po/%.po: %
+	$(call mkdir_p_parent_template)
+	$(QUIET_XGETTEXT) \
+	    if grep -q PRItime $<; then \
+		(\
+			sed -e 's|PRItime|PRIuMAX|g' <$< \
+				>.build/pot/po/$< && \
+			cd .build/pot/po && \
+			$(XGETTEXT) --omit-header \
+				-o $(@:.build/pot/po/%=%) \
+				$(XGETTEXT_FLAGS_C) $< && \
+			rm $<; \
+		); \
+	    else \
+		$(XGETTEXT) --omit-header \
+			-o $@ $(XGETTEXT_FLAGS_C) $<; \
+	    fi

-po/git.pot: $(GENERATED_H) FORCE
-	# All modifications will be reverted at the end, so we do not
-	# want to have any local change.
-	git diff --quiet HEAD && git diff --quiet --cached
+$(LOCALIZED_SH_GEN_PO): .build/pot/po/%.po: %
+	$(call mkdir_p_parent_template)
+	$(QUIET_XGETTEXT)$(XGETTEXT) --omit-header \
+		-o$@ $(XGETTEXT_FLAGS_SH) $<

-	@for s in $(LOCALIZED_C) $(LOCALIZED_SH) $(LOCALIZED_PERL); \
-	do \
-		sed -e 's|PRItime|PRIuMAX|g' <"$$s" >"$$s+" && \
-		cat "$$s+" >"$$s" && rm "$$s+"; \
-	done
+$(LOCALIZED_PERL_GEN_PO): .build/pot/po/%.po: %
+	$(call mkdir_p_parent_template)
+	$(QUIET_XGETTEXT)$(XGETTEXT) --omit-header \
+		-o$@ $(XGETTEXT_FLAGS_PERL) $<

-	$(QUIET_XGETTEXT)$(XGETTEXT) -o$@+ $(XGETTEXT_FLAGS_C) $(LOCALIZED_C)
-	$(QUIET_XGETTEXT)$(XGETTEXT) -o$@+ --join-existing $(XGETTEXT_FLAGS_SH) \
-		$(LOCALIZED_SH)
-	$(QUIET_XGETTEXT)$(XGETTEXT) -o$@+ --join-existing $(XGETTEXT_FLAGS_PERL) \
-		$(LOCALIZED_PERL)
+define gen_pot_header
+$(XGETTEXT) $(XGETTEXT_FLAGS_C) \
+	-o - /dev/null | \
+sed -e 's|charset=CHARSET|charset=UTF-8|' \
+    -e 's|\(Last-Translator: \)FULL NAME <.*>|\1make by the Makefile|' \
+    -e 's|\(Language-Team: \)LANGUAGE <.*>|\1Git Mailing List <git@vger.kernel.org>|' \
+    >$@ && \
+echo '"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\\n"' >>$@
+endef

-	# Reverting the munged source, leaving only the updated $@
-	git reset --hard
-	mv $@+ $@
+.build/pot/git.header: $(LOCALIZED_ALL_GEN_PO)
+	$(call mkdir_p_parent_template)
+	$(QUIET_GEN)$(gen_pot_header)
+
+po/git.pot: .build/pot/git.header $(LOCALIZED_ALL_GEN_PO)
+	$(QUIET_GEN)$(MSGCAT) $^ >$@

 .PHONY: pot
 pot: po/git.pot

+define check_po_file_envvar
+	$(if $(PO_FILE), \
+		$(if $(filter po/%.po,$(PO_FILE)), , \
+			$(error PO_FILE should match pattern: "po/%.po")), \
+		$(error PO_FILE is not defined))
+endef
+
+.PHONY: po-update
+po-update: po/git.pot
+	$(check_po_file_envvar)
+	@if test ! -e $(PO_FILE); then \
+		echo >&2 "error: $(PO_FILE) does not exist"; \
+		echo >&2 'To create an initial po file, use: "make po-init PO_FILE=po/XX.po"'; \
+		exit 1; \
+	fi
+	$(QUIET_MSGMERGE)$(MSGMERGE) $(MSGMERGE_FLAGS) $(PO_FILE) po/git.pot
+
+.PHONY: check-pot
+check-pot: $(LOCALIZED_ALL_GEN_PO)
+
+### TODO FIXME: Translating everything in these files is a bad
+### heuristic for "core", as we'll translate obscure error() messages
+### along with commonly seen i18n messages. A better heuristic would
+### be to e.g. use spatch to first remove error/die/warning
+### etc. messages.
+LOCALIZED_C_CORE =
+LOCALIZED_C_CORE += builtin/checkout.c
+LOCALIZED_C_CORE += builtin/clone.c
+LOCALIZED_C_CORE += builtin/index-pack.c
+LOCALIZED_C_CORE += builtin/push.c
+LOCALIZED_C_CORE += builtin/reset.c
+LOCALIZED_C_CORE += remote.c
+LOCALIZED_C_CORE += wt-status.c
+
+LOCALIZED_C_CORE_GEN_PO = $(LOCALIZED_C_CORE:%=.build/pot/po/%.po)
+
+.build/pot/git-core.header: $(LOCALIZED_C_CORE_GEN_PO)
+	$(call mkdir_p_parent_template)
+	$(QUIET_GEN)$(gen_pot_header)
+
+po/git-core.pot: .build/pot/git-core.header $(LOCALIZED_C_CORE_GEN_PO)
+	$(QUIET_GEN)$(MSGCAT) $^ >$@
+
+.PHONY: po-init
+po-init: po/git-core.pot
+	$(check_po_file_envvar)
+	@if test -e $(PO_FILE); then \
+		echo >&2 "error: $(PO_FILE) exists already"; \
+		exit 1; \
+	fi
+	$(QUIET_MSGINIT)msginit \
+		--input=$< \
+		--output=$(PO_FILE) \
+		--no-translator \
+		--locale=$(PO_FILE:po/%.po=%)
+
+## po/*.po files & their rules
 ifdef NO_GETTEXT
 POFILES :=
 MOFILES :=
@ -3293,6 +3388,8 @@ cocciclean:
 	$(RM) contrib/coccinelle/*.cocci.patch*

 clean: profile-clean coverage-clean cocciclean
+	$(RM) -r .build
+	$(RM) po/git.pot po/git-core.pot
 	$(RM) *.res
 	$(RM) $(OBJECTS)
 	$(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB)
--- a/builtin/submodule--helper.c
+++ b/builtin/submodule--helper.c
@ -3260,7 +3260,7 @@ static int module_add(int argc, const char **argv, const char *prefix)
 			   N_("reference repository")),
 		OPT_BOOL(0, "dissociate", &dissociate, N_("borrow the objects from reference repositories")),
 		OPT_STRING(0, "name", &add_data.sm_name, N_("name"),
-			   N_("sets the submodule’s name to the given string "
+			   N_("sets the submodule's name to the given string "
 			      "instead of defaulting to its path")),
 		OPT_INTEGER(0, "depth", &add_data.depth, N_("depth for shallow clones")),
 		OPT_END()
--- a/ci/run-static-analysis.sh
+++ b/ci/run-static-analysis.sh
@ -29,4 +29,6 @@ fi
 make hdr-check ||
 exit 1

+make check-pot
+
 save_good_tree
--- a/po/.gitignore
+++ b/po/.gitignore
@ -1 +1,3 @@
 /build
+/git.pot
+/git-core.pot
--- a/po/README.md
+++ b/po/README.md
@ -9,8 +9,14 @@ coordinates our localization effort in the l10 coordinator repository:

    https://github.com/git-l10n/git-po/

-The two character language translation codes are defined by ISO\_639-1, as
-stated in the gettext(1) full manual, appendix A.1, Usual Language Codes.
+We will use XX as an alias to refer to the language translation code in
+the following paragraphs, for example we use "po/XX.po" to refer to the
+translation file for a specific language. But this doesn't mean that
+the language code has only two letters. The language code can be in one
+of two forms: "ll" or "ll\_CC". Here "ll" is the ISO 639 two-letter
+language code and "CC" is the ISO 3166 two-letter code for country names
+and subdivisions. For example: "de" for German language code, "zh\_CN"
+for Simplified Chinese language code.


 ## Contributing to an existing translation
@ -39,72 +45,74 @@ language, so that the l10n coordinator only needs to interact with one
 person per language.


-## Core translation
-
-The core translation is the smallest set of work that must be completed
-for a new language translation. Because there are more than 5000 messages
-in the template message file "po/git.pot" that need to be translated,
-this is not a piece of cake for the contributor for a new language.
-
-The core template message file which contains a small set of messages
-will be generated in "po-core/core.pot" automatically by running a helper
-program named "git-po-helper" (described later).
-
-```shell
-git-po-helper init --core XX.po
-```
-
-After translating the generated "po-core/XX.po", you can merge it to
-"po/XX.po" using the following commands:
-
-```shell
-msgcat po-core/XX.po po/XX.po -s -o /tmp/XX.po
-mv /tmp/XX.po po/XX.po
-git-po-helper update XX.po
-```
-
-Edit "po/XX.po" by hand to fix "fuzzy" messages, which may have misplaced
-translated messages and duplicate messages.
-
-
 ## Translation Process Flow

 The overall data-flow looks like this:

-    +-------------------+            +------------------+
-    | Git source code   | ---(1)---> | L10n coordinator |
-    | repository        | <---(4)--- | repository       |
-    +-------------------+            +------------------+
-                                          |      ^
-                                         (2)    (3)
-                                          V      |
-                                     +------------------+
-                                     | Language Team XX |
-                                     +------------------+
+    +-------------------+             +------------------+
+    | Git source code   | ----(2)---> | L10n coordinator |
+    | repository        | <---(5)---- | repository       |
+    +-------------------+             +------------------+
+                    |                     |    ^
+                   (1)                   (3)  (4)
+                    V                     v    |
+               +----------------------------------+
+               |        Language Team XX          |
+               +----------------------------------+

 - Translatable strings are marked in the source file.
- L10n coordinator pulls from the source (1)
- L10n coordinator updates the message template "po/git.pot"
- Language team pulls from L10n coordinator (2)
- Language team updates the message file "po/XX.po"
- L10n coordinator pulls from Language team (3)
- L10n coordinator asks the result to be pulled (4).
+- Language teams can start translation iterations at any time, even
+  before the l10n window opens:
+
+  + Pull from the master branch of the source (1)
+  + Update the message file by running "make po-update PO\_FILE=po/XX.po"
+  + Translate the message file "po/XX.po"
+
+- The L10n coordinator pulls from source and announces the l10n window
+  open (2)
+- Language team pulls from the l10n coordinator, starts another
+  translation iteration against the l10n coordinator's tree (3)
+
+  + Run "git pull --rebase" from the l10n coordinator
+  + Update the message file by running "make po-update PO\_FILE=po/XX.po"
+  + Translate the message file "po/XX.po"
+  + Squash trivial l10n git commits using "git rebase -i"
+
+- Language team sends pull request to the l10n coordinator (4)
+- L10n coordinator checks and merges
+- L10n coordinator asks the result to be pulled (5).


-## Maintaining the "po/git.pot" file
+## Dynamically generated POT files

-(This is done by the l10n coordinator).
+POT files are templates for l10n contributors to create or update their
+translation files. We used to have the "po/git.pot" file which was
+generated by the l10n coordinator, but this file had been removed from
+the tree.

-The "po/git.pot" file contains a message catalog extracted from Git's
-sources. The l10n coordinator maintains it by adding new translations with
-msginit(1), or update existing ones with msgmerge(1).  In order to update
-the Git sources to extract the messages from, the l10n coordinator is
-expected to pull from the main git repository at strategic point in
-history (e.g. when a major release and release candidates are tagged),
-and then run "make pot" at the top-level directory.
+The two POT files "po/git.pot" and "po/git-core.pot" can be created
+dynamically when necessary.

-Language contributors use this file to prepare translations for their
-language, but they are not expected to modify it.
+L10n contributors use "po/git.pot" to prepare translations for their
+languages, but they are not expected to modify it. The "po/git.pot" file
+can be generated manually with the following command:
+
+```shell
+make po/git.pot
+```
+
+The "po/git-core.pot" file is the template for core translations. A core
+translation is the minimum set of work necessary to complete a
+translation of a new language. Since there are more than 5000 messages
+in the full set of template message file "po/git.pot" that need to be
+translated, this is not a piece of cake for new language contributors.
+
+The "core" template file "po/git-core.pot" can be generated manually
+by running:
+
+```shell
+make po/git-core.pot
+```


 ## Initializing a "XX.po" file
@ -115,32 +123,14 @@ If your language XX does not have translated message file "po/XX.po" yet,
 you add a translation for the first time by running:

 ```shell
-msginit --locale=XX
+make po-init PO_FILE=po/XX.po
 ```

-in the "po/" directory, where XX is the locale, e.g. "de", "is", "pt\_BR",
-"zh\_CN", etc.
+where XX is the locale, e.g. "de", "is", "pt\_BR", "zh\_CN", etc.

-Then edit the automatically generated copyright info in your new "XX.po"
-to be correct, e.g. for Icelandic:
-
-```diff
-@@ -1,6 +1,6 @@
-# Icelandic translations for PACKAGE package.
-# Copyright (C) 2010 THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
-+# Icelandic translations for Git.
-+# Copyright (C) 2010 Ævar Arnfjörð Bjarmason <avarab@gmail.com>
-+# This file is distributed under the same license as the Git package.
- # Ævar Arnfjörð Bjarmason <avarab@gmail.com>, 2010.
-```
-
-And change references to PACKAGE VERSION in the PO Header Entry to
-just "Git":
-
-```shell
-perl -pi -e 's/(?<="Project-Id-Version: )PACKAGE VERSION/Git/' XX.po
-```
+The newly generated message file "po/XX.po" is based on the core pot
+file "po/git-core.pot", so it contains only a minimal set of messages
+and it's a good start for a new language contribution.

 Once you are done testing the translation (see below), commit the result
 and ask the l10n coordinator to pull from you.
@ -153,19 +143,53 @@ and ask the l10n coordinator to pull from you.
 If you are replacing translation strings in an existing "XX.po" file to
 improve the translation, just edit the file.

-If there's an existing "XX.po" file for your language, but the repository
-of the l10n coordinator has newer "po/git.pot" file, you would need to first
-pull from the l10n coordinator (see the beginning of this document for its
-URL), and then update the existing translation by running:
+If you want to find new translatable strings in source files of upstream
+repository and propagate them to your "po/XX.po", run command:

 ```shell
-msgmerge --add-location --backup=off -U XX.po git.pot
+make po-update PO_FILE=po/XX.po
 ```

-in the "po/" directory, where "XX.po" is the file you want to update.
+It will:

-Once you are done testing the translation (see below), commit the result
-and ask the l10n coordinator to pull from you.
+- Call "make po/git.pot" to generate new "po/git.pot" file
+- Call "msgmerge --add-location --backup=off -U po/XX.po po/git.pot"
+  to update your "po/XX.po"
+- The "--add-location" option for msgmerge will add location lines,
+  and these location lines will help translation tools to locate
+  translation context easily.
+
+Once you are done testing the translation (see below), it's better
+to commit a location-less "po/XX.po" file to save repository space
+and make a user-friendly patch for review.
+
+To save a location-less "po/XX.po" automatically in repository, you
+can:
+
+First define a new attribute for "po/XX.po" by appending the following
+line in ".git/info/attributes":
+
+```
+/po/XX.po filter=gettext-no-location
+```
+
+Then define the driver for the "gettext-no-location" clean filter to
+strip out both filenames and locations from the contents as follows:
+
+```shell
+git config --global filter.gettext-no-location.clean \
+           "msgcat --no-location -"
+```
+
+For users who have gettext version 0.20 or higher, it is also possible
+to define a clean filter to preserve filenames but not locations:
+
+```shell
+git config --global filter.gettext-no-location.clean \
+           "msgcat --add-location=file -"
+```
+
+You're now ready to ask the l10n coordinator to pull from you.


 ## Fuzzy translation
@ -196,6 +220,14 @@ common errors, e.g. missing printf format strings, or translated
 messages that deviate from the originals in whether they begin/end
 with a newline or not.

+L10n coordinator will check your contributions using a helper program
+(see "PO helper" section below):
+
+```shell
+git-po-helper check-po po/XX.po
+git-po-helper check-commits <rev-list-opts>
+```
+

 ## Marking strings for translation

@ -370,29 +402,6 @@ l10n workflow.
 To build and install the helper program from source, see
 [git-po-helper/README][].

-Usage for git-po-helper:
-
- To start a new language translation:
-
-  ```shell
-  git-po-helper init XX.po
-  ```
-
- To update your "XX.po" file:
-
-  ```shell
-  git-po-helper update XX.po
-  ```
-
- To check commit log and syntax of "XX.po":
-
-  ```shell
-  git-po-helper check-po XX.po
-  git-po-helper check-commits
-  ```
-
-Run "git-po-helper" without arguments to show usage.
-

 ## Conventions

@ -436,13 +445,8 @@ additional conventions:
 - Initialize proper filename of the "XX.po" file conforming to
  iso-639 and iso-3166.

- Must complete a minimal translation based on the "po-core/core.pot"
-  template. Using the following command to initialize the minimal
-  "po-core/XX.po" file:
-
-  ```shell
-  git-po-helper init --core <your-language>
-  ```
+- Must complete a minimal translation based on the "Core
+  translation". See that section above.

 - Add a new entry in the "po/TEAMS" file with proper format, and check
  the syntax of "po/TEAMS" by running the following command:
--- a/po/git.pot
+++ b/po/git.pot
--- a/shared.mak
+++ b/shared.mak
@ -62,7 +62,9 @@ ifndef V
 	QUIET_BUILT_IN = @echo '   ' BUILTIN $@;
 	QUIET_LNCP     = @echo '   ' LN/CP $@;
 	QUIET_XGETTEXT = @echo '   ' XGETTEXT $@;
+	QUIET_MSGINIT  = @echo '   ' MSGINIT $@;
 	QUIET_MSGFMT   = @echo '   ' MSGFMT $@;
+	QUIET_MSGMERGE = @echo '   ' MSGMERGE $@;
 	QUIET_GCOV     = @echo '   ' GCOV $@;
 	QUIET_SP       = @echo '   ' SP $<;
 	QUIET_HDR      = @echo '   ' HDR $(<:hcc=h);