Merge pull request #6954 from freedomofpress/i18n-6917-c10s-l10n

build: eliminate "i18n_tool.py"

.circleci/config.yml

@@ -240,7 +240,9 @@ jobs:
           name: Run tests
           command: |
             sudo apt update && sudo apt install python3-sh python3-babel
-            export LOCALES="$(/usr/bin/python3 securedrop/i18n_tool.py list-locales --lines | circleci tests split | tr '\n' ' ')"
+            NUM_SUPPORTED_LOCALES="$(make count-supported-locales)"
+            [[ "$NUM_SUPPORTED_LOCALES" -eq "$CIRCLE_NODE_TOTAL" ]] || { echo "Parallelism (${CIRCLE_NODE_TOTAL} must equal the number of supported languages (${NUM_SUPPORTED_LOCALES})."; exit 1; }
+            export LOCALES="$(make supported-locales | jq --raw-output 'join("\n")' | circleci tests split)"
             fromtag=$(docker images | grep securedrop-test-focal-py3 | head -n1 | awk '{print $2}')
             DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make translation-test
 

.gitignore

@@ -95,7 +95,9 @@ coverage.xml
 .hypothesis/
 .mypy_cache/
 
-# Translations compiled during packaging:
+# Translation artifacts generated during testing and packaging:
+install_files/ansible-base/roles/tails-config/templates/locale/*.po
+securedrop/tests/i18n/messages.pot
 securedrop/translations/**/*.mo
 
 # Flask stuff:

Makefile

@@ -4,6 +4,7 @@ GCLOUD_VERSION := 222.0.0-1
 SDROOT := $(shell git rev-parse --show-toplevel)
 TAG ?= $(shell git rev-parse HEAD)
 STABLE_VER := $(shell cat molecule/shared/stable.ver)
+VERSION=$(shell python -c "import securedrop.version; print(securedrop.version.__version__)")
 
 SDBIN := $(SDROOT)/securedrop/bin
 DEVSHELL := $(SDBIN)/dev-shell
@@ -34,6 +35,7 @@ update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 		--output-file requirements/python3/develop-requirements.txt \
 		../admin/requirements-ansible.in \
 		../admin/requirements.in \
+		requirements/python3/translation-requirements.in \
 		requirements/python3/develop-requirements.in
 	@SLIM_BUILD=1 $(DEVSHELL) pip-compile --generate-hashes \
 		--allow-unsafe \
@@ -138,7 +140,7 @@ yamllint:  ## Lint YAML files (does not validate syntax!).
 	@echo
 
 .PHONY: lint
-lint: ansible-config-lint check-ruff app-lint check-black html-lint shellcheck typelint yamllint ## Runs all lint checks
+lint: ansible-config-lint app-lint check-black check-desktop-files check-strings check-ruff check-supported-locales html-lint shellcheck typelint yamllint ## Runs all lint checks
 
 .PHONY: safety
 safety:  ## Run `safety check` to check python dependencies for vulnerabilities.
@@ -225,7 +227,8 @@ securedrop/config.py: ## Generate the test SecureDrop application config.
 		 ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.items())); \
 		 ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))'
 	@echo >> securedrop/config.py
-	@echo "SUPPORTED_LOCALES = $$(if test -f /opt/venvs/securedrop-app-code/bin/python3; then ./securedrop/i18n_tool.py list-locales --python; else DOCKER_BUILD_VERBOSE=false $(DEVSHELL) ./i18n_tool.py list-locales --python; fi)" | sed 's/\r//' >> securedrop/config.py
+	@echo "SUPPORTED_LOCALES = $$(make --quiet supported-locales)" >> securedrop/config.py
+	@echo "SUPPORTED_LOCALES.append('en_US')" >> securedrop/config.py
 	@echo
 
 HOOKS_DIR=.githooks
@@ -339,32 +342,132 @@ upgrade-destroy:  ## Destroy an upgrade test environment.
 #
 ##############
 
-DESKTOP_LOCALE_BASE=install_files/ansible-base/roles/tails-config/templates
-DESKTOP_LOCALE_DIR=$(DESKTOP_LOCALE_BASE)/locale
-
-.PHONY: translate
-translate:  ## Update POT files from translated strings in source code.
-	@echo "Updating translations..."
-	@$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py translate-messages --extract-update
-	@$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py translate-desktop --extract-update
-	@echo
+# Global configuration:
+I18N_CONF=securedrop/i18n.json
+I18N_LIST=securedrop/i18n.rst
+
+# securedrop/securedrop configuration:
+LOCALE_DIR=securedrop/translations
+POT=$(LOCALE_DIR)/messages.pot
+
+# securedrop/desktop configuration:
+DESKTOP_BASE=install_files/ansible-base/roles/tails-config/templates
+DESKTOP_LOCALE_DIR=$(DESKTOP_BASE)/locale
+DESKTOP_I18N_CONF=$(DESKTOP_LOCALE_DIR)/LINGUAS
+DESKTOP_POT=$(DESKTOP_LOCALE_DIR)/messages.pot
+
+## Global
+
+.PHONY: check-strings
+check-strings: $(POT) $(DESKTOP_POT) ## Check that the translation catalogs are up to date with source code.
+	@$(MAKE) --no-print-directory extract-strings
+	@git diff --quiet $^ || { echo "Translation catalogs are out of date. Please run \"make extract-strings\" and commit the changes."; exit 1; }
+
+.PHONY: extract-strings
+extract-strings: $(POT) $(DESKTOP_POT) ## Extract translatable strings from source code.
+	@$(MAKE) --always-make --no-print-directory $^
+
+## securedrop/securedrop
+
+# Derive POT from sources.
+$(POT): securedrop
+	@echo "updating catalog template: $@"
+	@mkdir -p ${LOCALE_DIR}
+	@pybabel extract \
+		-F securedrop/babel.cfg \
+		--charset=utf-8 \
+		--output=${POT} \
+		--project="SecureDrop" \
+		--version=${VERSION} \
+		[email protected] \
+		--copyright-holder="Freedom of the Press Foundation" \
+		--add-comments="Translators:" \
+		--strip-comments \
+		--add-location=never \
+		--no-wrap \
+		--ignore-dirs tests \
+		$^
+	@sed -i -e '/^"POT-Creation-Date/d' $@
+
+## securedrop/desktop
+
+.PHONY: check-desktop-files
+check-desktop-files: ${DESKTOP_BASE}/*.j2
+	@$(MAKE) --always-make --no-print-directory update-desktop-files
+	@git diff --quiet $^ || { echo "Desktop files are out of date. Please run \"make update-desktop-files\" and commit the changes."; exit 1; }
+
+.PHONY: update-desktop-files
+update-desktop-files: ${DESKTOP_BASE}/*.j2
+	@$(MAKE) --always-make --no-print-directory $^
+
+# Derive POT from templates.
+$(DESKTOP_POT): ${DESKTOP_BASE}/*.in
+	pybabel extract \
+		-F securedrop/babel.cfg \
+		--output=${DESKTOP_POT} \
+		--project=SecureDrop \
+		--version=${VERSION} \
+		[email protected] \
+		--copyright-holder="Freedom of the Press Foundation" \
+		--add-location=never \
+		--sort-output \
+		$^
+	@sed -i -e '/^"POT-Creation-Date/d' $@
+
+# Render desktop files from templates.  msgfmt needs each
+# "$LANG/LC_MESSAGES/messages.po" file in "$LANG.po".
+%.j2: %.j2.in
+	@find ${DESKTOP_LOCALE_DIR}/* \
+		-maxdepth 0 \
+		-type d \
+		-exec bash -c 'locale="$$(basename {})"; cp ${DESKTOP_LOCALE_DIR}/$${locale}/LC_MESSAGES/messages.po $(DESKTOP_LOCALE_DIR)/$${locale}.po' \;
+	@msgfmt \
+		-d ${DESKTOP_LOCALE_DIR} \
+		--desktop \
+		--template $< \
+		--output-file $@
+	@rm ${DESKTOP_LOCALE_DIR}/*.po
+
+# Render desktop list from "i18n.json".
+$(DESKTOP_I18N_CONF):
+	@jq --raw-output '.supported_locales[].desktop' ${I18N_CONF} > $@
+
+## Supported locales
+
+.PHONY: check-supported-locales
+check-supported-locales: $(I18N_LIST) $(DESKTOP_I18N_CONF) ## Check that the desktop and documentation lists of supported locales are up to date.
+	@$(MAKE) --no-print-directory update-supported-locales
+	@git diff --quiet $^ || { echo "Desktop and/or documentation lists of supported locales are out of date. Please run \"make update-supported-locales\" and commit the changes."; exit 1; }
+
+.PHONY: count-supported-locales
+count-supported-locales: ## Return the number of supported locales.
+	@jq --raw-output '.supported_locales | length' ${I18N_CONF}
+
+.PHONY: update-supported-locales
+update-supported-locales: $(I18N_LIST) $(DESKTOP_I18N_CONF) ## Render the desktop and documentation list of supported locales.
+	@$(MAKE) --always-make --no-print-directory $^
+
+# Render documentation list from "i18n.json".
+${I18N_LIST}: ${I18N_CONF}
+	@echo '.. GENERATED BY "make update-supported-locales":' > $@
+	@jq --raw-output \
+		'.supported_locales | to_entries | map("* \(.value.name) (``\(.key)``)") | join("\n")' \
+		$< >> $@
+
+.PHONY: supported-locales
+supported-locales: ## List supported locales (languages).
+	@jq --compact-output '.supported_locales | keys' ${I18N_CONF}
+
+## Utilities
 
 .PHONY: translation-test
-translation-test:  ## Run page layout tests in all supported languages.
+translation-test: ## Run page layout tests in all supported languages.
 	@echo "Running translation tests..."
 	@$(DEVSHELL) $(SDBIN)/translation-test $${LOCALES}
 	@echo
 
-.PHONY: list-translators
-list-translators:  ## Collect the names of translators since the last merge from Weblate.
-	@$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py list-translators
-
-.PHONY: list-all-translators
-list-all-translators:  ## Collect the names of all translators in the project's history.
-	@$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py list-translators --all
-
 .PHONY: update-user-guides
-update-user-guides:  ## Regenerate docs screenshots. Set DOCS_REPO_DIR to repo checkout root.
+update-user-guides: ## Regenerate docs screenshots. Set DOCS_REPO_DIR to repo checkout root.
 ifndef DOCS_REPO_DIR
 	$(error DOCS_REPO_DIR must be set to the documentation repo checkout root.)
 endif
@@ -376,19 +479,9 @@ endif
 
 .PHONY: verify-mo
 verify-mo: ## Verify that all gettext machine objects (.mo) are reproducible from their catalogs (.po).
-	@# TODO(#6917): Once Weblate (rather than i18n_tool.py) is correctly filing
-	@# both .po and .mo under $DESKTOP_LOCALE_DIR, remove this step.  (See
-	@# also: 76f3adeed90f4aaadbf0685e09dec6314367d5c0.)
-	@find ${DESKTOP_LOCALE_BASE} \
-		-maxdepth 1 \
-		-name "*.po" \
-		-exec bash -c 'PO="$$(basename {} | sed \'s/.po//')"; cp ${DESKTOP_LOCALE_BASE}/$${PO}.po $(DESKTOP_LOCALE_DIR)/$${PO}/LC_MESSAGES/messages.po' \;
 	@TERM=dumb devops/scripts/verify-mo.py ${DESKTOP_LOCALE_DIR}/*
 	@# All good; now clean up.
-	@# TODO(#6917): git restore "${LOCALE_DIR}/**/*.po"
-	@find ${DESKTOP_LOCALE_DIR} \
-		-name "*.po" \
-		-delete
+	@git restore "${LOCALE_DIR}/**/*.po"
 
 
 ###########

install_files/ansible-base/roles/tails-config/templates/desktop-journalist-icon.j2

@@ -6,9 +6,6 @@ Type=Application
 Terminal=false
 StartupNotify=true
 Categories=Network;
-Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png
-Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }}
-Name=SecureDrop Journalist Interface
 Name[ar]=واجهة SecureDrop للصحفيين
 Name[ca]=Interfície de periodista del SecureDrop
 Name[cs]=SecureDrop rozhraní novináře
@@ -30,3 +27,6 @@ Name[sv]=SecureDrop journalistgränssnitt
 Name[tr]=SecureDrop Gazeteci Arayüzü
 Name[zh_Hans]=SecureDrop 记者界面
 Name[zh_Hant]=SecureDrop 記者使用介面
+Name=SecureDrop Journalist Interface
+Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png
+Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }}

install_files/ansible-base/roles/tails-config/templates/desktop-source-icon.j2

@@ -6,9 +6,6 @@ Type=Application
 Terminal=false
 StartupNotify=true
 Categories=Network;
-Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png
-Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }}
-Name=SecureDrop Source Interface
 Name[ar]=واجهة SecureDrop للمصدر
 Name[ca]=Interfície de font del SecureDrop
 Name[cs]=SecureDrop rozhraní zdroje
@@ -30,3 +27,6 @@ Name[sv]=SecureDrop källgränssnitt
 Name[tr]=SecureDrop Kaynak Arayüzü
 Name[zh_Hans]=SecureDrop 线人界面
 Name[zh_Hant]=SecureDrop 線人使用介面
+Name=SecureDrop Source Interface
+Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png
+Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }}

install_files/ansible-base/roles/tails-config/templates/locale/LINGUAS

@@ -0,0 +1,21 @@
+ar
+ca
+cs
+de_DE
+el
+es_ES
+fr
+hi
+is
+it
+nb_NO
+nl
+pt_BR
+pt_PT
+ro
+ru
+sk
+sv
+tr
+zh_Hans
+zh_Hant

install_files/ansible-base/roles/tails-config/templates/ar.po → install_files/ansible-base/roles/tails-config/templates/locale/ar/LC_MESSAGES/messages.po

@@ -9,14 +9,12 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2018-06-30 10:57+0000\n"
 "Last-Translator: erinm <[email protected]>\n"
-"Language-Team: Arabic <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/ar/>\n"
+"Language-Team: Arabic <https://weblate.securedrop.org/projects/securedrop/desktop/ar/>\n"
 "Language: ar\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 "
-"&& n%100<=10 ? 3 : n%100>=11 ? 4 : 5;\n"
+"Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5;\n"
 "X-Generator: Weblate 2.20\n"
 
 msgid "Launch Source Interface"

install_files/ansible-base/roles/tails-config/templates/ca.po → install_files/ansible-base/roles/tails-config/templates/locale/ca/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2023-06-14 13:56+0000\n"
 "Last-Translator: John Smith <[email protected]>\n"
-"Language-Team: Catalan <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/ca/>\n"
+"Language-Team: Catalan <https://weblate.securedrop.org/projects/securedrop/desktop/ca/>\n"
 "Language: ca\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/cs.po → install_files/ansible-base/roles/tails-config/templates/locale/cs/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2023-06-21 06:56+0000\n"
 "Last-Translator: Jan Papež <[email protected]>\n"
-"Language-Team: Czech <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/cs/>\n"
+"Language-Team: Czech <https://weblate.securedrop.org/projects/securedrop/desktop/cs/>\n"
 "Language: cs\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/de_DE.po → install_files/ansible-base/roles/tails-config/templates/locale/de_DE/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2023-06-18 10:56+0000\n"
 "Last-Translator: Curtis Baltimore <[email protected]>\n"
-"Language-Team: German <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/de/>\n"
+"Language-Team: German <https://weblate.securedrop.org/projects/securedrop/desktop/de/>\n"
 "Language: de_DE\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/el.po → install_files/ansible-base/roles/tails-config/templates/locale/el/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2018-08-22 11:02+0000\n"
 "Last-Translator: Adrian <[email protected]>\n"
-"Language-Team: Greek <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/el/>\n"
+"Language-Team: Greek <https://weblate.securedrop.org/projects/securedrop/desktop/el/>\n"
 "Language: el\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/es_ES.po → install_files/ansible-base/roles/tails-config/templates/locale/es_ES/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2018-12-05 08:18+0000\n"
 "Last-Translator: Adolfo Jayme-Barrientos <[email protected]>\n"
-"Language-Team: Spanish <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/es/>\n"
+"Language-Team: Spanish <https://weblate.securedrop.org/projects/securedrop/desktop/es/>\n"
 "Language: es_ES\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/fr.po → install_files/ansible-base/roles/tails-config/templates/locale/fr/LC_MESSAGES/messages.po

@@ -7,11 +7,9 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2017-09-04 16:26+0200\n"
 "PO-Revision-Date: 2023-06-13 16:56+0000\n"
 "Last-Translator: AO Localization Lab <[email protected]>\n"
-"Language-Team: French <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/fr/>\n"
+"Language-Team: French <https://weblate.securedrop.org/projects/securedrop/desktop/fr/>\n"
 "Language: fr\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/hi.po → install_files/ansible-base/roles/tails-config/templates/locale/hi/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2018-03-08 13:50+0000\n"
 "Last-Translator: Muhammad Usman <[email protected]>\n"
-"Language-Team: Hindi <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/hi/>\n"
+"Language-Team: Hindi <https://weblate.securedrop.org/projects/securedrop/desktop/hi/>\n"
 "Language: hi\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/is.po → install_files/ansible-base/roles/tails-config/templates/locale/is/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2023-06-13 09:56+0000\n"
 "Last-Translator: Sveinn í Felli <[email protected]>\n"
-"Language-Team: Icelandic <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/is/>\n"
+"Language-Team: Icelandic <https://weblate.securedrop.org/projects/securedrop/desktop/is/>\n"
 "Language: is\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"

install_files/ansible-base/roles/tails-config/templates/it.po → install_files/ansible-base/roles/tails-config/templates/locale/it/LC_MESSAGES/messages.po

@@ -9,8 +9,7 @@ msgstr ""
 "Report-Msgid-Bugs-To: [email protected]\n"
 "PO-Revision-Date: 2018-01-13 08:54+0000\n"
 "Last-Translator: Claudio Arseni <[email protected]>\n"
-"Language-Team: Italian <https://weblate.securedrop.org/projects/securedrop/"
-"desktop/it/>\n"
+"Language-Team: Italian <https://weblate.securedrop.org/projects/securedrop/desktop/it/>\n"
 "Language: it\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"