diff --git a/.circleci/config.yml b/.circleci/config.yml index 607bc62a40..eff3f852ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,7 +240,9 @@ jobs: name: Run tests command: | sudo apt update && sudo apt install python3-sh python3-babel - export LOCALES="$(/usr/bin/python3 securedrop/i18n_tool.py list-locales --lines | circleci tests split | tr '\n' ' ')" + NUM_SUPPORTED_LOCALES="$(make count-supported-locales)" + [[ "$NUM_SUPPORTED_LOCALES" -eq "$CIRCLE_NODE_TOTAL" ]] || { echo "Parallelism (${CIRCLE_NODE_TOTAL} must equal the number of supported languages (${NUM_SUPPORTED_LOCALES})."; exit 1; } + export LOCALES="$(make supported-locales | jq --raw-output 'join("\n")' | circleci tests split)" fromtag=$(docker images | grep securedrop-test-focal-py3 | head -n1 | awk '{print $2}') DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make translation-test diff --git a/.gitignore b/.gitignore index 3d2e6cf17d..935e3f3a01 100644 --- a/.gitignore +++ b/.gitignore @@ -95,7 +95,9 @@ coverage.xml .hypothesis/ .mypy_cache/ -# Translations compiled during packaging: +# Translation artifacts generated during testing and packaging: +install_files/ansible-base/roles/tails-config/templates/locale/*.po +securedrop/tests/i18n/messages.pot securedrop/translations/**/*.mo # Flask stuff: diff --git a/Makefile b/Makefile index 2e2eac2414..b528fcf257 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ GCLOUD_VERSION := 222.0.0-1 SDROOT := $(shell git rev-parse --show-toplevel) TAG ?= $(shell git rev-parse HEAD) STABLE_VER := $(shell cat molecule/shared/stable.ver) +VERSION=$(shell python -c "import securedrop.version; print(securedrop.version.__version__)") SDBIN := $(SDROOT)/securedrop/bin DEVSHELL := $(SDBIN)/dev-shell @@ -34,6 +35,7 @@ update-python3-requirements: ## Update Python 3 requirements with pip-compile. --output-file requirements/python3/develop-requirements.txt \ ../admin/requirements-ansible.in \ ../admin/requirements.in \ + requirements/python3/translation-requirements.in \ requirements/python3/develop-requirements.in @SLIM_BUILD=1 $(DEVSHELL) pip-compile --generate-hashes \ --allow-unsafe \ @@ -138,7 +140,7 @@ yamllint: ## Lint YAML files (does not validate syntax!). @echo .PHONY: lint -lint: ansible-config-lint check-ruff app-lint check-black html-lint shellcheck typelint yamllint ## Runs all lint checks +lint: ansible-config-lint app-lint check-black check-desktop-files check-strings check-ruff check-supported-locales html-lint shellcheck typelint yamllint ## Runs all lint checks .PHONY: safety safety: ## Run `safety check` to check python dependencies for vulnerabilities. @@ -225,7 +227,8 @@ securedrop/config.py: ## Generate the test SecureDrop application config. ctx.update(dict((k, {"stdout":v}) for k,v in os.environ.items())); \ ctx = open("config.py", "w").write(env.get_template("config.py.example").render(ctx))' @echo >> securedrop/config.py - @echo "SUPPORTED_LOCALES = $$(if test -f /opt/venvs/securedrop-app-code/bin/python3; then ./securedrop/i18n_tool.py list-locales --python; else DOCKER_BUILD_VERBOSE=false $(DEVSHELL) ./i18n_tool.py list-locales --python; fi)" | sed 's/\r//' >> securedrop/config.py + @echo "SUPPORTED_LOCALES = $$(make --quiet supported-locales)" >> securedrop/config.py + @echo "SUPPORTED_LOCALES.append('en_US')" >> securedrop/config.py @echo HOOKS_DIR=.githooks @@ -339,32 +342,132 @@ upgrade-destroy: ## Destroy an upgrade test environment. # ############## -DESKTOP_LOCALE_BASE=install_files/ansible-base/roles/tails-config/templates -DESKTOP_LOCALE_DIR=$(DESKTOP_LOCALE_BASE)/locale - -.PHONY: translate -translate: ## Update POT files from translated strings in source code. - @echo "Updating translations..." - @$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py translate-messages --extract-update - @$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py translate-desktop --extract-update - @echo +# Global configuration: +I18N_CONF=securedrop/i18n.json +I18N_LIST=securedrop/i18n.rst + +# securedrop/securedrop configuration: +LOCALE_DIR=securedrop/translations +POT=$(LOCALE_DIR)/messages.pot + +# securedrop/desktop configuration: +DESKTOP_BASE=install_files/ansible-base/roles/tails-config/templates +DESKTOP_LOCALE_DIR=$(DESKTOP_BASE)/locale +DESKTOP_I18N_CONF=$(DESKTOP_LOCALE_DIR)/LINGUAS +DESKTOP_POT=$(DESKTOP_LOCALE_DIR)/messages.pot + +## Global + +.PHONY: check-strings +check-strings: $(POT) $(DESKTOP_POT) ## Check that the translation catalogs are up to date with source code. + @$(MAKE) --no-print-directory extract-strings + @git diff --quiet $^ || { echo "Translation catalogs are out of date. Please run \"make extract-strings\" and commit the changes."; exit 1; } + +.PHONY: extract-strings +extract-strings: $(POT) $(DESKTOP_POT) ## Extract translatable strings from source code. + @$(MAKE) --always-make --no-print-directory $^ + +## securedrop/securedrop + +# Derive POT from sources. +$(POT): securedrop + @echo "updating catalog template: $@" + @mkdir -p ${LOCALE_DIR} + @pybabel extract \ + -F securedrop/babel.cfg \ + --charset=utf-8 \ + --output=${POT} \ + --project="SecureDrop" \ + --version=${VERSION} \ + --msgid-bugs-address=securedrop@freedom.press \ + --copyright-holder="Freedom of the Press Foundation" \ + --add-comments="Translators:" \ + --strip-comments \ + --add-location=never \ + --no-wrap \ + --ignore-dirs tests \ + $^ + @sed -i -e '/^"POT-Creation-Date/d' $@ + +## securedrop/desktop + +.PHONY: check-desktop-files +check-desktop-files: ${DESKTOP_BASE}/*.j2 + @$(MAKE) --always-make --no-print-directory update-desktop-files + @git diff --quiet $^ || { echo "Desktop files are out of date. Please run \"make update-desktop-files\" and commit the changes."; exit 1; } + +.PHONY: update-desktop-files +update-desktop-files: ${DESKTOP_BASE}/*.j2 + @$(MAKE) --always-make --no-print-directory $^ + +# Derive POT from templates. +$(DESKTOP_POT): ${DESKTOP_BASE}/*.in + pybabel extract \ + -F securedrop/babel.cfg \ + --output=${DESKTOP_POT} \ + --project=SecureDrop \ + --version=${VERSION} \ + --msgid-bugs-address=securedrop@freedom.press \ + --copyright-holder="Freedom of the Press Foundation" \ + --add-location=never \ + --sort-output \ + $^ + @sed -i -e '/^"POT-Creation-Date/d' $@ + +# Render desktop files from templates. msgfmt needs each +# "$LANG/LC_MESSAGES/messages.po" file in "$LANG.po". +%.j2: %.j2.in + @find ${DESKTOP_LOCALE_DIR}/* \ + -maxdepth 0 \ + -type d \ + -exec bash -c 'locale="$$(basename {})"; cp ${DESKTOP_LOCALE_DIR}/$${locale}/LC_MESSAGES/messages.po $(DESKTOP_LOCALE_DIR)/$${locale}.po' \; + @msgfmt \ + -d ${DESKTOP_LOCALE_DIR} \ + --desktop \ + --template $< \ + --output-file $@ + @rm ${DESKTOP_LOCALE_DIR}/*.po + +# Render desktop list from "i18n.json". +$(DESKTOP_I18N_CONF): + @jq --raw-output '.supported_locales[].desktop' ${I18N_CONF} > $@ + +## Supported locales + +.PHONY: check-supported-locales +check-supported-locales: $(I18N_LIST) $(DESKTOP_I18N_CONF) ## Check that the desktop and documentation lists of supported locales are up to date. + @$(MAKE) --no-print-directory update-supported-locales + @git diff --quiet $^ || { echo "Desktop and/or documentation lists of supported locales are out of date. Please run \"make update-supported-locales\" and commit the changes."; exit 1; } + +.PHONY: count-supported-locales +count-supported-locales: ## Return the number of supported locales. + @jq --raw-output '.supported_locales | length' ${I18N_CONF} + +.PHONY: update-supported-locales +update-supported-locales: $(I18N_LIST) $(DESKTOP_I18N_CONF) ## Render the desktop and documentation list of supported locales. + @$(MAKE) --always-make --no-print-directory $^ + +# Render documentation list from "i18n.json". +${I18N_LIST}: ${I18N_CONF} + @echo '.. GENERATED BY "make update-supported-locales":' > $@ + @jq --raw-output \ + '.supported_locales | to_entries | map("* \(.value.name) (``\(.key)``)") | join("\n")' \ + $< >> $@ + +.PHONY: supported-locales +supported-locales: ## List supported locales (languages). + @jq --compact-output '.supported_locales | keys' ${I18N_CONF} + +## Utilities .PHONY: translation-test -translation-test: ## Run page layout tests in all supported languages. +translation-test: ## Run page layout tests in all supported languages. @echo "Running translation tests..." @$(DEVSHELL) $(SDBIN)/translation-test $${LOCALES} @echo -.PHONY: list-translators -list-translators: ## Collect the names of translators since the last merge from Weblate. - @$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py list-translators - -.PHONY: list-all-translators -list-all-translators: ## Collect the names of all translators in the project's history. - @$(DEVSHELL) $(SDROOT)/securedrop/i18n_tool.py list-translators --all - .PHONY: update-user-guides -update-user-guides: ## Regenerate docs screenshots. Set DOCS_REPO_DIR to repo checkout root. +update-user-guides: ## Regenerate docs screenshots. Set DOCS_REPO_DIR to repo checkout root. ifndef DOCS_REPO_DIR $(error DOCS_REPO_DIR must be set to the documentation repo checkout root.) endif @@ -376,19 +479,9 @@ endif .PHONY: verify-mo verify-mo: ## Verify that all gettext machine objects (.mo) are reproducible from their catalogs (.po). - @# TODO(#6917): Once Weblate (rather than i18n_tool.py) is correctly filing - @# both .po and .mo under $DESKTOP_LOCALE_DIR, remove this step. (See - @# also: 76f3adeed90f4aaadbf0685e09dec6314367d5c0.) - @find ${DESKTOP_LOCALE_BASE} \ - -maxdepth 1 \ - -name "*.po" \ - -exec bash -c 'PO="$$(basename {} | sed \'s/.po//')"; cp ${DESKTOP_LOCALE_BASE}/$${PO}.po $(DESKTOP_LOCALE_DIR)/$${PO}/LC_MESSAGES/messages.po' \; @TERM=dumb devops/scripts/verify-mo.py ${DESKTOP_LOCALE_DIR}/* @# All good; now clean up. - @# TODO(#6917): git restore "${LOCALE_DIR}/**/*.po" - @find ${DESKTOP_LOCALE_DIR} \ - -name "*.po" \ - -delete + @git restore "${LOCALE_DIR}/**/*.po" ########### diff --git a/install_files/ansible-base/roles/tails-config/templates/desktop-journalist-icon.j2 b/install_files/ansible-base/roles/tails-config/templates/desktop-journalist-icon.j2 index 3bf8bd1ae7..3a9c2837b2 100644 --- a/install_files/ansible-base/roles/tails-config/templates/desktop-journalist-icon.j2 +++ b/install_files/ansible-base/roles/tails-config/templates/desktop-journalist-icon.j2 @@ -6,9 +6,6 @@ Type=Application Terminal=false StartupNotify=true Categories=Network; -Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png -Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }} -Name=SecureDrop Journalist Interface Name[ar]=واجهة SecureDrop للصحفيين Name[ca]=Interfície de periodista del SecureDrop Name[cs]=SecureDrop rozhraní novináře @@ -30,3 +27,6 @@ Name[sv]=SecureDrop journalistgränssnitt Name[tr]=SecureDrop Gazeteci Arayüzü Name[zh_Hans]=SecureDrop 记者界面 Name[zh_Hant]=SecureDrop 記者使用介面 +Name=SecureDrop Journalist Interface +Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png +Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }} diff --git a/install_files/ansible-base/roles/tails-config/templates/desktop-source-icon.j2 b/install_files/ansible-base/roles/tails-config/templates/desktop-source-icon.j2 index c4d985fc80..50efd29c25 100644 --- a/install_files/ansible-base/roles/tails-config/templates/desktop-source-icon.j2 +++ b/install_files/ansible-base/roles/tails-config/templates/desktop-source-icon.j2 @@ -6,9 +6,6 @@ Type=Application Terminal=false StartupNotify=true Categories=Network; -Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png -Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }} -Name=SecureDrop Source Interface Name[ar]=واجهة SecureDrop للمصدر Name[ca]=Interfície de font del SecureDrop Name[cs]=SecureDrop rozhraní zdroje @@ -30,3 +27,6 @@ Name[sv]=SecureDrop källgränssnitt Name[tr]=SecureDrop Kaynak Arayüzü Name[zh_Hans]=SecureDrop 线人界面 Name[zh_Hant]=SecureDrop 線人使用介面 +Name=SecureDrop Source Interface +Icon={{ tails_config_securedrop_dotfiles }}/securedrop_icon.png +Exec=/usr/local/bin/tor-browser {{ item.0.onion_url }} diff --git a/install_files/ansible-base/roles/tails-config/templates/locale/LINGUAS b/install_files/ansible-base/roles/tails-config/templates/locale/LINGUAS new file mode 100644 index 0000000000..ff60f06fa7 --- /dev/null +++ b/install_files/ansible-base/roles/tails-config/templates/locale/LINGUAS @@ -0,0 +1,21 @@ +ar +ca +cs +de_DE +el +es_ES +fr +hi +is +it +nb_NO +nl +pt_BR +pt_PT +ro +ru +sk +sv +tr +zh_Hans +zh_Hant diff --git a/install_files/ansible-base/roles/tails-config/templates/ar.po b/install_files/ansible-base/roles/tails-config/templates/locale/ar/LC_MESSAGES/messages.po similarity index 93% rename from install_files/ansible-base/roles/tails-config/templates/ar.po rename to install_files/ansible-base/roles/tails-config/templates/locale/ar/LC_MESSAGES/messages.po index ded47b45d4..75bffc05aa 100644 --- a/install_files/ansible-base/roles/tails-config/templates/ar.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/ar/LC_MESSAGES/messages.po @@ -9,14 +9,12 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-06-30 10:57+0000\n" "Last-Translator: erinm \n" -"Language-Team: Arabic \n" +"Language-Team: Arabic \n" "Language: ar\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 " -"&& n%100<=10 ? 3 : n%100>=11 ? 4 : 5;\n" +"Plural-Forms: nplurals=6; plural=n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 ? 4 : 5;\n" "X-Generator: Weblate 2.20\n" msgid "Launch Source Interface" diff --git a/install_files/ansible-base/roles/tails-config/templates/ca.po b/install_files/ansible-base/roles/tails-config/templates/locale/ca/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/ca.po rename to install_files/ansible-base/roles/tails-config/templates/locale/ca/LC_MESSAGES/messages.po index f489070d20..1e948a8655 100644 --- a/install_files/ansible-base/roles/tails-config/templates/ca.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/ca/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-14 13:56+0000\n" "Last-Translator: John Smith \n" -"Language-Team: Catalan \n" +"Language-Team: Catalan \n" "Language: ca\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/cs.po b/install_files/ansible-base/roles/tails-config/templates/locale/cs/LC_MESSAGES/messages.po similarity index 98% rename from install_files/ansible-base/roles/tails-config/templates/cs.po rename to install_files/ansible-base/roles/tails-config/templates/locale/cs/LC_MESSAGES/messages.po index 73c2d93bb5..729e9e24fb 100644 --- a/install_files/ansible-base/roles/tails-config/templates/cs.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/cs/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-21 06:56+0000\n" "Last-Translator: Jan Papež \n" -"Language-Team: Czech \n" +"Language-Team: Czech \n" "Language: cs\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/de_DE.po b/install_files/ansible-base/roles/tails-config/templates/locale/de_DE/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/de_DE.po rename to install_files/ansible-base/roles/tails-config/templates/locale/de_DE/LC_MESSAGES/messages.po index 3cb536d776..c82535d03b 100644 --- a/install_files/ansible-base/roles/tails-config/templates/de_DE.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/de_DE/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-18 10:56+0000\n" "Last-Translator: Curtis Baltimore \n" -"Language-Team: German \n" +"Language-Team: German \n" "Language: de_DE\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/el.po b/install_files/ansible-base/roles/tails-config/templates/locale/el/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/el.po rename to install_files/ansible-base/roles/tails-config/templates/locale/el/LC_MESSAGES/messages.po index 0fb7fce83e..c051584334 100644 --- a/install_files/ansible-base/roles/tails-config/templates/el.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/el/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-08-22 11:02+0000\n" "Last-Translator: Adrian \n" -"Language-Team: Greek \n" +"Language-Team: Greek \n" "Language: el\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/es_ES.po b/install_files/ansible-base/roles/tails-config/templates/locale/es_ES/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/es_ES.po rename to install_files/ansible-base/roles/tails-config/templates/locale/es_ES/LC_MESSAGES/messages.po index accdb186a9..c494ea4a85 100644 --- a/install_files/ansible-base/roles/tails-config/templates/es_ES.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/es_ES/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-12-05 08:18+0000\n" "Last-Translator: Adolfo Jayme-Barrientos \n" -"Language-Team: Spanish \n" +"Language-Team: Spanish \n" "Language: es_ES\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/fr.po b/install_files/ansible-base/roles/tails-config/templates/locale/fr/LC_MESSAGES/messages.po similarity index 95% rename from install_files/ansible-base/roles/tails-config/templates/fr.po rename to install_files/ansible-base/roles/tails-config/templates/locale/fr/LC_MESSAGES/messages.po index 428589cd63..10ac7a09e9 100644 --- a/install_files/ansible-base/roles/tails-config/templates/fr.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/fr/LC_MESSAGES/messages.po @@ -7,11 +7,9 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" -"POT-Creation-Date: 2017-09-04 16:26+0200\n" "PO-Revision-Date: 2023-06-13 16:56+0000\n" "Last-Translator: AO Localization Lab \n" -"Language-Team: French \n" +"Language-Team: French \n" "Language: fr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/hi.po b/install_files/ansible-base/roles/tails-config/templates/locale/hi/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/hi.po rename to install_files/ansible-base/roles/tails-config/templates/locale/hi/LC_MESSAGES/messages.po index 142c2c407b..64cf71bffc 100644 --- a/install_files/ansible-base/roles/tails-config/templates/hi.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/hi/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-03-08 13:50+0000\n" "Last-Translator: Muhammad Usman \n" -"Language-Team: Hindi \n" +"Language-Team: Hindi \n" "Language: hi\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/is.po b/install_files/ansible-base/roles/tails-config/templates/locale/is/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/is.po rename to install_files/ansible-base/roles/tails-config/templates/locale/is/LC_MESSAGES/messages.po index bea8db94a0..eb49a33ec4 100644 --- a/install_files/ansible-base/roles/tails-config/templates/is.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/is/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-13 09:56+0000\n" "Last-Translator: Sveinn í Felli \n" -"Language-Team: Icelandic \n" +"Language-Team: Icelandic \n" "Language: is\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/it.po b/install_files/ansible-base/roles/tails-config/templates/locale/it/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/it.po rename to install_files/ansible-base/roles/tails-config/templates/locale/it/LC_MESSAGES/messages.po index bca9c254cb..24300d9377 100644 --- a/install_files/ansible-base/roles/tails-config/templates/it.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/it/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-01-13 08:54+0000\n" "Last-Translator: Claudio Arseni \n" -"Language-Team: Italian \n" +"Language-Team: Italian \n" "Language: it\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/desktop.pot b/install_files/ansible-base/roles/tails-config/templates/locale/messages.pot similarity index 65% rename from install_files/ansible-base/roles/tails-config/templates/desktop.pot rename to install_files/ansible-base/roles/tails-config/templates/locale/messages.pot index c1ee0f43fe..8aa68fa351 100644 --- a/install_files/ansible-base/roles/tails-config/templates/desktop.pot +++ b/install_files/ansible-base/roles/tails-config/templates/locale/messages.pot @@ -1,46 +1,51 @@ -# SOME DESCRIPTIVE TITLE. -# Copyright (C) YEAR Freedom of the Press Foundation -# This file is distributed under the same license as the PACKAGE package. -# FIRST AUTHOR , YEAR. +# Translations template for SecureDrop. +# Copyright (C) 2023 Freedom of the Press Foundation +# This file is distributed under the same license as the SecureDrop project. +# FIRST AUTHOR , 2023. # #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" +"Project-Id-Version: SecureDrop 2.7.0~rc1\n" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" -"Language: \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.12.1\n" -msgid "Launch Source Interface" +msgid "Check for SecureDrop Updates" msgstr "" msgid "Launch Journalist Interface" msgstr "" -msgid "Check for SecureDrop Updates" +msgid "Launch Source Interface" msgstr "" -msgid "SSH into the App Server" +msgid "Open File Browser" msgstr "" -msgid "SSH into the Monitor Server" +msgid "Open KeePassXC Password Vault" msgstr "" -msgid "Open KeePassXC Password Vault" +msgid "SSH into the App Server" msgstr "" -msgid "Open File Browser" +msgid "SSH into the Monitor Server" msgstr "" -#: desktop-journalist-icon.j2.in:10 +#. Name msgid "SecureDrop Journalist Interface" msgstr "" -#: desktop-source-icon.j2.in:10 +#. Name msgid "SecureDrop Source Interface" msgstr "" + +#. Icon +msgid "{{ tails_config_securedrop_dotfiles }}/securedrop_icon.png" +msgstr "" + diff --git a/install_files/ansible-base/roles/tails-config/templates/nb_NO.po b/install_files/ansible-base/roles/tails-config/templates/locale/nb_NO/LC_MESSAGES/messages.po similarity index 94% rename from install_files/ansible-base/roles/tails-config/templates/nb_NO.po rename to install_files/ansible-base/roles/tails-config/templates/locale/nb_NO/LC_MESSAGES/messages.po index 51b2903f54..919339c853 100644 --- a/install_files/ansible-base/roles/tails-config/templates/nb_NO.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/nb_NO/LC_MESSAGES/messages.po @@ -7,11 +7,9 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" -"POT-Creation-Date: 2017-09-04 16:26+0200\n" "PO-Revision-Date: 2023-06-20 22:56+0000\n" "Last-Translator: Øyvind Bye Skille \n" -"Language-Team: Norwegian Bokmål \n" +"Language-Team: Norwegian Bokmål \n" "Language: nb_NO\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/nl.po b/install_files/ansible-base/roles/tails-config/templates/locale/nl/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/nl.po rename to install_files/ansible-base/roles/tails-config/templates/locale/nl/LC_MESSAGES/messages.po index e99ade9c8c..5aecb21c30 100644 --- a/install_files/ansible-base/roles/tails-config/templates/nl.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/nl/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2017-12-01 13:48+0000\n" "Last-Translator: kwadronaut \n" -"Language-Team: Dutch \n" +"Language-Team: Dutch \n" "Language: nl\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/pt_BR.po b/install_files/ansible-base/roles/tails-config/templates/locale/pt_BR/LC_MESSAGES/messages.po similarity index 96% rename from install_files/ansible-base/roles/tails-config/templates/pt_BR.po rename to install_files/ansible-base/roles/tails-config/templates/locale/pt_BR/LC_MESSAGES/messages.po index 71f1dbad2e..aeaa41bbc9 100644 --- a/install_files/ansible-base/roles/tails-config/templates/pt_BR.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/pt_BR/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-22 07:39+0000\n" "Last-Translator: notmuchtohide \n" -"Language-Team: Portuguese (Brazil) \n" +"Language-Team: Portuguese (Brazil) \n" "Language: pt_BR\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/pt_PT.po b/install_files/ansible-base/roles/tails-config/templates/locale/pt_PT/LC_MESSAGES/messages.po similarity index 96% rename from install_files/ansible-base/roles/tails-config/templates/pt_PT.po rename to install_files/ansible-base/roles/tails-config/templates/locale/pt_PT/LC_MESSAGES/messages.po index 6518359b3f..77abcabf5d 100644 --- a/install_files/ansible-base/roles/tails-config/templates/pt_PT.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/pt_PT/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-22 08:56+0000\n" "Last-Translator: deeplow \n" -"Language-Team: Portuguese (Portugal) \n" +"Language-Team: Portuguese (Portugal) \n" "Language: pt_PT\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/ro.po b/install_files/ansible-base/roles/tails-config/templates/locale/ro/LC_MESSAGES/messages.po similarity index 95% rename from install_files/ansible-base/roles/tails-config/templates/ro.po rename to install_files/ansible-base/roles/tails-config/templates/locale/ro/LC_MESSAGES/messages.po index ac3fde426d..f7c053a512 100644 --- a/install_files/ansible-base/roles/tails-config/templates/ro.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/ro/LC_MESSAGES/messages.po @@ -9,14 +9,12 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-03-11 16:05+0000\n" "Last-Translator: Jobava \n" -"Language-Team: Romanian \n" +"Language-Team: Romanian \n" "Language: ro\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Plural-Forms: nplurals=3; plural=n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < " -"20)) ? 1 : 2;\n" +"Plural-Forms: nplurals=3; plural=n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < 20)) ? 1 : 2;\n" "X-Generator: Weblate 2.18\n" msgid "Launch Source Interface" diff --git a/install_files/ansible-base/roles/tails-config/templates/ru.po b/install_files/ansible-base/roles/tails-config/templates/locale/ru/LC_MESSAGES/messages.po similarity index 94% rename from install_files/ansible-base/roles/tails-config/templates/ru.po rename to install_files/ansible-base/roles/tails-config/templates/locale/ru/LC_MESSAGES/messages.po index 3ac8648811..21349ce47f 100644 --- a/install_files/ansible-base/roles/tails-config/templates/ru.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/ru/LC_MESSAGES/messages.po @@ -9,14 +9,12 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-20 19:56+0000\n" "Last-Translator: Adham Kurbanov \n" -"Language-Team: Russian \n" +"Language-Team: Russian \n" "Language: ru\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 : n%10>=2 && n" -"%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2;\n" +"Plural-Forms: nplurals=3; plural=n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2;\n" "X-Generator: Weblate 4.14.1\n" msgid "Launch Source Interface" diff --git a/install_files/ansible-base/roles/tails-config/templates/sk.po b/install_files/ansible-base/roles/tails-config/templates/locale/sk/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/sk.po rename to install_files/ansible-base/roles/tails-config/templates/locale/sk/LC_MESSAGES/messages.po index 74a084dd08..b520501335 100644 --- a/install_files/ansible-base/roles/tails-config/templates/sk.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/sk/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2019-08-29 21:39+0000\n" "Last-Translator: 1000101 \n" -"Language-Team: Slovak \n" +"Language-Team: Slovak \n" "Language: sk\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/sv.po b/install_files/ansible-base/roles/tails-config/templates/locale/sv/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/sv.po rename to install_files/ansible-base/roles/tails-config/templates/locale/sv/LC_MESSAGES/messages.po index eacc5f8d2e..d50a4706de 100644 --- a/install_files/ansible-base/roles/tails-config/templates/sv.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/sv/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-13 07:56+0000\n" "Last-Translator: Jonas Waga \n" -"Language-Team: Swedish \n" +"Language-Team: Swedish \n" "Language: sv\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/tr.po b/install_files/ansible-base/roles/tails-config/templates/locale/tr/LC_MESSAGES/messages.po similarity index 97% rename from install_files/ansible-base/roles/tails-config/templates/tr.po rename to install_files/ansible-base/roles/tails-config/templates/locale/tr/LC_MESSAGES/messages.po index f7755086a4..e3e8e77840 100644 --- a/install_files/ansible-base/roles/tails-config/templates/tr.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/tr/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2018-01-11 15:26+0000\n" "Last-Translator: Volkan \n" -"Language-Team: Turkish \n" +"Language-Team: Turkish \n" "Language: tr\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/zh_Hans.po b/install_files/ansible-base/roles/tails-config/templates/locale/zh_Hans/LC_MESSAGES/messages.po similarity index 96% rename from install_files/ansible-base/roles/tails-config/templates/zh_Hans.po rename to install_files/ansible-base/roles/tails-config/templates/locale/zh_Hans/LC_MESSAGES/messages.po index 6faf2b57a4..2af89b6631 100644 --- a/install_files/ansible-base/roles/tails-config/templates/zh_Hans.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/zh_Hans/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-15 04:56+0000\n" "Last-Translator: Kishin Sagume \n" -"Language-Team: Chinese (Simplified) \n" +"Language-Team: Chinese (Simplified) \n" "Language: zh_Hans\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/install_files/ansible-base/roles/tails-config/templates/zh_Hant.po b/install_files/ansible-base/roles/tails-config/templates/locale/zh_Hant/LC_MESSAGES/messages.po similarity index 96% rename from install_files/ansible-base/roles/tails-config/templates/zh_Hant.po rename to install_files/ansible-base/roles/tails-config/templates/locale/zh_Hant/LC_MESSAGES/messages.po index 4d7acd2edd..db7ded2cf4 100644 --- a/install_files/ansible-base/roles/tails-config/templates/zh_Hant.po +++ b/install_files/ansible-base/roles/tails-config/templates/locale/zh_Hant/LC_MESSAGES/messages.po @@ -9,8 +9,7 @@ msgstr "" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: 2023-06-15 01:56+0000\n" "Last-Translator: Chi-Hsun Tsai \n" -"Language-Team: Chinese (Traditional) \n" +"Language-Team: Chinese (Traditional) \n" "Language: zh_Hant\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" diff --git a/securedrop/babel.cfg b/securedrop/babel.cfg index 061374b5b6..ebaa0d26ab 100644 --- a/securedrop/babel.cfg +++ b/securedrop/babel.cfg @@ -13,3 +13,5 @@ extensions=jinja2.ext.autoescape,jinja2.ext.with_,jinja2.ext.do [javascript: **.js.in] extract_messages = $._, jQuery._ + +[desktop: **.j2.in] diff --git a/securedrop/bin/dev-deps b/securedrop/bin/dev-deps index 6b5478f977..fd0753fa3a 100755 --- a/securedrop/bin/dev-deps +++ b/securedrop/bin/dev-deps @@ -133,8 +133,8 @@ function reset_demo() { chmod -f 700 /var/lib/securedrop/keys/private-keys-v1.d || true chmod -f 700 /var/lib/securedrop/keys/openpgp-revocs.d || true - # Generate translated strings - ./i18n_tool.py translate-messages --compile + # Compile translated strings + pybabel compile --directory translations/ # remove previously uploaded custom logos rm -f /var/www/securedrop/static/i/custom_logo.png diff --git a/securedrop/bin/generate-docs-screenshots b/securedrop/bin/generate-docs-screenshots index 93f8aff29d..40c859a2fc 100755 --- a/securedrop/bin/generate-docs-screenshots +++ b/securedrop/bin/generate-docs-screenshots @@ -13,5 +13,5 @@ urandom build_redwood maybe_create_config_py -./i18n_tool.py translate-messages --compile +pybabel compile --directory translations/ pytest -v --page-layout "${@:-tests/functional/pageslayout}" diff --git a/securedrop/bin/translation-test b/securedrop/bin/translation-test index 5cc6303bdc..0cb95be12f 100755 --- a/securedrop/bin/translation-test +++ b/securedrop/bin/translation-test @@ -13,11 +13,11 @@ run_x11vnc & urandom build_redwood maybe_create_config_py -./i18n_tool.py translate-messages --compile +pybabel compile --directory translations/ mkdir -p "/tmp/test-results/logs" -SUPPORTED_LOCALES=$(./i18n_tool.py list-locales) +SUPPORTED_LOCALES=$(make --directory .. --quiet supported-locales | jq --raw-output 'join (" ")') LOCALES="${*:-${SUPPORTED_LOCALES}}" function locale_is_supported { diff --git a/securedrop/debian/translations.sh b/securedrop/debian/translations.sh index 9f164c0566..b18e02e0cc 100644 --- a/securedrop/debian/translations.sh +++ b/securedrop/debian/translations.sh @@ -13,9 +13,6 @@ python3 -m venv /tmp/securedrop-app-code-i18n-ve # Install dependencies /tmp/securedrop-app-code-i18n-ve/bin/pip3 install --no-deps --no-binary :all: --require-hashes -r requirements/python3/translation-requirements.txt -# Compile the translations, need to have a placeholder config.py that we clean up -export PYTHONDONTWRITEBYTECODE="true" -cp config.py.example config.py +# Compile the translations . /tmp/securedrop-app-code-i18n-ve/bin/activate -/tmp/securedrop-app-code-i18n-ve/bin/python3 ./i18n_tool.py --verbose translate-messages --compile -rm config.py +pybabel compile --directory translations/ diff --git a/securedrop/dockerfiles/focal/python3/SlimDockerfile b/securedrop/dockerfiles/focal/python3/SlimDockerfile index 1893147b2b..5beb6d199c 100644 --- a/securedrop/dockerfiles/focal/python3/SlimDockerfile +++ b/securedrop/dockerfiles/focal/python3/SlimDockerfile @@ -8,7 +8,7 @@ ENV USER_ID ${USER_ID:-0} RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ apache2-dev coreutils vim \ python3-pip python3-all python3-venv virtualenv python3-dev libssl-dev \ - gnupg2 redis-server git curl wget \ + gnupg2 redis-server git curl wget jq \ enchant libffi-dev sqlite3 gettext sudo tor basez pkg-config # Install Rust using the same steps as diff --git a/securedrop/i18n.py b/securedrop/i18n.py index 15234800c9..596ca74076 100644 --- a/securedrop/i18n.py +++ b/securedrop/i18n.py @@ -211,11 +211,11 @@ def get_locale(config: SecureDropConfig) -> str: - config.DEFAULT_LOCALE - config.FALLBACK_LOCALE """ - preferences = [] + preferences: List[str] = [] if session and session.get("locale"): - preferences.append(session.get("locale")) + preferences.append(session["locale"]) if request.args.get("l"): - preferences.insert(0, request.args.get("l")) + preferences.insert(0, request.args["l"]) if not preferences: preferences.extend(get_accepted_languages()) preferences.append(config.DEFAULT_LOCALE) diff --git a/securedrop/i18n.rst b/securedrop/i18n.rst new file mode 100644 index 0000000000..5d4478eb25 --- /dev/null +++ b/securedrop/i18n.rst @@ -0,0 +1,22 @@ +.. GENERATED BY "make update-supported-locales": +* Arabic (``ar``) +* Catalan (``ca``) +* Czech (``cs``) +* German (``de_DE``) +* Greek (``el``) +* Spanish (``es_ES``) +* French (``fr_FR``) +* Hindi (``hi``) +* Icelandic (``is``) +* Italian (``it_IT``) +* Norwegian (``nb_NO``) +* Dutch (``nl``) +* Portuguese, Brasil (``pt_BR``) +* Portuguese, Portugal (``pt_PT``) +* Romanian (``ro``) +* Russian (``ru``) +* Slovak (``sk``) +* Swedish (``sv``) +* Turkish (``tr``) +* Chinese, Simplified (``zh_Hans``) +* Chinese, Traditional (``zh_Hant``) diff --git a/securedrop/i18n_tool.py b/securedrop/i18n_tool.py deleted file mode 100755 index 985b46015b..0000000000 --- a/securedrop/i18n_tool.py +++ /dev/null @@ -1,738 +0,0 @@ -#!/opt/venvs/securedrop-app-code/bin/python - -import argparse -import glob -import json -import logging -import os -import re -import signal -import subprocess -import sys -import textwrap -from argparse import _SubParsersAction -from os.path import abspath, dirname, join, realpath -from pathlib import Path -from typing import Dict, List, Optional, Set - -import version - -logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s") -log = logging.getLogger(__name__) - -I18N_CONF = os.path.join(os.path.dirname(__file__), "i18n.json") - -# Map components of the "securedrop" Weblate project (keys) to their filesystem -# paths (values) relative to the repository root. -LOCALE_DIR = { - "securedrop": "securedrop/translations", - "desktop": "install_files/ansible-base/roles/tails-config/templates", -} - - -class I18NTool: - # - # The database of support language, indexed by the language code - # used by weblate (i.e. whatever shows as CODE in - # https://weblate.securedrop.org/projects/securedrop/securedrop/CODE/ - # is the index of the SUPPORTED_LANGUAGES database below. - # - # name: English name of the language to the documentation, not for - # display in the interface. - # desktop: The language code used for desktop icons. - # - with open(I18N_CONF) as i18n_conf: - conf = json.load(i18n_conf) - supported_languages = conf["supported_locales"] - release_tag_re = re.compile(r"^\d+\.\d+\.\d+$") - translated_commit_re = re.compile("Translated using Weblate") - updated_commit_re = re.compile(r"(?:updated from| (?:revision|commit):) (\w+)") - - def file_is_modified(self, path: str) -> bool: - return bool(subprocess.call(["git", "-C", dirname(path), "diff", "--quiet", path])) - - def ensure_i18n_remote(self, args: argparse.Namespace) -> None: - """ - Make sure we have a git remote for the i18n repo. - """ - - k = {"cwd": args.root} - if "i18n" not in subprocess.check_output(["git", "remote"], encoding="utf-8", **k): - subprocess.check_call(["git", "remote", "add", "i18n", args.url], **k) - subprocess.check_call(["git", "fetch", "i18n"], **k) - - def translate_messages(self, args: argparse.Namespace) -> None: - messages_file = Path(args.translations_dir).absolute() / "messages.pot" - - if args.extract_update: - if not os.path.exists(args.translations_dir): - os.makedirs(args.translations_dir) - sources = args.sources.split(",") - subprocess.check_call( - [ - "pybabel", - "extract", - "--charset=utf-8", - "--mapping", - args.mapping, - "--output", - messages_file, - "--project=SecureDrop", - "--version", - args.version, - "--msgid-bugs-address=securedrop@freedom.press", - "--copyright-holder=Freedom of the Press Foundation", - "--add-comments=Translators:", - "--strip-comments", - "--add-location=never", - "--no-wrap", - *sources, - ] - ) - - msg_file_content = messages_file.read_text() - updated_content = _remove_from_content_line_with_text( - text='"POT-Creation-Date:', content=msg_file_content - ) - messages_file.write_text(updated_content) - - if ( - self.file_is_modified(str(messages_file)) - and len(os.listdir(args.translations_dir)) > 1 - ): - tglob = f"{args.translations_dir}/*/LC_MESSAGES/*.po" - for translation in glob.iglob(tglob): - subprocess.check_call( - [ - "msgattrib", - "--no-fuzzy", - "--output-file", - translation, - translation, - ] - ) - subprocess.check_call( - [ - "msgmerge", - "--previous", - "--update", - "--no-fuzzy-matching", - "--no-wrap", - translation, - messages_file, - ] - ) - log.warning(f"messages translations updated in {messages_file}") - else: - log.warning("messages translations are already up to date") - - if args.compile and len(os.listdir(args.translations_dir)) > 1: - # Suppress all pybabel to hide warnings (e.g. - # https://github.com/python-babel/babel/issues/566) and verbose "compiling..." messages - subprocess.run( - ["pybabel", "compile", "--directory", args.translations_dir], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - def translate_desktop(self, args: argparse.Namespace) -> None: - messages_file = Path(args.translations_dir).absolute() / "desktop.pot" - - if args.extract_update: - sources = args.sources.split(",") - xgettext_flags = [] - - # First extract from JavaScript sources via Babel ("xgettext - # --language=JavaScript" can't parse gettext as exposed by GNOME - # JavaScript): - js_sources = [source for source in sources if ".js" in source] - if js_sources: - xgettext_flags.append("--join-existing") - subprocess.check_call( - [ - "pybabel", - "extract", - "--charset=utf-8", - "--mapping", - args.mapping, - "--output", - "desktop.pot", - "--project=SecureDrop", - "--version", - args.version, - "--msgid-bugs-address=securedrop@freedom.press", - "--copyright-holder=Freedom of the Press Foundation", - "--add-comments=Translators:", - "--strip-comments", - "--add-location=never", - "--no-wrap", - *js_sources, - ], - cwd=args.translations_dir, - ) - - # Then extract from desktop templates via xgettext in appending - # "--join-existing" mode: - desktop_sources = [source for source in sources if ".js" not in source] - subprocess.check_call( - [ - "xgettext", - *xgettext_flags, - "--output=desktop.pot", - "--language=Desktop", - "--keyword", - "--keyword=Name", - "--package-version", - args.version, - "--msgid-bugs-address=securedrop@freedom.press", - "--copyright-holder=Freedom of the Press Foundation", - *desktop_sources, - ], - cwd=args.translations_dir, - ) - msg_file_content = messages_file.read_text() - updated_content = _remove_from_content_line_with_text( - text='"POT-Creation-Date:', content=msg_file_content - ) - messages_file.write_text(updated_content) - - if self.file_is_modified(str(messages_file)): - for f in os.listdir(args.translations_dir): - if not f.endswith(".po"): - continue - po_file = os.path.join(args.translations_dir, f) - subprocess.check_call( - ["msgmerge", "--no-fuzzy-matching", "--update", po_file, messages_file] - ) - log.warning(f"messages translations updated in {messages_file}") - else: - log.warning("desktop translations are already up to date") - - if args.compile: - pos = [f for f in os.listdir(args.translations_dir) if f.endswith(".po")] - linguas = [lingua[:-3] for lingua in pos] - content = "\n".join(linguas) + "\n" - linguas_file = join(args.translations_dir, "LINGUAS") - try: - open(linguas_file, "w").write(content) - - # First, compile each message catalog for normal gettext use. - # We have to iterate over them, rather than using "pybabel - # compile --directory", in order to replicate gettext's - # standard "{locale}/LC_MESSAGES/{domain}.mo" structure (like - # "securedrop/translations"). - locale_dir = os.path.join(args.translations_dir, "locale") - for po in pos: - locale = po.replace(".po", "") - output_dir = os.path.join(locale_dir, locale, "LC_MESSAGES") - subprocess.run(["mkdir", "--parents", output_dir], check=True) - subprocess.run( - [ - "msgfmt", - "--output-file", - os.path.join(output_dir, "messages.mo"), - po, - ], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - cwd=args.translations_dir, - ) - - # Then use msgfmt to update the desktop files as usual. - desktop_sources = [ - source for source in args.sources.split(",") if ".js" not in source - ] - for source in desktop_sources: - target = source.rstrip(".in") - subprocess.check_call( - [ - "msgfmt", - "--desktop", - "--template", - source, - "-o", - target, - "-d", - ".", - ], - cwd=args.translations_dir, - ) - self.sort_desktop_template(join(args.translations_dir, target)) - finally: - if os.path.exists(linguas_file): - os.unlink(linguas_file) - - def sort_desktop_template(self, template: str) -> None: - """ - Sorts the lines containing the icon names. - """ - lines = open(template).readlines() - names = sorted(line for line in lines if line.startswith("Name")) - others = (line for line in lines if not line.startswith("Name")) - with open(template, "w") as new_template: - for line in others: - new_template.write(line) - for line in names: - new_template.write(line) - - def set_translate_parser( - self, parser: argparse.ArgumentParser, translations_dir: str, sources: str - ) -> None: - parser.add_argument( - "--extract-update", - action="store_true", - help=("extract strings to translate and " "update existing translations"), - ) - parser.add_argument("--compile", action="store_true", help="compile translations") - parser.add_argument( - "--translations-dir", - default=translations_dir, - help=f"Base directory for translation files (default {translations_dir})", - ) - parser.add_argument( - "--version", - default=version.__version__, - help=( - "SecureDrop version " - "to store in pot files (default {})".format(version.__version__) - ), - ) - parser.add_argument( - "--sources", - default=sources, - help=f"Source files and directories to extract (default {sources})", - ) - - def set_translate_messages_parser(self, subps: _SubParsersAction) -> None: - parser = subps.add_parser( - "translate-messages", help=("Update and compile " "source and template translations") - ) - translations_dir = join(dirname(realpath(__file__)), "translations") - sources = ".,source_templates,journalist_templates" - self.set_translate_parser(parser, translations_dir, sources) - mapping = "babel.cfg" - parser.add_argument( - "--mapping", - default=mapping, - help=f"Mapping of files to consider (default {mapping})", - ) - parser.set_defaults(func=self.translate_messages) - - def set_translate_desktop_parser(self, subps: _SubParsersAction) -> None: - parser = subps.add_parser( - "translate-desktop", help=("Update and compile " "desktop icons translations") - ) - translations_dir = join( - dirname(realpath(__file__)), - "..", - LOCALE_DIR["desktop"], - ) - sources = ",".join( - [ - "desktop-journalist-icon.j2.in", - "desktop-source-icon.j2.in", - "extension.js.in", - ] - ) - mapping = join(dirname(realpath(__file__)), "babel.cfg") - parser.add_argument( - "--mapping", - default=mapping, - help=f"Mapping of files to consider (default {mapping})", - ) - self.set_translate_parser(parser, translations_dir, sources) - parser.set_defaults(func=self.translate_desktop) - - @staticmethod - def require_git_email_name(git_dir: str) -> bool: - cmd = ( - "git -C {d} config --get user.name > /dev/null && " - "git -C {d} config --get user.email > /dev/null".format(d=git_dir) - ) - # nosemgrep: python.lang.security.audit.subprocess-shell-true.subprocess-shell-true - if subprocess.call(cmd, shell=True): # nosec - if "docker" in open("/proc/1/cgroup").read(): - log.error( - "remember ~/.gitconfig does not exist " - "in the dev-shell Docker container, " - "only .git/config does" - ) - raise Exception(cmd + " returned false, please set name and email") - return True - - def update_docs(self, args: argparse.Namespace) -> None: - l10n_content = ".. GENERATED BY i18n_tool.py DO NOT EDIT:\n\n" - for (code, info) in sorted(self.supported_languages.items()): - l10n_content += "* " + info["name"] + " (``" + code + "``)\n" - includes = abspath(join(args.docs_repo_dir, "docs/includes")) - l10n_txt = join(includes, "l10n.txt") - open(l10n_txt, mode="w").write(l10n_content) - self.require_git_email_name(includes) - if self.file_is_modified(l10n_txt): - subprocess.check_call(["git", "add", "l10n.txt"], cwd=includes) - msg = "docs: update the list of supported languages" - subprocess.check_call(["git", "commit", "-m", msg, "l10n.txt"], cwd=includes) - log.warning(l10n_txt + " updated") - git_show_out = subprocess.check_output(["git", "show"], encoding="utf-8", cwd=includes) - log.warning(git_show_out) - else: - log.warning(l10n_txt + " already up to date") - - def set_update_docs_parser(self, subps: _SubParsersAction) -> None: - parser = subps.add_parser("update-docs", help=("Update the documentation")) - parser.add_argument( - "--docs-repo-dir", - required=True, - help=("root directory of the SecureDrop documentation repository"), - ) - parser.set_defaults(func=self.update_docs) - - def update_from_weblate(self, args: argparse.Namespace) -> None: - """ - Pull in updated translations from the i18n repo. - """ - self.ensure_i18n_remote(args) - - # Check out *all* remote changes to the LOCALE_DIRs. - subprocess.check_call( - [ - "git", - "checkout", - args.target, - "--", - *LOCALE_DIR.values(), - ], - cwd=args.root, - ) - - # Use the LOCALE_DIR corresponding to the "securedrop" component to - # determine which locales are present. - locale_dir = os.path.join(args.root, LOCALE_DIR["securedrop"]) - locales = self.translated_locales(locale_dir) - if args.supported_languages: - codes = args.supported_languages.split(",") - locales = {code: locales[code] for code in locales if code in codes} - - # Then iterate over all locales present and decide which to stage and commit. - for code, path in locales.items(): - paths = [] - - def add(path: str) -> None: - """ - Add the file to the git index. - """ - subprocess.check_call(["git", "add", path], cwd=args.root) - paths.append(path) - - # Any translated locale may have changes that need to be staged from the - # securedrop/securedrop component. - add(path) - - # Only supported locales may have changes that need to be staged from the - # securedrop/desktop component, because the link between the two components - # is defined in I18N_CONF when a language is marked supported. - try: - info = self.supported_languages[code] - name = info["name"] - desktop_code = info["desktop"] - path = join( - LOCALE_DIR["desktop"], - f"{desktop_code}.po", - ) - add(path) - except KeyError: - log.info( - f"{code} has translations but is not marked as supported; " - f"skipping desktop translation" - ) - name = code - - # Try to commit changes for this locale no matter what, even if it turns out to be a - # no-op. - self.commit_changes(args, code, name, paths) - - def translators( - self, args: argparse.Namespace, path: str, since_commit: Optional[str] - ) -> Set[str]: - """ - Return the set of people who've modified a file in Weblate. - - Extracts all the authors of translation changes to the given - path since the given commit. Translation changes are - identified by the presence of "Translated using Weblate" in - the commit message. - """ - - log_command = ["git", "--no-pager", "-C", args.root, "log", "--format=%aN\x1e%s\x1e%H"] - - if since_commit: - since = self.get_commit_timestamp(args.root, since_commit) - log_command.extend(["--since", since]) - - log_command.extend([args.target, "--", path]) - - # NB. We use an explicit str.split("\n") here because str.splitlines() splits on a - # set of characters that includes the \x1e "record separator" we pass to "git log - # --format" in log_command. See #6648. - log_lines = subprocess.check_output(log_command, encoding="utf-8").strip().split("\n") - path_changes = [c.split("\x1e") for c in log_lines] - path_changes = [ - c - for c in path_changes - if len(c) > 1 and c[2] != since_commit and self.translated_commit_re.match(c[1]) - ] - log.debug("Path changes for %s: %s", path, path_changes) - translators = {c[0] for c in path_changes} - log.debug("Translators for %s: %s", path, translators) - return translators - - def get_path_commits(self, root: str, path: str) -> List[str]: - """ - Returns the list of commit hashes involving the path, most recent first. - """ - cmd = ["git", "--no-pager", "log", "--format=%H", path] - return subprocess.check_output(cmd, encoding="utf-8", cwd=root).splitlines() - - def translated_locales(self, path: str) -> Dict[str, str]: - """Return a dictionary of all locale directories present in `path`, where the keys - are the base (directory) names and the values are the full paths.""" - p = Path(path) - return {x.name: str(x) for x in p.iterdir() if x.is_dir()} - - def commit_changes( - self, args: argparse.Namespace, code: str, name: str, paths: List[str] - ) -> None: - """Check if any of the given paths have had changed staged. If so, commit them.""" - self.require_git_email_name(args.root) - authors: Set[str] = set() - cmd = ["git", "--no-pager", "diff", "--name-only", "--cached", *paths] - diffs = subprocess.check_output(cmd, cwd=args.root, encoding="utf-8") - - # If nothing was changed, "git commit" will return nonzero as a no-op. - if len(diffs) == 0: - return - - # for each modified file, find the last commit made by this - # function, then collect all the contributors since that - # commit, so they can be credited in this one. if no commit - # with the right message is found, just use the most recent - # commit that touched the file. - for path in paths: - path_commits = self.get_path_commits(args.root, path) - since_commit = None - for path_commit in path_commits: - commit_message = subprocess.check_output( - ["git", "--no-pager", "show", path_commit], - encoding="utf-8", - cwd=args.root, - ) - m = self.updated_commit_re.search(commit_message) - if m: - since_commit = m.group(1) - break - log.debug("Crediting translators of %s since %s", path, since_commit) - authors |= self.translators(args, path, since_commit) - - authors_as_str = "\n ".join(sorted(authors)) - - current = subprocess.check_output( - ["git", "rev-parse", args.target], cwd=args.root, encoding="utf-8" - ) - message = textwrap.dedent( - """ - l10n: updated {name} ({code}) - - contributors: - {authors} - - updated from: - repo: {remote} - commit: {current} - """ - ).format(remote=args.url, name=name, authors=authors_as_str, code=code, current=current) - subprocess.check_call(["git", "commit", "-m", message, *paths], cwd=args.root) - log.debug(f"Committing with this message: {message}") - - def set_update_from_weblate_parser(self, subps: _SubParsersAction) -> None: - parser = subps.add_parser("update-from-weblate", help=("Import translations from weblate")) - root = join(dirname(realpath(__file__)), "..") - parser.add_argument( - "--root", - default=root, - help=("root of the SecureDrop git repository" " (default {})".format(root)), - ) - url = "https://github.com/freedomofpress/securedrop-i18n" - parser.add_argument( - "--url", default=url, help=("URL of the weblate repository" " (default {})".format(url)) - ) - parser.add_argument( - "--target", - default="i18n/i18n", - help=( - "Commit on i18n branch at which to stop gathering translator contributions " - "(default: i18n/i18n)" - ), - ) - parser.add_argument( - "--supported-languages", help="comma separated list of supported languages" - ) - parser.set_defaults(func=self.update_from_weblate) - - def set_list_locales_parser(self, subps: _SubParsersAction) -> None: - parser = subps.add_parser("list-locales", help="List supported locales") - parser.add_argument( - "--python", - action="store_true", - help=("Print the locales as a Python list suitable for config.py"), - ) - parser.add_argument("--lines", action="store_true", help=("List one locale per line")) - parser.set_defaults(func=self.list_locales) - - def list_locales(self, args: argparse.Namespace) -> None: - if args.lines: - for lang in sorted(list(self.supported_languages.keys()) + ["en_US"]): - print(lang) - elif args.python: - print(sorted(list(self.supported_languages.keys()) + ["en_US"])) - else: - print(" ".join(sorted(list(self.supported_languages.keys()) + ["en_US"]))) - - def set_list_translators_parser(self, subps: _SubParsersAction) -> None: - parser = subps.add_parser("list-translators", help=("List contributing translators")) - root = join(dirname(realpath(__file__)), "..") - parser.add_argument( - "--root", - default=root, - help=("root of the SecureDrop git repository" " (default {})".format(root)), - ) - url = "https://github.com/freedomofpress/securedrop-i18n" - parser.add_argument( - "--url", default=url, help=("URL of the weblate repository" " (default {})".format(url)) - ) - parser.add_argument( - "--target", - default="i18n/i18n", - help=( - "Commit on i18n branch at which to stop gathering translator contributions " - "(default: i18n/i18n)" - ), - ) - parser.add_argument( - "--since", - help=( - "Gather translator contributions from the time of this commit " - "(default: last release tag)" - ), - ) - parser.add_argument( - "--all", - action="store_true", - help=( - "List everyone who's ever contributed, instead of just since the last " - "release or specified commit." - ), - ) - parser.set_defaults(func=self.list_translators) - - def get_last_release(self, root: str) -> str: - """ - Returns the last release tag, e.g. 1.5.0. - """ - tags = ( - subprocess.check_output(["git", "-C", root, "tag", "--list"]) - .decode("utf-8") - .splitlines() - ) - release_tags = sorted([t.strip() for t in tags if self.release_tag_re.match(t)]) - if not release_tags: - raise ValueError("Could not find a release tag!") - return release_tags[-1] - - def get_commit_timestamp(self, root: str, commit: Optional[str]) -> str: - """ - Returns the UNIX timestamp of the given commit. - """ - cmd = ["git", "-C", root, "log", "-n", "1", "--pretty=format:%ct"] - if commit: - cmd.append(commit) - - timestamp = subprocess.check_output(cmd) - return timestamp.decode("utf-8").strip() - - def list_translators(self, args: argparse.Namespace) -> None: - self.ensure_i18n_remote(args) - app_template = "{}/{}/LC_MESSAGES/messages.po" - desktop_template = LOCALE_DIR["desktop"] + "/{}.po" - since = None - if args.all: - print("Listing all translators who have ever helped") - else: - since = args.since if args.since else self.get_last_release(args.root) - print(f"Listing translators who have helped since {since}") - for code, info in sorted(self.supported_languages.items()): - translators = set() - paths = [ - app_template.format(LOCALE_DIR["securedrop"], code), - desktop_template.format(info["desktop"]), - ] - for path in paths: - try: - t = self.translators(args, path, since) - translators.update(t) - except Exception as e: - print(f"Could not check git history of {path}: {e}", file=sys.stderr) - print( - "{} ({}):{}".format( - code, - info["name"], - "\n {}\n".format("\n ".join(sorted(translators))) if translators else "\n", - ) - ) - - def get_args(self) -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog=__file__, description="i18n tool for SecureDrop.") - parser.set_defaults(func=lambda args: parser.print_help()) - - parser.add_argument("-v", "--verbose", action="store_true") - subps = parser.add_subparsers() - - self.set_translate_messages_parser(subps) - self.set_translate_desktop_parser(subps) - self.set_update_docs_parser(subps) - self.set_update_from_weblate_parser(subps) - self.set_list_translators_parser(subps) - self.set_list_locales_parser(subps) - - return parser - - def setup_verbosity(self, args: argparse.Namespace) -> None: - if args.verbose: - logging.getLogger("sh.command").setLevel(logging.INFO) - log.setLevel(logging.DEBUG) - else: - log.setLevel(logging.INFO) - - def main(self, argv: List[str]) -> int: - try: - args = self.get_args().parse_args(argv) - self.setup_verbosity(args) - return args.func(args) - except KeyboardInterrupt: - return signal.SIGINT - - -def _remove_from_content_line_with_text(text: str, content: str) -> str: - # Split where the text is located; assume there is only one instance of the text - split_content = content.split(text) - assert len(split_content) == 2 - - # Remove the while line containing the text - content_before_line = split_content[0] - content_after_line = split_content[1].split("\n", maxsplit=1)[1] - return content_before_line + content_after_line - - -if __name__ == "__main__": # pragma: no cover - sys.exit(I18NTool().main(sys.argv[1:])) diff --git a/securedrop/requirements/python3/develop-requirements.txt b/securedrop/requirements/python3/develop-requirements.txt index b165ae829c..6d073073fc 100644 --- a/securedrop/requirements/python3/develop-requirements.txt +++ b/securedrop/requirements/python3/develop-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --allow-unsafe --generate-hashes --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python3/develop-requirements.in +# pip-compile --allow-unsafe --generate-hashes --output-file=requirements/python3/develop-requirements.txt ../admin/requirements-ansible.in ../admin/requirements.in requirements/python3/develop-requirements.in requirements/python3/translation-requirements.in # ansible-lint==4.2.0 \ --hash=sha256:b9fc9a6564f5d60a4284497f966f38ef78f0e2505edbe2bd1225f1ade31c2d8a \ @@ -57,6 +57,15 @@ attrs==21.4.0 \ # jsonschema # pytest # semgrep +babel==2.12.1 \ + --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \ + --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455 + # via + # -r requirements/python3/translation-requirements.in + # babelgladeextractor +babelgladeextractor==0.7.0 \ + --hash=sha256:bcf805e28b4bb18c8b6909a65a7cf5c7c2bcbf4ae50b164878c9682d22271798 + # via -r requirements/python3/translation-requirements.in bandit==1.7.0 \ --hash=sha256:216be4d044209fa06cf2a3e51b319769a51be8318140659719aa7a115c35ed07 \ --hash=sha256:8a4c7415254d75df8ff3c3b15cfe9042ecee628a1e40b44c15a98890fbfc2608 @@ -801,6 +810,10 @@ python-vagrant==0.5.15 \ # via # -r requirements/python3/develop-requirements.in # molecule-vagrant +pytz==2022.2.1 \ + --hash=sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197 \ + --hash=sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5 + # via babel pyyaml==5.4.1 \ --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \ diff --git a/securedrop/requirements/python3/translation-requirements.in b/securedrop/requirements/python3/translation-requirements.in index 299d264097..ee86f39665 100644 --- a/securedrop/requirements/python3/translation-requirements.in +++ b/securedrop/requirements/python3/translation-requirements.in @@ -1 +1,2 @@ -babel>=2.9.1 +babel>=2.10 +BabelGladeExtractor diff --git a/securedrop/requirements/python3/translation-requirements.txt b/securedrop/requirements/python3/translation-requirements.txt index dd438f840d..ec7a9c35cb 100644 --- a/securedrop/requirements/python3/translation-requirements.txt +++ b/securedrop/requirements/python3/translation-requirements.txt @@ -4,9 +4,14 @@ # # pip-compile --allow-unsafe --generate-hashes --output-file=requirements/python3/translation-requirements.txt requirements/python3/translation-requirements.in # -babel==2.9.1 \ - --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ - --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 +babel==2.12.1 \ + --hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \ + --hash=sha256:cc2d99999cd01d44420ae725a21c9e3711b3aadc7976d6147f622d8581963455 + # via + # -r requirements/python3/translation-requirements.in + # babelgladeextractor +babelgladeextractor==0.7.0 \ + --hash=sha256:bcf805e28b4bb18c8b6909a65a7cf5c7c2bcbf4ae50b164878c9682d22271798 # via -r requirements/python3/translation-requirements.in pytz==2022.2.1 \ --hash=sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197 \ diff --git a/securedrop/tests/test_i18n.py b/securedrop/tests/test_i18n.py index 0dca558df0..1d457527b5 100644 --- a/securedrop/tests/test_i18n.py +++ b/securedrop/tests/test_i18n.py @@ -21,7 +21,6 @@ from typing import List import i18n -import i18n_tool import journalist_app as journalist_app_module import pytest import source_app @@ -247,22 +246,31 @@ def test_i18n(): i18n_dir = Path(__file__).absolute().parent / "i18n" sources = [str(i18n_dir / "code.py"), str(i18n_dir / "template.html")] - i18n_tool.I18NTool().main( + pot = i18n_dir / "messages.pot" + subprocess.check_call( [ - "--verbose", - "translate-messages", + "pybabel", + "extract", "--mapping", str(i18n_dir / "babel.cfg"), - "--translations-dir", - str(translation_dirs), - "--sources", - ",".join(sources), - "--extract-update", + "--output", + pot, + *sources, ] ) - pot = translation_dirs / "messages.pot" - subprocess.check_call(["pybabel", "init", "-i", pot, "-d", translation_dirs, "-l", "en_US"]) + subprocess.check_call( + [ + "pybabel", + "init", + "--input-file", + pot, + "--output-dir", + translation_dirs, + "--locale", + "en_US", + ] + ) for (locale, translated_msg) in ( ("fr_FR", "code bonjour"), @@ -271,7 +279,18 @@ def test_i18n(): ("nb_NO", "code norwegian"), ("es_ES", "code spanish"), ): - subprocess.check_call(["pybabel", "init", "-i", pot, "-d", translation_dirs, "-l", locale]) + subprocess.check_call( + [ + "pybabel", + "init", + "--input-file", + pot, + "--output-dir", + translation_dirs, + "--locale", + locale, + ] + ) # Populate the po file with a translation po_file = translation_dirs / locale / "LC_MESSAGES" / "messages.po" @@ -281,15 +300,18 @@ def test_i18n(): msgstr=translated_msg, ) - i18n_tool.I18NTool().main( - [ - "--verbose", - "translate-messages", - "--translations-dir", - str(translation_dirs), - "--compile", - ] - ) + subprocess.check_call( + [ + "pybabel", + "compile", + "--directory", + translation_dirs, + "--locale", + locale, + "--input-file", + po_file, + ] + ) # Use our config (and not an app fixture) because the i18n module # grabs values at init time and we can't inject them later. diff --git a/securedrop/tests/test_i18n_tool.py b/securedrop/tests/test_i18n_tool.py deleted file mode 100644 index a6c6494539..0000000000 --- a/securedrop/tests/test_i18n_tool.py +++ /dev/null @@ -1,442 +0,0 @@ -import os -import shutil -import signal -import subprocess -import time -from os.path import abspath, dirname, exists, getmtime, join, realpath -from pathlib import Path -from unittest.mock import patch - -import i18n_tool -import pytest -from tests.test_i18n import set_msg_translation_in_po_file - - -class TestI18NTool: - def setup(self): - self.dir = abspath(dirname(realpath(__file__))) - - def test_main(self, tmpdir, caplog): - with pytest.raises(SystemExit): - i18n_tool.I18NTool().main(["--help"]) - - tool = i18n_tool.I18NTool() - with patch.object(tool, "setup_verbosity", side_effect=KeyboardInterrupt): - assert ( - tool.main(["translate-messages", "--translations-dir", str(tmpdir)]) - == signal.SIGINT - ) - - def test_translate_desktop_l10n(self, tmpdir): - in_files = {} - for what in ("source", "journalist"): - in_files[what] = join(str(tmpdir), what + ".desktop.in") - shutil.copy(join(self.dir, "i18n/" + what + ".desktop.in"), in_files[what]) - i18n_tool.I18NTool().main( - [ - "--verbose", - "translate-desktop", - "--translations-dir", - str(tmpdir), - "--sources", - in_files["source"], - "--extract-update", - ] - ) - messages_file = join(str(tmpdir), "desktop.pot") - assert exists(messages_file) - with open(messages_file) as fobj: - pot = fobj.read() - assert "SecureDrop Source Interfaces" in pot - # pretend this happened a few seconds ago - few_seconds_ago = time.time() - 60 - os.utime(messages_file, (few_seconds_ago, few_seconds_ago)) - - i18n_file = join(str(tmpdir), "source.desktop") - - # Extract+update but do not compile - old_messages_mtime = getmtime(messages_file) - assert not exists(i18n_file) - i18n_tool.I18NTool().main( - [ - "--verbose", - "translate-desktop", - "--translations-dir", - str(tmpdir), - "--sources", - ",".join(list(in_files.values())), - "--extract-update", - ] - ) - assert not exists(i18n_file) - current_messages_mtime = getmtime(messages_file) - assert old_messages_mtime < current_messages_mtime - - for locale, translation in [ - ("fr_FR", "SOURCE FR"), - # Regression test for #4192; bug when adding Romanian as an accepted language - ("ro", "SOURCE RO"), - ]: - po_file = Path(tmpdir) / f"{locale}.po" - subprocess.check_call( - [ - "msginit", - "--no-translator", - "--locale", - locale, - "--output", - po_file, - "--input", - messages_file, - ] - ) - set_msg_translation_in_po_file( - po_file=po_file, - msgid_to_translate="SecureDrop Source Interfaces", - msgstr=translation, - ) - - # Compile but do not extract+update - old_messages_mtime = current_messages_mtime - i18n_tool.I18NTool().main( - [ - "--verbose", - "translate-desktop", - "--translations-dir", - str(tmpdir), - "--sources", - ",".join(list(in_files.values()) + ["BOOM"]), - "--compile", - ] - ) - assert old_messages_mtime == getmtime(messages_file) - with open(po_file) as fobj: - po = fobj.read() - assert "SecureDrop Source Interfaces" in po - assert "SecureDrop Journalist Interfaces" not in po - with open(i18n_file) as fobj: - i18n = fobj.read() - assert "SOURCE FR" in i18n - - def test_translate_messages_l10n(self, tmpdir): - source = [ - join(self.dir, "i18n/code.py"), - join(self.dir, "i18n/template.html"), - ] - args = [ - "--verbose", - "translate-messages", - "--translations-dir", - str(tmpdir), - "--mapping", - join(self.dir, "i18n/babel.cfg"), - "--sources", - ",".join(source), - "--extract-update", - "--compile", - ] - i18n_tool.I18NTool().main(args) - messages_file = join(str(tmpdir), "messages.pot") - assert exists(messages_file) - with open(messages_file, "rb") as fobj: - pot = fobj.read() - assert b"code hello i18n" in pot - assert b"template hello i18n" in pot - - locale = "en_US" - locale_dir = join(str(tmpdir), locale) - subprocess.check_call( - [ - "pybabel", - "init", - "-i", - messages_file, - "-d", - str(tmpdir), - "-l", - locale, - ] - ) - - # Add a dummy translation - po_file = Path(locale_dir) / "LC_MESSAGES/messages.po" - for msgid in ["code hello i18n", "template hello i18n"]: - set_msg_translation_in_po_file( - po_file=po_file, - msgid_to_translate=msgid, - msgstr=msgid, - ) - - mo_file = join(locale_dir, "LC_MESSAGES/messages.mo") - assert not exists(mo_file) - i18n_tool.I18NTool().main(args) - assert exists(mo_file) - with open(mo_file, mode="rb") as fobj: - mo = fobj.read() - assert b"code hello i18n" in mo - assert b"template hello i18n" in mo - - def test_translate_messages_compile_arg(self, tmpdir): - args = [ - "--verbose", - "translate-messages", - "--translations-dir", - str(tmpdir), - "--mapping", - join(self.dir, "i18n/babel.cfg"), - ] - i18n_tool.I18NTool().main( - args - + [ - "--sources", - join(self.dir, "i18n/code.py"), - "--extract-update", - ] - ) - messages_file = join(str(tmpdir), "messages.pot") - assert exists(messages_file) - with open(messages_file) as fobj: - pot = fobj.read() - assert "code hello i18n" in pot - - locale = "en_US" - locale_dir = join(str(tmpdir), locale) - po_file = join(locale_dir, "LC_MESSAGES/messages.po") - subprocess.check_call( - ["pybabel", "init", "-i", messages_file, "-d", str(tmpdir), "-l", locale] - ) - assert exists(po_file) - # pretend this happened a few seconds ago - few_seconds_ago = time.time() - 60 - os.utime(po_file, (few_seconds_ago, few_seconds_ago)) - - mo_file = join(locale_dir, "LC_MESSAGES/messages.mo") - - # - # Extract+update but do not compile - # - old_po_mtime = getmtime(po_file) - assert not exists(mo_file) - i18n_tool.I18NTool().main( - args - + [ - "--sources", - join(self.dir, "i18n/code.py"), - "--extract-update", - ] - ) - assert not exists(mo_file) - current_po_mtime = getmtime(po_file) - assert old_po_mtime < current_po_mtime - - # - # Translation would occur here - let's fake it - set_msg_translation_in_po_file( - po_file=Path(po_file), - msgid_to_translate="code hello i18n", - msgstr="code hello i18n", - ) - - # - # Compile but do not extract+update - # - source = [ - join(self.dir, "i18n/code.py"), - join(self.dir, "i18n/template.html"), - ] - current_po_mtime = getmtime(po_file) - i18n_tool.I18NTool().main( - args - + [ - "--sources", - ",".join(source), - "--compile", - ] - ) - assert current_po_mtime == getmtime(po_file) - with open(mo_file, mode="rb") as fobj: - mo = fobj.read() - assert b"code hello i18n" in mo - assert b"template hello i18n" not in mo - - def test_require_git_email_name(self, tmpdir): - k = {"cwd": str(tmpdir)} - subprocess.check_call(["git", "init"], **k) - with pytest.raises(Exception) as excinfo: - i18n_tool.I18NTool.require_git_email_name(str(tmpdir)) - assert "please set name" in str(excinfo.value) - - subprocess.check_call(["git", "config", "user.email", "you@example.com"], **k) - subprocess.check_call(["git", "config", "user.name", "Your Name"], **k) - assert i18n_tool.I18NTool.require_git_email_name(str(tmpdir)) - - def test_update_docs(self, tmpdir, caplog): - k = {"cwd": str(tmpdir)} - subprocess.check_call(["git", "init"], **k) - subprocess.check_call(["git", "config", "user.email", "you@example.com"], **k) - subprocess.check_call(["git", "config", "user.name", "Your Name"], **k) - os.makedirs(join(str(tmpdir), "docs/includes")) - subprocess.check_call(["touch", "docs/includes/l10n.txt"], **k) - subprocess.check_call(["git", "add", "docs/includes/l10n.txt"], **k) - subprocess.check_call(["git", "commit", "-m", "init"], **k) - - i18n_tool.I18NTool().main(["--verbose", "update-docs", "--docs-repo-dir", str(tmpdir)]) - assert "l10n.txt updated" in caplog.text - caplog.clear() - i18n_tool.I18NTool().main(["--verbose", "update-docs", "--docs-repo-dir", str(tmpdir)]) - assert "l10n.txt already up to date" in caplog.text - - def test_update_from_weblate(self, tmpdir, caplog): - d = str(tmpdir) - for repo in ("i18n", "securedrop"): - os.mkdir(join(d, repo)) - k = {"cwd": join(d, repo)} - subprocess.check_call(["git", "init"], **k) - subprocess.check_call(["git", "config", "user.email", "you@example.com"], **k) - subprocess.check_call(["git", "config", "user.name", "Loïc Nordhøy"], **k) - subprocess.check_call(["touch", "README.md"], **k) - subprocess.check_call(["git", "add", "README.md"], **k) - subprocess.check_call(["git", "commit", "-m", "README", "README.md"], **k) - for o in os.listdir(join(self.dir, "i18n")): - f = join(self.dir, "i18n", o) - if os.path.isfile(f): - shutil.copyfile(f, join(d, "i18n", o)) - else: - shutil.copytree(f, join(d, "i18n", o)) - k = {"cwd": join(d, "i18n")} - subprocess.check_call(["git", "add", "securedrop", "install_files"], **k) - subprocess.check_call(["git", "commit", "-m", "init", "-a"], **k) - subprocess.check_call(["git", "checkout", "-b", "i18n", "master"], **k) - - def r(): - return "".join([str(record) for record in caplog.records]) - - # - # de_DE is not amount the supported languages, it is not taken - # into account despite the fact that it exists in weblate. - # - caplog.clear() - i18n_tool.I18NTool().main( - [ - "--verbose", - "update-from-weblate", - "--root", - join(str(tmpdir), "securedrop"), - "--url", - join(str(tmpdir), "i18n"), - "--supported-languages", - "nl", - ] - ) - assert "l10n: updated Dutch (nl)" in r() - assert "l10n: updated German (de_DE)" not in r() - - # - # de_DE is added but there is no change in the nl translation - # therefore nothing is done for nl - # - caplog.clear() - i18n_tool.I18NTool().main( - [ - "--verbose", - "update-from-weblate", - "--root", - join(str(tmpdir), "securedrop"), - "--url", - join(str(tmpdir), "i18n"), - "--supported-languages", - "nl,de_DE", - ] - ) - assert "l10n: updated Dutch (nl)" not in r() - assert "l10n: updated German (de_DE)" in r() - - # - # nothing new for nl or de_DE: nothing is done - # - caplog.clear() - i18n_tool.I18NTool().main( - [ - "--verbose", - "update-from-weblate", - "--root", - join(str(tmpdir), "securedrop"), - "--url", - join(str(tmpdir), "i18n"), - "--supported-languages", - "nl,de_DE", - ] - ) - assert "l10n: updated Dutch (nl)" not in r() - assert "l10n: updated German (de_DE)" not in r() - message = subprocess.check_output( - ["git", "--no-pager", "-C", "securedrop", "show"], - cwd=d, - encoding="utf-8", - ) - assert "Loïc" in message - - # an update is done to nl in weblate - i18n_dir = Path(d) / "i18n" - po_file = i18n_dir / "securedrop/translations/nl/LC_MESSAGES/messages.po" - content = po_file.read_text() - text_to_update = "inactiviteit" - assert text_to_update in content - updated_content = content.replace(text_to_update, "INACTIVITEIT") - po_file.write_text(updated_content) - - k = {"cwd": join(d, "i18n")} - subprocess.check_call(["git", "add", str(po_file)], **k) - subprocess.check_call(["git", "config", "user.email", "somone@else.com"], **k) - subprocess.check_call(["git", "config", "user.name", "Someone Else"], **k) - subprocess.check_call( - ["git", "commit", "-m", "Translated using Weblate", str(po_file)], **k - ) - - k = {"cwd": join(d, "securedrop")} - subprocess.check_call(["git", "config", "user.email", "somone@else.com"], **k) - subprocess.check_call(["git", "config", "user.name", "Someone Else"], **k) - - # - # the nl translation update from weblate is copied - # over. - # - caplog.clear() - i18n_tool.I18NTool().main( - [ - "--verbose", - "update-from-weblate", - "--root", - join(str(tmpdir), "securedrop"), - "--url", - join(str(tmpdir), "i18n"), - "--supported-languages", - "nl,de_DE", - ] - ) - assert "l10n: updated Dutch (nl)" in r() - assert "l10n: updated German (de_DE)" not in r() - - # The translator is credited in Git history. - message = subprocess.check_output( - ["git", "--no-pager", "-C", "securedrop", "show"], - cwd=d, - encoding="utf-8", - ) - assert "Someone Else" in message - assert "Loïc" not in message - - # The "list-translators" command correctly reads the translator from Git history. - caplog.clear() - i18n_tool.I18NTool().main( - [ - "--verbose", - "list-translators", - "--all", - "--root", - join(str(tmpdir), "securedrop"), - "--url", - join(str(tmpdir), "i18n"), - ] - ) - assert "Someone Else" in caplog.text diff --git a/securedrop/tests/test_template_filters.py b/securedrop/tests/test_template_filters.py index e4d5b094a9..797ebd4220 100644 --- a/securedrop/tests/test_template_filters.py +++ b/securedrop/tests/test_template_filters.py @@ -2,7 +2,6 @@ from datetime import datetime, timedelta from pathlib import Path -import i18n_tool import journalist_app import source_app import template_filters @@ -87,23 +86,24 @@ def do_test(create_app): test_config = create_config_for_i18n_test(supported_locales=["en_US", "fr_FR"]) i18n_dir = Path(__file__).absolute().parent / "i18n" - i18n_tool.I18NTool().main( + pot = Path(test_config.TEMP_DIR) / "messages.pot" + subprocess.check_call( [ - "--verbose", - "translate-messages", + "pybabel", + "extract", "--mapping", str(i18n_dir / "babel.cfg"), - "--translations-dir", - str(test_config.TEMP_DIR), - "--sources", + "--output", + pot, str(i18n_dir / "code.py"), - "--extract-update", - "--compile", ] ) + # To be able to test template filters for a given language, its message + # catalog must exist, but it doesn't have to contain any actual + # translations. So we can just initialize it based on the template created + # by "pybabel extract". for lang in ("en_US", "fr_FR"): - pot = Path(test_config.TEMP_DIR) / "messages.pot" subprocess.check_call( ["pybabel", "init", "-i", pot, "-d", test_config.TEMP_DIR, "-l", lang] ) diff --git a/securedrop/translations/messages.pot b/securedrop/translations/messages.pot index 4438587a4d..012a0279d2 100644 --- a/securedrop/translations/messages.pot +++ b/securedrop/translations/messages.pot @@ -6,7 +6,7 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: SecureDrop 2.6.0~rc1\n" +"Project-Id-Version: SecureDrop 2.7.0~rc1\n" "Report-Msgid-Bugs-To: securedrop@freedom.press\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" @@ -14,7 +14,7 @@ msgstr "" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.9.1\n" +"Generated-By: Babel 2.12.1\n" msgid "Name too long" msgstr "" @@ -729,8 +729,10 @@ msgstr "" msgid "LOG IN" msgstr "" -msgid "Field must be between 1 and {max_codename_len} characters long." -msgstr "" +msgid "Field must be between 1 and {max_codename_len} character long." +msgid_plural "Field must be between 1 and {max_codename_len} characters long." +msgstr[0] "" +msgstr[1] "" msgid "Invalid input." msgstr ""