From 94e4de032334e4a5a79a2e390abf81f5e7ec924a Mon Sep 17 00:00:00 2001 From: Georg Leciejewski Date: Sun, 25 Sep 2022 16:36:44 +0200 Subject: [PATCH 1/3] split translation into parts so deepl-api does not complain, Handle deepl's target locale requirement for EN and PT which need their sub-version defined e.g. en-us --- config/locales/en.yml | 1 + config/locales/ru.yml | 1 + .../tasks/translators/deepl_translator.rb | 36 +++++++++++++++---- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index f254c813..d5c0bcac 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -96,6 +96,7 @@ en: Setup DeepL Pro API key via DEEPL_AUTH_KEY environment variable or translation.deepl_api_key in config/i18n-tasks.yml. Get the key at https://www.deepl.com/pro. no_results: DeepL returned no results. + specific_target_missing: You must supply a specific variant for the given target language e.g. en-us instead of en. google_translate: errors: no_api_key: >- diff --git a/config/locales/ru.yml b/config/locales/ru.yml index feb82f13..e904bc8a 100644 --- a/config/locales/ru.yml +++ b/config/locales/ru.yml @@ -94,6 +94,7 @@ ru: Задайте ключ API DeepL через переменную окружения DEEPL_AUTH_KEY или translation.deepl_api_key Получите ключ через https://www.deepl.com/pro. no_results: DeepL не дал результатов. + specific_target_missing: You must supply a specific variant for the given target language e.g. en-us instead of en. google_translate: errors: no_api_key: >- diff --git a/lib/i18n/tasks/translators/deepl_translator.rb b/lib/i18n/tasks/translators/deepl_translator.rb index 12fd8fd2..8c45e826 100644 --- a/lib/i18n/tasks/translators/deepl_translator.rb +++ b/lib/i18n/tasks/translators/deepl_translator.rb @@ -4,6 +4,11 @@ module I18n::Tasks::Translators class DeeplTranslator < BaseTranslator + # max allowed texts per request + BATCH_SIZE = 50 + # those languages must be specified with their sub-kind e.g en-us + SPECIFIC_TARGETS= ['en', 'pt'].freeze + def initialize(*) begin require 'deepl' @@ -17,12 +22,16 @@ def initialize(*) protected def translate_values(list, from:, to:, **options) - result = DeepL.translate(list, to_deepl_compatible_locale(from), to_deepl_compatible_locale(to), options) - if result.is_a?(DeepL::Resources::Text) - [result.text] - else - result.map(&:text) + results = [] + list.each_slice(BATCH_SIZE) do |parts| + res = DeepL.translate(parts, to_deepl_source_locale(from), to_deepl_target_locale(to), options) + if res.is_a?(DeepL::Resources::Text) + results << res.text + else + results += res.map(&:text) + end end + results end def options_for_translate_values(**options) @@ -60,11 +69,24 @@ def no_results_error_message private - # Convert 'es-ES' to 'ES' - def to_deepl_compatible_locale(locale) + # Convert 'es-ES' to 'ES', en-us to EN + def to_deepl_source_locale(locale) locale.to_s.split('-', 2).first.upcase end + # Convert 'es-ES' to 'ES' but skip specific locales + # This and the above should probably be done by the deepl gem + def to_deepl_target_locale(locale) + loc, sub = locale.to_s.split('-') + if SPECIFIC_TARGETS.include?(loc) + fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub + + locale.to_s.upcase + else + loc.upcase + end + end + def configure_api_key! api_key = @i18n_tasks.translation_config[:deepl_api_key] host = @i18n_tasks.translation_config[:deepl_host] From 93b8849f35eba29709d926cbae0cd7587e25c150 Mon Sep 17 00:00:00 2001 From: Georg Leciejewski Date: Mon, 26 Sep 2022 13:07:19 +0200 Subject: [PATCH 2/3] normalize translations, issue a warning for deprecated deepl target locales --- config/locales/en.yml | 3 ++- config/locales/ru.yml | 3 ++- lib/i18n/tasks/translators/deepl_translator.rb | 7 +++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index d5c0bcac..46bd3bd4 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -96,7 +96,8 @@ en: Setup DeepL Pro API key via DEEPL_AUTH_KEY environment variable or translation.deepl_api_key in config/i18n-tasks.yml. Get the key at https://www.deepl.com/pro. no_results: DeepL returned no results. - specific_target_missing: You must supply a specific variant for the given target language e.g. en-us instead of en. + specific_target_missing: You must supply a specific variant for the given target language + e.g. en-us instead of en. google_translate: errors: no_api_key: >- diff --git a/config/locales/ru.yml b/config/locales/ru.yml index e904bc8a..88524a8a 100644 --- a/config/locales/ru.yml +++ b/config/locales/ru.yml @@ -94,7 +94,8 @@ ru: Задайте ключ API DeepL через переменную окружения DEEPL_AUTH_KEY или translation.deepl_api_key Получите ключ через https://www.deepl.com/pro. no_results: DeepL не дал результатов. - specific_target_missing: You must supply a specific variant for the given target language e.g. en-us instead of en. + specific_target_missing: You must supply a specific variant for the given target language + e.g. en-us instead of en. google_translate: errors: no_api_key: >- diff --git a/lib/i18n/tasks/translators/deepl_translator.rb b/lib/i18n/tasks/translators/deepl_translator.rb index 8c45e826..3a47c207 100644 --- a/lib/i18n/tasks/translators/deepl_translator.rb +++ b/lib/i18n/tasks/translators/deepl_translator.rb @@ -74,13 +74,12 @@ def to_deepl_source_locale(locale) locale.to_s.split('-', 2).first.upcase end - # Convert 'es-ES' to 'ES' but skip specific locales - # This and the above should probably be done by the deepl gem + # Convert 'es-ES' to 'ES' but warn about locales requiring a specific variant def to_deepl_target_locale(locale) loc, sub = locale.to_s.split('-') if SPECIFIC_TARGETS.include?(loc) - fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub - + # Must see how the deepl api evolves, so this could be an error in the future + @i18n_tasks.warn_deprecated I18n.t('i18n_tasks.deepl_translate.errors.specific_target_missing') unless sub locale.to_s.upcase else loc.upcase From 3818509b2b8fadad4b3e5fff7eadd8ed4d2f02dc Mon Sep 17 00:00:00 2001 From: Georg Leciejewski Date: Mon, 26 Sep 2022 14:08:09 +0200 Subject: [PATCH 3/3] fix one deepl test, left one failing due to mangling with html-tag, could be a bug on their side of we may need another tag-handling param --- spec/deepl_translate_spec.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spec/deepl_translate_spec.rb b/spec/deepl_translate_spec.rb index 0f6b0634..afcd60a8 100644 --- a/spec/deepl_translate_spec.rb +++ b/spec/deepl_translate_spec.rb @@ -6,12 +6,14 @@ RSpec.describe 'DeepL Translation' do nil_value_test = ['nil-value-key', nil, nil] - text_test = ['key', "Hello, %{user} O'Neill! How are you?", "Hola, %{user} ¡O'Neill! Como estas?"] - html_test = ['html-key.html', "Hello, %{user} big O'neill ❤︎", "Hola, %{user} gran O'neill ❤︎"] + text_test = ['key', "Hello, %{user} O'Neill! How are you?", "¡Hola, %{user} O'Neill! ¿Cómo estás?"] html_test_plrl = ['html-key.html.one', 'Hello %{count}', 'Hola %{count}'] array_test = ['array-key', ['Hello.', nil, '', 'Goodbye.'], ['Hola.', nil, '', 'Adiós.']] fixnum_test = ['numeric-key', 1, 1] ref_key_test = ['ref-key', :reference, :reference] + # this test fails atm due to moving of the bold tag => "Hola, %{user} gran O'neill ❤︎ " + # it could be a bug, but the api also allows to ignore certain tags and there is the new html-markup version which could be used to + html_test = ['html-key.html', "Hello, %{user} big O'neill ❤︎", "Hola, %{user} gran O'neill ❤︎ "] describe 'real world test' do delegate :i18n_task, :in_test_app_dir, :run_cmd, to: :TestCodebase @@ -53,13 +55,13 @@ run_cmd 'translate-missing', '--backend=deepl' expect(task.t('common.hello', 'es')).to eq(text_test[2]) - expect(task.t('common.hello_html', 'es')).to eq(html_test[2]) expect(task.t('common.hello_plural_html.one', 'es')).to eq(html_test_plrl[2]) expect(task.t('common.array_key', 'es')).to eq(array_test[2]) expect(task.t('common.nil-value-key', 'es')).to eq(nil_value_test[2]) expect(task.t('common.fixnum-key', 'es')).to eq(fixnum_test[2]) expect(task.t('common.ref-key', 'es')).to eq(ref_key_test[2]) expect(task.t('common.a', 'es')).to eq('λ') + expect(task.t('common.hello_html', 'es')).to eq(html_test[2]) end end end