From 2c0d2fa3b50ac4e053068b565d141aa5017988ed Mon Sep 17 00:00:00 2001 From: John Meiss Date: Mon, 28 Aug 2023 15:25:34 +0200 Subject: [PATCH 1/4] feat: add OpenAI translator --- README.md | 22 +++++ config/locales/en.yml | 6 ++ config/locales/ru.yml | 6 ++ lib/i18n/tasks/configuration.rb | 1 + lib/i18n/tasks/translation.rb | 5 +- .../tasks/translators/openai_translator.rb | 99 +++++++++++++++++++ 6 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 lib/i18n/tasks/translators/openai_translator.rb diff --git a/README.md b/README.md index 6489c711..07eb5ab0 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,17 @@ $ i18n-tasks translate-missing --backend=yandex $ i18n-tasks translate-missing --from=en es fr ``` +### OpenAI Translate missing keys + +Translate missing values with OpenAI ([more below on the API key](#openai-translation-config)). + +```console +$ i18n-tasks translate-missing --backend=openai + +# accepts from and locales options: +$ i18n-tasks translate-missing --from=en es fr +``` + ### Find usages See where the keys are used with `i18n-tasks find`: @@ -424,6 +435,17 @@ translation: yandex_api_key: ``` + +### OpenAI Translate + +`i18n-tasks translate-missing` requires a OpenAI API key, get it at [OpenAI](https://openai.com/). + +```yaml +# config/i18n-tasks.yml +translation: + openai_api_key: +``` + ## Interactive console `i18n-tasks irb` starts an IRB session in i18n-tasks context. Type `guide` for more information. diff --git a/config/locales/en.yml b/config/locales/en.yml index 46bd3bd4..46aafb08 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -112,6 +112,12 @@ en: missing: details_title: Value in other locales or source none: No translations are missing. + openai_translate: + errors: + no_api_key: >- + Set OpenAI API key via OPENAI_API_KEY environment variable or translation.openai_api_key + in config/i18n-tasks.yml. Get the key at https://openai.com/. + no_results: OpenAI returned no results. remove_unused: confirm: one: "%{count} translation will be removed from %{locales}." diff --git a/config/locales/ru.yml b/config/locales/ru.yml index 88524a8a..18d44a66 100644 --- a/config/locales/ru.yml +++ b/config/locales/ru.yml @@ -111,6 +111,12 @@ ru: missing: details_title: На других языках или в коде none: Всё переведено. + openai_translate: + errors: + no_api_key: |- + Установить ключ API Яндекса с помощью переменной среды OPENAI_API_KEY или translation.openai_api_key + в config / i18n-tasks.yml. Получите ключ по адресу https://openai.com/. + no_results: Яндекс не дал результатов. remove_unused: confirm: few: Переводы (%{count}) будут удалены из %{locales}. diff --git a/lib/i18n/tasks/configuration.rb b/lib/i18n/tasks/configuration.rb index 7bcf371d..8d7719d3 100644 --- a/lib/i18n/tasks/configuration.rb +++ b/lib/i18n/tasks/configuration.rb @@ -66,6 +66,7 @@ def translation_config conf[:deepl_api_key] = ENV['DEEPL_AUTH_KEY'] if ENV.key?('DEEPL_AUTH_KEY') conf[:deepl_host] = ENV['DEEPL_HOST'] if ENV.key?('DEEPL_HOST') conf[:deepl_version] = ENV['DEEPL_VERSION'] if ENV.key?('DEEPL_VERSION') + conf[:openai_api_key] = ENV['OPENAI_API_KEY'] if ENV.key?('OPENAI_API_KEY') conf[:yandex_api_key] = ENV['YANDEX_API_KEY'] if ENV.key?('YANDEX_API_KEY') conf end diff --git a/lib/i18n/tasks/translation.rb b/lib/i18n/tasks/translation.rb index 51296f45..31edfdf5 100644 --- a/lib/i18n/tasks/translation.rb +++ b/lib/i18n/tasks/translation.rb @@ -2,13 +2,14 @@ require 'i18n/tasks/translators/deepl_translator' require 'i18n/tasks/translators/google_translator' +require 'i18n/tasks/translators/openai_translator' require 'i18n/tasks/translators/yandex_translator' module I18n::Tasks module Translation # @param [I18n::Tasks::Tree::Siblings] forest to translate to the locales of its root nodes # @param [String] from locale - # @param [:deepl, :google, :yandex] backend + # @param [:deepl, :openai, :google, :yandex] backend # @return [I18n::Tasks::Tree::Siblings] translated forest def translate_forest(forest, from:, backend: :google) case backend @@ -16,6 +17,8 @@ def translate_forest(forest, from:, backend: :google) Translators::DeeplTranslator.new(self).translate_forest(forest, from) when :google Translators::GoogleTranslator.new(self).translate_forest(forest, from) + when :openai + Translators::OpenAiTranslator.new(self).translate_forest(forest, from) when :yandex Translators::YandexTranslator.new(self).translate_forest(forest, from) else diff --git a/lib/i18n/tasks/translators/openai_translator.rb b/lib/i18n/tasks/translators/openai_translator.rb new file mode 100644 index 00000000..2849011d --- /dev/null +++ b/lib/i18n/tasks/translators/openai_translator.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'i18n/tasks/translators/base_translator' + +module I18n::Tasks::Translators + class OpenAiTranslator < BaseTranslator + # max allowed texts per request + BATCH_SIZE = 50 + + def initialize(*) + begin + require 'openai' + rescue LoadError + raise ::I18n::Tasks::CommandError, "Add gem 'ruby-openai' to your Gemfile to use this command" + end + super + end + + def options_for_translate_values(from:, to:, **options) + options.merge( + from:, + to: + ) + end + + def options_for_html + {} + end + + def options_for_plain + {} + end + + def no_results_error_message + I18n.t('i18n_tasks.openai_translate.errors.no_results') + end + + private + + def translator + @translator ||= OpenAI::Client.new(access_token: api_key) + end + + def api_key + @api_key ||= begin + key = @i18n_tasks.translation_config[:openai_api_key] + fail ::I18n::Tasks::CommandError, I18n.t('i18n_tasks.openai_translate.errors.no_api_key') if key.blank? + + key + end + end + + def translate_values(list, from:, to:, **options) + results = [] + + list.each_slice(BATCH_SIZE) do |batch| + translations = translate(batch, from, to) + + results << JSON.parse(translations) + end + + results.flatten + end + + def translate(values, from, to) + messages = [ + { + role: "system", + content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n locale array. + The array has a structured format and contains multiple strings. Your task is to translate each of these strings and create a new array with the translated strings. + Keep in mind the context of all the strings for a more accurate translation.\n", + }, + { + role: "user", + content: "Translate this array: \n\n\n", + }, + { + role: "user", + content: values.to_json, + } + ] + + response = translator.chat( + parameters: { + model: "gpt-3.5-turbo", + messages:, + temperature: 0.7 + } + ) + + translations = response.dig("choices", 0, "message", "content") + error = response.dig("error") + + raise "AI error: #{error}" if error.present? + + translations + end + end +end From cdee227654e3761cd5fa9dfb01aa29dfe59a0714 Mon Sep 17 00:00:00 2001 From: John Meiss Date: Mon, 28 Aug 2023 16:15:28 +0200 Subject: [PATCH 2/4] fix rubocop errors --- lib/i18n/tasks/translators/openai_translator.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/i18n/tasks/translators/openai_translator.rb b/lib/i18n/tasks/translators/openai_translator.rb index 2849011d..31303613 100644 --- a/lib/i18n/tasks/translators/openai_translator.rb +++ b/lib/i18n/tasks/translators/openai_translator.rb @@ -18,8 +18,8 @@ def initialize(*) def options_for_translate_values(from:, to:, **options) options.merge( - from:, - to: + from: from, + to: to ) end @@ -83,7 +83,7 @@ def translate(values, from, to) response = translator.chat( parameters: { model: "gpt-3.5-turbo", - messages:, + messages: messages, temperature: 0.7 } ) @@ -96,4 +96,4 @@ def translate(values, from, to) translations end end -end +end \ No newline at end of file From 1b88f71374e0bb4d7e6b504b9629038c13823789 Mon Sep 17 00:00:00 2001 From: John Meiss Date: Mon, 28 Aug 2023 16:17:33 +0200 Subject: [PATCH 3/4] Add prompt credit --- lib/i18n/tasks/translators/openai_translator.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/i18n/tasks/translators/openai_translator.rb b/lib/i18n/tasks/translators/openai_translator.rb index 31303613..22b08d95 100644 --- a/lib/i18n/tasks/translators/openai_translator.rb +++ b/lib/i18n/tasks/translators/openai_translator.rb @@ -63,6 +63,8 @@ def translate_values(list, from:, to:, **options) end def translate(values, from, to) + # Prompt from + # https://github.com/ObservedObserver/chatgpt-i18n/blob/main/src/services/translate.ts messages = [ { role: "system", From b9864d13ed14aaa8958ed5d14f7757805ddd1f95 Mon Sep 17 00:00:00 2001 From: John Meiss Date: Tue, 29 Aug 2023 10:57:52 +0200 Subject: [PATCH 4/4] Fix Rubocop violation --- .../tasks/translators/openai_translator.rb | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/lib/i18n/tasks/translators/openai_translator.rb b/lib/i18n/tasks/translators/openai_translator.rb index 22b08d95..60ef8d67 100644 --- a/lib/i18n/tasks/translators/openai_translator.rb +++ b/lib/i18n/tasks/translators/openai_translator.rb @@ -50,7 +50,7 @@ def api_key end end - def translate_values(list, from:, to:, **options) + def translate_values(list, from:, to:) results = [] list.each_slice(BATCH_SIZE) do |batch| @@ -62,40 +62,39 @@ def translate_values(list, from:, to:, **options) results.flatten end - def translate(values, from, to) - # Prompt from - # https://github.com/ObservedObserver/chatgpt-i18n/blob/main/src/services/translate.ts + def translate(values, from, to) # rubocop:disable Metrics/MethodLength messages = [ { - role: "system", - content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n locale array. - The array has a structured format and contains multiple strings. Your task is to translate each of these strings and create a new array with the translated strings. - Keep in mind the context of all the strings for a more accurate translation.\n", + role: 'system', + content: "You are a helpful assistant that translates content from the #{from} to #{to} locale in an i18n + locale array. The array has a structured format and contains multiple strings. Your task is to translate + each of these strings and create a new array with the translated strings. Keep in mind the context of all + the strings for a more accurate translation.\n" }, { - role: "user", - content: "Translate this array: \n\n\n", + role: 'user', + content: "Translate this array: \n\n\n" }, { - role: "user", - content: values.to_json, + role: 'user', + content: values.to_json } ] response = translator.chat( parameters: { - model: "gpt-3.5-turbo", + model: 'gpt-3.5-turbo', messages: messages, temperature: 0.7 } ) - translations = response.dig("choices", 0, "message", "content") - error = response.dig("error") + translations = response.dig('choices', 0, 'message', 'content') + error = response['error'] - raise "AI error: #{error}" if error.present? + fail "AI error: #{error}" if error.present? translations end end -end \ No newline at end of file +end