Add tooling to localize Brave extension with Transifex

DEPS

@@ -5,7 +5,7 @@ deps = {
   "vendor/tracking-protection": "https://github.com/brave/tracking-protection.git@0931529eba33109c6b3946a83295577fea540045",
   "vendor/hashset-cpp": "https://github.com/bbondy/hashset-cpp.git@f427324d667d7188a9e0975cca7f3a8c06226b4d",
   "vendor/bloom-filter-cpp": "https://github.com/bbondy/bloom-filter-cpp.git@6faa14ececa33badad149c40f94ff9867159681c",
-  "vendor/brave-extension": "https://github.com/brave/brave-extension.git@a1e99dabc60f4055ae095ca8e6cc86f719c9f429",
+  "vendor/brave-extension": "https://github.com/brave/brave-extension.git@887f1c0c3c65df41613dcfc094b57cb8054161b6",
   "vendor/requests": "https://github.com/kennethreitz/requests@e4d59bedfd3c7f4f254f4f5d036587bcd8152458",
   "vendor/boto": "https://github.com/boto/boto@f7574aa6cc2c819430c1f05e9a1a1a666ef8169b",
   "vendor/python-patch": "https://github.com/svn2github/python-patch@a336a458016ced89aba90dfc3f4c8222ae3b1403",

script/lib/transifex.py

@@ -2,9 +2,11 @@
 from lib.config import get_env_var
 from xml.sax.saxutils import escape, unescape
 import HTMLParser
+import io
 import os
 import requests
 import sys
+import tempfile
 import lxml.etree
 import FP
 
@@ -91,30 +93,29 @@ def get_strings_dict_from_xtb_file(xtb_file_path):
   return { translation_tag.get('id'): textify(translation_tag) for translation_tag in translation_tags }
 
 
-def update_source_string_file_to_transifex(filename, xml_content):
+def update_source_string_file_to_transifex(filename, content):
   """Uploads the specified source string file to transifex"""
   print 'Updating existing known resource'
   url_part = 'project/%s/resource/%s/content' % (transifex_project_name, filename)
   url = base_url + url_part
   payload = {
-    'content': xml_content
+    'content': content
   }
   headers = { 'Content-Type': 'application/json' }
   r = requests.put(url, json=payload, auth=get_auth(), headers=headers)
   assert r.status_code >= 200 and r.status_code <= 299, 'Aborting. Status code %d: %s' % (r.status_code, r.content)
   return True
 
 
-def upload_source_string_file_to_transifex(filename, xml_content):
+def upload_source_string_file_to_transifex(filename, xml_content, i18n_type):
   """Uploads the specified source string file to transifex"""
   url_part = 'project/%s/resources/' % transifex_project_name
   url = base_url + url_part
-  localization_format = 'ANDROID'
   payload = {
     'name': filename,
     'slug': filename,
     'content': xml_content,
-    'i18n_type': localization_format
+    'i18n_type': i18n_type
   }
   headers = { 'Content-Type': 'application/json' }
   #r = requests.post(url, json=payload, auth=get_auth(), headers=headers)
@@ -402,3 +403,53 @@ def check_missing_source_grd_strings_to_transifex(grd_file_path):
   x_transifex_extra_strings = transifex_string_ids - grd_string_names
   assert len(x_transifex_extra_strings) == 0, ('Transifex has extra strings over GRD %s' %
           list(x_transifex_extra_strings))
+
+
+def upload_source_files_to_transifex(source_file_path, filename):
+  uploaded = False
+  i18n_type = ''
+  content = ''
+  ext = os.path.splitext(source_file_path)[1]
+  if ext == '.grd':
+    # Generate the intermediate Transifex format for the source translations
+    output_xml_file_handle, output_xml_path = tempfile.mkstemp('.xml')
+    content = generate_source_strings_xml_from_grd(output_xml_file_handle, source_file_path)
+    os.close(output_xml_file_handle)
+    i18n_type = 'ANDROID'
+  elif ext == '.json':
+    i18n_type = 'CHROME'
+    with io.open(source_file_path, mode='r', encoding='utf-8') as json_file:
+      content = json_file.read()
+  else:
+    assert False, 'Unsupported source file ext %s: %s' % (ext, source_file_path)
+
+  uploaded = upload_source_string_file_to_transifex(filename, content, i18n_type)
+  assert uploaded, 'Could not upload xml file'
+
+
+def pull_source_files_from_transifex(source_file_path, filename):
+  ext = os.path.splitext(source_file_path)[1]
+  if ext == '.grd':
+      # Generate the intermediate Transifex format
+      xtb_files = get_xtb_files(source_file_path)
+      base_path = os.path.dirname(source_file_path)
+      grd_strings = get_grd_strings(source_file_path)
+      for (lang_code, xtb_rel_path) in xtb_files:
+        xtb_file_path = os.path.join(base_path, xtb_rel_path)
+        xml_content = get_transifex_translation_file_content(filename, lang_code)
+        translations = get_strings_dict_from_xml_content(xml_content)
+        xtb_content = generate_xtb_content(lang_code, grd_strings, translations)
+        print 'Updated: ', xtb_file_path
+        with open(xtb_file_path, mode='w') as f:
+          f.write(xtb_content)
+  elif ext == '.json':
+    langs_dir_path = os.path.dirname(os.path.dirname(source_file_path))
+    lang_codes = set(os.listdir(langs_dir_path))
+    lang_codes.discard('en_US')
+    lang_codes.discard('.DS_Store')
+    for lang_code in lang_codes:
+      print 'getting filename %s for lang_code %s' % (filename, lang_code)
+      content = get_transifex_translation_file_content(filename, lang_code)
+      localized_translation_path = os.path.join(langs_dir_path, lang_code, 'message.json')
+      with open(localized_translation_path, mode='w') as f:
+        f.write(content)

script/pull-l10n.py

@@ -4,15 +4,15 @@
 import sys
 import xml.etree.ElementTree
 from lib.config import get_env_var
-from lib.transifex import get_xtb_files, get_grd_strings, textify, get_transifex_translation_file_content, get_strings_dict_from_xml_content, generate_xtb_content
+from lib.transifex import pull_source_files_from_transifex
 
 
 SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 
 
 def parse_args():
   parser = argparse.ArgumentParser(description='Push strings to Transifex')
-  parser.add_argument('--grd_path',
+  parser.add_argument('--source_string_path',
                       nargs=1)
   return parser.parse_args()
 
@@ -27,27 +27,14 @@ def check_args():
 def main():
   args = parse_args()
   check_args()
-  grd_file_path = os.path.join(SOURCE_ROOT, args.grd_path[0])
-  filename = os.path.basename(grd_file_path).split('.')[0]
+  source_string_path = os.path.join(SOURCE_ROOT, args.source_string_path[0])
+  filename = os.path.basename(source_string_path).split('.')[0]
 
-  grd_strings = get_grd_strings(grd_file_path)
 
   print '-----------'
-  print 'Source GRD:', grd_file_path
+  print 'Source string file:', source_string_path
   print 'Transifex resource slug: ', filename
-
-  # Generate the intermediate Transifex format
-  xtb_files = get_xtb_files(grd_file_path)
-  base_path = os.path.dirname(grd_file_path)
-  for (lang_code, xtb_rel_path) in xtb_files:
-    xtb_file_path = os.path.join(base_path, xtb_rel_path)
-    xml_content = get_transifex_translation_file_content(filename, lang_code)
-    translations = get_strings_dict_from_xml_content(xml_content)
-    xtb_content = generate_xtb_content(lang_code, grd_strings, translations)
-    print 'Updated: ', xtb_file_path
-    f = open(xtb_file_path, 'w')
-    f.write(xtb_content)
-    f.close()
+  pull_source_files_from_transifex(source_string_path, filename)
 
 
 if __name__ == '__main__':

script/push-l10n.py

@@ -1,10 +1,9 @@
 import argparse
 import os
 import sys
-import tempfile
 import xml.etree.ElementTree
 from lib.config import get_env_var
-from lib.transifex import upload_source_string_file_to_transifex, generate_source_strings_xml_from_grd, get_transifex_languages, get_grd_strings, check_for_chromium_upgrade, check_missing_source_grd_strings_to_transifex
+from lib.transifex import check_for_chromium_upgrade, check_missing_source_grd_strings_to_transifex, upload_source_files_to_transifex
 
 
 BRAVE_SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
@@ -13,7 +12,7 @@
 
 def parse_args():
   parser = argparse.ArgumentParser(description='Push strings to Transifex')
-  parser.add_argument('--grd_path',
+  parser.add_argument('--source_string_path',
                       nargs=1)
   return parser.parse_args()
 
@@ -25,28 +24,18 @@ def check_args():
   assert transifex_info, message
 
 
-def upload_source_files_to_transifex(grd_file_path, filename):
-  # Generate the intermediate Transifex format for the source translations
-  output_xml_file_handle, output_xml_path = tempfile.mkstemp('.xml')
-  xml_content = generate_source_strings_xml_from_grd(output_xml_file_handle, grd_file_path)
-
-  # Update Transifex
-  uploaded = upload_source_string_file_to_transifex(filename, xml_content)
-  os.close(output_xml_file_handle)
-  if not uploaded:
-    sys.exit('Could not upload xml file')
-
-
 def main():
   args = parse_args()
   check_args()
 
-  grd_file_path = os.path.join(BRAVE_SOURCE_ROOT, args.grd_path[0])
-  filename = os.path.basename(grd_file_path).split('.')[0]
+  source_string_path = os.path.join(BRAVE_SOURCE_ROOT, args.source_string_path[0])
+  filename = os.path.basename(source_string_path).split('.')[0]
 
-  upload_source_files_to_transifex(grd_file_path, filename)
-  check_for_chromium_upgrade(SOURCE_ROOT, grd_file_path)
-  check_missing_source_grd_strings_to_transifex(grd_file_path)
+  upload_source_files_to_transifex(source_string_path, filename)
+  ext = os.path.splitext(source_string_path)[1]
+  if ext == '.grd':
+    check_for_chromium_upgrade(SOURCE_ROOT, source_string_path)
+    check_missing_source_grd_strings_to_transifex(source_string_path)
 
 
 if __name__ == '__main__':