feat: support auto generate i18n translate (langgenius#6964)

Co-authored-by: crazywoola <[email protected]>

.github/workflows/translate-i18n-base-on-english.yml

@@ -0,0 +1,52 @@
+name: Check i18n Files and Create PR
+
+on:
+  pull_request:
+    types: [closed]
+    branches: [main]
+
+jobs:
+  check-and-update:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: web
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for file changes in i18n/en-US
+        id: check_files
+        run: |
+          changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -- 'i18n/en-US/*.ts')
+          echo "Changed files: $changed_files"
+          if [ -n "$changed_files" ]; then
+            echo "FILES_CHANGED=true" >> $GITHUB_ENV
+          else
+            echo "FILES_CHANGED=false" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Node.js
+        if: env.FILES_CHANGED == 'true'
+        uses: actions/setup-node@v2
+        with:
+          node-version: 'lts/*'
+
+      - name: Install dependencies
+        if: env.FILES_CHANGED == 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Run npm script
+        if: env.FILES_CHANGED == 'true'
+        run: npm run auto-gen-i18n
+
+      - name: Create Pull Request
+        if: env.FILES_CHANGED == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: Update i18n files based on en-US changes
+          title: 'chore: translate i18n files'
+          body: This PR was automatically created to update i18n files based on changes in en-US locale.
+          branch: chore/automated-i18n-updates

web/i18n/auto-gen-i18n.js

@@ -0,0 +1,82 @@
+/* eslint-disable no-eval */
+const fs = require('node:fs')
+const path = require('node:path')
+const transpile = require('typescript').transpile
+const magicast = require('magicast')
+const { parseModule, generateCode, loadFile } = magicast
+const bingTranslate = require('bing-translate-api')
+const { translate } = bingTranslate
+const data = require('./languages.json')
+
+const targetLanguage = 'en-US'
+// https://github.com/plainheart/bing-translate-api/blob/master/src/met/lang.json
+const languageKeyMap = data.languages.reduce((map, language) => {
+  if (language.supported) {
+    if (language.value === 'zh-Hans' || language.value === 'zh-Hant')
+      map[language.value] = language.value
+    else
+      map[language.value] = language.value.split('-')[0]
+  }
+
+  return map
+}, {})
+
+async function translateMissingKeyDeeply(sourceObj, targetObject, toLanguage) {
+  await Promise.all(Object.keys(sourceObj).map(async (key) => {
+    if (targetObject[key] === undefined) {
+      if (typeof sourceObj[key] === 'object') {
+        targetObject[key] = {}
+        await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage)
+      }
+      else {
+        const { translation } = await translate(sourceObj[key], null, languageKeyMap[toLanguage])
+        targetObject[key] = translation
+        // console.log(translation)
+      }
+    }
+    else if (typeof sourceObj[key] === 'object') {
+      targetObject[key] = targetObject[key] || {}
+      await translateMissingKeyDeeply(sourceObj[key], targetObject[key], toLanguage)
+    }
+  }))
+}
+
+async function autoGenTrans(fileName, toGenLanguage) {
+  const fullKeyFilePath = path.join(__dirname, targetLanguage, `${fileName}.ts`)
+  const toGenLanguageFilePath = path.join(__dirname, toGenLanguage, `${fileName}.ts`)
+  const fullKeyContent = eval(transpile(fs.readFileSync(fullKeyFilePath, 'utf8')))
+  // To keep object format and format it for magicast to work: const translation = { ... } => export default {...}
+  const readContent = await loadFile(toGenLanguageFilePath)
+  const { code: toGenContent } = generateCode(readContent)
+  const mod = await parseModule(`export default ${toGenContent.replace('export default translation', '').replace('const translation = ', '')}`)
+  const toGenOutPut = mod.exports.default
+
+  await translateMissingKeyDeeply(fullKeyContent, toGenOutPut, toGenLanguage)
+  const { code } = generateCode(mod)
+  const res = `const translation =${code.replace('export default', '')}
+
+export default translation
+`.replace(/,\n\n/g, ',\n').replace('};', '}')
+
+  fs.writeFileSync(toGenLanguageFilePath, res)
+}
+
+async function main() {
+  // const fileName = 'workflow'
+  // Promise.all(Object.keys(languageKeyMap).map(async (toLanguage) => {
+  //   await autoGenTrans(fileName, toLanguage)
+  // }))
+
+  const files = fs
+    .readdirSync(path.join(__dirname, targetLanguage))
+    .map(file => file.replace(/\.ts/, ''))
+    .filter(f => f !== 'app-debug') // ast parse error in app-debug
+
+  await Promise.all(files.map(async (file) => {
+    await Promise.all(Object.keys(languageKeyMap).map(async (language) => {
+      await autoGenTrans(file, language)
+    }))
+  }))
+}
+
+main()

web/i18n/script.js → web/i18n/check-i18n.js

@@ -27,6 +27,7 @@ async function getKeysFromLanuage(language) {
         // console.log(camelCaseFileName)
         const content = fs.readFileSync(filePath, 'utf8')
         const translation = eval(transpile(content))
+        // console.log(translation)
         const keys = Object.keys(translation)
         const nestedKeys = []
         const iterateKeys = (obj, prefix = '') => {

web/package.json

@@ -15,7 +15,8 @@
     "prepare": "cd ../ && node -e \"if (process.env.NODE_ENV !== 'production'){process.exit(1)} \" || husky install ./web/.husky",
     "gen-icons": "node ./app/components/base/icons/script.js",
     "uglify-embed": "node ./bin/uglify-embed",
-    "check-i18n": "node ./i18n/script.js",
+    "check-i18n": "node ./i18n/check-i18n.js",
+    "auto-gen-i18n": "node ./i18n/auto-gen-i18n.js",
     "test": "jest",
     "test:watch": "jest --watch"
   },
@@ -126,6 +127,7 @@
     "@types/sortablejs": "^1.15.1",
     "@types/uuid": "^9.0.8",
     "autoprefixer": "^10.4.14",
+    "bing-translate-api": "^4.0.2",
     "code-inspector-plugin": "^0.13.0",
     "cross-env": "^7.0.3",
     "eslint": "^8.36.0",
@@ -134,6 +136,7 @@
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "lint-staged": "^13.2.2",
+    "magicast": "^0.3.4",
     "postcss": "^8.4.31",
     "sass": "^1.61.0",
     "tailwindcss": "^3.4.4",