Implement a build tool for default messages extraction (#19620) (#21580)

* Implement a build tool for default messages extraction

* Refactor extraction tool, resolve review comments, fix bugs

* Resolve comments, refactor extraction tool, fix bugs

* Add context to messages extraction tool

* Resolve comments

* Fix bugs

* Add messages extraction from .jade files, refactor code

* Add template literals parsing

* Return defaultMessages.json to plain structure

* Refactor utils

* Fix bugs

* Refactor code, resolve review comments, fix bugs

* Fix minor bug

* Get rid of '@babel/traverse' and add its native implementation

* Add handlebars messages extraction

* Fix mkdir on macOS

* Fix bugs, inject default formats, finalize the tool

* Fix filsystem permissions

* Refactor nodes traversal

* Update code style

* Downgrade @babel/types to fix build issues

* Resolve comments

* Fix minor bugs

package.json

@@ -224,6 +224,8 @@
     "yauzl": "2.7.0"
   },
   "devDependencies": {
+    "@babel/parser": "7.0.0-beta.52",
+    "@babel/types": "7.0.0-beta.31",
     "@elastic/eslint-config-kibana": "link:packages/eslint-config-kibana",
     "@elastic/eslint-plugin-kibana-custom": "link:packages/eslint-plugin-kibana-custom",
     "@kbn/es": "link:packages/kbn-es",
@@ -308,6 +310,7 @@
     "jest-raw-loader": "^1.0.1",
     "jimp": "0.2.28",
     "jsdom": "9.9.1",
+    "json5": "^1.0.1",
     "karma": "1.7.0",
     "karma-chrome-launcher": "2.1.1",
     "karma-coverage": "1.1.1",

scripts/extract_default_translations.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/run_extract_default_translations');

src/dev/i18n/constants.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const DEFAULT_MESSAGE_KEY = 'defaultMessage';
+export const CONTEXT_KEY = 'context';

src/dev/i18n/extract_code_messages.js

@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { parse } from '@babel/parser';
+import {
+  isCallExpression,
+  isIdentifier,
+  isJSXIdentifier,
+  isJSXOpeningElement,
+  isMemberExpression,
+} from '@babel/types';
+
+import { extractI18nCallMessages } from './extract_i18n_call_messages';
+import { isI18nTranslateFunction, traverseNodes } from './utils';
+import { extractIntlMessages, extractFormattedMessages } from './extract_react_messages';
+
+/**
+ * Detect Intl.formatMessage() function call (React).
+ *
+ * Example: `intl.formatMessage({ id: 'message-id', defaultMessage: 'Message text' });`
+ */
+function isIntlFormatMessageFunction(node) {
+  return (
+    isCallExpression(node) &&
+    isMemberExpression(node.callee) &&
+    isIdentifier(node.callee.object, { name: 'intl' }) &&
+    isIdentifier(node.callee.property, { name: 'formatMessage' })
+  );
+}
+
+/**
+ * Detect <FormattedMessage> elements in JSX.
+ *
+ * Example: `<FormattedMessage id="message-id" defaultMessage="Message text"/>`
+ */
+function isFormattedMessageElement(node) {
+  return isJSXOpeningElement(node) && isJSXIdentifier(node.name, { name: 'FormattedMessage' });
+}
+
+export function* extractCodeMessages(buffer) {
+  const content = parse(buffer.toString(), {
+    sourceType: 'module',
+    plugins: ['jsx', 'typescript', 'objectRestSpread', 'classProperties', 'asyncGenerators'],
+  });
+
+  for (const node of traverseNodes(content.program.body)) {
+    if (isI18nTranslateFunction(node)) {
+      yield extractI18nCallMessages(node);
+    } else if (isIntlFormatMessageFunction(node)) {
+      yield extractIntlMessages(node);
+    } else if (isFormattedMessageElement(node)) {
+      yield extractFormattedMessages(node);
+    }
+  }
+}

src/dev/i18n/extract_default_translations.js

@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve } from 'path';
+import { formats } from '@kbn/i18n';
+import JSON5 from 'json5';
+
+import { extractHtmlMessages } from './extract_html_messages';
+import { extractCodeMessages } from './extract_code_messages';
+import { extractJadeMessages } from './extract_jade_messages';
+import { extractHandlebarsMessages } from './extract_handlebars_messages';
+import { globAsync, makeDirAsync, accessAsync, readFileAsync, writeFileAsync } from './utils';
+
+function addMessageToMap(targetMap, key, value) {
+  const existingValue = targetMap.get(key);
+  if (targetMap.has(key) && existingValue.message !== value.message) {
+    throw new Error(
+      `There is more than one default message for the same id "${key}": "${existingValue}" and "${value}"`
+    );
+  }
+  targetMap.set(key, value);
+}
+
+export async function extractDefaultTranslations(inputPath) {
+  const entries = await globAsync('*.{js,jsx,jade,ts,tsx,html,hbs,handlebars}', {
+    cwd: inputPath,
+    matchBase: true,
+  });
+
+  const { htmlEntries, codeEntries, jadeEntries, hbsEntries } = entries.reduce(
+    (paths, entry) => {
+      const resolvedPath = resolve(inputPath, entry);
+
+      if (resolvedPath.endsWith('.html')) {
+        paths.htmlEntries.push(resolvedPath);
+      } else if (resolvedPath.endsWith('.jade')) {
+        paths.jadeEntries.push(resolvedPath);
+      } else if (resolvedPath.endsWith('.hbs') || resolvedPath.endsWith('.handlebars')) {
+        paths.hbsFiles.push(resolvedPath);
+      } else {
+        paths.codeEntries.push(resolvedPath);
+      }
+
+      return paths;
+    },
+    { htmlEntries: [], codeEntries: [], jadeEntries: [], hbsEntries: [] }
+  );
+
+  const defaultMessagesMap = new Map();
+
+  await Promise.all(
+    [
+      [htmlEntries, extractHtmlMessages],
+      [codeEntries, extractCodeMessages],
+      [jadeEntries, extractJadeMessages],
+      [hbsEntries, extractHandlebarsMessages],
+    ].map(async ([entries, extractFunction]) => {
+      const files = await Promise.all(
+        entries.map(async entry => {
+          return {
+            name: entry,
+            content: await readFileAsync(entry),
+          };
+        })
+      );
+
+      for (const { name, content } of files) {
+        try {
+          for (const [id, value] of extractFunction(content)) {
+            addMessageToMap(defaultMessagesMap, id, value);
+          }
+        } catch (error) {
+          throw new Error(`Error in ${name}\n${error.message || error}`);
+        }
+      }
+    })
+  );
+
+  // .slice(0, -1): remove closing curly brace from json to append messages
+  let jsonBuffer = Buffer.from(JSON5.stringify({ formats }, { quote: `'`, space: 2 }).slice(0, -1));
+
+  const defaultMessages = [...defaultMessagesMap].sort(([key1], [key2]) => {
+    return key1 < key2 ? -1 : 1;
+  });
+
+  for (const [mapKey, mapValue] of defaultMessages) {
+    jsonBuffer = Buffer.concat([
+      jsonBuffer,
+      Buffer.from(`  '${mapKey}': '${mapValue.message}',`),
+      Buffer.from(mapValue.context ? ` // ${mapValue.context}\n` : '\n'),
+    ]);
+  }
+
+  // append previously removed closing curly brace
+  jsonBuffer = Buffer.concat([jsonBuffer, Buffer.from('}\n')]);
+
+  try {
+    await accessAsync(resolve(inputPath, 'translations'));
+  } catch (_) {
+    await makeDirAsync(resolve(inputPath, 'translations'));
+  }
+
+  await writeFileAsync(resolve(inputPath, 'translations', 'en.json'), jsonBuffer);
+}

src/dev/i18n/extract_handlebars_messages.js

@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { formatJSString } from './utils';
+
+const HBS_REGEX = /(?<=\{\{)([\s\S]*?)(?=\}\})/g;
+const TOKENS_REGEX = /[^'\s]+|(?:'([^'\\]|\\[\s\S])*')/g;
+
+/**
+ * Example: `'{{i18n 'message-id' '{"defaultMessage": "Message text"}'}}'`
+ */
+export function* extractHandlebarsMessages(buffer) {
+  for (const expression of buffer.toString().match(HBS_REGEX) || []) {
+    const tokens = expression.match(TOKENS_REGEX);
+
+    const [functionName, idString, propertiesString] = tokens;
+
+    if (functionName !== 'i18n') {
+      continue;
+    }
+
+    if (tokens.length !== 3) {
+      throw new Error('Wrong arguments amount for handlebars i18n call.');
+    }
+
+    if (!idString.startsWith(`'`) || !idString.endsWith(`'`)) {
+      throw new Error('Message id should be a string literal.');
+    }
+
+    const messageId = formatJSString(idString.slice(1, -1));
+
+    if (!propertiesString.startsWith(`'`) || !propertiesString.endsWith(`'`)) {
+      throw new Error(
+        `Cannot parse "${messageId}" message: properties string should be a string literal.`
+      );
+    }
+
+    const properties = JSON.parse(propertiesString.slice(1, -1));
+    const message = formatJSString(properties.defaultMessage);
+
+    if (typeof message !== 'string') {
+      throw new Error(
+        `Cannot parse "${messageId}" message: defaultMessage value should be a string.`
+      );
+    }
+
+    const context = formatJSString(properties.context);
+
+    if (context != null && typeof context !== 'string') {
+      throw new Error(`Cannot parse "${messageId}" message: context value should be a string.`);
+    }
+
+    yield [messageId, { message, context }];
+  }
+}