diff --git a/changelogs/fragments/8423.yml b/changelogs/fragments/8423.yml new file mode 100644 index 000000000000..82f9992dcc4c --- /dev/null +++ b/changelogs/fragments/8423.yml @@ -0,0 +1,2 @@ +infra: +- Add precommit hook to validate i18n ([#8423](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8423)) \ No newline at end of file diff --git a/src/dev/i18n/extract_default_translations.d.ts b/src/dev/i18n/extract_default_translations.d.ts new file mode 100644 index 000000000000..e9651403646a --- /dev/null +++ b/src/dev/i18n/extract_default_translations.d.ts @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { I18nConfig } from './config'; + +export declare function validateMessageNamespace( + id: string, + filePath: string, + allowedPaths: Record, + reporter: unknown +): void; + +export declare function matchEntriesWithExctractors( + inputPath: string, + options: Record +): Promise<[[string[], unknown]]>; + +export declare function extractMessagesFromPathToMap( + inputPath: string, + targetMap: Map, + config: I18nConfig, + reporter: any +): Promise; diff --git a/src/dev/i18n/extractors/index.d.ts b/src/dev/i18n/extractors/index.d.ts new file mode 100644 index 000000000000..452ad9e3cb16 --- /dev/null +++ b/src/dev/i18n/extractors/index.d.ts @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export declare function extractCodeMessages( + buffer: Buffer, + reporter: unknown +): Generator<[string, { message: string; description?: string }], void>; diff --git a/src/dev/i18n/index.ts b/src/dev/i18n/index.ts index c59bee6fcb8d..0ea1253efb13 100644 --- a/src/dev/i18n/index.ts +++ b/src/dev/i18n/index.ts @@ -28,10 +28,10 @@ * under the License. */ -// @ts-ignore -export { extractMessagesFromPathToMap } from './extract_default_translations'; -// @ts-ignore -export { matchEntriesWithExctractors } from './extract_default_translations'; +export { + extractMessagesFromPathToMap, + matchEntriesWithExctractors, +} from './extract_default_translations'; export { arrayify, writeFileAsync, @@ -39,6 +39,7 @@ export { accessAsync, normalizePath, ErrorReporter, + FailReporter, } from './utils'; export { serializeToJson, serializeToJson5 } from './serializers'; export { diff --git a/src/dev/i18n/tasks/check_default_messages_in_files.ts b/src/dev/i18n/tasks/check_default_messages_in_files.ts new file mode 100644 index 000000000000..842a4eb9dc62 --- /dev/null +++ b/src/dev/i18n/tasks/check_default_messages_in_files.ts @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { glob } from 'glob'; +import { createFailError, isFailError, REPO_ROOT } from '@osd/dev-utils'; +import { ErrorReporter, filterConfigPaths, I18nConfig, normalizePath, readFileAsync } from '..'; +import { extractCodeMessages } from '../extractors'; +import { validateMessageNamespace } from '../extract_default_translations'; + +function filterEntries(entries: string[], exclude: string[]) { + return entries.filter((entry: string) => + exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath)) + ); +} + +function addMessageToMap( + targetMap: Map, + key: string, + value: { message: string }, + reporter: ErrorReporter +) { + const existingValue = targetMap.get(key); + + if (targetMap.has(key) && existingValue!.message !== value.message) { + reporter.report( + createFailError(`There is more than one default message for the same id "${key}": +"${existingValue!.message}" and "${value.message}"`) + ); + } else { + targetMap.set(key, value); + } +} + +async function extractMessagesFromFilesToMap( + files: string[], + targetMap: Map, + config: I18nConfig, + reporter: any +) { + const availablePaths = Object.values(config.paths).flat(); + const ignoredPatterns = [ + '**/node_modules/**', + '**/__tests__/**', + '**/dist/**', + '**/target/**', + '**/vendor/**', + '**/*.test.{js,jsx,ts,tsx}', + '**/*.d.ts', + ]; + + const filesToCheck = files.filter( + (file) => glob.sync(file, { ignore: ignoredPatterns, root: REPO_ROOT }).length > 0 + ); + + const fileContents = await Promise.all( + filterEntries(filesToCheck, config.exclude) + .filter((entry) => { + const normalizedEntry = normalizePath(entry); + return availablePaths.some( + (availablePath) => + normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) || + normalizePath(availablePath) === normalizedEntry + ); + }) + .map(async (entry: any) => ({ + name: entry, + content: await readFileAsync(entry), + })) + ); + + for (const { name, content } of fileContents) { + const reporterWithContext = reporter.withContext({ name }); + + try { + for (const [id, value] of extractCodeMessages(content, reporterWithContext)) { + validateMessageNamespace(id, name, config.paths, reporterWithContext); + addMessageToMap(targetMap, id, value, reporterWithContext); + } + } catch (error) { + if (!isFailError(error)) { + throw error; + } + + reporterWithContext.report(error); + } + } +} + +export function checkDefaultMessagesInFiles(config: I18nConfig | undefined, inputPaths: string[]) { + if (!config) { + throw new Error('Config is missing'); + } + const filteredPaths = filterConfigPaths(inputPaths, config) as string[]; + if (filteredPaths.length === 0) return; + return [ + { + title: 'Checking default messages in files', + task: async (context: { + messages: Map; + reporter: ErrorReporter; + }) => { + const { messages, reporter } = context; + const initialErrorsNumber = reporter.errors.length; + + // Return result if no new errors were reported for this path. + const result = await extractMessagesFromFilesToMap( + filteredPaths, + messages, + config, + reporter + ); + if (reporter.errors.length === initialErrorsNumber) { + return result; + } + + throw reporter; + }, + }, + ]; +} diff --git a/src/dev/i18n/tasks/check_files_for_untracked_translations.ts b/src/dev/i18n/tasks/check_files_for_untracked_translations.ts new file mode 100644 index 000000000000..841260fea0c4 --- /dev/null +++ b/src/dev/i18n/tasks/check_files_for_untracked_translations.ts @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Any modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { glob } from 'glob'; +import { REPO_ROOT } from '@osd/dev-utils'; +import { ListrContext } from '.'; +import { I18nConfig, normalizePath, readFileAsync } from '..'; +// @ts-ignore +import { extractCodeMessages } from '../extractors'; + +function filterEntries(entries: string[], exclude: string[]) { + return entries.filter((entry: string) => + exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath)) + ); +} + +async function checkFilesForUntrackedMessagesTask({ + files, + config, + reporter, +}: { + files: string[]; + config: I18nConfig; + reporter: any; +}) { + const availablePaths = Object.values(config.paths).flat(); + const ignoredPatterns = availablePaths.concat([ + '**/*.d.ts', + '**/*.test.{js,jsx,ts,tsx}', + '**/__fixtures__/**', + '**/__tests__/**', + '**/build/**', + '**/dist/**', + '**/node_modules/**', + '**/packages/osd-i18n/**', + '**/packages/osd-plugin-generator/template/**', + '**/scripts/**', + '**/src/dev/**', + '**/target/**', + '**/test/**', + '**/vendor/**', + ]); + + const filesToCheck = files.filter( + (file) => glob.sync(file, { ignore: ignoredPatterns, root: REPO_ROOT }).length > 0 + ); + + const fileContents = await Promise.all( + filterEntries(filesToCheck, config.exclude) + .filter((entry) => { + const normalizedEntry = normalizePath(entry); + return !availablePaths.some( + (availablePath) => + normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) || + normalizePath(availablePath) === normalizedEntry + ); + }) + .map(async (entry: any) => ({ + name: entry, + content: await readFileAsync(entry), + })) + ); + + for (const { name, content } of fileContents) { + const reporterWithContext = reporter.withContext({ name }); + for (const [id] of extractCodeMessages(content, reporterWithContext)) { + const errorMessage = `Untracked file contains i18n label (${id}).`; + reporterWithContext.report(errorMessage); + } + } +} + +export function checkFilesForUntrackedMessages(files: string[]) { + return [ + { + title: `Checking untracked messages in files`, + task: async (context: ListrContext) => { + const { reporter, config } = context; + if (!config) { + throw new Error('Config is not defined'); + } + const initialErrorsNumber = reporter.errors.length; + const result = await checkFilesForUntrackedMessagesTask({ files, config, reporter }); + if (reporter.errors.length === initialErrorsNumber) { + return result; + } + + throw reporter; + }, + }, + ]; +} diff --git a/src/dev/i18n/tasks/index.ts b/src/dev/i18n/tasks/index.ts index 6b9c899b5649..4418b5cfb5ec 100644 --- a/src/dev/i18n/tasks/index.ts +++ b/src/dev/i18n/tasks/index.ts @@ -35,6 +35,8 @@ export { extractUntrackedMessages } from './extract_untracked_translations'; export { checkCompatibility } from './check_compatibility'; export { mergeConfigs } from './merge_configs'; export { checkConfigs } from './check_configs'; +export { checkFilesForUntrackedMessages } from './check_files_for_untracked_translations'; +export { checkDefaultMessagesInFiles } from './check_default_messages_in_files'; export interface ListrContext { config?: I18nConfig; diff --git a/src/dev/i18n/utils/index.ts b/src/dev/i18n/utils/index.ts index fa057c6a8d67..75af963d144a 100644 --- a/src/dev/i18n/utils/index.ts +++ b/src/dev/i18n/utils/index.ts @@ -54,6 +54,7 @@ export { arrayify, // classes ErrorReporter, // @ts-ignore + FailReporter, // @ts-ignore } from './utils'; export { verifyICUMessage } from './verify_icu_message'; diff --git a/src/dev/i18n/utils/utils.js b/src/dev/i18n/utils/utils.js index 79868ddd6314..ce95f3d3c3f3 100644 --- a/src/dev/i18n/utils/utils.js +++ b/src/dev/i18n/utils/utils.js @@ -342,6 +342,18 @@ export class ErrorReporter { } } +export class FailReporter { + errors = []; + + withContext(context) { + return { report: (error) => this.report(error, context) }; + } + + report(error, context) { + this.errors.push(createFailError(`Error in ${normalizePath(context.name)}\n${error}`)); + } +} + // export function arrayify(subj: Subj | Subj[]): Subj[] { export function arrayify(subj) { return Array.isArray(subj) ? subj : [subj]; diff --git a/src/dev/precommit_hook/check_i18n.js b/src/dev/precommit_hook/check_i18n.js new file mode 100644 index 000000000000..a2b344580835 --- /dev/null +++ b/src/dev/precommit_hook/check_i18n.js @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import Listr from 'listr'; + +import { + checkConfigs, + mergeConfigs, + checkFilesForUntrackedMessages, + checkDefaultMessagesInFiles, +} from '../i18n/tasks'; +import { FailReporter } from '../i18n'; + +export async function checkI18n(log, files) { + const relativeFiles = files + .filter((file) => file.isJs() || (file.isTypescript() && !file.isTypescriptAmbient())) + .map((file) => file.getRelativePath()); + + const list = new Listr( + [ + { + title: 'Checking .i18nrc.json files', + task: () => new Listr(checkConfigs()), + }, + { + title: 'Merging .i18nrc.json files', + task: () => new Listr(mergeConfigs()), + }, + { + title: 'Checking For Untracked Messages based on .i18nrc.json', + task: () => new Listr(checkFilesForUntrackedMessages(relativeFiles)), + }, + { + title: 'Validating Default Messages', + task: ({ config }) => new Listr(checkDefaultMessagesInFiles(config, relativeFiles)), + }, + ], + { + concurrent: false, + renderer: 'silent', + } + ); + + try { + const reporter = new FailReporter(); + const messages = new Map(); + await list.run({ messages, reporter }); + + const num = relativeFiles.length; + console.log(` succ [i18n-check] ${num} file${num === 1 ? '' : 's'} checked successfully`); + } catch (error) { + if (error instanceof FailReporter) { + return error.errors; + } else { + throw error; + } + } +} diff --git a/src/dev/precommit_hook/index.js b/src/dev/precommit_hook/index.js index 898b4393d860..04358e8bfd33 100644 --- a/src/dev/precommit_hook/index.js +++ b/src/dev/precommit_hook/index.js @@ -31,3 +31,4 @@ export { checkFileCasing } from './check_file_casing'; export { getFilesForCommit, getUnstagedFiles } from './get_files_for_commit'; export { checkDevDocs } from './check_dev_docs'; +export { checkI18n } from './check_i18n'; diff --git a/src/dev/run_precommit_hook.js b/src/dev/run_precommit_hook.js index b840fca99752..aa81d71c3903 100644 --- a/src/dev/run_precommit_hook.js +++ b/src/dev/run_precommit_hook.js @@ -36,6 +36,7 @@ import { getUnstagedFiles, checkFileCasing, checkDevDocs, + checkI18n, } from './precommit_hook'; run( @@ -57,6 +58,13 @@ run( errors.push(error); } + try { + const result = await checkI18n(log, files); + if (Array.isArray(result)) errors.push(...result); + } catch (error) { + errors.push(error); + } + for (const Linter of [Eslint, Stylelint]) { const filesToLint = Linter.pickFilesToLint(log, files); if (filesToLint.length > 0) {