From 8152b20fbf9c10a7c1bcc0ab66094c38d038c5db Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 19 Feb 2019 04:00:44 +0200 Subject: [PATCH] [6.7] [Telemetry] Localization Stats (#29213) (#31243) Backports the following commits to 6.7: - [Telemetry] Localization Stats (#29213) --- src/server/i18n/index.js | 1 + .../status/lib/get_kibana_info_for_stats.js | 1 + x-pack/plugins/xpack_main/common/constants.js | 6 ++ x-pack/plugins/xpack_main/index.js | 2 + .../server/lib/file_integrity.test.ts | 56 +++++++++++++++++++ .../xpack_main/server/lib/file_integrity.ts | 39 +++++++++++++ .../get_localization_usage_collector.test.ts | 39 +++++++++++++ .../lib/get_localization_usage_collector.ts | 53 ++++++++++++++++++ .../local/__tests__/get_local_stats.js | 12 +++- .../xpack_main/telemetry/telemetry_local.js | 2 + 10 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/xpack_main/server/lib/file_integrity.test.ts create mode 100644 x-pack/plugins/xpack_main/server/lib/file_integrity.ts create mode 100644 x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.test.ts create mode 100644 x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts diff --git a/src/server/i18n/index.js b/src/server/i18n/index.js index fff394f5bb21c..f07c3886bd3a9 100644 --- a/src/server/i18n/index.js +++ b/src/server/i18n/index.js @@ -62,4 +62,5 @@ export async function i18nMixin(kbnServer, server, config) { locale, ...translations, })); + server.decorate('server', 'getTranslationsFilePaths', () => translationPaths); } diff --git a/src/server/status/lib/get_kibana_info_for_stats.js b/src/server/status/lib/get_kibana_info_for_stats.js index bac800c03b2b1..1ddde211da94c 100644 --- a/src/server/status/lib/get_kibana_info_for_stats.js +++ b/src/server/status/lib/get_kibana_info_for_stats.js @@ -38,6 +38,7 @@ export function getKibanaInfoForStats(server, kbnServer) { name: config.get('server.name'), index: config.get('kibana.index'), host: config.get('server.host'), + locale: config.get('i18n.locale'), transport_address: `${config.get('server.host')}:${config.get('server.port')}`, version: kbnServer.version.replace(snapshotRegex, ''), snapshot: snapshotRegex.test(kbnServer.version), diff --git a/x-pack/plugins/xpack_main/common/constants.js b/x-pack/plugins/xpack_main/common/constants.js index 5f0b956fb1da6..892d89bee4b8a 100644 --- a/x-pack/plugins/xpack_main/common/constants.js +++ b/x-pack/plugins/xpack_main/common/constants.js @@ -63,3 +63,9 @@ export const LOCALSTORAGE_KEY = 'xpack.data'; * Link to the Elastic Telemetry privacy statement. */ export const PRIVACY_STATEMENT_URL = `https://www.elastic.co/legal/telemetry-privacy-statement`; + +/** + * The type name used within the Monitoring index to publish localization stats. + * @type {string} + */ +export const KIBANA_LOCALIZATION_STATS_TYPE = 'localization'; diff --git a/x-pack/plugins/xpack_main/index.js b/x-pack/plugins/xpack_main/index.js index 032558c30765a..af4d8cdcab0bd 100644 --- a/x-pack/plugins/xpack_main/index.js +++ b/x-pack/plugins/xpack_main/index.js @@ -13,6 +13,7 @@ import { import { mirrorPluginStatus } from '../../server/lib/mirror_plugin_status'; import { replaceInjectedVars } from './server/lib/replace_injected_vars'; import { setupXPackMain } from './server/lib/setup_xpack_main'; +import { getLocalizationUsageCollector } from './server/lib/get_localization_usage_collector'; import { xpackInfoRoute, telemetryRoute, @@ -133,6 +134,7 @@ export const xpackMain = (kibana) => { xpackInfoRoute(server); telemetryRoute(server); settingsRoute(server, this.kbnServer); + server.usage.collectorSet.register(getLocalizationUsageCollector(server)); } }); }; diff --git a/x-pack/plugins/xpack_main/server/lib/file_integrity.test.ts b/x-pack/plugins/xpack_main/server/lib/file_integrity.test.ts new file mode 100644 index 0000000000000..5e554cd77d849 --- /dev/null +++ b/x-pack/plugins/xpack_main/server/lib/file_integrity.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Readable } from 'stream'; + +jest.mock('fs', () => ({ + createReadStream(filepath: string): Readable { + if (filepath === 'ERROR') { + throw new Error('MOCK ERROR - Invalid Path'); + } + const readableStream = new Readable(); + const streamData = filepath.split(''); + let cursor = 0; + + readableStream._read = function(size) { + const current = streamData[cursor++]; + if (typeof current === 'undefined') { + return this.push(null); + } + this.push(current); + }; + + return readableStream; + }, +})); + +import { getIntegrityHash, getIntegrityHashes } from './file_integrity'; + +describe('Integrity Hash', () => { + it('creates a hash from a file given a file path', async () => { + const filePath = 'somepath.json'; + const expectedHash = '3295d40d2f35ac27145d37fcd5cdc80b'; + const integrityHash = await getIntegrityHash(filePath); + expect(integrityHash).toEqual(expectedHash); + }); + + it('returns null on error', async () => { + const filePath = 'ERROR'; + const integrityHash = await getIntegrityHash(filePath); + expect(integrityHash).toEqual(null); + }); +}); + +describe('Integrity Hashes', () => { + it('returns an object with each filename and its hash', async () => { + const filePaths = ['somepath1.json', 'somepath2.json']; + const integrityHashes = await getIntegrityHashes(filePaths); + expect(integrityHashes).toEqual({ + 'somepath1.json': '8cbfe6a9f8174b2d7e77c2111a84f0e6', + 'somepath2.json': '4177c075ade448d6e69fd94b39d0be15', + }); + }); +}); diff --git a/x-pack/plugins/xpack_main/server/lib/file_integrity.ts b/x-pack/plugins/xpack_main/server/lib/file_integrity.ts new file mode 100644 index 0000000000000..1db3795828fff --- /dev/null +++ b/x-pack/plugins/xpack_main/server/lib/file_integrity.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createHash } from 'crypto'; +import * as fs from 'fs'; +import { zipObject } from 'lodash'; +import * as stream from 'stream'; +import * as util from 'util'; + +const pipeline = util.promisify(stream.pipeline); + +export type Hash = string; + +export interface Integrities { + [filePath: string]: Hash; +} + +export async function getIntegrityHashes(filepaths: string[]): Promise { + const hashes = await Promise.all(filepaths.map(getIntegrityHash)); + return zipObject(filepaths, hashes); +} + +export async function getIntegrityHash(filepath: string): Promise { + try { + const output = createHash('md5'); + + await pipeline(fs.createReadStream(filepath), output); + const data = output.read(); + if (data instanceof Buffer) { + return data.toString('hex'); + } + return data; + } catch (err) { + return null; + } +} diff --git a/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.test.ts b/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.test.ts new file mode 100644 index 0000000000000..cd2296a73499f --- /dev/null +++ b/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +interface TranslationsMock { + [title: string]: string; +} + +const createI18nLoaderMock = (translations: TranslationsMock) => { + return { + getTranslationsByLocale() { + return { + messages: translations, + }; + }, + }; +}; + +import { getTranslationCount } from './get_localization_usage_collector'; + +describe('getTranslationCount', () => { + it('returns 0 if no translations registered', async () => { + const i18nLoaderMock = createI18nLoaderMock({}); + const count = await getTranslationCount(i18nLoaderMock, 'en'); + expect(count).toEqual(0); + }); + + it('returns number of translations', async () => { + const i18nLoaderMock = createI18nLoaderMock({ + a: '1', + b: '2', + 'b.a': '3', + }); + const count = await getTranslationCount(i18nLoaderMock, 'en'); + expect(count).toEqual(3); + }); +}); diff --git a/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts b/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts new file mode 100644 index 0000000000000..61c7b8e77458c --- /dev/null +++ b/x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18nLoader } from '@kbn/i18n'; +import { size } from 'lodash'; +// @ts-ignore +import { KIBANA_LOCALIZATION_STATS_TYPE } from '../../common/constants'; +import { getIntegrityHashes, Integrities } from './file_integrity'; + +export interface UsageStats { + locale: string; + integrities: Integrities; + labelsCount?: number; +} + +export async function getTranslationCount(loader: any, locale: string): Promise { + const translations = await loader.getTranslationsByLocale(locale); + return size(translations.messages); +} + +export function createCollectorFetch(server: any) { + return async function fetchUsageStats(): Promise { + const config = server.config(); + const locale: string = config.get('i18n.locale'); + const translationFilePaths: string[] = server.getTranslationsFilePaths(); + + const [labelsCount, integrities] = await Promise.all([ + getTranslationCount(i18nLoader, locale), + getIntegrityHashes(translationFilePaths), + ]); + + return { + locale, + integrities, + labelsCount, + }; + }; +} + +/* + * @param {Object} server + * @return {Object} kibana usage stats type collection object + */ +export function getLocalizationUsageCollector(server: any) { + const { collectorSet } = server.usage; + return collectorSet.makeUsageCollector({ + type: KIBANA_LOCALIZATION_STATS_TYPE, + fetch: createCollectorFetch(server), + }); +} diff --git a/x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_local_stats.js b/x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_local_stats.js index eca758d1a2d8a..03056f8c9e26b 100644 --- a/x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_local_stats.js +++ b/x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_local_stats.js @@ -64,7 +64,12 @@ describe('get_local_stats', () => { os: { platform: 'rocky', platformRelease: 'iv', - } + }, + }, + localization: { + locale: 'en', + labelsCount: 0, + integrities: {} }, sun: { chances: 5 }, clouds: { chances: 95 }, @@ -92,6 +97,11 @@ describe('get_local_stats', () => { }, versions: [{ version: '8675309', count: 1 }], plugins: { + localization: { + locale: 'en', + labelsCount: 0, + integrities: {} + }, sun: { chances: 5 }, clouds: { chances: 95 }, rain: { chances: 2 }, diff --git a/x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry_local.js b/x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry_local.js index 13b23443e19a9..582ea6232265b 100644 --- a/x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry_local.js +++ b/x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry_local.js @@ -165,6 +165,8 @@ export default function ({ getService }) { 'stack_stats.kibana.plugins.kql.defaultQueryLanguage', 'stack_stats.kibana.plugins.kql.optInCount', 'stack_stats.kibana.plugins.kql.optOutCount', + "stack_stats.kibana.plugins.localization.labelsCount", + "stack_stats.kibana.plugins.localization.locale", 'stack_stats.kibana.plugins.maps.attributesPerMap.dataSourcesCount.avg', 'stack_stats.kibana.plugins.maps.attributesPerMap.dataSourcesCount.max', 'stack_stats.kibana.plugins.maps.attributesPerMap.dataSourcesCount.min',