diff --git a/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts new file mode 100644 index 0000000000000..833344fa368b0 --- /dev/null +++ b/packages/kbn-telemetry-tools/src/tools/__fixture__/parsed_schema_defined_with_spreads_collector.ts @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { SyntaxKind } from 'typescript'; +import { ParsedUsageCollection } from '../ts_parser'; + +export const parsedSchemaDefinedWithSpreadsCollector: ParsedUsageCollection = [ + 'src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts', + { + collectorName: 'schema_defined_with_spreads', + schema: { + value: { + flat: { + type: 'keyword', + }, + my_str: { + type: 'text', + }, + my_objects: { + total: { + type: 'number', + }, + type: { + type: 'boolean', + }, + }, + }, + }, + fetch: { + typeName: 'Usage', + typeDescriptor: { + flat: { + kind: SyntaxKind.StringKeyword, + type: 'StringKeyword', + }, + my_str: { + kind: SyntaxKind.StringKeyword, + type: 'StringKeyword', + }, + my_objects: { + total: { + kind: SyntaxKind.NumberKeyword, + type: 'NumberKeyword', + }, + type: { + kind: SyntaxKind.BooleanKeyword, + type: 'BooleanKeyword', + }, + }, + }, + }, + }, +]; diff --git a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap index 9868a7d31d498..0b619dd70bb45 100644 --- a/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap +++ b/packages/kbn-telemetry-tools/src/tools/__snapshots__/extract_collectors.test.ts.snap @@ -142,6 +142,53 @@ Array [ }, }, ], + Array [ + "src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts", + Object { + "collectorName": "schema_defined_with_spreads", + "fetch": Object { + "typeDescriptor": Object { + "flat": Object { + "kind": 146, + "type": "StringKeyword", + }, + "my_objects": Object { + "total": Object { + "kind": 143, + "type": "NumberKeyword", + }, + "type": Object { + "kind": 131, + "type": "BooleanKeyword", + }, + }, + "my_str": Object { + "kind": 146, + "type": "StringKeyword", + }, + }, + "typeName": "Usage", + }, + "schema": Object { + "value": Object { + "flat": Object { + "type": "keyword", + }, + "my_objects": Object { + "total": Object { + "type": "number", + }, + "type": Object { + "type": "boolean", + }, + }, + "my_str": Object { + "type": "text", + }, + }, + }, + }, + ], Array [ "src/fixtures/telemetry_collectors/working_collector.ts", Object { diff --git a/packages/kbn-telemetry-tools/src/tools/extract_collectors.test.ts b/packages/kbn-telemetry-tools/src/tools/extract_collectors.test.ts index 0517cb9034d0a..b03db75b219f6 100644 --- a/packages/kbn-telemetry-tools/src/tools/extract_collectors.test.ts +++ b/packages/kbn-telemetry-tools/src/tools/extract_collectors.test.ts @@ -34,7 +34,7 @@ describe('extractCollectors', () => { const programPaths = await getProgramPaths(configs[0]); const results = [...extractCollectors(programPaths, tsConfig)]; - expect(results).toHaveLength(7); + expect(results).toHaveLength(8); expect(results).toMatchSnapshot(); }); }); diff --git a/packages/kbn-telemetry-tools/src/tools/ts_parser.test.ts b/packages/kbn-telemetry-tools/src/tools/ts_parser.test.ts index b7ca33a7bcd74..d036b93a7bbf9 100644 --- a/packages/kbn-telemetry-tools/src/tools/ts_parser.test.ts +++ b/packages/kbn-telemetry-tools/src/tools/ts_parser.test.ts @@ -25,6 +25,7 @@ import { parsedNestedCollector } from './__fixture__/parsed_nested_collector'; import { parsedExternallyDefinedCollector } from './__fixture__/parsed_externally_defined_collector'; import { parsedImportedUsageInterface } from './__fixture__/parsed_imported_usage_interface'; import { parsedImportedSchemaCollector } from './__fixture__/parsed_imported_schema'; +import { parsedSchemaDefinedWithSpreadsCollector } from './__fixture__/parsed_schema_defined_with_spreads_collector'; export function loadFixtureProgram(fixtureName: string) { const fixturePath = path.resolve( @@ -62,6 +63,12 @@ describe('parseUsageCollection', () => { expect(result).toEqual([parsedWorkingCollector]); }); + it('parses collector with schema defined as union of spreads', () => { + const { program, sourceFile } = loadFixtureProgram('schema_defined_with_spreads_collector'); + const result = [...parseUsageCollection(sourceFile, program)]; + expect(result).toEqual([parsedSchemaDefinedWithSpreadsCollector]); + }); + it('parses nested collectors', () => { const { program, sourceFile } = loadFixtureProgram('nested_collector'); const result = [...parseUsageCollection(sourceFile, program)]; diff --git a/packages/kbn-telemetry-tools/src/tools/utils.ts b/packages/kbn-telemetry-tools/src/tools/utils.ts index e8e1b3fed1aef..ac6edcb363fb6 100644 --- a/packages/kbn-telemetry-tools/src/tools/utils.ts +++ b/packages/kbn-telemetry-tools/src/tools/utils.ts @@ -100,42 +100,55 @@ export function getIdentifierDeclaration(node: ts.Node) { return getIdentifierDeclarationFromSource(node, source); } -export function getVariableValue(node: ts.Node): string | Record { +export function getVariableValue(node: ts.Node, program: ts.Program): string | Record { if (ts.isStringLiteral(node) || ts.isNumericLiteral(node)) { return node.text; } if (ts.isObjectLiteralExpression(node)) { - return serializeObject(node); + return serializeObject(node, program); } if (ts.isIdentifier(node)) { const declaration = getIdentifierDeclaration(node); if (ts.isVariableDeclaration(declaration) && declaration.initializer) { - return getVariableValue(declaration.initializer); + return getVariableValue(declaration.initializer, program); + } else { + // Go fetch it in another file + return getIdentifierValue(node, node, program, { chaseImport: true }); } - // TODO: If this is another imported value from another file, we'll need to go fetch it like in getPropertyValue } - throw Error(`Unsuppored Node: cannot get value of node (${node.getText()}) of kind ${node.kind}`); + if (ts.isSpreadAssignment(node)) { + return getVariableValue(node.expression, program); + } + + throw Error( + `Unsupported Node: cannot get value of node (${node.getText()}) of kind ${node.kind}` + ); } -export function serializeObject(node: ts.Node) { +export function serializeObject(node: ts.Node, program: ts.Program) { if (!ts.isObjectLiteralExpression(node)) { throw new Error(`Expecting Object literal Expression got ${node.getText()}`); } - const value: Record = {}; + let value: Record = {}; for (const property of node.properties) { const propertyName = property.name?.getText(); + const val = ts.isPropertyAssignment(property) + ? getVariableValue(property.initializer, program) + : getVariableValue(property, program); + if (typeof propertyName === 'undefined') { - throw new Error(`Unable to get property name ${property.getText()}`); - } - const cleanPropertyName = propertyName.replace(/["']/g, ''); - if (ts.isPropertyAssignment(property)) { - value[cleanPropertyName] = getVariableValue(property.initializer); + if (typeof val === 'object') { + value = { ...value, ...val }; + } else { + throw new Error(`Unable to get property name ${property.getText()}`); + } } else { - value[cleanPropertyName] = getVariableValue(property); + const cleanPropertyName = propertyName.replace(/["']/g, ''); + value[cleanPropertyName] = val; } } @@ -155,45 +168,53 @@ export function getResolvedModuleSourceFile( return resolvedModuleSourceFile; } -export function getPropertyValue( +export function getIdentifierValue( node: ts.Node, + initializer: ts.Identifier, program: ts.Program, config: Optional<{ chaseImport: boolean }> = {} ) { const { chaseImport = false } = config; + const identifierName = initializer.getText(); + const declaration = getIdentifierDeclaration(initializer); + if (ts.isImportSpecifier(declaration)) { + if (!chaseImport) { + throw new Error( + `Value of node ${identifierName} is imported from another file. Chasing imports is not allowed.` + ); + } - if (ts.isPropertyAssignment(node)) { - const { initializer } = node; + const importedModuleName = getModuleSpecifier(declaration); - if (ts.isIdentifier(initializer)) { - const identifierName = initializer.getText(); - const declaration = getIdentifierDeclaration(initializer); - if (ts.isImportSpecifier(declaration)) { - if (!chaseImport) { - throw new Error( - `Value of node ${identifierName} is imported from another file. Chasing imports is not allowed.` - ); - } + const source = node.getSourceFile(); + const declarationSource = getResolvedModuleSourceFile(source, program, importedModuleName); + const declarationNode = getIdentifierDeclarationFromSource(initializer, declarationSource); + if (!ts.isVariableDeclaration(declarationNode)) { + throw new Error(`Expected ${identifierName} to be variable declaration.`); + } + if (!declarationNode.initializer) { + throw new Error(`Expected ${identifierName} to be initialized.`); + } + const serializedObject = serializeObject(declarationNode.initializer, program); + return serializedObject; + } - const importedModuleName = getModuleSpecifier(declaration); + return getVariableValue(declaration, program); +} - const source = node.getSourceFile(); - const declarationSource = getResolvedModuleSourceFile(source, program, importedModuleName); - const declarationNode = getIdentifierDeclarationFromSource(initializer, declarationSource); - if (!ts.isVariableDeclaration(declarationNode)) { - throw new Error(`Expected ${identifierName} to be variable declaration.`); - } - if (!declarationNode.initializer) { - throw new Error(`Expected ${identifierName} to be initialized.`); - } - const serializedObject = serializeObject(declarationNode.initializer); - return serializedObject; - } +export function getPropertyValue( + node: ts.Node, + program: ts.Program, + config: Optional<{ chaseImport: boolean }> = {} +) { + if (ts.isPropertyAssignment(node)) { + const { initializer } = node; - return getVariableValue(declaration); + if (ts.isIdentifier(initializer)) { + return getIdentifierValue(node, initializer, program, config); } - return getVariableValue(initializer); + return getVariableValue(initializer, program); } } diff --git a/src/fixtures/telemetry_collectors/indexed_interface_with_not_matching_schema.ts b/src/fixtures/telemetry_collectors/indexed_interface_with_not_matching_schema.ts index 0ec8d2e15c34a..b925696c96563 100644 --- a/src/fixtures/telemetry_collectors/indexed_interface_with_not_matching_schema.ts +++ b/src/fixtures/telemetry_collectors/indexed_interface_with_not_matching_schema.ts @@ -41,8 +41,9 @@ export const myCollector = makeUsageCollector({ return { something: { count_2: 2 } }; }, schema: { + // @ts-expect-error Intentionally missing count_2 something: { - count_1: { type: 'long' }, // Intentionally missing count_2 + count_1: { type: 'long' }, }, }, }); diff --git a/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts b/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts new file mode 100644 index 0000000000000..87306f502f71d --- /dev/null +++ b/src/fixtures/telemetry_collectors/schema_defined_with_spreads_collector.ts @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { CollectorSet, MakeSchemaFrom } from '../../plugins/usage_collection/server/collector'; +import { loggerMock } from '../../core/server/logging/logger.mock'; + +const { makeUsageCollector } = new CollectorSet({ + logger: loggerMock.create(), + maximumWaitTimeForAllCollectorsInS: 0, +}); + +interface MyObject { + total: number; + type: boolean; +} + +interface Usage { + flat?: string; + my_str?: string; + my_objects: MyObject; +} + +const SOME_NUMBER: number = 123; + +const someSchema: MakeSchemaFrom> = { + flat: { + type: 'keyword', + }, + my_str: { + type: 'text', + }, +}; + +const someOtherSchema: MakeSchemaFrom> = { + my_objects: { + total: { + type: 'number', + }, + type: { type: 'boolean' }, + }, +}; + +export const myCollector = makeUsageCollector({ + type: 'schema_defined_with_spreads', + isReady: () => true, + fetch() { + const testString = '123'; + // query ES and get some data + + // summarize the data into a model + // return the modeled object that includes whatever you want to track + try { + return { + flat: 'hello', + my_str: testString, + my_objects: { + total: SOME_NUMBER, + type: true, + }, + }; + } catch (err) { + return { + my_objects: { + total: 0, + type: true, + }, + }; + } + }, + schema: { + ...someSchema, + ...someOtherSchema, + }, +}); diff --git a/src/plugins/telemetry/server/collectors/usage/schema.ts b/src/plugins/telemetry/server/collectors/usage/schema.ts index 8f4d555d75c49..4bfb6f75c7c8f 100644 --- a/src/plugins/telemetry/server/collectors/usage/schema.ts +++ b/src/plugins/telemetry/server/collectors/usage/schema.ts @@ -31,7 +31,7 @@ const licenseSchema: MakeSchemaFrom = { max_resource_units: { type: 'long' }, }; -export const staticTelemetrySchema: MakeSchemaFrom> = { +export const staticTelemetrySchema: MakeSchemaFrom = { ece: { kb_uuid: { type: 'keyword' }, es_uuid: { type: 'keyword' }, diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 365e1ce201337..3bd91bec6d34b 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -38,16 +38,17 @@ export type RecursiveMakeSchemaFrom = U extends object ? MakeSchemaFrom : { type: AllowedSchemaTypes }; +// Using Required to enforce all optional keys in the object export type MakeSchemaFrom = { - [Key in keyof Base]: Base[Key] extends Array + [Key in keyof Required]: Required[Key] extends Array ? RecursiveMakeSchemaFrom - : RecursiveMakeSchemaFrom; + : RecursiveMakeSchemaFrom[Key]>; }; export interface CollectorOptions { type: string; init?: Function; - schema?: MakeSchemaFrom>; // Using Required to enforce all optional keys in the object + schema?: MakeSchemaFrom; fetch: (callCluster: LegacyAPICaller, esClient?: ElasticsearchClient) => Promise | T; /* * A hook for allowing the fetched data payload to be organized into a typed diff --git a/x-pack/.telemetryrc.json b/x-pack/.telemetryrc.json index 30b2178259d68..a7aa408ba68ef 100644 --- a/x-pack/.telemetryrc.json +++ b/x-pack/.telemetryrc.json @@ -5,7 +5,6 @@ "plugins/actions/server/usage/actions_usage_collector.ts", "plugins/alerts/server/usage/alerts_usage_collector.ts", "plugins/apm/server/lib/apm_telemetry/index.ts", - "plugins/canvas/server/collectors/collector.ts", "plugins/infra/server/usage/usage_collector.ts", "plugins/reporting/server/usage/reporting_usage_collector.ts", "plugins/maps/server/maps_telemetry/collectors/register.ts" diff --git a/x-pack/plugins/canvas/server/collectors/collector.ts b/x-pack/plugins/canvas/server/collectors/collector.ts index eb650ca5ad152..39a8262a5deec 100644 --- a/x-pack/plugins/canvas/server/collectors/collector.ts +++ b/x-pack/plugins/canvas/server/collectors/collector.ts @@ -5,11 +5,16 @@ */ import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; -import { LegacyAPICaller } from 'kibana/server'; import { TelemetryCollector } from '../../types'; -import { workpadCollector } from './workpad_collector'; -import { customElementCollector } from './custom_element_collector'; +import { workpadCollector, workpadSchema, WorkpadTelemetry } from './workpad_collector'; +import { + customElementCollector, + CustomElementTelemetry, + customElementSchema, +} from './custom_element_collector'; + +type CanvasUsage = WorkpadTelemetry & CustomElementTelemetry; const collectors: TelemetryCollector[] = [workpadCollector, customElementCollector]; @@ -29,18 +34,19 @@ export function registerCanvasUsageCollector( return; } - const canvasCollector = usageCollection.makeUsageCollector({ + const canvasCollector = usageCollection.makeUsageCollector({ type: 'canvas', isReady: () => true, - fetch: async (callCluster: LegacyAPICaller) => { + fetch: async (callCluster) => { const collectorResults = await Promise.all( collectors.map((collector) => collector(kibanaIndex, callCluster)) ); return collectorResults.reduce((reduction, usage) => { return { ...reduction, ...usage }; - }, {}); + }, {}) as CanvasUsage; // We need the casting because `TelemetryCollector` claims it returns `Record` }, + schema: { ...workpadSchema, ...customElementSchema }, }); usageCollection.registerCollector(canvasCollector); diff --git a/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts b/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts index 7b39e8b83b045..08770452a01b0 100644 --- a/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts +++ b/x-pack/plugins/canvas/server/collectors/custom_element_collector.ts @@ -6,6 +6,7 @@ import { SearchParams } from 'elasticsearch'; import { get } from 'lodash'; +import { MakeSchemaFrom } from 'src/plugins/usage_collection/server'; import { collectFns } from './collector_helpers'; import { TelemetryCollector, @@ -19,7 +20,7 @@ interface CustomElementSearch { [CUSTOM_ELEMENT_TYPE]: TelemetryCustomElementDocument; } -interface CustomElementTelemetry { +export interface CustomElementTelemetry { custom_elements?: { count: number; elements: { @@ -31,6 +32,18 @@ interface CustomElementTelemetry { }; } +export const customElementSchema: MakeSchemaFrom = { + custom_elements: { + count: { type: 'long' }, + elements: { + min: { type: 'long' }, + max: { type: 'long' }, + avg: { type: 'float' }, + }, + functions_in_use: { type: 'keyword' }, + }, +}; + function isCustomElement(maybeCustomElement: any): maybeCustomElement is TelemetryCustomElement { return ( maybeCustomElement !== null && diff --git a/x-pack/plugins/canvas/server/collectors/workpad_collector.ts b/x-pack/plugins/canvas/server/collectors/workpad_collector.ts index 9fa39c580962d..b269613b6d66f 100644 --- a/x-pack/plugins/canvas/server/collectors/workpad_collector.ts +++ b/x-pack/plugins/canvas/server/collectors/workpad_collector.ts @@ -6,6 +6,7 @@ import { SearchParams } from 'elasticsearch'; import { sum as arraySum, min as arrayMin, max as arrayMax, get } from 'lodash'; +import { MakeSchemaFrom } from 'src/plugins/usage_collection/server'; import { CANVAS_TYPE } from '../../common/lib/constants'; import { collectFns } from './collector_helpers'; import { TelemetryCollector, CanvasWorkpad } from '../../types'; @@ -15,7 +16,7 @@ interface WorkpadSearch { [CANVAS_TYPE]: CanvasWorkpad; } -interface WorkpadTelemetry { +export interface WorkpadTelemetry { workpads?: { total: number; }; @@ -54,6 +55,43 @@ interface WorkpadTelemetry { }; } +export const workpadSchema: MakeSchemaFrom = { + workpads: { total: { type: 'long' } }, + pages: { + total: { type: 'long' }, + per_workpad: { + avg: { type: 'float' }, + min: { type: 'long' }, + max: { type: 'long' }, + }, + }, + elements: { + total: { type: 'long' }, + per_page: { + avg: { type: 'float' }, + min: { type: 'long' }, + max: { type: 'long' }, + }, + }, + functions: { + total: { type: 'long' }, + in_use: { type: 'keyword' }, + per_element: { + avg: { type: 'float' }, + min: { type: 'long' }, + max: { type: 'long' }, + }, + }, + variables: { + total: { type: 'long' }, + per_workpad: { + avg: { type: 'float' }, + min: { type: 'long' }, + max: { type: 'long' }, + }, + }, +}; + /** Gather statistic about the given workpads @param workpadDocs a collection of workpad documents diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 1236f2ad9b559..e3bb9c7c4d532 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -1,5 +1,122 @@ { "properties": { + "canvas": { + "properties": { + "workpads": { + "properties": { + "total": { + "type": "long" + } + } + }, + "pages": { + "properties": { + "total": { + "type": "long" + }, + "per_workpad": { + "properties": { + "avg": { + "type": "float" + }, + "min": { + "type": "long" + }, + "max": { + "type": "long" + } + } + } + } + }, + "elements": { + "properties": { + "total": { + "type": "long" + }, + "per_page": { + "properties": { + "avg": { + "type": "float" + }, + "min": { + "type": "long" + }, + "max": { + "type": "long" + } + } + } + } + }, + "functions": { + "properties": { + "total": { + "type": "long" + }, + "in_use": { + "type": "keyword" + }, + "per_element": { + "properties": { + "avg": { + "type": "float" + }, + "min": { + "type": "long" + }, + "max": { + "type": "long" + } + } + } + } + }, + "variables": { + "properties": { + "total": { + "type": "long" + }, + "per_workpad": { + "properties": { + "avg": { + "type": "float" + }, + "min": { + "type": "long" + }, + "max": { + "type": "long" + } + } + } + } + }, + "custom_elements": { + "properties": { + "count": { + "type": "long" + }, + "elements": { + "properties": { + "min": { + "type": "long" + }, + "max": { + "type": "long" + }, + "avg": { + "type": "float" + } + } + }, + "functions_in_use": { + "type": "keyword" + } + } + } + } + }, "cloud": { "properties": { "isCloudEnabled": {