From c5cea8da3049b2195996cc3695f6c09bb9bdf678 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 17 Aug 2023 15:41:20 -0700 Subject: [PATCH 01/79] Stub catalog reader interface Signed-off-by: Simeon Widdis --- .../integrations/repository/catalog_reader.ts | 10 +++++++ .../repository/local_catalog_reader.ts | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 server/adaptors/integrations/repository/catalog_reader.ts create mode 100644 server/adaptors/integrations/repository/local_catalog_reader.ts diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_reader.ts new file mode 100644 index 0000000000..c8cdbb39a3 --- /dev/null +++ b/server/adaptors/integrations/repository/catalog_reader.ts @@ -0,0 +1,10 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +interface CatalogReader { + readFile: (filename: string) => Promise; + readDir: (filename: string) => Promise; + isDir: (filename: string) => Promise; +} diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/local_catalog_reader.ts new file mode 100644 index 0000000000..b3d9d796dd --- /dev/null +++ b/server/adaptors/integrations/repository/local_catalog_reader.ts @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * A CatalogReader that reads from the local filesystem. + * Used to read Integration information when the user uploads their own catalog. + */ +class LocalCatalogReader implements CatalogReader { + directory: string; + + constructor(directory: string) { + this.directory = directory; + } + + async readFile(_dirname: string): Promise { + return ''; + } + + async readDir(_dirname: string): Promise { + return []; + } + + async isDir(_dirname: string): Promise { + return false; + } +} From 77ad89b68e22295095f9382e300321738267f783 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 17 Aug 2023 16:00:13 -0700 Subject: [PATCH 02/79] Add basic catalog functionality to new catalog reader Signed-off-by: Simeon Widdis --- .../integrations/repository/catalog_reader.ts | 3 +- .../repository/local_catalog_reader.ts | 41 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_reader.ts index c8cdbb39a3..9710ce7fd2 100644 --- a/server/adaptors/integrations/repository/catalog_reader.ts +++ b/server/adaptors/integrations/repository/catalog_reader.ts @@ -6,5 +6,6 @@ interface CatalogReader { readFile: (filename: string) => Promise; readDir: (filename: string) => Promise; - isDir: (filename: string) => Promise; + isIntegration: (filename: string) => Promise; + isRepository: (filename: string) => Promise; } diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/local_catalog_reader.ts index b3d9d796dd..2634e7e62d 100644 --- a/server/adaptors/integrations/repository/local_catalog_reader.ts +++ b/server/adaptors/integrations/repository/local_catalog_reader.ts @@ -3,6 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ +import * as fs from 'fs/promises'; +import path from 'path'; +import sanitize from 'sanitize-filename'; +import { Integration } from './integration'; + /** * A CatalogReader that reads from the local filesystem. * Used to read Integration information when the user uploads their own catalog. @@ -14,15 +19,41 @@ class LocalCatalogReader implements CatalogReader { this.directory = directory; } - async readFile(_dirname: string): Promise { - return ''; + // Use before any call to `fs` + _prepare(filename: string): string { + return path.join(this.directory, sanitize(filename)); + } + + async readFile(filename: string): Promise { + return await fs.readFile(this._prepare(filename), { encoding: 'utf-8' }); } - async readDir(_dirname: string): Promise { - return []; + async readDir(dirname: string): Promise { + // TODO return empty list if not a directory + return await fs.readdir(this._prepare(dirname)); } - async isDir(_dirname: string): Promise { + async isDirectory(dirname: string): Promise { + return (await fs.lstat(this._prepare(dirname))).isDirectory(); + } + + async isRepository(dirname: string): Promise { + if (await this.isIntegration(dirname)) { + return false; + } + // If there is at least one integration in a directory, it's a repository. + for (const item of await this.readDir(dirname)) { + if (await this.isIntegration(item)) { + return true; + } + } return false; } + + async isIntegration(dirname: string): Promise { + if (!(await this.isDirectory(dirname))) { + return false; + } + return new Integration(this._prepare(dirname)).check(); + } } From 676591df5c6d22a05cbf0121a444ce04c8af270b Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 21 Aug 2023 11:36:02 -0700 Subject: [PATCH 03/79] Refactor validation logic with a deeper interface Signed-off-by: Simeon Widdis --- .../repository/__test__/integration.test.ts | 11 ++++-- .../integrations/repository/integration.ts | 21 +--------- server/adaptors/integrations/validators.ts | 39 ++++++++++++++++++- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 4474fc48ff..2002ad04a9 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -16,8 +16,13 @@ describe('Integration', () => { name: 'sample', version: '2.0.0', license: 'Apache-2.0', - type: '', - components: [], + type: 'logs', + components: [ + { + name: 'logs', + version: '1.0.0', + }, + ], assets: { savedObjects: { name: 'sample', @@ -105,7 +110,7 @@ describe('Integration', () => { const result = await integration.getConfig(sampleIntegration.version); expect(result).toBeNull(); - expect(logValidationErrorsMock).toHaveBeenCalledWith(expect.any(String), expect.any(Array)); + expect(logValidationErrorsMock).toHaveBeenCalled(); }); it('should return null and log syntax errors if the config file has syntax errors', async () => { diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index fe9ddd0ef3..1350cb6537 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -6,7 +6,7 @@ import * as fs from 'fs/promises'; import path from 'path'; import { ValidateFunction } from 'ajv'; -import { templateValidator } from '../validators'; +import { validateTemplate } from '../validators'; /** * Helper function to compare version numbers. @@ -49,18 +49,6 @@ async function isDirectory(dirPath: string): Promise { } } -/** - * Helper function to log validation errors. - * Relies on the `ajv` package for validation error logs.. - * - * @param integration The name of the component that failed validation. - * @param validator A failing ajv validator. - */ -function logValidationErrors(integration: string, validator: ValidateFunction) { - const errors = validator.errors?.map((e) => e.message); - console.error(`Validation errors in ${integration}`, errors); -} - /** * The Integration class represents the data for Integration Templates. * It is backed by the repository file system. @@ -165,12 +153,7 @@ export class Integration { const config = await fs.readFile(configPath, { encoding: 'utf-8' }); const possibleTemplate = JSON.parse(config); - if (!templateValidator(possibleTemplate)) { - logValidationErrors(configFile, templateValidator); - return null; - } - - return possibleTemplate; + return validateTemplate(possibleTemplate, true) ? possibleTemplate : null; } catch (err: any) { if (err instanceof SyntaxError) { console.error(`Syntax errors in ${configFile}`, err); diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index 3cb24212d2..ab51c17cb9 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -107,5 +107,40 @@ const instanceSchema: JSONSchemaType = { required: ['name', 'templateName', 'dataSource', 'creationDate', 'assets'], }; -export const templateValidator = ajv.compile(templateSchema); -export const instanceValidator = ajv.compile(instanceSchema); +const templateValidator = ajv.compile(templateSchema); +const instanceValidator = ajv.compile(instanceSchema); + +// AJV validators use side effects for errors, so we provide a more conventional wrapper. +// The wrapper optionally handles error logging with the `logErrors` parameter. +export const validateTemplate = (data: { name?: unknown }, logErrors?: true): boolean => { + if (!templateValidator(data)) { + if (logErrors) { + console.error( + `The integration '${data.name ?? 'config'}' is invalid:`, + ajv.errorsText(templateValidator.errors) + ); + } + return false; + } + // We assume an invariant that the type of an integration is connected with its component. + if (data.components.findIndex((x) => x.name === data.type) < 0) { + if (logErrors) { + console.error(`The integration type '${data.type}' must be included as a component`); + } + return false; + } + return true; +}; + +export const validateInstance = (data: { name?: unknown }, logErrors?: true): boolean => { + if (!instanceValidator(data)) { + if (logErrors) { + console.error( + `The integration '${data.name ?? 'instance'} is invalid:`, + ajv.errorsText(instanceValidator.errors) + ); + } + return false; + } + return true; +}; From 12c4bcf8b72a2a067827b3958da4ca24595b9985 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 21 Aug 2023 11:36:02 -0700 Subject: [PATCH 04/79] Refactor validation logic with a deeper interface Signed-off-by: Simeon Widdis --- .../repository/__test__/integration.test.ts | 11 ++++-- .../integrations/repository/integration.ts | 21 +--------- server/adaptors/integrations/validators.ts | 39 ++++++++++++++++++- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 4474fc48ff..2002ad04a9 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -16,8 +16,13 @@ describe('Integration', () => { name: 'sample', version: '2.0.0', license: 'Apache-2.0', - type: '', - components: [], + type: 'logs', + components: [ + { + name: 'logs', + version: '1.0.0', + }, + ], assets: { savedObjects: { name: 'sample', @@ -105,7 +110,7 @@ describe('Integration', () => { const result = await integration.getConfig(sampleIntegration.version); expect(result).toBeNull(); - expect(logValidationErrorsMock).toHaveBeenCalledWith(expect.any(String), expect.any(Array)); + expect(logValidationErrorsMock).toHaveBeenCalled(); }); it('should return null and log syntax errors if the config file has syntax errors', async () => { diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index fe9ddd0ef3..1350cb6537 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -6,7 +6,7 @@ import * as fs from 'fs/promises'; import path from 'path'; import { ValidateFunction } from 'ajv'; -import { templateValidator } from '../validators'; +import { validateTemplate } from '../validators'; /** * Helper function to compare version numbers. @@ -49,18 +49,6 @@ async function isDirectory(dirPath: string): Promise { } } -/** - * Helper function to log validation errors. - * Relies on the `ajv` package for validation error logs.. - * - * @param integration The name of the component that failed validation. - * @param validator A failing ajv validator. - */ -function logValidationErrors(integration: string, validator: ValidateFunction) { - const errors = validator.errors?.map((e) => e.message); - console.error(`Validation errors in ${integration}`, errors); -} - /** * The Integration class represents the data for Integration Templates. * It is backed by the repository file system. @@ -165,12 +153,7 @@ export class Integration { const config = await fs.readFile(configPath, { encoding: 'utf-8' }); const possibleTemplate = JSON.parse(config); - if (!templateValidator(possibleTemplate)) { - logValidationErrors(configFile, templateValidator); - return null; - } - - return possibleTemplate; + return validateTemplate(possibleTemplate, true) ? possibleTemplate : null; } catch (err: any) { if (err instanceof SyntaxError) { console.error(`Syntax errors in ${configFile}`, err); diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index 3cb24212d2..ab51c17cb9 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -107,5 +107,40 @@ const instanceSchema: JSONSchemaType = { required: ['name', 'templateName', 'dataSource', 'creationDate', 'assets'], }; -export const templateValidator = ajv.compile(templateSchema); -export const instanceValidator = ajv.compile(instanceSchema); +const templateValidator = ajv.compile(templateSchema); +const instanceValidator = ajv.compile(instanceSchema); + +// AJV validators use side effects for errors, so we provide a more conventional wrapper. +// The wrapper optionally handles error logging with the `logErrors` parameter. +export const validateTemplate = (data: { name?: unknown }, logErrors?: true): boolean => { + if (!templateValidator(data)) { + if (logErrors) { + console.error( + `The integration '${data.name ?? 'config'}' is invalid:`, + ajv.errorsText(templateValidator.errors) + ); + } + return false; + } + // We assume an invariant that the type of an integration is connected with its component. + if (data.components.findIndex((x) => x.name === data.type) < 0) { + if (logErrors) { + console.error(`The integration type '${data.type}' must be included as a component`); + } + return false; + } + return true; +}; + +export const validateInstance = (data: { name?: unknown }, logErrors?: true): boolean => { + if (!instanceValidator(data)) { + if (logErrors) { + console.error( + `The integration '${data.name ?? 'instance'} is invalid:`, + ajv.errorsText(instanceValidator.errors) + ); + } + return false; + } + return true; +}; From 412480e74ab4cfbcf2cdf42a689f46d0c8eb3487 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 21 Aug 2023 12:02:27 -0700 Subject: [PATCH 05/79] Remove redundant test. This test is unneeded after 12c4bcf Signed-off-by: Simeon Widdis --- .../integrations/__test__/local_repository.test.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/adaptors/integrations/__test__/local_repository.test.ts b/server/adaptors/integrations/__test__/local_repository.test.ts index 722711710b..6b0ddc72fd 100644 --- a/server/adaptors/integrations/__test__/local_repository.test.ts +++ b/server/adaptors/integrations/__test__/local_repository.test.ts @@ -31,14 +31,4 @@ describe('The local repository', () => { const integrations: Integration[] = await repository.getIntegrationList(); await Promise.all(integrations.map((i) => expect(i.deepCheck()).resolves.toBeTruthy())); }); - - it('Should not have a type that is not imported in the config', async () => { - const repository: Repository = new Repository(path.join(__dirname, '../__data__/repository')); - const integrations: Integration[] = await repository.getIntegrationList(); - for (const integration of integrations) { - const config = await integration.getConfig(); - const components = config!.components.map((x) => x.name); - expect(components).toContain(config!.type); - } - }); }); From 6d2bad103cb35ec5934e3e614b022040b61a6931 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 21 Aug 2023 14:12:44 -0700 Subject: [PATCH 06/79] Add tests for new validators Signed-off-by: Simeon Widdis --- .../integrations/__test__/validators.test.ts | 79 +++++++++++++++++++ .../integrations/repository/integration.ts | 1 - 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 server/adaptors/integrations/__test__/validators.test.ts diff --git a/server/adaptors/integrations/__test__/validators.test.ts b/server/adaptors/integrations/__test__/validators.test.ts new file mode 100644 index 0000000000..f998324af0 --- /dev/null +++ b/server/adaptors/integrations/__test__/validators.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { validateTemplate, validateInstance } from '../validators'; + +const validTemplate: IntegrationTemplate = { + name: 'test', + version: '1.0.0', + license: 'Apache-2.0', + type: 'logs', + components: [ + { + name: 'logs', + version: '1.0.0', + }, + ], + assets: {}, +}; + +const validInstance: IntegrationInstance = { + name: 'test', + templateName: 'test', + dataSource: 'test', + creationDate: new Date(0).toISOString(), + assets: [], +}; + +describe('validateTemplate', () => { + it('Returns true for a valid Integration Template', () => { + expect(validateTemplate(validTemplate)).toBe(true); + }); + + it('Returns false if a template is missing a license', () => { + const sample: any = structuredClone(validTemplate); + sample.license = undefined; + expect(validateTemplate(sample)).toBe(false); + }); + + it('Returns false if a template has an invalid type', () => { + const sample: any = structuredClone(validTemplate); + sample.components[0].name = 'not-logs'; + expect(validateTemplate(sample)).toBe(false); + }); + + it('Respects logErrors', () => { + const logValidationErrorsMock = jest.spyOn(console, 'error'); + const sample1: any = structuredClone(validTemplate); + sample1.license = undefined; + const sample2: any = structuredClone(validTemplate); + sample2.components[0].name = 'not-logs'; + + expect(validateTemplate(sample1, true)).toBe(false); + expect(validateTemplate(sample2, true)).toBe(false); + expect(logValidationErrorsMock).toBeCalledTimes(2); + }); +}); + +describe('validateInstance', () => { + it('Returns true for a valid Integration Instance', () => { + expect(validateInstance(validInstance)).toBe(true); + }); + + it('Returns false if an instance is missing a template', () => { + const sample: any = structuredClone(validInstance); + sample.templateName = undefined; + expect(validateInstance(sample)).toBe(false); + }); + + it('Respects logErrors', () => { + const logValidationErrorsMock = jest.spyOn(console, 'error'); + const sample1: any = structuredClone(validInstance); + sample1.templateName = undefined; + + expect(validateInstance(sample1, true)).toBe(false); + expect(logValidationErrorsMock).toBeCalled(); + }); +}); diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index 1350cb6537..7c20db6e49 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -5,7 +5,6 @@ import * as fs from 'fs/promises'; import path from 'path'; -import { ValidateFunction } from 'ajv'; import { validateTemplate } from '../validators'; /** From 55d649097dd89c98c8cad91d0f9640d88fe755d6 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 21 Aug 2023 14:22:35 -0700 Subject: [PATCH 07/79] Make better failure mode for invalid objects Signed-off-by: Simeon Widdis --- .../adaptors/integrations/__test__/validators.test.ts | 10 ++++++++++ server/adaptors/integrations/validators.ts | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/server/adaptors/integrations/__test__/validators.test.ts b/server/adaptors/integrations/__test__/validators.test.ts index f998324af0..b52128d540 100644 --- a/server/adaptors/integrations/__test__/validators.test.ts +++ b/server/adaptors/integrations/__test__/validators.test.ts @@ -55,6 +55,11 @@ describe('validateTemplate', () => { expect(validateTemplate(sample2, true)).toBe(false); expect(logValidationErrorsMock).toBeCalledTimes(2); }); + + it("Doesn't crash if given a non-object", () => { + // May happen in some user-provided JSON parsing scenarios. + expect(validateTemplate([] as any, true)).toBe(false); + }); }); describe('validateInstance', () => { @@ -76,4 +81,9 @@ describe('validateInstance', () => { expect(validateInstance(sample1, true)).toBe(false); expect(logValidationErrorsMock).toBeCalled(); }); + + it("Doesn't crash if given a non-object", () => { + // May happen in some user-provided JSON parsing scenarios. + expect(validateInstance([] as any, true)).toBe(false); + }); }); diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index ab51c17cb9..7621cf7176 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -116,7 +116,7 @@ export const validateTemplate = (data: { name?: unknown }, logErrors?: true): bo if (!templateValidator(data)) { if (logErrors) { console.error( - `The integration '${data.name ?? 'config'}' is invalid:`, + `The integration config '${data.name ?? data}' is invalid:`, ajv.errorsText(templateValidator.errors) ); } @@ -136,7 +136,7 @@ export const validateInstance = (data: { name?: unknown }, logErrors?: true): bo if (!instanceValidator(data)) { if (logErrors) { console.error( - `The integration '${data.name ?? 'instance'} is invalid:`, + `The integration instance '${data.name ?? data}' is invalid:`, ajv.errorsText(instanceValidator.errors) ); } From ba56fb8050385c88ab4cad41a8c1722eff1a9da1 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 22 Aug 2023 14:16:54 -0700 Subject: [PATCH 08/79] Generalize Result type Signed-off-by: Simeon Widdis --- .../integrations/__test__/validators.test.ts | 12 ++++++------ server/adaptors/integrations/types.ts | 2 ++ server/adaptors/integrations/validators.ts | 10 ++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/adaptors/integrations/__test__/validators.test.ts b/server/adaptors/integrations/__test__/validators.test.ts index ba573c4c47..3c6e133f5c 100644 --- a/server/adaptors/integrations/__test__/validators.test.ts +++ b/server/adaptors/integrations/__test__/validators.test.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { validateTemplate, validateInstance, ValidationResult } from '../validators'; +import { validateTemplate, validateInstance } from '../validators'; const validTemplate: IntegrationTemplate = { name: 'test', @@ -29,7 +29,7 @@ const validInstance: IntegrationInstance = { describe('validateTemplate', () => { it('Returns a success value for a valid Integration Template', () => { - const result: ValidationResult = validateTemplate(validTemplate); + const result: Result = validateTemplate(validTemplate); expect(result.ok).toBe(true); expect((result as any).value).toBe(validTemplate); }); @@ -38,7 +38,7 @@ describe('validateTemplate', () => { const sample: any = structuredClone(validTemplate); sample.license = undefined; - const result: ValidationResult = validateTemplate(sample); + const result: Result = validateTemplate(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); @@ -48,7 +48,7 @@ describe('validateTemplate', () => { const sample: any = structuredClone(validTemplate); sample.components[0].name = 'not-logs'; - const result: ValidationResult = validateTemplate(sample); + const result: Result = validateTemplate(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); @@ -62,7 +62,7 @@ describe('validateTemplate', () => { describe('validateInstance', () => { it('Returns true for a valid Integration Instance', () => { - const result: ValidationResult = validateInstance(validInstance); + const result: Result = validateInstance(validInstance); expect(result.ok).toBe(true); expect((result as any).value).toBe(validInstance); }); @@ -71,7 +71,7 @@ describe('validateInstance', () => { const sample: any = structuredClone(validInstance); sample.templateName = undefined; - const result: ValidationResult = validateInstance(sample); + const result: Result = validateInstance(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index c12476909f..d84dc3c5a4 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +type Result = { ok: true; value: T } | { ok: false; error: E }; + interface IntegrationTemplate { name: string; version: string; diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index b40871eefb..b9c7f33f65 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -5,8 +5,6 @@ import Ajv, { JSONSchemaType } from 'ajv'; -export type ValidationResult = { ok: true; value: T } | { ok: false; error: E }; - const ajv = new Ajv(); const staticAsset: JSONSchemaType = { @@ -118,11 +116,11 @@ const instanceValidator = ajv.compile(instanceSchema); * this is a more conventional wrapper that simplifies calling. * * @param data The data to be validated as an IntegrationTemplate. - * @return A ValidationResult indicating whether the validation was successful or not. + * @return A Result indicating whether the validation was successful or not. * If validation succeeds, returns an object with 'ok' set to true and the validated data. * If validation fails, returns an object with 'ok' set to false and an Error object describing the validation error. */ -export const validateTemplate = (data: unknown): ValidationResult => { +export const validateTemplate = (data: unknown): Result => { if (!templateValidator(data)) { return { ok: false, error: new Error(ajv.errorsText(templateValidator.errors)) }; } @@ -143,11 +141,11 @@ export const validateTemplate = (data: unknown): ValidationResult => { +export const validateInstance = (data: unknown): Result => { if (!instanceValidator(data)) { return { ok: false, From cb29208bea86407beea76042b9238d8f994084b1 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 22 Aug 2023 15:14:42 -0700 Subject: [PATCH 09/79] Convert backend to use catalog reader (unstable) Signed-off-by: Simeon Widdis --- .../integrations_kibana_backend.ts | 61 +++++- .../integrations/repository/catalog_reader.ts | 4 +- .../integrations/repository/integration.ts | 188 ++++++++---------- .../repository/local_catalog_reader.ts | 27 +-- 4 files changed, 137 insertions(+), 143 deletions(-) diff --git a/server/adaptors/integrations/integrations_kibana_backend.ts b/server/adaptors/integrations/integrations_kibana_backend.ts index f28c883ecf..b503390a24 100644 --- a/server/adaptors/integrations/integrations_kibana_backend.ts +++ b/server/adaptors/integrations/integrations_kibana_backend.ts @@ -53,17 +53,33 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { return result; }; + // Internal; use getIntegrationTemplates. + _getAllIntegrationTemplates = async (): Promise => { + const integrationList = await this.repository.getIntegrationList(); + const configResults = await Promise.all(integrationList.map((x) => x.getConfig())); + const configs = configResults.filter((cfg) => cfg.ok) as Array<{ value: IntegrationTemplate }>; + return Promise.resolve({ hits: configs.map((cfg) => cfg.value) }); + }; + + // Internal; use getIntegrationTemplates. + _getIntegrationTemplatesByName = async ( + name: string + ): Promise => { + const integration = await this.repository.getIntegration(name); + const config = await integration?.getConfig(); + if (!config || !config.ok) { + return Promise.resolve({ hits: [] }); + } + return Promise.resolve({ hits: [config.value] }); + }; + getIntegrationTemplates = async ( query?: IntegrationTemplateQuery ): Promise => { if (query?.name) { - const integration = await this.repository.getIntegration(query.name); - const config = await integration?.getConfig(); - return Promise.resolve({ hits: config ? [config] : [] }); + return this._getIntegrationTemplatesByName(query.name); } - const integrationList = await this.repository.getIntegrationList(); - const configList = await Promise.all(integrationList.map((x) => x.getConfig())); - return Promise.resolve({ hits: configList.filter((x) => x !== null) as IntegrationTemplate[] }); + return this._getAllIntegrationTemplates(); }; getIntegrationInstances = async ( @@ -159,14 +175,21 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { }; getStatic = async (templateName: string, staticPath: string): Promise => { - const data = await (await this.repository.getIntegration(templateName))?.getStatic(staticPath); - if (!data) { + const integration = await this.repository.getIntegration(templateName); + if (integration === null) { + return Promise.reject({ + message: `Template ${templateName} not found`, + statusCode: 404, + }); + } + const data = await integration.getStatic(staticPath); + if (!data.ok) { return Promise.reject({ message: `Asset ${staticPath} not found`, statusCode: 404, }); } - return Promise.resolve(data); + return Promise.resolve(data.value); }; getSchemas = async (templateName: string): Promise => { @@ -188,7 +211,15 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { statusCode: 404, }); } - return Promise.resolve(integration.getAssets()); + const assets = await integration.getAssets(); + if (assets.ok) { + return assets.value; + } + const is404 = (assets.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: assets.error.message, + statusCode: is404 ? 404 : 500, + }); }; getSampleData = async (templateName: string): Promise<{ sampleData: object[] | null }> => { @@ -199,6 +230,14 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { statusCode: 404, }); } - return Promise.resolve(integration.getSampleData()); + const sampleData = await integration.getSampleData(); + if (sampleData.ok) { + return sampleData.value; + } + const is404 = (sampleData.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: sampleData.error.message, + statusCode: is404 ? 404 : 500, + }); }; } diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_reader.ts index 9710ce7fd2..b48938227a 100644 --- a/server/adaptors/integrations/repository/catalog_reader.ts +++ b/server/adaptors/integrations/repository/catalog_reader.ts @@ -5,7 +5,7 @@ interface CatalogReader { readFile: (filename: string) => Promise; + readFileRaw: (filename: string) => Promise; readDir: (filename: string) => Promise; - isIntegration: (filename: string) => Promise; - isRepository: (filename: string) => Promise; + isDirectory: (filename: string) => Promise; } diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index 21f187bdec..ca48c606fc 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as fs from 'fs/promises'; import path from 'path'; import { validateTemplate } from '../validators'; +import { LocalCatalogReader } from './local_catalog_reader'; /** * Helper function to compare version numbers. @@ -33,77 +33,47 @@ function compareVersions(a: string, b: string): number { return 0; // a == b } -/** - * Helper function to check if the given path is a directory - * - * @param dirPath The directory to check. - * @returns True if the path is a directory. - */ -async function isDirectory(dirPath: string): Promise { - try { - const stats = await fs.stat(dirPath); - return stats.isDirectory(); - } catch { - return false; - } -} - /** * The Integration class represents the data for Integration Templates. * It is backed by the repository file system. * It includes accessor methods for integration configs, as well as helpers for nested components. */ export class Integration { + reader: CatalogReader; directory: string; name: string; - constructor(directory: string) { + constructor(directory: string, reader?: CatalogReader) { this.directory = directory; this.name = path.basename(directory); + this.reader = reader ?? new LocalCatalogReader(directory); } /** - * Check the integration for validity. - * This is not a deep check, but a quick check to verify that the integration is a valid directory and has a config file. - * - * @returns true if the integration is valid. - */ - async check(): Promise { - if (!(await isDirectory(this.directory))) { - return false; - } - return (await this.getConfig()) !== null; - } - - /** - * Like check(), but thoroughly checks all nested integration dependencies. + * Like getConfig(), but thoroughly checks all nested integration dependencies for validity. * - * @returns true if the integration is valid. + * @returns a Result indicating whether the integration is valid. */ - async deepCheck(): Promise { - if (!(await this.check())) { - console.error('check failed'); - return false; + async deepCheck(): Promise> { + const configResult = await this.getConfig(); + if (!configResult.ok) { + return configResult; } try { - // An integration must have at least one mapping const schemas = await this.getSchemas(); - if (Object.keys(schemas.mappings).length === 0) { - return false; + if (!schemas.ok || Object.keys(schemas.value.mappings).length === 0) { + return { ok: false, error: new Error('The integration has no schemas available') }; } - // An integration must have at least one asset const assets = await this.getAssets(); - if (Object.keys(assets).length === 0) { - return false; + if (!assets.ok || Object.keys(assets).length === 0) { + return { ok: false, error: new Error('An integration must have at least one asset') }; } } catch (err: any) { - // Any loading errors are considered invalid - console.error('Deep check failed for exception', err); - return false; + return { ok: false, error: err }; } - return true; + return configResult; } /** @@ -114,7 +84,7 @@ export class Integration { * @returns A string with the latest version, or null if no versions are available. */ async getLatestVersion(): Promise { - const files = await fs.readdir(this.directory); + const files = await this.reader.readDir(this.directory); const versions: string[] = []; for (const file of files) { @@ -138,35 +108,34 @@ export class Integration { * @param version The version of the config to retrieve. * @returns The config if a valid config matching the version is present, otherwise null. */ - async getConfig(version?: string): Promise { + async getConfig(version?: string): Promise> { + if (!this.reader.isDirectory(this.directory)) { + return { ok: false, error: new Error(`${this.directory} is not a valid directory`) }; + } + const maybeVersion: string | null = version ? version : await this.getLatestVersion(); if (maybeVersion === null) { - return null; + return { + ok: false, + error: new Error(`No valid config matching version ${version} is available`), + }; } const configFile = `${this.name}-${maybeVersion}.json`; - const configPath = path.join(this.directory, configFile); try { - const config = await fs.readFile(configPath, { encoding: 'utf-8' }); + const config = await this.reader.readFile(configFile); const possibleTemplate = JSON.parse(config); - const template = validateTemplate(possibleTemplate); - if (template.ok) { - return template.value; - } - console.error(template.error); - return null; + return validateTemplate(possibleTemplate); } catch (err: any) { if (err instanceof SyntaxError) { console.error(`Syntax errors in ${configFile}`, err); - return null; } if (err instanceof Error && (err as { code?: string }).code === 'ENOENT') { console.error(`Attempted to retrieve non-existent config ${configFile}`); - return null; } - throw new Error('Could not load integration', { cause: err }); + return { ok: false, error: err }; } } @@ -181,30 +150,33 @@ export class Integration { */ async getAssets( version?: string - ): Promise<{ - savedObjects?: object[]; - }> { - const config = await this.getConfig(version); - if (config === null) { - return Promise.reject(new Error('Attempted to get assets of invalid config')); + ): Promise< + Result<{ + savedObjects?: object[]; + }> + > { + const configResult = await this.getConfig(version); + if (!configResult.ok) { + return configResult; } - const result: { savedObjects?: object[] } = {}; + const config = configResult.value; + + const resultValue: { savedObjects?: object[] } = {}; if (config.assets.savedObjects) { const sobjPath = path.join( - this.directory, 'assets', `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson` ); try { - const ndjson = await fs.readFile(sobjPath, { encoding: 'utf-8' }); + const ndjson = await this.reader.readFile(sobjPath); const asJson = '[' + ndjson.trim().replace(/\n/g, ',') + ']'; const parsed = JSON.parse(asJson); - result.savedObjects = parsed; + resultValue.savedObjects = parsed; } catch (err: any) { - console.error("Failed to load saved object assets, proceeding as if it's absent", err); + return { ok: false, error: err }; } } - return result; + return { ok: true, value: resultValue }; } /** @@ -217,18 +189,22 @@ export class Integration { */ async getSampleData( version?: string - ): Promise<{ - sampleData: object[] | null; - }> { - const config = await this.getConfig(version); - if (config === null) { - return Promise.reject(new Error('Attempted to get assets of invalid config')); + ): Promise< + Result<{ + sampleData: object[] | null; + }> + > { + const configResult = await this.getConfig(version); + if (!configResult.ok) { + return configResult; } - const result: { sampleData: object[] | null } = { sampleData: null }; + const config = configResult.value; + + const resultValue: { sampleData: object[] | null } = { sampleData: null }; if (config.sampleData) { - const sobjPath = path.join(this.directory, 'data', config.sampleData?.path); + const sobjPath = path.join('data', config.sampleData?.path); try { - const jsonContent = await fs.readFile(sobjPath, { encoding: 'utf-8' }); + const jsonContent = await this.reader.readFile(sobjPath); const parsed = JSON.parse(jsonContent) as object[]; for (const value of parsed) { if (!('@timestamp' in value)) { @@ -243,12 +219,12 @@ export class Integration { ).toISOString(), }); } - result.sampleData = parsed; + resultValue.sampleData = parsed; } catch (err: any) { - console.error("Failed to load saved object assets, proceeding as if it's absent", err); + return { ok: false, error: err }; } } - return result; + return { ok: true, value: resultValue }; } /** @@ -263,32 +239,31 @@ export class Integration { */ async getSchemas( version?: string - ): Promise<{ - mappings: { [key: string]: any }; - }> { - const config = await this.getConfig(version); - if (config === null) { - return Promise.reject(new Error('Attempted to get assets of invalid config')); + ): Promise< + Result<{ + mappings: { [key: string]: any }; + }> + > { + const configResult = await this.getConfig(version); + if (!configResult.ok) { + return configResult; } - const result: { mappings: { [key: string]: any } } = { + const config = configResult.value; + + const resultValue: { mappings: { [key: string]: object } } = { mappings: {}, }; try { for (const component of config.components) { const schemaFile = `${component.name}-${component.version}.mapping.json`; - const rawSchema = await fs.readFile(path.join(this.directory, 'schemas', schemaFile), { - encoding: 'utf-8', - }); + const rawSchema = await this.reader.readFile(path.join('schemas', schemaFile)); const parsedSchema = JSON.parse(rawSchema); - result.mappings[component.name] = parsedSchema; + resultValue.mappings[component.name] = parsedSchema; } } catch (err: any) { - // It's not clear that an invalid schema can be recovered from. - // For integrations to function, we need schemas to be valid. - console.error('Error loading schema', err); - return Promise.reject(new Error('Could not load schema', { cause: err })); + return { ok: false, error: err }; } - return result; + return { ok: true, value: resultValue }; } /** @@ -297,16 +272,13 @@ export class Integration { * @param staticPath The path of the static to retrieve. * @returns A buffer with the static's data if present, otherwise null. */ - async getStatic(staticPath: string): Promise { - const fullStaticPath = path.join(this.directory, 'static', staticPath); + async getStatic(staticPath: string): Promise> { + const fullStaticPath = path.join('static', staticPath); try { - return await fs.readFile(fullStaticPath); + const buffer = await this.reader.readFileRaw(fullStaticPath); + return { ok: true, value: buffer }; } catch (err: any) { - if (err instanceof Error && (err as { code?: string }).code === 'ENOENT') { - console.error(`Static not found: ${staticPath}`); - return null; - } - throw err; + return { ok: false, error: err }; } } } diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/local_catalog_reader.ts index 2634e7e62d..2422ce127a 100644 --- a/server/adaptors/integrations/repository/local_catalog_reader.ts +++ b/server/adaptors/integrations/repository/local_catalog_reader.ts @@ -6,13 +6,12 @@ import * as fs from 'fs/promises'; import path from 'path'; import sanitize from 'sanitize-filename'; -import { Integration } from './integration'; /** * A CatalogReader that reads from the local filesystem. * Used to read Integration information when the user uploads their own catalog. */ -class LocalCatalogReader implements CatalogReader { +export class LocalCatalogReader implements CatalogReader { directory: string; constructor(directory: string) { @@ -28,6 +27,10 @@ class LocalCatalogReader implements CatalogReader { return await fs.readFile(this._prepare(filename), { encoding: 'utf-8' }); } + async readFileRaw(filename: string): Promise { + return await fs.readFile(this._prepare(filename)); + } + async readDir(dirname: string): Promise { // TODO return empty list if not a directory return await fs.readdir(this._prepare(dirname)); @@ -36,24 +39,4 @@ class LocalCatalogReader implements CatalogReader { async isDirectory(dirname: string): Promise { return (await fs.lstat(this._prepare(dirname))).isDirectory(); } - - async isRepository(dirname: string): Promise { - if (await this.isIntegration(dirname)) { - return false; - } - // If there is at least one integration in a directory, it's a repository. - for (const item of await this.readDir(dirname)) { - if (await this.isIntegration(item)) { - return true; - } - } - return false; - } - - async isIntegration(dirname: string): Promise { - if (!(await this.isDirectory(dirname))) { - return false; - } - return new Integration(this._prepare(dirname)).check(); - } } From c9de36e20924ffd5973e33820ce3ce240cb36880 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 22 Aug 2023 15:56:43 -0700 Subject: [PATCH 10/79] Repair tests for integrations class (unstable) Signed-off-by: Simeon Widdis --- .../repository/__test__/integration.test.ts | 120 +++++++----------- .../integrations/repository/catalog_reader.ts | 6 +- .../integrations/repository/integration.ts | 17 +-- .../repository/local_catalog_reader.ts | 13 +- 4 files changed, 66 insertions(+), 90 deletions(-) diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 2002ad04a9..e6611e3a5b 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -33,35 +33,7 @@ describe('Integration', () => { beforeEach(() => { integration = new Integration('./sample'); - }); - - describe('check', () => { - it('should return false if the directory does not exist', async () => { - const spy = jest.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => false } as Stats); - - const result = await integration.check(); - - expect(spy).toHaveBeenCalled(); - expect(result).toBe(false); - }); - - it('should return true if the directory exists and getConfig returns a valid template', async () => { - jest.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Stats); - integration.getConfig = jest.fn().mockResolvedValue(sampleIntegration); - - const result = await integration.check(); - - expect(result).toBe(true); - }); - - it('should return false if the directory exists but getConfig returns null', async () => { - jest.spyOn(fs, 'stat').mockResolvedValue({ isDirectory: () => true } as Stats); - integration.getConfig = jest.fn().mockResolvedValue(null); - - const result = await integration.check(); - - expect(result).toBe(false); - }); + jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); }); describe('getLatestVersion', () => { @@ -94,39 +66,45 @@ describe('Integration', () => { }); describe('getConfig', () => { + it('should return an error if the directory does not exist', async () => { + const spy = jest + .spyOn(fs, 'lstat') + .mockResolvedValueOnce({ isDirectory: () => false } as Stats); + + const result = await integration.getConfig(); + + expect(spy).toHaveBeenCalled(); + expect(result.ok).toBe(false); + }); + it('should return the parsed config template if it is valid', async () => { jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(sampleIntegration)); const result = await integration.getConfig(sampleIntegration.version); - expect(result).toEqual(sampleIntegration); + expect(result).toEqual({ ok: true, value: sampleIntegration }); }); - it('should return null and log validation errors if the config template is invalid', async () => { + it('should return an error if the config template is invalid', async () => { const invalidTemplate = { ...sampleIntegration, version: 2 }; jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(invalidTemplate)); - const logValidationErrorsMock = jest.spyOn(console, 'error'); const result = await integration.getConfig(sampleIntegration.version); - expect(result).toBeNull(); - expect(logValidationErrorsMock).toHaveBeenCalled(); + expect(result.ok).toBe(false); }); - it('should return null and log syntax errors if the config file has syntax errors', async () => { + it('should return an error if the config file has syntax errors', async () => { jest.spyOn(fs, 'readFile').mockResolvedValue('Invalid JSON'); - const logSyntaxErrorsMock = jest.spyOn(console, 'error'); const result = await integration.getConfig(sampleIntegration.version); - expect(result).toBeNull(); - expect(logSyntaxErrorsMock).toHaveBeenCalledWith(expect.any(String), expect.any(SyntaxError)); + expect(result.ok).toBe(false); }); - it('should return null and log errors if the integration config does not exist', async () => { - integration.directory = './non-existing-directory'; - const logErrorsMock = jest.spyOn(console, 'error'); - jest.spyOn(fs, 'readFile').mockImplementation((..._args) => { + it('should return an error if the integration config does not exist', async () => { + integration.directory = './empty-directory'; + const readFileMock = jest.spyOn(fs, 'readFile').mockImplementation((..._args) => { // Can't find any information on how to mock an actual file not found error, // But at least according to the current implementation this should be equivalent. const error: any = new Error('ENOENT: File not found'); @@ -136,37 +114,38 @@ describe('Integration', () => { const result = await integration.getConfig(sampleIntegration.version); - expect(jest.spyOn(fs, 'readFile')).toHaveBeenCalled(); - expect(logErrorsMock).toHaveBeenCalledWith(expect.any(String)); - expect(result).toBeNull(); + expect(readFileMock).toHaveBeenCalled(); + expect(result.ok).toBe(false); }); }); describe('getAssets', () => { it('should return linked saved object assets when available', async () => { - integration.getConfig = jest.fn().mockResolvedValue(sampleIntegration); + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleIntegration }); jest.spyOn(fs, 'readFile').mockResolvedValue('{"name":"asset1"}\n{"name":"asset2"}'); const result = await integration.getAssets(sampleIntegration.version); - expect(result.savedObjects).toEqual([{ name: 'asset1' }, { name: 'asset2' }]); + expect(result.ok).toBe(true); + expect((result as any).value.savedObjects).toStrictEqual([ + { name: 'asset1' }, + { name: 'asset2' }, + ]); }); - it('should reject a return if the provided version has no config', async () => { - integration.getConfig = jest.fn().mockResolvedValue(null); + it('should return an error if the provided version has no config', async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: false, error: new Error() }); - expect(integration.getAssets()).rejects.toThrowError(); + expect(integration.getAssets()).resolves.toHaveProperty('ok', false); }); - it('should log an error if the saved object assets are invalid', async () => { - const logErrorsMock = jest.spyOn(console, 'error'); - integration.getConfig = jest.fn().mockResolvedValue(sampleIntegration); + it('should return an error if the saved object assets are invalid', async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleIntegration }); jest.spyOn(fs, 'readFile').mockResolvedValue('{"unclosed":'); const result = await integration.getAssets(sampleIntegration.version); - expect(logErrorsMock).toHaveBeenCalledWith(expect.any(String), expect.any(Error)); - expect(result.savedObjects).toBeUndefined(); + expect(result.ok).toBe(false); }); }); @@ -178,7 +157,7 @@ describe('Integration', () => { { name: 'component2', version: '2.0.0' }, ], }; - integration.getConfig = jest.fn().mockResolvedValue(sampleConfig); + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); const mappingFile1 = 'component1-1.0.0.mapping.json'; const mappingFile2 = 'component2-2.0.0.mapping.json'; @@ -190,7 +169,8 @@ describe('Integration', () => { const result = await integration.getSchemas(); - expect(result).toEqual({ + expect(result.ok).toBe(true); + expect((result as any).value).toStrictEqual({ mappings: { component1: { mapping: 'mapping1' }, component2: { mapping: 'mapping2' }, @@ -207,22 +187,20 @@ describe('Integration', () => { ); }); - it('should reject with an error if the config is null', async () => { - integration.getConfig = jest.fn().mockResolvedValue(null); + it('should reject with an error if the config is invalid', async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: false, error: new Error() }); - await expect(integration.getSchemas()).rejects.toThrowError( - 'Attempted to get assets of invalid config' - ); + await expect(integration.getSchemas()).resolves.toHaveProperty('ok', false); }); it('should reject with an error if a mapping file is invalid', async () => { const sampleConfig = { components: [{ name: 'component1', version: '1.0.0' }], }; - integration.getConfig = jest.fn().mockResolvedValue(sampleConfig); + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); jest.spyOn(fs, 'readFile').mockRejectedValueOnce(new Error('Could not load schema')); - await expect(integration.getSchemas()).rejects.toThrowError('Could not load schema'); + await expect(integration.getSchemas()).resolves.toHaveProperty('ok', false); }); }); @@ -231,21 +209,21 @@ describe('Integration', () => { const readFileMock = jest .spyOn(fs, 'readFile') .mockResolvedValue(Buffer.from('logo data', 'ascii')); - expect(await integration.getStatic('/logo.png')).toStrictEqual( - Buffer.from('logo data', 'ascii') - ); + + const result = await integration.getStatic('logo.png'); + + expect(result.ok).toBe(true); + expect((result as any).value).toStrictEqual(Buffer.from('logo data', 'ascii')); expect(readFileMock).toBeCalledWith(path.join('sample', 'static', 'logo.png')); }); - it('should return null and log an error if the static file is not found', async () => { - const logErrorsMock = jest.spyOn(console, 'error'); + it('should return an error if the static file is not found', async () => { jest.spyOn(fs, 'readFile').mockImplementation((..._args) => { const error: any = new Error('ENOENT: File not found'); error.code = 'ENOENT'; return Promise.reject(error); }); - expect(await integration.getStatic('/logo.png')).toBeNull(); - expect(logErrorsMock).toBeCalledWith(expect.any(String)); + expect(integration.getStatic('/logo.png')).resolves.toHaveProperty('ok', false); }); }); }); diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_reader.ts index b48938227a..db6a70ee62 100644 --- a/server/adaptors/integrations/repository/catalog_reader.ts +++ b/server/adaptors/integrations/repository/catalog_reader.ts @@ -3,9 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ +type IntegrationPart = 'assets' | 'data' | 'schemas' | 'static'; + interface CatalogReader { - readFile: (filename: string) => Promise; - readFileRaw: (filename: string) => Promise; + readFile: (filename: string, type?: IntegrationPart) => Promise; + readFileRaw: (filename: string, type?: IntegrationPart) => Promise; readDir: (filename: string) => Promise; isDirectory: (filename: string) => Promise; } diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index ca48c606fc..e3514b1edf 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -109,7 +109,7 @@ export class Integration { * @returns The config if a valid config matching the version is present, otherwise null. */ async getConfig(version?: string): Promise> { - if (!this.reader.isDirectory(this.directory)) { + if (!(await this.reader.isDirectory(this.directory))) { return { ok: false, error: new Error(`${this.directory} is not a valid directory`) }; } @@ -163,12 +163,9 @@ export class Integration { const resultValue: { savedObjects?: object[] } = {}; if (config.assets.savedObjects) { - const sobjPath = path.join( - 'assets', - `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson` - ); + const sobjPath = `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson`; try { - const ndjson = await this.reader.readFile(sobjPath); + const ndjson = await this.reader.readFile(sobjPath, 'assets'); const asJson = '[' + ndjson.trim().replace(/\n/g, ',') + ']'; const parsed = JSON.parse(asJson); resultValue.savedObjects = parsed; @@ -202,9 +199,8 @@ export class Integration { const resultValue: { sampleData: object[] | null } = { sampleData: null }; if (config.sampleData) { - const sobjPath = path.join('data', config.sampleData?.path); try { - const jsonContent = await this.reader.readFile(sobjPath); + const jsonContent = await this.reader.readFile(config.sampleData.path, 'data'); const parsed = JSON.parse(jsonContent) as object[]; for (const value of parsed) { if (!('@timestamp' in value)) { @@ -256,7 +252,7 @@ export class Integration { try { for (const component of config.components) { const schemaFile = `${component.name}-${component.version}.mapping.json`; - const rawSchema = await this.reader.readFile(path.join('schemas', schemaFile)); + const rawSchema = await this.reader.readFile(schemaFile, 'schemas'); const parsedSchema = JSON.parse(rawSchema); resultValue.mappings[component.name] = parsedSchema; } @@ -273,9 +269,8 @@ export class Integration { * @returns A buffer with the static's data if present, otherwise null. */ async getStatic(staticPath: string): Promise> { - const fullStaticPath = path.join('static', staticPath); try { - const buffer = await this.reader.readFileRaw(fullStaticPath); + const buffer = await this.reader.readFileRaw(staticPath, 'static'); return { ok: true, value: buffer }; } catch (err: any) { return { ok: false, error: err }; diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/local_catalog_reader.ts index 2422ce127a..e042592298 100644 --- a/server/adaptors/integrations/repository/local_catalog_reader.ts +++ b/server/adaptors/integrations/repository/local_catalog_reader.ts @@ -19,16 +19,17 @@ export class LocalCatalogReader implements CatalogReader { } // Use before any call to `fs` - _prepare(filename: string): string { - return path.join(this.directory, sanitize(filename)); + // Sanitizes filenames by default, manually prepend directories with a prefix if necessary + _prepare(filename: string, prefix?: string): string { + return path.join(this.directory, prefix ?? '.', sanitize(filename)); } - async readFile(filename: string): Promise { - return await fs.readFile(this._prepare(filename), { encoding: 'utf-8' }); + async readFile(filename: string, type?: IntegrationPart): Promise { + return await fs.readFile(this._prepare(filename, type), { encoding: 'utf-8' }); } - async readFileRaw(filename: string): Promise { - return await fs.readFile(this._prepare(filename)); + async readFileRaw(filename: string, type?: IntegrationPart): Promise { + return await fs.readFile(this._prepare(filename, type)); } async readDir(dirname: string): Promise { From 37c798000d0cf9748b2a42d029943d12c6baf2fb Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 22 Aug 2023 16:33:32 -0700 Subject: [PATCH 11/79] Refactor repository for new integration interface Signed-off-by: Simeon Widdis --- .../repository/__test__/repository.test.ts | 24 +++++++------ .../integrations/repository/repository.ts | 34 ++++++++++--------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/server/adaptors/integrations/repository/__test__/repository.test.ts b/server/adaptors/integrations/repository/__test__/repository.test.ts index 913968f495..0e08199e9f 100644 --- a/server/adaptors/integrations/repository/__test__/repository.test.ts +++ b/server/adaptors/integrations/repository/__test__/repository.test.ts @@ -20,14 +20,11 @@ describe('Repository', () => { describe('getIntegrationList', () => { it('should return an array of Integration instances', async () => { - // Mock fs.readdir to return a list of folders jest.spyOn(fs, 'readdir').mockResolvedValue((['folder1', 'folder2'] as unknown) as Dirent[]); - - // Mock fs.lstat to return a directory status jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); - - // Mock Integration check method to always return true - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(true); + jest + .spyOn(Integration.prototype, 'getConfig') + .mockResolvedValue({ ok: true, value: {} as any }); const integrations = await repository.getIntegrationList(); @@ -48,7 +45,9 @@ describe('Repository', () => { } }); - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(true); + jest + .spyOn(Integration.prototype, 'getConfig') + .mockResolvedValue({ ok: true, value: {} as any }); const integrations = await repository.getIntegrationList(); @@ -67,15 +66,20 @@ describe('Repository', () => { describe('getIntegration', () => { it('should return an Integration instance if it exists and passes the check', async () => { - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(true); + jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); + jest + .spyOn(Integration.prototype, 'getConfig') + .mockResolvedValue({ ok: true, value: {} as any }); const integration = await repository.getIntegration('integrationName'); expect(integration).toBeInstanceOf(Integration); }); - it('should return null if the integration does not exist or fails the check', async () => { - jest.spyOn(Integration.prototype, 'check').mockResolvedValue(false); + it('should return null if the integration does not exist or fails checks', async () => { + jest + .spyOn(Integration.prototype, 'getConfig') + .mockResolvedValue({ ok: false, error: new Error() }); const integration = await repository.getIntegration('invalidIntegration'); diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index 00d241327d..00039851b7 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -3,31 +3,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as fs from 'fs/promises'; import * as path from 'path'; import { Integration } from './integration'; +import { LocalCatalogReader } from './local_catalog_reader'; export class Repository { + reader: CatalogReader; directory: string; - constructor(directory: string) { + constructor(directory: string, reader?: CatalogReader) { this.directory = directory; + this.reader = reader ?? new LocalCatalogReader(directory); } async getIntegrationList(): Promise { try { - const folders = await fs.readdir(this.directory); - const integrations = Promise.all( - folders.map(async (folder) => { - const integPath = path.join(this.directory, folder); - if (!(await fs.lstat(integPath)).isDirectory()) { - return null; - } - const integ = new Integration(integPath); - return (await integ.check()) ? integ : null; - }) - ); - return (await integrations).filter((x) => x !== null) as Integration[]; + const folders = await this.reader.readDir(this.directory); + const integrations = await Promise.all(folders.map((i) => this.getIntegration(i))); + return integrations.filter((x) => x !== null) as Integration[]; } catch (error) { console.error(`Error reading integration directories in: ${this.directory}`, error); return []; @@ -35,7 +28,16 @@ export class Repository { } async getIntegration(name: string): Promise { - const integ = new Integration(path.join(this.directory, name)); - return (await integ.check()) ? integ : null; + if (!(await this.reader.isDirectory(name))) { + console.error(`Requested integration '${name}' does not exist`); + return null; + } + const integ = new Integration(path.join(this.directory, name), this.reader); + const checkResult = await integ.getConfig(); + if (!checkResult.ok) { + console.error(`Integration '${name}' is invalid:`, checkResult.error); + return null; + } + return integ; } } From b331090bc3423639a2d70e4729cca74460e6c148 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 22 Aug 2023 17:03:51 -0700 Subject: [PATCH 12/79] Fix outer repository and backend tests Signed-off-by: Simeon Widdis --- .../integrations/__test__/kibana_backend.test.ts | 16 +++++++++------- .../__test__/local_repository.test.ts | 12 ++++++++++-- .../integrations/repository/catalog_reader.ts | 1 + .../integrations/repository/integration.ts | 4 ++-- .../repository/local_catalog_reader.ts | 4 ++++ .../integrations/repository/repository.ts | 9 ++++++--- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/server/adaptors/integrations/__test__/kibana_backend.test.ts b/server/adaptors/integrations/__test__/kibana_backend.test.ts index 63d62764ce..c14ef6f53f 100644 --- a/server/adaptors/integrations/__test__/kibana_backend.test.ts +++ b/server/adaptors/integrations/__test__/kibana_backend.test.ts @@ -147,21 +147,23 @@ describe('IntegrationsKibanaBackend', () => { describe('getIntegrationTemplates', () => { it('should get integration templates by name', async () => { const query = { name: 'template1' }; - const integration = { getConfig: jest.fn().mockResolvedValue({ name: 'template1' }) }; + const integration = { + getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template1' } }), + }; mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); const result = await backend.getIntegrationTemplates(query); expect(mockRepository.getIntegration).toHaveBeenCalledWith(query.name); expect(integration.getConfig).toHaveBeenCalled(); - expect(result).toEqual({ hits: [await integration.getConfig()] }); + expect(result).toEqual({ hits: [{ name: 'template1' }] }); }); it('should get all integration templates', async () => { const integrationList = [ - { getConfig: jest.fn().mockResolvedValue({ name: 'template1' }) }, - { getConfig: jest.fn().mockResolvedValue(null) }, - { getConfig: jest.fn().mockResolvedValue({ name: 'template2' }) }, + { getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template1' } }) }, + { getConfig: jest.fn().mockResolvedValue({ ok: false, error: new Error() }) }, + { getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template2' } }) }, ]; mockRepository.getIntegrationList.mockResolvedValue( (integrationList as unknown) as Integration[] @@ -174,7 +176,7 @@ describe('IntegrationsKibanaBackend', () => { expect(integrationList[1].getConfig).toHaveBeenCalled(); expect(integrationList[2].getConfig).toHaveBeenCalled(); expect(result).toEqual({ - hits: [await integrationList[0].getConfig(), await integrationList[2].getConfig()], + hits: [{ name: 'template1' }, { name: 'template2' }], }); }); }); @@ -277,7 +279,7 @@ describe('IntegrationsKibanaBackend', () => { const staticPath = 'path/to/static'; const assetData = Buffer.from('asset data'); const integration = { - getStatic: jest.fn().mockResolvedValue(assetData), + getStatic: jest.fn().mockResolvedValue({ ok: true, value: assetData }), }; mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); diff --git a/server/adaptors/integrations/__test__/local_repository.test.ts b/server/adaptors/integrations/__test__/local_repository.test.ts index 6b0ddc72fd..162b95414c 100644 --- a/server/adaptors/integrations/__test__/local_repository.test.ts +++ b/server/adaptors/integrations/__test__/local_repository.test.ts @@ -21,7 +21,7 @@ describe('The local repository', () => { } // Otherwise, all directories must be integrations const integ = new Integration(integPath); - await expect(integ.check()).resolves.toBe(true); + expect(integ.getConfig()).resolves.toHaveProperty('ok', true); }) ); }); @@ -29,6 +29,14 @@ describe('The local repository', () => { it('Should pass deep validation for all local integrations.', async () => { const repository: Repository = new Repository(path.join(__dirname, '../__data__/repository')); const integrations: Integration[] = await repository.getIntegrationList(); - await Promise.all(integrations.map((i) => expect(i.deepCheck()).resolves.toBeTruthy())); + await Promise.all( + integrations.map(async (i) => { + const result = await i.deepCheck(); + if (!result.ok) { + console.error(result.error); + } + expect(result.ok).toBe(true); + }) + ); }); }); diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_reader.ts index db6a70ee62..9683bcaf9b 100644 --- a/server/adaptors/integrations/repository/catalog_reader.ts +++ b/server/adaptors/integrations/repository/catalog_reader.ts @@ -10,4 +10,5 @@ interface CatalogReader { readFileRaw: (filename: string, type?: IntegrationPart) => Promise; readDir: (filename: string) => Promise; isDirectory: (filename: string) => Promise; + join: (filename: string) => CatalogReader; } diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index e3514b1edf..ac7d3d16ee 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -84,7 +84,7 @@ export class Integration { * @returns A string with the latest version, or null if no versions are available. */ async getLatestVersion(): Promise { - const files = await this.reader.readDir(this.directory); + const files = await this.reader.readDir(''); const versions: string[] = []; for (const file of files) { @@ -109,7 +109,7 @@ export class Integration { * @returns The config if a valid config matching the version is present, otherwise null. */ async getConfig(version?: string): Promise> { - if (!(await this.reader.isDirectory(this.directory))) { + if (!(await this.reader.isDirectory(''))) { return { ok: false, error: new Error(`${this.directory} is not a valid directory`) }; } diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/local_catalog_reader.ts index e042592298..e95f3c46c1 100644 --- a/server/adaptors/integrations/repository/local_catalog_reader.ts +++ b/server/adaptors/integrations/repository/local_catalog_reader.ts @@ -40,4 +40,8 @@ export class LocalCatalogReader implements CatalogReader { async isDirectory(dirname: string): Promise { return (await fs.lstat(this._prepare(dirname))).isDirectory(); } + + join(filename: string): LocalCatalogReader { + return new LocalCatalogReader(path.join(this.directory, filename)); + } } diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index 00039851b7..acea2d4ed5 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -18,8 +18,11 @@ export class Repository { async getIntegrationList(): Promise { try { - const folders = await this.reader.readDir(this.directory); - const integrations = await Promise.all(folders.map((i) => this.getIntegration(i))); + // TODO in the future, we want to support traversing nested directory structures. + const folders = await this.reader.readDir(''); + const integrations = await Promise.all( + folders.map((i) => this.getIntegration(path.basename(i))) + ); return integrations.filter((x) => x !== null) as Integration[]; } catch (error) { console.error(`Error reading integration directories in: ${this.directory}`, error); @@ -32,7 +35,7 @@ export class Repository { console.error(`Requested integration '${name}' does not exist`); return null; } - const integ = new Integration(path.join(this.directory, name), this.reader); + const integ = new Integration(name, this.reader.join(name)); const checkResult = await integ.getConfig(); if (!checkResult.ok) { console.error(`Integration '${name}' is invalid:`, checkResult.error); From 4b29e17c72f381e0a1277564548d4132044c6675 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 22 Aug 2023 17:16:30 -0700 Subject: [PATCH 13/79] Add tests for sample data Signed-off-by: Simeon Widdis --- .../repository/__test__/integration.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index e6611e3a5b..105f81cb49 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -226,4 +226,39 @@ describe('Integration', () => { expect(integration.getStatic('/logo.png')).resolves.toHaveProperty('ok', false); }); }); + + describe('getSampleData', () => { + it('should return sample data', async () => { + const sampleConfig = { sampleData: { path: 'sample.json' } }; + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); + const readFileMock = jest.spyOn(fs, 'readFile').mockResolvedValue('[{"sample": true}]'); + + const result = await integration.getSampleData(); + + expect(result.ok).toBe(true); + expect((result as any).value.sampleData).toStrictEqual([{ sample: true }]); + expect(readFileMock).toBeCalledWith(path.join('sample', 'data', 'sample.json'), { + encoding: 'utf-8', + }); + }); + + it("should return null if there's no sample data", async () => { + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: {} }); + + const result = await integration.getSampleData(); + + expect(result.ok).toBe(true); + expect((result as any).value.sampleData).toBeNull(); + }); + + it('should catch and fail gracefully on invalid sample data', async () => { + const sampleConfig = { sampleData: { path: 'sample.json' } }; + integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleConfig }); + jest.spyOn(fs, 'readFile').mockResolvedValue('[{"closingBracket": false]'); + + const result = await integration.getSampleData(); + + expect(result.ok).toBe(false); + }); + }); }); From 1bef29eacd686134f1f22a10c6c8dd31d3545cee Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 10:43:18 -0700 Subject: [PATCH 14/79] Add CatalogReader JavaDocs Signed-off-by: Simeon Widdis --- .../integrations/repository/catalog_reader.ts | 44 +++++++++++++++++-- .../repository/local_catalog_reader.ts | 19 ++++++-- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_reader.ts index 9683bcaf9b..c52fb9d0f0 100644 --- a/server/adaptors/integrations/repository/catalog_reader.ts +++ b/server/adaptors/integrations/repository/catalog_reader.ts @@ -6,9 +6,47 @@ type IntegrationPart = 'assets' | 'data' | 'schemas' | 'static'; interface CatalogReader { + /** + * Reads a file from the data source. + * + * @param filename The name of the file to read. + * @param type Optional. The type of integration part to read ('assets', 'data', 'schemas', or 'static'). + * @returns A Promise that resolves with the content of the file as a string. + */ readFile: (filename: string, type?: IntegrationPart) => Promise; + + /** + * Reads a file from the data source as raw binary data. + * + * @param filename The name of the file to read. + * @param type Optional. The type of integration part to read ('assets', 'data', 'schemas', or 'static'). + * @returns A Promise that resolves with the content of the file as a Buffer. + */ readFileRaw: (filename: string, type?: IntegrationPart) => Promise; - readDir: (filename: string) => Promise; - isDirectory: (filename: string) => Promise; - join: (filename: string) => CatalogReader; + + /** + * Reads the contents of a directory from the data source. + * + * @param dirname The name of the directory to read. + * @returns A Promise that resolves with an array of filenames within the directory. + */ + readDir: (dirname: string) => Promise; + + /** + * Checks if a given path on the data source is a directory. + * + * @param dirname The path to check. + * @returns A Promise that resolves with a boolean indicating whether the path is a directory or not. + */ + isDirectory: (dirname: string) => Promise; + + /** + * Creates a new CatalogReader instance with the specified subdirectory appended to the current directory. + * Since CatalogReaders sanitize given paths by default, + * this is useful for exploring nested data. + * + * @param subdirectory The path to append to the current directory. + * @returns A new CatalogReader instance. + */ + join: (subdirectory: string) => CatalogReader; } diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/local_catalog_reader.ts index e95f3c46c1..4f473022c5 100644 --- a/server/adaptors/integrations/repository/local_catalog_reader.ts +++ b/server/adaptors/integrations/repository/local_catalog_reader.ts @@ -14,14 +14,25 @@ import sanitize from 'sanitize-filename'; export class LocalCatalogReader implements CatalogReader { directory: string; + /** + * Creates a new LocalCatalogReader instance. + * + * @param directory The base directory from which to read files. This is not sanitized. + */ constructor(directory: string) { this.directory = directory; } - // Use before any call to `fs` - // Sanitizes filenames by default, manually prepend directories with a prefix if necessary - _prepare(filename: string, prefix?: string): string { - return path.join(this.directory, prefix ?? '.', sanitize(filename)); + /** + * Prepares a filename for use in filesystem operations by sanitizing and joining it with the base directory. + * This method is intended to be used before any filesystem-related call. + * + * @param filename The name of the file to prepare. + * @param subdir Optional. A subdirectory to prepend to the filename. Not sanitized. + * @returns The prepared path for the file, including the base directory and optional prefix. + */ + _prepare(filename: string, subdir?: string): string { + return path.join(this.directory, subdir ?? '.', sanitize(filename)); } async readFile(filename: string, type?: IntegrationPart): Promise { From a18cf47912e22f38a23cbe58f57b71272defe9a9 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 11:07:46 -0700 Subject: [PATCH 15/79] Repair integrations builder Signed-off-by: Simeon Widdis --- .../integrations/__test__/builder.test.ts | 65 ++++++++++++------- .../integrations/integrations_builder.ts | 24 +++++-- 2 files changed, 59 insertions(+), 30 deletions(-) diff --git a/server/adaptors/integrations/__test__/builder.test.ts b/server/adaptors/integrations/__test__/builder.test.ts index 81af1d6f69..6fa0d79185 100644 --- a/server/adaptors/integrations/__test__/builder.test.ts +++ b/server/adaptors/integrations/__test__/builder.test.ts @@ -93,13 +93,23 @@ describe('IntegrationInstanceBuilder', () => { ], }; - // Mock the implementation of the methods in the Integration class - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); - sampleIntegration.getAssets = jest.fn().mockResolvedValue({ savedObjects: remappedAssets }); - sampleIntegration.getConfig = jest.fn().mockResolvedValue({ + const mockTemplate: Partial = { name: 'integration-template', type: 'integration-type', - }); + assets: { + savedObjects: { + name: 'assets', + version: '1.0.0', + }, + }, + }; + + // Mock the implementation of the methods in the Integration class + sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: mockTemplate }); + sampleIntegration.getAssets = jest + .fn() + .mockResolvedValue({ ok: true, value: { savedObjects: remappedAssets } }); + sampleIntegration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: mockTemplate }); // Mock builder sub-methods const remapIDsSpy = jest.spyOn(builder, 'remapIDs'); @@ -121,22 +131,24 @@ describe('IntegrationInstanceBuilder', () => { dataSource: 'instance-datasource', name: 'instance-name', }; - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(false); + sampleIntegration.deepCheck = jest + .fn() + .mockResolvedValue({ ok: false, error: new Error('Mock error') }); - await expect(builder.build(sampleIntegration, options)).rejects.toThrowError( - 'Integration is not valid' - ); + await expect(builder.build(sampleIntegration, options)).rejects.toThrowError('Mock error'); }); - it('should reject with an error if getAssets throws an error', async () => { + it('should reject with an error if getAssets rejects', async () => { const options = { dataSource: 'instance-datasource', name: 'instance-name', }; const errorMessage = 'Failed to get assets'; - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); - sampleIntegration.getAssets = jest.fn().mockRejectedValue(new Error(errorMessage)); + sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: {} }); + sampleIntegration.getAssets = jest + .fn() + .mockResolvedValue({ ok: false, error: new Error(errorMessage) }); await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); }); @@ -153,22 +165,24 @@ describe('IntegrationInstanceBuilder', () => { }, ]; const errorMessage = 'Failed to post assets'; - sampleIntegration.deepCheck = jest.fn().mockResolvedValue(true); - sampleIntegration.getAssets = jest.fn().mockResolvedValue({ savedObjects: remappedAssets }); + sampleIntegration.deepCheck = jest.fn().mockResolvedValue({ ok: true, value: {} }); + sampleIntegration.getAssets = jest + .fn() + .mockResolvedValue({ ok: true, value: { savedObjects: remappedAssets } }); builder.postAssets = jest.fn().mockRejectedValue(new Error(errorMessage)); await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); }); - it('should reject with an error if getConfig returns null', async () => { - const options = { - dataSource: 'instance-datasource', - name: 'instance-name', - }; - sampleIntegration.getConfig = jest.fn().mockResolvedValue(null); + // it('should reject with an error if getConfig returns null', async () => { + // const options = { + // dataSource: 'instance-datasource', + // name: 'instance-name', + // }; + // sampleIntegration.getConfig = jest.fn().mockResolvedValue(null); - await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(); - }); + // await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(); + // }); }); describe('remapIDs', () => { @@ -264,8 +278,11 @@ describe('IntegrationInstanceBuilder', () => { it('should build an integration instance', async () => { const integration = { getConfig: jest.fn().mockResolvedValue({ - name: 'integration-template', - type: 'integration-type', + ok: true, + value: { + name: 'integration-template', + type: 'integration-type', + }, }), }; const refs = [ diff --git a/server/adaptors/integrations/integrations_builder.ts b/server/adaptors/integrations/integrations_builder.ts index b12e1a1321..117816187a 100644 --- a/server/adaptors/integrations/integrations_builder.ts +++ b/server/adaptors/integrations/integrations_builder.ts @@ -21,15 +21,21 @@ export class IntegrationInstanceBuilder { this.client = client; } - async build(integration: Integration, options: BuilderOptions): Promise { + build(integration: Integration, options: BuilderOptions): Promise { const instance = integration .deepCheck() .then((result) => { - if (!result) { - return Promise.reject(new Error('Integration is not valid')); + if (!result.ok) { + return Promise.reject(result.error); } + return integration.getAssets(); + }) + .then((assets) => { + if (!assets.ok) { + return Promise.reject(assets.error); + } + return assets.value; }) - .then(() => integration.getAssets()) .then((assets) => this.remapIDs(assets.savedObjects!)) .then((assets) => this.remapDataSource(assets, options.dataSource)) .then((assets) => this.postAssets(assets)) @@ -90,10 +96,16 @@ export class IntegrationInstanceBuilder { refs: AssetReference[], options: BuilderOptions ): Promise { - const config: IntegrationTemplate = (await integration.getConfig())!; + const config: Result = await integration.getConfig(); + console.error(config); + if (!config.ok) { + return Promise.reject( + new Error('Attempted to create instance with invalid template', config.error) + ); + } return Promise.resolve({ name: options.name, - templateName: config.name, + templateName: config.value.name, dataSource: options.dataSource, creationDate: new Date().toISOString(), assets: refs, From 047b14a96b71d62646cfc07e58bd2d304ff1afcd Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 11:09:12 -0700 Subject: [PATCH 16/79] Remove extra commented test Signed-off-by: Simeon Widdis --- server/adaptors/integrations/__test__/builder.test.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/adaptors/integrations/__test__/builder.test.ts b/server/adaptors/integrations/__test__/builder.test.ts index 6fa0d79185..84387ca8c6 100644 --- a/server/adaptors/integrations/__test__/builder.test.ts +++ b/server/adaptors/integrations/__test__/builder.test.ts @@ -173,16 +173,6 @@ describe('IntegrationInstanceBuilder', () => { await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(errorMessage); }); - - // it('should reject with an error if getConfig returns null', async () => { - // const options = { - // dataSource: 'instance-datasource', - // name: 'instance-name', - // }; - // sampleIntegration.getConfig = jest.fn().mockResolvedValue(null); - - // await expect(builder.build(sampleIntegration, options)).rejects.toThrowError(); - // }); }); describe('remapIDs', () => { From e65fe8b568135f26ccef3fa0a2eaf6597a1e6572 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 11:11:25 -0700 Subject: [PATCH 17/79] Remove unnecessary log statement Signed-off-by: Simeon Widdis --- server/adaptors/integrations/integrations_builder.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/adaptors/integrations/integrations_builder.ts b/server/adaptors/integrations/integrations_builder.ts index 117816187a..960912e121 100644 --- a/server/adaptors/integrations/integrations_builder.ts +++ b/server/adaptors/integrations/integrations_builder.ts @@ -97,7 +97,6 @@ export class IntegrationInstanceBuilder { options: BuilderOptions ): Promise { const config: Result = await integration.getConfig(); - console.error(config); if (!config.ok) { return Promise.reject( new Error('Attempted to create instance with invalid template', config.error) From 00ddb5ba35e15868d9966b70a05a6f1bb5ff4cfe Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 11:35:12 -0700 Subject: [PATCH 18/79] Repair getSchemas behavior to return correct type Let it be known at on this day, with this commit, I have truly grokked why we don't use `any` in typescript. Signed-off-by: Simeon Widdis --- server/adaptors/integrations/integrations_adaptor.ts | 4 +--- .../adaptors/integrations/integrations_kibana_backend.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/adaptors/integrations/integrations_adaptor.ts b/server/adaptors/integrations/integrations_adaptor.ts index cf7f4853e3..574a4d25dd 100644 --- a/server/adaptors/integrations/integrations_adaptor.ts +++ b/server/adaptors/integrations/integrations_adaptor.ts @@ -24,9 +24,7 @@ export interface IntegrationsAdaptor { getStatic: (templateName: string, path: string) => Promise; - getSchemas: ( - templateName: string - ) => Promise<{ mappings: { [key: string]: unknown }; schemas: { [key: string]: unknown } }>; + getSchemas: (templateName: string) => Promise<{ mappings: { [key: string]: unknown } }>; getAssets: (templateName: string) => Promise<{ savedObjects?: unknown }>; diff --git a/server/adaptors/integrations/integrations_kibana_backend.ts b/server/adaptors/integrations/integrations_kibana_backend.ts index b503390a24..494b6d1741 100644 --- a/server/adaptors/integrations/integrations_kibana_backend.ts +++ b/server/adaptors/integrations/integrations_kibana_backend.ts @@ -192,7 +192,7 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { return Promise.resolve(data.value); }; - getSchemas = async (templateName: string): Promise => { + getSchemas = async (templateName: string): Promise<{ mappings: { [key: string]: unknown } }> => { const integration = await this.repository.getIntegration(templateName); if (integration === null) { return Promise.reject({ @@ -200,7 +200,11 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { statusCode: 404, }); } - return Promise.resolve(integration.getSchemas()); + const result = await integration.getSchemas(); + if (result.ok) { + return result.value; + } + return Promise.reject(result.error); }; getAssets = async (templateName: string): Promise<{ savedObjects?: any }> => { From 8af7f4ab824577c07e39ac7aeb96972cacb49729 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 11:52:50 -0700 Subject: [PATCH 19/79] Add tests for getSchemas Signed-off-by: Simeon Widdis --- .../__test__/kibana_backend.test.ts | 55 ++++++++++++++++++- .../integrations_kibana_backend.ts | 19 ++++--- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/server/adaptors/integrations/__test__/kibana_backend.test.ts b/server/adaptors/integrations/__test__/kibana_backend.test.ts index c14ef6f53f..da8ade9615 100644 --- a/server/adaptors/integrations/__test__/kibana_backend.test.ts +++ b/server/adaptors/integrations/__test__/kibana_backend.test.ts @@ -290,7 +290,7 @@ describe('IntegrationsKibanaBackend', () => { expect(result).toEqual(assetData); }); - it('should reject with a 404 if asset is not found', async () => { + it('should reject with a 404 if integration is not found', async () => { const templateName = 'template1'; const staticPath = 'path/to/static'; mockRepository.getIntegration.mockResolvedValue(null); @@ -300,6 +300,59 @@ describe('IntegrationsKibanaBackend', () => { 404 ); }); + + it('should reject with a 404 if static data is not found', async () => { + const templateName = 'template1'; + const staticPath = 'path/to/static'; + mockRepository.getIntegration.mockResolvedValue({ + getStatic: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getStatic(templateName, staticPath)).rejects.toHaveProperty( + 'statusCode', + 404 + ); + }); + }); + + describe('getSchemas', () => { + it('should get schema data', async () => { + const templateName = 'template1'; + const staticPath = 'path/to/static'; + const schemaData = { mappings: { test: {} } }; + const integration = { + getSchemas: jest.fn().mockResolvedValue({ ok: true, value: schemaData }), + }; + mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + + const result = await backend.getSchemas(templateName); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getSchemas).toHaveBeenCalled(); + expect(result).toEqual(schemaData); + }); + + it('should reject with a 404 if integration is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getSchemas(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with a 404 if schema data is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue({ + getSchemas: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getSchemas(templateName)).rejects.toHaveProperty('statusCode', 404); + }); }); describe('getAssetStatus', () => { diff --git a/server/adaptors/integrations/integrations_kibana_backend.ts b/server/adaptors/integrations/integrations_kibana_backend.ts index 494b6d1741..da99efa7d7 100644 --- a/server/adaptors/integrations/integrations_kibana_backend.ts +++ b/server/adaptors/integrations/integrations_kibana_backend.ts @@ -183,13 +183,14 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { }); } const data = await integration.getStatic(staticPath); - if (!data.ok) { - return Promise.reject({ - message: `Asset ${staticPath} not found`, - statusCode: 404, - }); + if (data.ok) { + return data.value; } - return Promise.resolve(data.value); + const is404 = (data.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: data.error.message, + statusCode: is404 ? 404 : 500, + }); }; getSchemas = async (templateName: string): Promise<{ mappings: { [key: string]: unknown } }> => { @@ -204,7 +205,11 @@ export class IntegrationsKibanaBackend implements IntegrationsAdaptor { if (result.ok) { return result.value; } - return Promise.reject(result.error); + const is404 = (result.error as { code?: string }).code === 'ENOENT'; + return Promise.reject({ + message: result.error.message, + statusCode: is404 ? 404 : 500, + }); }; getAssets = async (templateName: string): Promise<{ savedObjects?: any }> => { From a31963da5b631427647568b3b4a767a8199992ad Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 11:57:32 -0700 Subject: [PATCH 20/79] Add tests for asset and sample data backend methods Signed-off-by: Simeon Widdis --- .../__test__/kibana_backend.test.ts | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/server/adaptors/integrations/__test__/kibana_backend.test.ts b/server/adaptors/integrations/__test__/kibana_backend.test.ts index da8ade9615..4b333e081d 100644 --- a/server/adaptors/integrations/__test__/kibana_backend.test.ts +++ b/server/adaptors/integrations/__test__/kibana_backend.test.ts @@ -321,7 +321,6 @@ describe('IntegrationsKibanaBackend', () => { describe('getSchemas', () => { it('should get schema data', async () => { const templateName = 'template1'; - const staticPath = 'path/to/static'; const schemaData = { mappings: { test: {} } }; const integration = { getSchemas: jest.fn().mockResolvedValue({ ok: true, value: schemaData }), @@ -355,6 +354,78 @@ describe('IntegrationsKibanaBackend', () => { }); }); + describe('getAssets', () => { + it('should get asset data', async () => { + const templateName = 'template1'; + const assetData = { savedObjects: [{ test: true }] }; + const integration = { + getAssets: jest.fn().mockResolvedValue({ ok: true, value: assetData }), + }; + mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + + const result = await backend.getAssets(templateName); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getAssets).toHaveBeenCalled(); + expect(result).toEqual(assetData); + }); + + it('should reject with a 404 if integration is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getAssets(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with a 404 if asset data is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue({ + getAssets: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getAssets(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + }); + + describe('getSampleData', () => { + it('should get sample data', async () => { + const templateName = 'template1'; + const sampleData = { sampleData: [{ test: true }] }; + const integration = { + getSampleData: jest.fn().mockResolvedValue({ ok: true, value: sampleData }), + }; + mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + + const result = await backend.getSampleData(templateName); + + expect(mockRepository.getIntegration).toHaveBeenCalledWith(templateName); + expect(integration.getSampleData).toHaveBeenCalled(); + expect(result).toEqual(sampleData); + }); + + it('should reject with a 404 if integration is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue(null); + + await expect(backend.getSampleData(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + + it('should reject with a 404 if sample data is not found', async () => { + const templateName = 'template1'; + mockRepository.getIntegration.mockResolvedValue({ + getSampleData: jest.fn().mockResolvedValue({ + ok: false, + error: { message: 'Not found', code: 'ENOENT' }, + }), + } as any); + + await expect(backend.getSampleData(templateName)).rejects.toHaveProperty('statusCode', 404); + }); + }); + describe('getAssetStatus', () => { it('should return "available" if all assets are available', async () => { const assets = [ From feeaef59e3146ca4fbd9d29995b6d2ab4e22c90a Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 23 Aug 2023 17:08:46 -0700 Subject: [PATCH 21/79] Break flyout validation methods out of constructing method Signed-off-by: Simeon Widdis --- .../components/add_integration_flyout.tsx | 262 ++++++++++-------- 1 file changed, 145 insertions(+), 117 deletions(-) diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index 29cc8921e6..0cc0924545 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -20,7 +20,7 @@ import { EuiTitle, } from '@elastic/eui'; import React, { useState } from 'react'; -import { HttpStart } from '../../../../../../src/core/public'; +import { HttpSetup, HttpStart } from '../../../../../../src/core/public'; import { useToast } from '../../../../public/components/common/toast'; interface IntegrationFlyoutProps { @@ -31,48 +31,65 @@ interface IntegrationFlyoutProps { http: HttpStart; } -export const doTypeValidation = (toCheck: any, required: any): boolean => { +type ValidationResult = { ok: true } | { ok: false; errors: string[] }; + +export const doTypeValidation = ( + toCheck: { type?: string; properties?: object }, + required: { type?: string; properties?: object } +): ValidationResult => { if (!required.type) { - return true; + return { ok: true }; } if (required.type === 'object') { - return Boolean(toCheck.properties); + if (Boolean(toCheck.properties)) { + return { ok: true }; + } + return { ok: false, errors: ["'object' type must have properties."] }; } - return required.type === toCheck.type; + if (required.type !== toCheck.type) { + return { ok: false, errors: [`Type mismatch: '${required.type}' and '${toCheck.type}'`] }; + } + return { ok: true }; }; export const doNestedPropertyValidation = ( - toCheck: { type?: string; properties?: any }, - required: { type?: string; properties?: any } -): boolean => { - if (!doTypeValidation(toCheck, required)) { - return false; + toCheck: { type?: string; properties?: { [key: string]: object } }, + required: { type?: string; properties?: { [key: string]: object } } +): ValidationResult => { + const typeCheck = doTypeValidation(toCheck, required); + if (!typeCheck.ok) { + return typeCheck; } - if (required.properties) { - return Object.keys(required.properties).every((property: string) => { - if (!toCheck.properties[property]) { - return false; - } - return doNestedPropertyValidation( - toCheck.properties[property], - required.properties[property] - ); - }); + for (const property of Object.keys(required.properties ?? {})) { + if (!Object.hasOwn(toCheck.properties ?? {}, property)) { + return { ok: false, errors: [`Missing field '${property}'`] }; + } + // Both are safely non-null after above checks. + const nested = doNestedPropertyValidation( + toCheck.properties![property], + required.properties![property] + ); + if (!nested.ok) { + return nested; + } } - return true; + return { ok: true }; }; export const doPropertyValidation = ( rootType: string, dataSourceProps: { [key: string]: { properties?: any } }, requiredMappings: { [key: string]: { template: { mappings: { properties?: any } } } } -): boolean => { +): ValidationResult => { // Check root object type (without dependencies) for (const [key, value] of Object.entries( requiredMappings[rootType].template.mappings.properties )) { - if (!dataSourceProps[key] || !doNestedPropertyValidation(dataSourceProps[key], value as any)) { - return false; + if ( + !dataSourceProps[key] || + !doNestedPropertyValidation(dataSourceProps[key], value as any).ok + ) { + return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; } } // Check nested dependencies @@ -82,12 +99,104 @@ export const doPropertyValidation = ( } if ( !dataSourceProps[key] || - !doNestedPropertyValidation(dataSourceProps[key], value.template.mappings.properties) + !doNestedPropertyValidation(dataSourceProps[key], value.template.mappings.properties).ok ) { - return false; + return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; } } - return true; + return { ok: true }; +}; + +// Returns true if the data stream is a legal name. +// Appends any additional validation errors to the provided errors array. +const checkDataSourceName = ( + targetDataSource: string, + integrationType: string +): ValidationResult => { + let errors: string[] = []; + if (!Boolean(targetDataSource.match(/^[a-z\d\.][a-z\d\._\-\*]*$/))) { + errors = errors.concat('This is not a valid index name.'); + return { ok: false, errors }; + } + const nameValidity: boolean = Boolean( + targetDataSource.match(new RegExp(`^ss4o_${integrationType}-[^\\-]+-[^\\-]+`)) + ); + if (!nameValidity) { + errors = errors.concat('This index does not match the suggested naming convention.'); + return { ok: false, errors }; + } + return { ok: true }; +}; + +const fetchDataSourceMappings = async ( + targetDataSource: string, + http: HttpSetup +): Promise<{ [key: string]: { properties: any } } | null> => { + return http + .post('/api/console/proxy', { + query: { + path: `${targetDataSource}/_mapping`, + method: 'GET', + }, + }) + .then((response) => { + // Un-nest properties by a level for caller convenience + Object.keys(response).forEach((key) => { + response[key].properties = response[key].mappings.properties; + }); + return response; + }) + .catch((err: any) => { + console.error(err); + return null; + }); +}; + +const fetchIntegrationMappings = async ( + targetName: string, + http: HttpSetup +): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { + return http + .get(`/api/integrations/repository/${targetName}/schema`) + .then((response) => { + if (response.statusCode && response.statusCode !== 200) { + throw new Error('Failed to retrieve Integration schema', { cause: response }); + } + return response.data.mappings; + }) + .catch((err: any) => { + console.error(err); + return null; + }); +}; + +const doExistingDataSourceValidation = async ( + targetDataSource: string, + integrationName: string, + integrationType: string, + http: HttpSetup +): Promise => { + const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); + if (!dataSourceNameCheck.ok) { + return dataSourceNameCheck.errors; + } + const [dataSourceMappings, integrationMappings] = await Promise.all([ + fetchDataSourceMappings(targetDataSource, http), + fetchIntegrationMappings(integrationName, http), + ]); + if (!dataSourceMappings) { + return ['Provided data stream could not be retrieved']; + } + if (!integrationMappings) { + return ['Failed to retrieve integration schema information']; + } + const validationResult = Object.values(dataSourceMappings).every((value) => + doPropertyValidation(integrationType, value.properties, integrationMappings) + ); + if (!validationResult) { + return ['The provided index does not match the schema']; + } + return []; }; export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { @@ -110,93 +219,6 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { setName(e.target.value); }; - // Returns true if the data stream is a legal name. - // Appends any additional validation errors to the provided errors array. - const checkDataSourceName = (targetDataSource: string, validationErrors: string[]): boolean => { - if (!Boolean(targetDataSource.match(/^[a-z\d\.][a-z\d\._\-\*]*$/))) { - validationErrors.push('This is not a valid index name.'); - setErrors(validationErrors); - return false; - } - const nameValidity: boolean = Boolean( - targetDataSource.match(new RegExp(`^ss4o_${integrationType}-[^\\-]+-[^\\-]+`)) - ); - if (!nameValidity) { - validationErrors.push('This index does not match the suggested naming convention.'); - setErrors(validationErrors); - } - return true; - }; - - const fetchDataSourceMappings = async ( - targetDataSource: string - ): Promise<{ [key: string]: { properties: any } } | null> => { - return http - .post('/api/console/proxy', { - query: { - path: `${targetDataSource}/_mapping`, - method: 'GET', - }, - }) - .then((response) => { - // Un-nest properties by a level for caller convenience - Object.keys(response).forEach((key) => { - response[key].properties = response[key].mappings.properties; - }); - return response; - }) - .catch((err: any) => { - console.error(err); - return null; - }); - }; - - const fetchIntegrationMappings = async ( - targetName: string - ): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { - return http - .get(`/api/integrations/repository/${targetName}/schema`) - .then((response) => { - if (response.statusCode && response.statusCode !== 200) { - throw new Error('Failed to retrieve Integration schema', { cause: response }); - } - return response.data.mappings; - }) - .catch((err: any) => { - console.error(err); - return null; - }); - }; - - const doExistingDataSourceValidation = async (targetDataSource: string): Promise => { - const validationErrors: string[] = []; - if (!checkDataSourceName(targetDataSource, validationErrors)) { - return false; - } - const [dataSourceMappings, integrationMappings] = await Promise.all([ - fetchDataSourceMappings(targetDataSource), - fetchIntegrationMappings(integrationName), - ]); - if (!dataSourceMappings) { - validationErrors.push('Provided data stream could not be retrieved'); - setErrors(validationErrors); - return false; - } - if (!integrationMappings) { - validationErrors.push('Failed to retrieve integration schema information'); - setErrors(validationErrors); - return false; - } - const validationResult = Object.values(dataSourceMappings).every((value) => - doPropertyValidation(integrationType, value.properties, integrationMappings) - ); - if (!validationResult) { - validationErrors.push('The provided index does not match the schema'); - setErrors(validationErrors); - } - return validationResult; - }; - const formContent = () => { return (
@@ -227,11 +249,17 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { { - const validationResult = await doExistingDataSourceValidation(dataSource); - if (validationResult) { + const validationResult = await doExistingDataSourceValidation( + dataSource, + integrationName, + integrationType, + http + ); + setErrors(validationResult); + if (validationResult.length === 0) { setToast('Index name or wildcard pattern is valid', 'success'); } - setDataSourceValid(validationResult); + setDataSourceValid(validationResult.length === 0); }} disabled={dataSource.length === 0} > From b644d112929a4018642964e7ee94bf37368a60f9 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 24 Aug 2023 10:36:35 -0700 Subject: [PATCH 22/79] Add tests for extracted flyout methods Signed-off-by: Simeon Widdis --- .../added_integration_flyout.test.tsx.snap | 7 + .../added_integration_flyout.test.tsx | 343 +++++++++++++++++- .../__tests__/mapping_validation.test.ts | 176 --------- .../components/add_integration_flyout.tsx | 12 +- 4 files changed, 354 insertions(+), 184 deletions(-) delete mode 100644 public/components/integrations/components/__tests__/mapping_validation.test.ts diff --git a/public/components/integrations/components/__tests__/__snapshots__/added_integration_flyout.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/added_integration_flyout.test.tsx.snap index 9ae4ce75ec..3ba54c92b4 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/added_integration_flyout.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/added_integration_flyout.test.tsx.snap @@ -2,7 +2,14 @@ exports[`Add Integration Flyout Test Renders add integration flyout with dummy integration name 1`] = ` diff --git a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx index 7f5280652a..26548065e2 100644 --- a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx +++ b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx @@ -6,15 +6,37 @@ import { configure, mount } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import { waitFor } from '@testing-library/react'; -import { AddIntegrationFlyout } from '../add_integration_flyout'; +import { + AddIntegrationFlyout, + checkDataSourceName, + doTypeValidation, + doNestedPropertyValidation, + doPropertyValidation, + fetchDataSourceMappings, + fetchIntegrationMappings, + doExistingDataSourceValidation, +} from '../add_integration_flyout'; +import * as add_integration_flyout from '../add_integration_flyout'; import React from 'react'; +import { HttpSetup } from '../../../../../../../src/core/public'; describe('Add Integration Flyout Test', () => { configure({ adapter: new Adapter() }); it('Renders add integration flyout with dummy integration name', async () => { const wrapper = mount( - + ) as HttpSetup + } + /> ); await waitFor(() => { @@ -22,3 +44,320 @@ describe('Add Integration Flyout Test', () => { }); }); }); + +describe('doTypeValidation', () => { + it('should return true if required type is not specified', () => { + const toCheck = { type: 'string' }; + const required = {}; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return true if types match', () => { + const toCheck = { type: 'string' }; + const required = { type: 'string' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return true if object has properties', () => { + const toCheck = { properties: { prop1: { type: 'string' } } }; + const required = { type: 'object' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return false if types do not match', () => { + const toCheck = { type: 'string' }; + const required = { type: 'number' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); +}); + +describe('doNestedPropertyValidation', () => { + it('should return true if type validation passes and no properties are required', () => { + const toCheck = { type: 'string' }; + const required = { type: 'string' }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return false if type validation fails', () => { + const toCheck = { type: 'string' }; + const required = { type: 'number' }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); + + it('should return false if a required property is missing', () => { + const toCheck = { type: 'object', properties: { prop1: { type: 'string' } } }; + const required = { + type: 'object', + properties: { prop1: { type: 'string' }, prop2: { type: 'number' } }, + }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); + + it('should return true if all required properties pass validation', () => { + const toCheck = { + type: 'object', + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }; + const required = { + type: 'object', + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); +}); + +describe('doPropertyValidation', () => { + it('should return true if all properties pass validation', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(true); + }); + + it('should return false if a property fails validation', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'boolean' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(false); + }); + + it('should return false if a required nested property is missing', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(false); + }); +}); + +describe('checkDataSourceName', () => { + it('Filters out invalid index names', () => { + const result = checkDataSourceName('ss4o_logs-no-exclams!', 'logs'); + + expect(result.ok).toBe(false); + }); + + it('Filters out incorrectly typed indices', () => { + const result = checkDataSourceName('ss4o_metrics-test-test', 'logs'); + + expect(result.ok).toBe(false); + }); + + it('Accepts correct indices', () => { + const result = checkDataSourceName('ss4o_logs-test-test', 'logs'); + + expect(result.ok).toBe(true); + }); +}); + +describe('fetchDataSourceMappings', () => { + it('Retrieves mappings', async () => { + const mockHttp = { + post: jest.fn().mockResolvedValue({ + source1: { mappings: { properties: { test: true } } }, + source2: { mappings: { properties: { test: true } } }, + }), + } as Partial; + + const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); + + await expect(result).resolves.toMatchObject({ + source1: { properties: { test: true } }, + source2: { properties: { test: true } }, + }); + }); + + it('Catches errors', async () => { + const mockHttp = { + post: jest.fn().mockRejectedValue(new Error('Mock error')), + } as Partial; + + const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); +}); + +describe('fetchIntegrationMappings', () => { + it('Returns schema mappings', async () => { + const mockHttp = { + get: jest.fn().mockResolvedValue({ data: { mappings: { test: true } }, statusCode: 200 }), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toStrictEqual({ test: true }); + }); + + it('Returns null if response fails', async () => { + const mockHttp = { + get: jest.fn().mockResolvedValue({ statusCode: 404 }), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); + + it('Catches request error', async () => { + const mockHttp = { + get: jest.fn().mockRejectedValue(new Error('mock error')), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); +}); + +describe('doExistingDataSourceValidation', () => { + it('Catches and returns checkDataSourceName errors', async () => { + const mockHttp = {} as Partial; + jest + .spyOn(add_integration_flyout, 'checkDataSourceName') + .mockReturnValue({ ok: false, errors: ['mock'] }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveLength(1); + }); + + it('Catches data stream fetch errors', async () => { + const mockHttp = {} as Partial; + jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest.spyOn(add_integration_flyout, 'fetchDataSourceMappings').mockResolvedValue(null); + jest + .spyOn(add_integration_flyout, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveLength(1); + }); + + it('Catches integration fetch errors', async () => { + const mockHttp = {} as Partial; + jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(add_integration_flyout, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest.spyOn(add_integration_flyout, 'fetchIntegrationMappings').mockResolvedValue(null); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveLength(1); + }); + + it('Catches type validation issues', async () => { + const mockHttp = {} as Partial; + jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(add_integration_flyout, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest + .spyOn(add_integration_flyout, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + jest + .spyOn(add_integration_flyout, 'doPropertyValidation') + .mockReturnValue({ ok: false, errors: ['mock'] }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveLength(1); + }); + + it('Returns no errors if everything passes', async () => { + const mockHttp = {} as Partial; + jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(add_integration_flyout, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest + .spyOn(add_integration_flyout, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + jest.spyOn(add_integration_flyout, 'doPropertyValidation').mockReturnValue({ ok: true }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveLength(0); + }); +}); diff --git a/public/components/integrations/components/__tests__/mapping_validation.test.ts b/public/components/integrations/components/__tests__/mapping_validation.test.ts deleted file mode 100644 index 4a02058cf4..0000000000 --- a/public/components/integrations/components/__tests__/mapping_validation.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import { - doTypeValidation, - doNestedPropertyValidation, - doPropertyValidation, -} from '../add_integration_flyout'; - -describe('Validation', () => { - describe('doTypeValidation', () => { - it('should return true if required type is not specified', () => { - const toCheck = { type: 'string' }; - const required = {}; - - const result = doTypeValidation(toCheck, required); - - expect(result).toBe(true); - }); - - it('should return true if types match', () => { - const toCheck = { type: 'string' }; - const required = { type: 'string' }; - - const result = doTypeValidation(toCheck, required); - - expect(result).toBe(true); - }); - - it('should return true if object has properties', () => { - const toCheck = { properties: { prop1: { type: 'string' } } }; - const required = { type: 'object' }; - - const result = doTypeValidation(toCheck, required); - - expect(result).toBe(true); - }); - - it('should return false if types do not match', () => { - const toCheck = { type: 'string' }; - const required = { type: 'number' }; - - const result = doTypeValidation(toCheck, required); - - expect(result).toBe(false); - }); - }); - - describe('doNestedPropertyValidation', () => { - it('should return true if type validation passes and no properties are required', () => { - const toCheck = { type: 'string' }; - const required = { type: 'string' }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result).toBe(true); - }); - - it('should return false if type validation fails', () => { - const toCheck = { type: 'string' }; - const required = { type: 'number' }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result).toBe(false); - }); - - it('should return false if a required property is missing', () => { - const toCheck = { type: 'object', properties: { prop1: { type: 'string' } } }; - const required = { - type: 'object', - properties: { prop1: { type: 'string' }, prop2: { type: 'number' } }, - }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result).toBe(false); - }); - - it('should return true if all required properties pass validation', () => { - const toCheck = { - type: 'object', - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }; - const required = { - type: 'object', - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result).toBe(true); - }); - }); - - describe('doPropertyValidation', () => { - it('should return true if all properties pass validation', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result).toBe(true); - }); - - it('should return false if a property fails validation', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'boolean' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result).toBe(false); - }); - - it('should return false if a required nested property is missing', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result).toBe(false); - }); - }); -}); diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index 0cc0924545..515d237ee6 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -109,7 +109,7 @@ export const doPropertyValidation = ( // Returns true if the data stream is a legal name. // Appends any additional validation errors to the provided errors array. -const checkDataSourceName = ( +export const checkDataSourceName = ( targetDataSource: string, integrationType: string ): ValidationResult => { @@ -128,7 +128,7 @@ const checkDataSourceName = ( return { ok: true }; }; -const fetchDataSourceMappings = async ( +export const fetchDataSourceMappings = async ( targetDataSource: string, http: HttpSetup ): Promise<{ [key: string]: { properties: any } } | null> => { @@ -152,7 +152,7 @@ const fetchDataSourceMappings = async ( }); }; -const fetchIntegrationMappings = async ( +export const fetchIntegrationMappings = async ( targetName: string, http: HttpSetup ): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { @@ -170,7 +170,7 @@ const fetchIntegrationMappings = async ( }); }; -const doExistingDataSourceValidation = async ( +export const doExistingDataSourceValidation = async ( targetDataSource: string, integrationName: string, integrationType: string, @@ -190,8 +190,8 @@ const doExistingDataSourceValidation = async ( if (!integrationMappings) { return ['Failed to retrieve integration schema information']; } - const validationResult = Object.values(dataSourceMappings).every((value) => - doPropertyValidation(integrationType, value.properties, integrationMappings) + const validationResult = Object.values(dataSourceMappings).every( + (value) => doPropertyValidation(integrationType, value.properties, integrationMappings).ok ); if (!validationResult) { return ['The provided index does not match the schema']; From 614613ac267af188607f7daf3c0e8c6b66596dda Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 24 Aug 2023 10:48:05 -0700 Subject: [PATCH 23/79] Switch validation method to use ValidationResult Signed-off-by: Simeon Widdis --- .../added_integration_flyout.test.tsx | 10 +++++----- .../components/add_integration_flyout.tsx | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx index 26548065e2..ae3b609f87 100644 --- a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx +++ b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx @@ -298,7 +298,7 @@ describe('doExistingDataSourceValidation', () => { const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - await expect(result).resolves.toHaveLength(1); + await expect(result).resolves.toHaveProperty('ok', false); }); it('Catches data stream fetch errors', async () => { @@ -311,7 +311,7 @@ describe('doExistingDataSourceValidation', () => { const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - await expect(result).resolves.toHaveLength(1); + await expect(result).resolves.toHaveProperty('ok', false); }); it('Catches integration fetch errors', async () => { @@ -324,7 +324,7 @@ describe('doExistingDataSourceValidation', () => { const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - await expect(result).resolves.toHaveLength(1); + await expect(result).resolves.toHaveProperty('ok', false); }); it('Catches type validation issues', async () => { @@ -342,7 +342,7 @@ describe('doExistingDataSourceValidation', () => { const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - await expect(result).resolves.toHaveLength(1); + await expect(result).resolves.toHaveProperty('ok', false); }); it('Returns no errors if everything passes', async () => { @@ -358,6 +358,6 @@ describe('doExistingDataSourceValidation', () => { const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - await expect(result).resolves.toHaveLength(0); + await expect(result).resolves.toHaveProperty('ok', true); }); }); diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index 515d237ee6..19544a9efc 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -175,28 +175,28 @@ export const doExistingDataSourceValidation = async ( integrationName: string, integrationType: string, http: HttpSetup -): Promise => { +): Promise => { const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); if (!dataSourceNameCheck.ok) { - return dataSourceNameCheck.errors; + return dataSourceNameCheck; } const [dataSourceMappings, integrationMappings] = await Promise.all([ fetchDataSourceMappings(targetDataSource, http), fetchIntegrationMappings(integrationName, http), ]); if (!dataSourceMappings) { - return ['Provided data stream could not be retrieved']; + return { ok: false, errors: ['Provided data stream could not be retrieved'] }; } if (!integrationMappings) { - return ['Failed to retrieve integration schema information']; + return { ok: false, errors: ['Failed to retrieve integration schema information'] }; } const validationResult = Object.values(dataSourceMappings).every( (value) => doPropertyValidation(integrationType, value.properties, integrationMappings).ok ); if (!validationResult) { - return ['The provided index does not match the schema']; + return { ok: false, errors: ['The provided index does not match the schema'] }; } - return []; + return { ok: true }; }; export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { @@ -255,11 +255,11 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { integrationType, http ); - setErrors(validationResult); - if (validationResult.length === 0) { + if (validationResult.ok) { setToast('Index name or wildcard pattern is valid', 'success'); } - setDataSourceValid(validationResult.length === 0); + setDataSourceValid(validationResult.ok); + setErrors(!validationResult.ok ? validationResult.errors : []); }} disabled={dataSource.length === 0} > From ab30ed0f002b94e773dde7ddffcdebcf3db1208c Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 6 Sep 2023 15:03:25 -0700 Subject: [PATCH 24/79] Swap out flyout for hello-world setup page Signed-off-by: Simeon Widdis --- .../integrations/components/integration.tsx | 16 +--------------- .../components/setup_integration.tsx | 11 +++++++++++ public/components/integrations/home.tsx | 4 +++- 3 files changed, 15 insertions(+), 16 deletions(-) create mode 100644 public/components/integrations/components/setup_integration.tsx diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index 28257400b9..e4404b7e30 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -27,7 +27,6 @@ import { useToast } from '../../../../public/components/common/toast'; export function Integration(props: AvailableIntegrationProps) { const { http, integrationTemplateId, chrome } = props; - const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); const { setToast } = useToast(); const [integration, setIntegration] = useState({} as { name: string; type: string }); @@ -280,7 +279,7 @@ export function Integration(props: AvailableIntegrationProps) { {IntegrationOverview({ integration, showFlyout: () => { - setIsFlyoutVisible(true); + window.location.hash = '#/available/nginx/setup'; }, setUpSample: () => { addIntegrationRequest(true, integrationTemplateId); @@ -299,19 +298,6 @@ export function Integration(props: AvailableIntegrationProps) { : IntegrationFields({ integration, integrationMapping })} - {isFlyoutVisible && ( - { - setIsFlyoutVisible(false); - }} - onCreate={(name, dataSource) => { - addIntegrationRequest(false, integrationTemplateId, name, dataSource); - }} - integrationName={integrationTemplateId} - integrationType={integration.type} - http={http} - /> - )} ); } diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx new file mode 100644 index 0000000000..9af6d0efcb --- /dev/null +++ b/public/components/integrations/components/setup_integration.tsx @@ -0,0 +1,11 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EuiPageHeader } from '@elastic/eui'; +import React from 'react'; + +export function SetupIntegrationStepsView() { + return Hello, World!; +} diff --git a/public/components/integrations/home.tsx b/public/components/integrations/home.tsx index 3fa85b98dc..875497cb0c 100644 --- a/public/components/integrations/home.tsx +++ b/public/components/integrations/home.tsx @@ -13,6 +13,7 @@ import { ChromeBreadcrumb } from '../../../../../src/core/public'; import { AvailableIntegrationOverviewPage } from './components/available_integration_overview_page'; import { AddedIntegrationOverviewPage } from './components/added_integration_overview_page'; import { AddedIntegration } from './components/added_integration'; +import { SetupIntegrationStepsView } from './components/setup_integration'; export type AppAnalyticsCoreDeps = TraceAnalyticsCoreDeps; @@ -54,7 +55,7 @@ export const Home = (props: HomeProps) => { /> ( { /> )} /> + } />
From 85361f80aa5987c1424fc8db50c0280a753b76ab Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 6 Sep 2023 17:09:45 -0700 Subject: [PATCH 25/79] Add basic step incrementing Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 90 ++++++++++++++++++- public/components/integrations/home.tsx | 4 +- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 9af6d0efcb..8b65e59b53 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -3,9 +3,91 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiPageHeader } from '@elastic/eui'; -import React from 'react'; +import { + EuiBottomBar, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiPage, + EuiPageBody, + EuiSteps, + EuiText, + OuiFlexGroup, + OuiFlexItem, +} from '@elastic/eui'; +import { EuiHeader } from '@opensearch-project/oui'; +import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; +import React, { useState } from 'react'; -export function SetupIntegrationStepsView() { - return Hello, World!; +const STEPS: EuiContainedStepProps[] = [ + { title: 'Name Integration', children: }, + { title: 'Select index or data source for integration', children: }, + { title: 'Review associated index with data from table', children: }, + { title: 'Select integration assets', children: }, +]; + +const getSteps = (activeStep: number): EuiContainedStepProps[] => { + return STEPS.map((step, idx) => { + let status: string = ''; + if (idx < activeStep) { + status = 'complete'; + } + if (idx > activeStep) { + status = 'disabled'; + } + return Object.assign({}, step, { status }); + }); +}; + +function SetupIntegrationStepOne() { + return This is step one.; +} + +function SetupIntegrationStepTwo() { + return This is step two.; +} + +function SetupIntegrationStepThree() { + return This is step three.; +} + +function SetupIntegrationStepFour() { + return This is step four.; +} + +function SetupIntegrationStep(activeStep: number) { + switch (activeStep) { + case 0: + return SetupIntegrationStepOne(); + case 1: + return SetupIntegrationStepTwo(); + case 2: + return SetupIntegrationStepThree(); + case 3: + return SetupIntegrationStepFour(); + default: + return Something went wrong...; + } +} + +export function SetupIntegrationStepsPage() { + const [step, setStep] = useState(0); + + return ( + + + + + + + {SetupIntegrationStep(step)} + + + Step Navigation + setStep(Math.max(step - 1, 0))}>Previous Step + setStep(Math.min(step + 1, 3))}>Next Step + + + + ); } diff --git a/public/components/integrations/home.tsx b/public/components/integrations/home.tsx index 875497cb0c..67ec7e74f1 100644 --- a/public/components/integrations/home.tsx +++ b/public/components/integrations/home.tsx @@ -13,7 +13,7 @@ import { ChromeBreadcrumb } from '../../../../../src/core/public'; import { AvailableIntegrationOverviewPage } from './components/available_integration_overview_page'; import { AddedIntegrationOverviewPage } from './components/added_integration_overview_page'; import { AddedIntegration } from './components/added_integration'; -import { SetupIntegrationStepsView } from './components/setup_integration'; +import { SetupIntegrationStepsPage } from './components/setup_integration'; export type AppAnalyticsCoreDeps = TraceAnalyticsCoreDeps; @@ -63,7 +63,7 @@ export const Home = (props: HomeProps) => { /> )} /> - } /> + } /> From af2183112e1482afa52e0c83bd57073385d04e10 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 7 Sep 2023 11:15:57 -0700 Subject: [PATCH 26/79] Add basic field skeleton for each step Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 207 ++++++++++++++++-- 1 file changed, 193 insertions(+), 14 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 8b65e59b53..da1e3d73db 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -6,16 +6,24 @@ import { EuiBottomBar, EuiButton, + EuiFieldText, EuiFlexGroup, EuiFlexItem, + EuiForm, + EuiFormRow, + EuiLink, + EuiHeader, EuiPage, EuiPageBody, + EuiSelect, + EuiSelectOption, EuiSteps, + EuiSpacer, EuiText, - OuiFlexGroup, - OuiFlexItem, + EuiTitle, + EuiRadioGroup, + EuiTextColor, } from '@elastic/eui'; -import { EuiHeader } from '@opensearch-project/oui'; import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; import React, { useState } from 'react'; @@ -26,6 +34,11 @@ const STEPS: EuiContainedStepProps[] = [ { title: 'Select integration assets', children: }, ]; +const ALLOWED_FILE_TYPES: EuiSelectOption[] = [ + { value: 'parquet', text: 'parquet' }, + { value: 'json', text: 'json' }, +]; + const getSteps = (activeStep: number): EuiContainedStepProps[] => { return STEPS.map((step, idx) => { let status: string = ''; @@ -40,22 +53,173 @@ const getSteps = (activeStep: number): EuiContainedStepProps[] => { }; function SetupIntegrationStepOne() { - return This is step one.; + return ( + + +

{STEPS[0].title}

+ + + + + + ); } function SetupIntegrationStepTwo() { - return This is step two.; + return ( + + +

{STEPS[1].title}

+ + + + + + + + + + + + + + + ); } function SetupIntegrationStepThree() { - return This is step three.; + return ( + + +

{STEPS[2].title}

+ + + + + + View table + + ); } -function SetupIntegrationStepFour() { - return This is step four.; +function SetupIntegrationStepFour( + selectAsset: string, + setSelectAsset: React.Dispatch>, + selectQuery: string, + setSelectQuery: React.Dispatch> +) { + return ( + + +

{STEPS[3].title}

+ + + + None{': '} + + Set up indices, but don't install any assets. + + + ), + }, + { + id: 'queries', + label: ( + + Minimal{': '} + + Set up indices and include provided saved queries. + + + ), + }, + { + id: 'visualizations', + label: ( + + Complete{': '} + + Indices, queries, and visualizations for the data. + + + ), + }, + { + id: 'all', + label: ( + + Everything{': '} + + Includes additional assets such as detectors or geospatial. + + + ), + }, + ]} + idSelected={selectAsset} + onChange={setSelectAsset} + /> + + + + + None{': '} + No acceleration. Cheap, but slow. + + ), + }, + { + id: 'basic', + label: ( + + Basic{': '} + + Basic optimizations balancing performance and cost. + + + ), + }, + { + id: 'advanced', + label: Advanced, + }, + { + id: 'ultra', + label: ( + + Ultra{': '} + + Ideal for performance-critical indices. + + + ), + }, + ]} + idSelected={selectQuery} + onChange={setSelectQuery} + /> + + + ); } function SetupIntegrationStep(activeStep: number) { + const [selectAsset, setSelectAsset] = useState('visualizations'); + const [selectQuery, setSelectQuery] = useState('basic'); + switch (activeStep) { case 0: return SetupIntegrationStepOne(); @@ -64,12 +228,31 @@ function SetupIntegrationStep(activeStep: number) { case 2: return SetupIntegrationStepThree(); case 3: - return SetupIntegrationStepFour(); + return SetupIntegrationStepFour(selectAsset, setSelectAsset, selectQuery, setSelectQuery); default: return Something went wrong...; } } +function SetupBottomBar(step: number, setStep: React.Dispatch>) { + return ( + + + + setStep(Math.max(step - 1, 0))}> + Cancel + + + + setStep(Math.min(step + 1, 3))}> + {step === 3 ? 'Save' : 'Next'} + + + + + ); +} + export function SetupIntegrationStepsPage() { const [step, setStep] = useState(0); @@ -82,11 +265,7 @@ export function SetupIntegrationStepsPage() { {SetupIntegrationStep(step)} - - Step Navigation - setStep(Math.max(step - 1, 0))}>Previous Step - setStep(Math.min(step + 1, 3))}>Next Step - + {SetupBottomBar(step, setStep)} ); From 42dbb544b4f367d92c270b4340766eb37a152239 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 7 Sep 2023 11:48:59 -0700 Subject: [PATCH 27/79] Add a cancel button Signed-off-by: Simeon Widdis --- .cypress/.DS_Store | Bin 6148 -> 6148 bytes .../components/setup_integration.tsx | 32 ++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.cypress/.DS_Store b/.cypress/.DS_Store index ab1b61f97e8de91e1f5ea7ca39c7e9ac4e976bce..6e31e66a45d9f2321b8bc37aefea379cbf2761e3 100644 GIT binary patch delta 334 zcmZoMXfc=|#>B)qu~2NHo+2aj!~pA!7aACWj2_K;ZiZrpWQHP!R3OX)vNIU+8A=$6 zlgf(=l5+Bs7#J9KCKcpl7MByppuyI%)FHRa;N;#`n;54u;yTh3#v+>|}=E?jbx*{O`ATt{vG}!pfAtD=? ICpNGE0G6Rur~m)} delta 72 zcmZoMXfc=|#>CJzu~2NHo+2aT!~knX#>qTPnwzbd&$4U|U~XmF%+A5j0aUWtk@-9G aWPTA{PDTa>h66y%FxiGjdUK4(5@rCW=n;kh diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index da1e3d73db..14faa9dee1 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -187,15 +187,15 @@ function SetupIntegrationStepFour( Basic{': '} - Basic optimizations balancing performance and cost. + Minimal optimizations balancing performance and cost. ), }, - { - id: 'advanced', - label: Advanced, - }, + // { + // id: 'advanced', + // label: Advanced, + // }, { id: 'ultra', label: ( @@ -239,10 +239,30 @@ function SetupBottomBar(step: number, setStep: React.Dispatch - setStep(Math.max(step - 1, 0))}> + { + // TODO evil hack because props aren't set up + let hash = window.location.hash; + hash = hash.trim(); + hash = hash.substring(0, hash.lastIndexOf('/setup')); + window.location.hash = hash; + }} + > Cancel + + + + {step > 0 ? ( + + setStep(Math.max(step - 1, 0))}> + Back + + + ) : null} setStep(Math.min(step + 1, 3))}> {step === 3 ? 'Save' : 'Next'} From 533e6c7ebba7fd5d05d11e96eef55215477d6d05 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 11 Sep 2023 13:26:25 -0700 Subject: [PATCH 28/79] Add config type to developing form Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 59 +++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 14faa9dee1..8aebea81c5 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -27,6 +27,25 @@ import { import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; import React, { useState } from 'react'; +interface IntegrationConfig { + metadata: { + name: string; + }; + dataSource: { + name: string; + description: string; + fileType: string; + tableLocation: string; + }; + connection: { + name: string; + }; + acceleration: { + assets: string; + queries: string; + }; +} + const STEPS: EuiContainedStepProps[] = [ { title: 'Name Integration', children: }, { title: 'Select index or data source for integration', children: }, @@ -106,10 +125,8 @@ function SetupIntegrationStepThree() { } function SetupIntegrationStepFour( - selectAsset: string, - setSelectAsset: React.Dispatch>, - selectQuery: string, - setSelectQuery: React.Dispatch> + integConfig: IntegrationConfig, + setConfig: React.Dispatch> ) { return ( @@ -164,8 +181,10 @@ function SetupIntegrationStepFour( ), }, ]} - idSelected={selectAsset} - onChange={setSelectAsset} + idSelected={integConfig.acceleration.assets} + onChange={(id) => + setConfig(Object.assign({}, integConfig, { acceleration: { assets: id } })) + } /> @@ -208,8 +227,10 @@ function SetupIntegrationStepFour( ), }, ]} - idSelected={selectQuery} - onChange={setSelectQuery} + idSelected={integConfig.acceleration.queries} + onChange={(id) => + setConfig(Object.assign({}, integConfig, { acceleration: { query: id } })) + } /> @@ -217,8 +238,24 @@ function SetupIntegrationStepFour( } function SetupIntegrationStep(activeStep: number) { - const [selectAsset, setSelectAsset] = useState('visualizations'); - const [selectQuery, setSelectQuery] = useState('basic'); + const [integConfig, setConfig] = useState({ + metadata: { + name: 'NginX Access 2.0', + }, + dataSource: { + name: 'ss4o_logs-nginx-*-*', + description: 'Integration for viewing Nginx logs in S3.', + fileType: 'parquet', + tableLocation: 'ss4o_logs-nginx-*-*', + }, + connection: { + name: 'S3 connection name', + }, + acceleration: { + assets: 'visualizations', + queries: 'basic', + }, + }); switch (activeStep) { case 0: @@ -228,7 +265,7 @@ function SetupIntegrationStep(activeStep: number) { case 2: return SetupIntegrationStepThree(); case 3: - return SetupIntegrationStepFour(selectAsset, setSelectAsset, selectQuery, setSelectQuery); + return SetupIntegrationStepFour(integConfig, setConfig); default: return Something went wrong...; } From 467488c499a6b648be403fdd528ee0f1c4bf7544 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 11 Sep 2023 15:39:44 -0700 Subject: [PATCH 29/79] Flatten integration config Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 8aebea81c5..9b40272b6f 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -28,22 +28,14 @@ import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/st import React, { useState } from 'react'; interface IntegrationConfig { - metadata: { - name: string; - }; - dataSource: { - name: string; - description: string; - fileType: string; - tableLocation: string; - }; - connection: { - name: string; - }; - acceleration: { - assets: string; - queries: string; - }; + instance_name: string; + datasource_name: string; + datasource_description: string; + datasource_filetype: string; + datasourcee_location: string; + connection_name: string; + asset_accel: string; + query_accel: string; } const STEPS: EuiContainedStepProps[] = [ @@ -181,10 +173,8 @@ function SetupIntegrationStepFour( ), }, ]} - idSelected={integConfig.acceleration.assets} - onChange={(id) => - setConfig(Object.assign({}, integConfig, { acceleration: { assets: id } })) - } + idSelected={integConfig.asset_accel} + onChange={(id) => setConfig(Object.assign({}, integConfig, { asset_accel: id }))} /> @@ -227,10 +217,8 @@ function SetupIntegrationStepFour( ), }, ]} - idSelected={integConfig.acceleration.queries} - onChange={(id) => - setConfig(Object.assign({}, integConfig, { acceleration: { query: id } })) - } + idSelected={integConfig.query_accel} + onChange={(id) => setConfig(Object.assign({}, integConfig, { query_accel: id }))} /> @@ -239,22 +227,14 @@ function SetupIntegrationStepFour( function SetupIntegrationStep(activeStep: number) { const [integConfig, setConfig] = useState({ - metadata: { - name: 'NginX Access 2.0', - }, - dataSource: { - name: 'ss4o_logs-nginx-*-*', - description: 'Integration for viewing Nginx logs in S3.', - fileType: 'parquet', - tableLocation: 'ss4o_logs-nginx-*-*', - }, - connection: { - name: 'S3 connection name', - }, - acceleration: { - assets: 'visualizations', - queries: 'basic', - }, + instance_name: 'NginX Access 2.0', + datasource_name: 'ss4o_logs-nginx-*-*', + datasource_description: 'Integration for viewing Nginx logs in S3.', + datasource_filetype: 'parquet', + datasourcee_location: 'ss4o_logs-nginx-*-*', + connection_name: 'S3 connection name', + asset_accel: 'visualizations', + query_accel: 'basic', }); switch (activeStep) { From 670d2290a268ff199a8d1c52239c13869d1c305a Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 12 Sep 2023 11:10:35 -0700 Subject: [PATCH 30/79] Add sample data table modal Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 90 +++++++++++++++++-- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 9b40272b6f..e357bcfa3d 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -23,6 +23,10 @@ import { EuiTitle, EuiRadioGroup, EuiTextColor, + EuiModal, + EuiModalHeader, + EuiModalBody, + EuiBasicTable, } from '@elastic/eui'; import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; import React, { useState } from 'react'; @@ -101,7 +105,66 @@ function SetupIntegrationStepTwo() { ); } -function SetupIntegrationStepThree() { +function IntegrationDataModal( + isDataModalVisible: boolean, + setDataModalVisible: React.Dispatch> +): React.JSX.Element | null { + let dataModal = null; + if (isDataModalVisible) { + dataModal = ( + setDataModalVisible(false)}> + +

Data Table

+ + + + + setDataModalVisible(false)} size="s"> + Close + + + + ); + } + return dataModal; +} + +function SetupIntegrationStepThree( + isDataModalVisible: boolean, + setDataModalVisible: React.Dispatch> +) { return ( @@ -111,7 +174,8 @@ function SetupIntegrationStepThree() { - View table + setDataModalVisible(true)}>View table + {IntegrationDataModal(isDataModalVisible, setDataModalVisible)} ); } @@ -125,7 +189,7 @@ function SetupIntegrationStepFour(

{STEPS[3].title}

 - + - + ), }, - // { - // id: 'advanced', - // label: Advanced, - // }, + { + id: 'advanced', + label: ( + + Advanced{': '} + + More intensive optimization for better performance. + + + ), + }, { id: 'ultra', label: ( @@ -236,6 +307,7 @@ function SetupIntegrationStep(activeStep: number) { asset_accel: 'visualizations', query_accel: 'basic', }); + const [isDataModalVisible, setDataModalVisible] = useState(false); switch (activeStep) { case 0: @@ -243,7 +315,7 @@ function SetupIntegrationStep(activeStep: number) { case 1: return SetupIntegrationStepTwo(); case 2: - return SetupIntegrationStepThree(); + return SetupIntegrationStepThree(isDataModalVisible, setDataModalVisible); case 3: return SetupIntegrationStepFour(integConfig, setConfig); default: From 0a997a487df3e64290a8b77c43668307ace1537e Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 13 Sep 2023 11:01:52 -0700 Subject: [PATCH 31/79] Add toggle for standard and advanced asset config Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 209 ++++++++++++++---- 1 file changed, 162 insertions(+), 47 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index e357bcfa3d..1aa7566002 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -3,6 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { EuiTabs } from '@elastic/eui'; +import { EuiTab } from '@elastic/eui'; +import { EuiIcon } from '@elastic/eui'; import { EuiBottomBar, EuiButton, @@ -27,6 +30,8 @@ import { EuiModalHeader, EuiModalBody, EuiBasicTable, + EuiSwitch, + EuiCallOut, } from '@elastic/eui'; import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; import React, { useState } from 'react'; @@ -45,7 +50,6 @@ interface IntegrationConfig { const STEPS: EuiContainedStepProps[] = [ { title: 'Name Integration', children: }, { title: 'Select index or data source for integration', children: }, - { title: 'Review associated index with data from table', children: }, { title: 'Select integration assets', children: }, ]; @@ -54,6 +58,39 @@ const ALLOWED_FILE_TYPES: EuiSelectOption[] = [ { value: 'json', text: 'json' }, ]; +const INTEGRATION_DATA_TABLE_COLUMNS = [ + { + field: 'field', + name: 'Field Name', + }, + { + field: 'type', + name: 'Field Type', + }, + { + field: 'isTimestamp', + name: 'Timestamp', + }, +]; + +const integrationDataTableData = [ + { + field: 'spanId', + type: 'string', + isTimestamp: false, + }, + { + field: 'severity.number', + type: 'long', + isTimestamp: false, + }, + { + field: '@timestamp', + type: 'date', + isTimestamp: true, + }, +]; + const getSteps = (activeStep: number): EuiContainedStepProps[] => { return STEPS.map((step, idx) => { let status: string = ''; @@ -67,7 +104,7 @@ const getSteps = (activeStep: number): EuiContainedStepProps[] => { }); }; -function SetupIntegrationStepOne() { +function SetupIntegrationMetadata() { return ( @@ -83,12 +120,16 @@ function SetupIntegrationStepOne() { ); } -function SetupIntegrationStepTwo() { +function SetupIntegrationNewTable() { return (

{STEPS[1].title}

 + +

No problem, we can help. Tell us about your data.

+ + @@ -118,37 +159,8 @@ function IntegrationDataModal( setDataModalVisible(false)} size="s"> @@ -161,14 +173,14 @@ function IntegrationDataModal( return dataModal; } -function SetupIntegrationStepThree( +function SetupIntegrationExistingTable( isDataModalVisible: boolean, setDataModalVisible: React.Dispatch> ) { return ( -

{STEPS[2].title}

+

{STEPS[1].title}

 @@ -180,15 +192,12 @@ function SetupIntegrationStepThree( ); } -function SetupIntegrationStepFour( +function SetupIntegrationAccelerationStandard( integConfig: IntegrationConfig, setConfig: React.Dispatch> ) { return ( - -

{STEPS[3].title}

- > +) { + return ( + {}, + }, + { + name: 'configure', + description: 'Configure Asset', + type: 'icon', + icon: 'indexSettings', + color: 'primary', + onClick: () => {}, + }, + ], + }, + ]} + items={[ + { + name: '[NGINX Core Logs 1.0] Overview', + type: 'dashboard', + acceleration: 'Enhanced', + }, + { + name: 'ss4o_logs-*-*', + type: 'index-pattern', + acceleration: 'Status', + }, + { + name: 'Top Paths', + type: 'visualization', + acceleration: 'Query', + }, + ]} + hasActions={true} + /> + ); +} + +function SetupIntegrationAcceleration( + integConfig: IntegrationConfig, + setConfig: React.Dispatch>, + isStandard: boolean, + setIsStandard: React.Dispatch> +) { + return ( + + +

{STEPS[2].title}

+ + + setIsStandard(true)}> + Standard + + setIsStandard(false)}> + Advanced + + + + {isStandard + ? SetupIntegrationAccelerationStandard(integConfig, setConfig) + : SetupIntegrationAccelerationAdvanced(integConfig, setConfig)} + + ); +} + function SetupIntegrationStep(activeStep: number) { const [integConfig, setConfig] = useState({ instance_name: 'NginX Access 2.0', @@ -308,16 +407,32 @@ function SetupIntegrationStep(activeStep: number) { query_accel: 'basic', }); const [isDataModalVisible, setDataModalVisible] = useState(false); + const [tableDetected, setTableDetected] = useState(false); + const [isStandard, setIsStandard] = useState(true); switch (activeStep) { case 0: - return SetupIntegrationStepOne(); + return SetupIntegrationMetadata(); case 1: - return SetupIntegrationStepTwo(); + let tableForm; + if (tableDetected) { + tableForm = SetupIntegrationExistingTable(isDataModalVisible, setDataModalVisible); + } else { + tableForm = SetupIntegrationNewTable(); + } + return ( +
+ {tableForm} + + setTableDetected(event.target.checked)} + /> +
+ ); case 2: - return SetupIntegrationStepThree(isDataModalVisible, setDataModalVisible); - case 3: - return SetupIntegrationStepFour(integConfig, setConfig); + return SetupIntegrationAcceleration(integConfig, setConfig, isStandard, setIsStandard); default: return Something went wrong...; } @@ -353,8 +468,8 @@ function SetupBottomBar(step: number, setStep: React.Dispatch ) : null} - setStep(Math.min(step + 1, 3))}> - {step === 3 ? 'Save' : 'Next'} + setStep(Math.min(step + 1, 2))}> + {step === STEPS.length - 1 ? 'Save' : 'Next'} From 8c7b0f222bcbd597969dbaeeedef6967b4c9b587 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 13 Sep 2023 11:07:43 -0700 Subject: [PATCH 32/79] Simplify imports Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 300 ++++++++---------- 1 file changed, 138 insertions(+), 162 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 1aa7566002..436ac5af52 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -3,36 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiTabs } from '@elastic/eui'; -import { EuiTab } from '@elastic/eui'; -import { EuiIcon } from '@elastic/eui'; -import { - EuiBottomBar, - EuiButton, - EuiFieldText, - EuiFlexGroup, - EuiFlexItem, - EuiForm, - EuiFormRow, - EuiLink, - EuiHeader, - EuiPage, - EuiPageBody, - EuiSelect, - EuiSelectOption, - EuiSteps, - EuiSpacer, - EuiText, - EuiTitle, - EuiRadioGroup, - EuiTextColor, - EuiModal, - EuiModalHeader, - EuiModalBody, - EuiBasicTable, - EuiSwitch, - EuiCallOut, -} from '@elastic/eui'; +import * as Eui from '@elastic/eui'; import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; import React, { useState } from 'react'; @@ -48,12 +19,12 @@ interface IntegrationConfig { } const STEPS: EuiContainedStepProps[] = [ - { title: 'Name Integration', children: }, - { title: 'Select index or data source for integration', children: }, - { title: 'Select integration assets', children: }, + { title: 'Name Integration', children: }, + { title: 'Select index or data source for integration', children: }, + { title: 'Select integration assets', children: }, ]; -const ALLOWED_FILE_TYPES: EuiSelectOption[] = [ +const ALLOWED_FILE_TYPES: Eui.EuiSelectOption[] = [ { value: 'parquet', text: 'parquet' }, { value: 'json', text: 'json' }, ]; @@ -106,43 +77,43 @@ const getSteps = (activeStep: number): EuiContainedStepProps[] => { function SetupIntegrationMetadata() { return ( - - + +

{STEPS[0].title}

- - + - - - + + + ); } function SetupIntegrationNewTable() { return ( - - + +

{STEPS[1].title}

- - + +

No problem, we can help. Tell us about your data.

- - - - - - - - - - - - - - - + + + + + + + + + + + + + + + ); } @@ -153,21 +124,21 @@ function IntegrationDataModal( let dataModal = null; if (isDataModalVisible) { dataModal = ( - setDataModalVisible(false)}> - + setDataModalVisible(false)}> +

Data Table

- - - + + - - setDataModalVisible(false)} size="s"> + + setDataModalVisible(false)} size="s"> Close - - - + + + ); } return dataModal; @@ -178,17 +149,17 @@ function SetupIntegrationExistingTable( setDataModalVisible: React.Dispatch> ) { return ( - - + +

{STEPS[1].title}

- - - - - - setDataModalVisible(true)}>View table + + + + + + setDataModalVisible(true)}>View table {IntegrationDataModal(isDataModalVisible, setDataModalVisible)} - + ); } @@ -197,111 +168,116 @@ function SetupIntegrationAccelerationStandard( setConfig: React.Dispatch> ) { return ( - - - + + + None{': '} - + Set up indices, but don't install any assets. - - + + ), }, { id: 'queries', label: ( - + Minimal{': '} - + Set up indices and include provided saved queries. - - + + ), }, { id: 'visualizations', label: ( - + Complete{': '} - + Indices, queries, and visualizations for the data. - - + + ), }, { id: 'all', label: ( - + Everything{': '} - + Includes additional assets such as detectors or geospatial. - - + + ), }, ]} idSelected={integConfig.asset_accel} onChange={(id) => setConfig(Object.assign({}, integConfig, { asset_accel: id }))} /> - + - - + + None{': '} - No acceleration. Cheap, but slow. - + + No acceleration. Cheap, but slow. + + ), }, { id: 'basic', label: ( - + Basic{': '} - + Minimal optimizations balancing performance and cost. - - + + ), }, { id: 'advanced', label: ( - + Advanced{': '} - + More intensive optimization for better performance. - - + + ), }, { id: 'ultra', label: ( - + Ultra{': '} - + Ideal for performance-critical indices. - - + + ), }, ]} idSelected={integConfig.query_accel} onChange={(id) => setConfig(Object.assign({}, integConfig, { query_accel: id }))} /> - - + + ); } @@ -310,7 +286,7 @@ function SetupIntegrationAccelerationAdvanced( setConfig: React.Dispatch> ) { return ( - > ) { return ( - - + +

{STEPS[2].title}

- - - setIsStandard(true)}> + + + setIsStandard(true)}> Standard - - setIsStandard(false)}> + + setIsStandard(false)}> Advanced - - - + + + {isStandard ? SetupIntegrationAccelerationStandard(integConfig, setConfig) : SetupIntegrationAccelerationAdvanced(integConfig, setConfig)} - + ); } @@ -423,8 +399,8 @@ function SetupIntegrationStep(activeStep: number) { return (
{tableForm} - - + setTableDetected(event.target.checked)} @@ -434,16 +410,16 @@ function SetupIntegrationStep(activeStep: number) { case 2: return SetupIntegrationAcceleration(integConfig, setConfig, isStandard, setIsStandard); default: - return Something went wrong...; + return Something went wrong...; } } function SetupBottomBar(step: number, setStep: React.Dispatch>) { return ( - - - - + + + { @@ -455,25 +431,25 @@ function SetupBottomBar(step: number, setStep: React.Dispatch Cancel - - - - - + + + + + {step > 0 ? ( - - setStep(Math.max(step - 1, 0))}> + + setStep(Math.max(step - 1, 0))}> Back - - + + ) : null} - - setStep(Math.min(step + 1, 2))}> + + setStep(Math.min(step + 1, 2))}> {step === STEPS.length - 1 ? 'Save' : 'Next'} - - - - + + + + ); } @@ -481,16 +457,16 @@ export function SetupIntegrationStepsPage() { const [step, setStep] = useState(0); return ( - - - - - - - {SetupIntegrationStep(step)} - + + + + + + + {SetupIntegrationStep(step)} + {SetupBottomBar(step, setStep)} - - + + ); } From 339fb0934cf598e2485765fbd7a901facfe471c8 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 13 Sep 2023 11:32:49 -0700 Subject: [PATCH 33/79] Refactor major class names Signed-off-by: Simeon Widdis --- .../{kibana_backend.test.ts => manager.test.ts} | 6 +++--- ...tions_kibana_backend.ts => integrations_manager.ts} | 4 ++-- .../{catalog_reader.ts => catalog_data_adaptor.ts} | 4 ++-- .../{local_catalog_reader.ts => fs_data_adaptor.ts} | 10 +++++----- server/adaptors/integrations/repository/integration.ts | 8 ++++---- server/adaptors/integrations/repository/repository.ts | 8 ++++---- server/routes/integrations/integrations_router.ts | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) rename server/adaptors/integrations/__test__/{kibana_backend.test.ts => manager.test.ts} (99%) rename server/adaptors/integrations/{integrations_kibana_backend.ts => integrations_manager.ts} (98%) rename server/adaptors/integrations/repository/{catalog_reader.ts => catalog_data_adaptor.ts} (95%) rename server/adaptors/integrations/repository/{local_catalog_reader.ts => fs_data_adaptor.ts} (83%) diff --git a/server/adaptors/integrations/__test__/kibana_backend.test.ts b/server/adaptors/integrations/__test__/manager.test.ts similarity index 99% rename from server/adaptors/integrations/__test__/kibana_backend.test.ts rename to server/adaptors/integrations/__test__/manager.test.ts index 4b333e081d..7d1972cbf7 100644 --- a/server/adaptors/integrations/__test__/kibana_backend.test.ts +++ b/server/adaptors/integrations/__test__/manager.test.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IntegrationsKibanaBackend } from '../integrations_kibana_backend'; +import { IntegrationsManager } from '../integrations_manager'; import { SavedObject, SavedObjectsClientContract } from '../../../../../../src/core/server/types'; import { Repository } from '../repository/repository'; import { IntegrationInstanceBuilder } from '../integrations_builder'; @@ -13,7 +13,7 @@ import { SavedObjectsFindResponse } from '../../../../../../src/core/server'; describe('IntegrationsKibanaBackend', () => { let mockSavedObjectsClient: jest.Mocked; let mockRepository: jest.Mocked; - let backend: IntegrationsKibanaBackend; + let backend: IntegrationsManager; beforeEach(() => { mockSavedObjectsClient = { @@ -26,7 +26,7 @@ describe('IntegrationsKibanaBackend', () => { getIntegration: jest.fn(), getIntegrationList: jest.fn(), } as any; - backend = new IntegrationsKibanaBackend(mockSavedObjectsClient, mockRepository); + backend = new IntegrationsManager(mockSavedObjectsClient, mockRepository); }); describe('deleteIntegrationInstance', () => { diff --git a/server/adaptors/integrations/integrations_kibana_backend.ts b/server/adaptors/integrations/integrations_manager.ts similarity index 98% rename from server/adaptors/integrations/integrations_kibana_backend.ts rename to server/adaptors/integrations/integrations_manager.ts index da99efa7d7..857e223b76 100644 --- a/server/adaptors/integrations/integrations_kibana_backend.ts +++ b/server/adaptors/integrations/integrations_manager.ts @@ -4,13 +4,13 @@ */ import path from 'path'; -import { addRequestToMetric } from '../../../server/common/metrics/metrics_helper'; +import { addRequestToMetric } from '../../common/metrics/metrics_helper'; import { IntegrationsAdaptor } from './integrations_adaptor'; import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types'; import { IntegrationInstanceBuilder } from './integrations_builder'; import { Repository } from './repository/repository'; -export class IntegrationsKibanaBackend implements IntegrationsAdaptor { +export class IntegrationsManager implements IntegrationsAdaptor { client: SavedObjectsClientContract; instanceBuilder: IntegrationInstanceBuilder; repository: Repository; diff --git a/server/adaptors/integrations/repository/catalog_reader.ts b/server/adaptors/integrations/repository/catalog_data_adaptor.ts similarity index 95% rename from server/adaptors/integrations/repository/catalog_reader.ts rename to server/adaptors/integrations/repository/catalog_data_adaptor.ts index c52fb9d0f0..c791909b41 100644 --- a/server/adaptors/integrations/repository/catalog_reader.ts +++ b/server/adaptors/integrations/repository/catalog_data_adaptor.ts @@ -5,7 +5,7 @@ type IntegrationPart = 'assets' | 'data' | 'schemas' | 'static'; -interface CatalogReader { +interface CatalogDataAdaptor { /** * Reads a file from the data source. * @@ -48,5 +48,5 @@ interface CatalogReader { * @param subdirectory The path to append to the current directory. * @returns A new CatalogReader instance. */ - join: (subdirectory: string) => CatalogReader; + join: (subdirectory: string) => CatalogDataAdaptor; } diff --git a/server/adaptors/integrations/repository/local_catalog_reader.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts similarity index 83% rename from server/adaptors/integrations/repository/local_catalog_reader.ts rename to server/adaptors/integrations/repository/fs_data_adaptor.ts index 4f473022c5..fd6c312908 100644 --- a/server/adaptors/integrations/repository/local_catalog_reader.ts +++ b/server/adaptors/integrations/repository/fs_data_adaptor.ts @@ -8,14 +8,14 @@ import path from 'path'; import sanitize from 'sanitize-filename'; /** - * A CatalogReader that reads from the local filesystem. + * A CatalogDataAdaptor that reads from the local filesystem. * Used to read Integration information when the user uploads their own catalog. */ -export class LocalCatalogReader implements CatalogReader { +export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor { directory: string; /** - * Creates a new LocalCatalogReader instance. + * Creates a new FileSystemCatalogDataAdaptor instance. * * @param directory The base directory from which to read files. This is not sanitized. */ @@ -52,7 +52,7 @@ export class LocalCatalogReader implements CatalogReader { return (await fs.lstat(this._prepare(dirname))).isDirectory(); } - join(filename: string): LocalCatalogReader { - return new LocalCatalogReader(path.join(this.directory, filename)); + join(filename: string): FileSystemCatalogDataAdaptor { + return new FileSystemCatalogDataAdaptor(path.join(this.directory, filename)); } } diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index ac7d3d16ee..3f5c63677d 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -5,7 +5,7 @@ import path from 'path'; import { validateTemplate } from '../validators'; -import { LocalCatalogReader } from './local_catalog_reader'; +import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; /** * Helper function to compare version numbers. @@ -39,14 +39,14 @@ function compareVersions(a: string, b: string): number { * It includes accessor methods for integration configs, as well as helpers for nested components. */ export class Integration { - reader: CatalogReader; + reader: CatalogDataAdaptor; directory: string; name: string; - constructor(directory: string, reader?: CatalogReader) { + constructor(directory: string, reader?: CatalogDataAdaptor) { this.directory = directory; this.name = path.basename(directory); - this.reader = reader ?? new LocalCatalogReader(directory); + this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); } /** diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index acea2d4ed5..f3ff15688b 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -5,15 +5,15 @@ import * as path from 'path'; import { Integration } from './integration'; -import { LocalCatalogReader } from './local_catalog_reader'; +import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; export class Repository { - reader: CatalogReader; + reader: CatalogDataAdaptor; directory: string; - constructor(directory: string, reader?: CatalogReader) { + constructor(directory: string, reader?: CatalogDataAdaptor) { this.directory = directory; - this.reader = reader ?? new LocalCatalogReader(directory); + this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); } async getIntegrationList(): Promise { diff --git a/server/routes/integrations/integrations_router.ts b/server/routes/integrations/integrations_router.ts index 5a4813127c..46fe47768f 100644 --- a/server/routes/integrations/integrations_router.ts +++ b/server/routes/integrations/integrations_router.ts @@ -13,7 +13,7 @@ import { OpenSearchDashboardsRequest, OpenSearchDashboardsResponseFactory, } from '../../../../../src/core/server/http/router'; -import { IntegrationsKibanaBackend } from '../../adaptors/integrations/integrations_kibana_backend'; +import { IntegrationsManager } from '../../adaptors/integrations/integrations_manager'; /** * Handle an `OpenSearchDashboardsRequest` using the provided `callback` function. @@ -54,7 +54,7 @@ const getAdaptor = ( context: RequestHandlerContext, _request: OpenSearchDashboardsRequest ): IntegrationsAdaptor => { - return new IntegrationsKibanaBackend(context.core.savedObjects.client); + return new IntegrationsManager(context.core.savedObjects.client); }; export function registerIntegrationsRoute(router: IRouter) { From fae680f3e7eea1b9615bbcc5a3d71bcc7580cfb2 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 13 Sep 2023 16:26:44 -0700 Subject: [PATCH 34/79] (WIP) begin refactoring functionality into adaptor Signed-off-by: Simeon Widdis --- .../repository/__test__/integration.test.ts | 2 + .../repository/catalog_data_adaptor.ts | 29 ++-- .../repository/fs_data_adaptor.ts | 133 +++++++++++++++--- .../integrations/repository/integration.ts | 133 +++++------------- .../integrations/repository/repository.ts | 19 ++- 5 files changed, 180 insertions(+), 136 deletions(-) diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 105f81cb49..7898f485b0 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -7,6 +7,7 @@ import * as fs from 'fs/promises'; import { Integration } from '../integration'; import { Dirent, Stats } from 'fs'; import * as path from 'path'; +import { FileSystemCatalogDataAdaptor } from '../fs_data_adaptor'; jest.mock('fs/promises'); @@ -79,6 +80,7 @@ describe('Integration', () => { it('should return the parsed config template if it is valid', async () => { jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(sampleIntegration)); + jest.spyOn(fs, 'lstat').mockResolvedValueOnce({ isDirectory: () => true } as Stats); const result = await integration.getConfig(sampleIntegration.version); diff --git a/server/adaptors/integrations/repository/catalog_data_adaptor.ts b/server/adaptors/integrations/repository/catalog_data_adaptor.ts index c791909b41..6373fee4d4 100644 --- a/server/adaptors/integrations/repository/catalog_data_adaptor.ts +++ b/server/adaptors/integrations/repository/catalog_data_adaptor.ts @@ -7,13 +7,13 @@ type IntegrationPart = 'assets' | 'data' | 'schemas' | 'static'; interface CatalogDataAdaptor { /** - * Reads a file from the data source. + * Reads a Json or NDJson file from the data source. * * @param filename The name of the file to read. * @param type Optional. The type of integration part to read ('assets', 'data', 'schemas', or 'static'). * @returns A Promise that resolves with the content of the file as a string. */ - readFile: (filename: string, type?: IntegrationPart) => Promise; + readFile: (filename: string, type?: IntegrationPart) => Promise>; /** * Reads a file from the data source as raw binary data. @@ -22,31 +22,38 @@ interface CatalogDataAdaptor { * @param type Optional. The type of integration part to read ('assets', 'data', 'schemas', or 'static'). * @returns A Promise that resolves with the content of the file as a Buffer. */ - readFileRaw: (filename: string, type?: IntegrationPart) => Promise; + readFileRaw: (filename: string, type?: IntegrationPart) => Promise>; /** - * Reads the contents of a directory from the data source. + * Reads the contents of a repository directory from the data source to find integrations. * * @param dirname The name of the directory to read. * @returns A Promise that resolves with an array of filenames within the directory. */ - readDir: (dirname: string) => Promise; + findIntegrations: (dirname?: string) => Promise>; /** - * Checks if a given path on the data source is a directory. + * Reads the contents of an integration version to find available versions. + * + * @param dirname The name of the directory to read. + * @returns A Promise that resolves with an array of filenames within the directory. + */ + findIntegrationVersions: (dirname?: string) => Promise>; + + /** + * Determine whether a directory is an integration, repository, or otherwise. * * @param dirname The path to check. * @returns A Promise that resolves with a boolean indicating whether the path is a directory or not. */ - isDirectory: (dirname: string) => Promise; + getDirectoryType: (dirname?: string) => Promise<'integration' | 'repository' | 'unknown'>; /** - * Creates a new CatalogReader instance with the specified subdirectory appended to the current directory. - * Since CatalogReaders sanitize given paths by default, - * this is useful for exploring nested data. + * Creates a new CatalogDataAdaptor instance with the specified subdirectory appended to the current directory. + * Useful for exploring nested data without needing to know the instance type. * * @param subdirectory The path to append to the current directory. - * @returns A new CatalogReader instance. + * @returns A new CatalogDataAdaptor instance. */ join: (subdirectory: string) => CatalogDataAdaptor; } diff --git a/server/adaptors/integrations/repository/fs_data_adaptor.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts index fd6c312908..df1c6781af 100644 --- a/server/adaptors/integrations/repository/fs_data_adaptor.ts +++ b/server/adaptors/integrations/repository/fs_data_adaptor.ts @@ -5,7 +5,48 @@ import * as fs from 'fs/promises'; import path from 'path'; -import sanitize from 'sanitize-filename'; + +/** + * Helper function to compare version numbers. + * Assumes that the version numbers are valid, produces undefined behavior otherwise. + * + * @param a Left-hand number + * @param b Right-hand number + * @returns -1 if a > b, 1 if a < b, 0 otherwise. + */ +function compareVersions(a: string, b: string): number { + const aParts = a.split('.').map(Number.parseInt); + const bParts = b.split('.').map(Number.parseInt); + + for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) { + const aValue = i < aParts.length ? aParts[i] : 0; + const bValue = i < bParts.length ? bParts[i] : 0; + + if (aValue > bValue) { + return -1; // a > b + } else if (aValue < bValue) { + return 1; // a < b + } + } + + return 0; // a == b +} + +function tryParseNDJson(content: string): object[] | null { + try { + const objects = []; + for (const line of content.split('\n')) { + if (line.trim() === '') { + // Other OSD ndjson parsers skip whitespace lines + continue; + } + objects.push(JSON.parse(line)); + } + return objects; + } catch (err: any) { + return null; + } +} /** * A CatalogDataAdaptor that reads from the local filesystem. @@ -23,33 +64,85 @@ export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor { this.directory = directory; } - /** - * Prepares a filename for use in filesystem operations by sanitizing and joining it with the base directory. - * This method is intended to be used before any filesystem-related call. - * - * @param filename The name of the file to prepare. - * @param subdir Optional. A subdirectory to prepend to the filename. Not sanitized. - * @returns The prepared path for the file, including the base directory and optional prefix. - */ - _prepare(filename: string, subdir?: string): string { - return path.join(this.directory, subdir ?? '.', sanitize(filename)); + async readFile(filename: string, type?: IntegrationPart): Promise> { + let content; + try { + content = await fs.readFile(path.join(this.directory, type ?? '.', filename), { + encoding: 'utf-8', + }); + } catch (err: any) { + return { ok: false, error: err }; + } + // First try to parse as JSON, then NDJSON, then fail. + try { + const parsed = JSON.parse(content); + return { ok: true, value: parsed }; + } catch (err: any) { + const parsed = tryParseNDJson(content); + if (parsed) { + return { ok: true, value: parsed }; + } + return { + ok: false, + error: new Error('Unable to parse file as JSON or NDJson', { cause: err }), + }; + } } - async readFile(filename: string, type?: IntegrationPart): Promise { - return await fs.readFile(this._prepare(filename, type), { encoding: 'utf-8' }); + async readFileRaw(filename: string, type?: IntegrationPart): Promise> { + try { + const buffer = await fs.readFile(path.join(this.directory, type ?? '.', filename)); + return { ok: true, value: buffer }; + } catch (err: any) { + return { ok: false, error: err }; + } } - async readFileRaw(filename: string, type?: IntegrationPart): Promise { - return await fs.readFile(this._prepare(filename, type)); + async findIntegrations(dirname: string = '.'): Promise> { + try { + const files = await fs.readdir(path.join(this.directory, dirname)); + return { ok: true, value: files }; + } catch (err: any) { + return { ok: false, error: err }; + } } - async readDir(dirname: string): Promise { - // TODO return empty list if not a directory - return await fs.readdir(this._prepare(dirname)); + async findIntegrationVersions(dirname: string = '.'): Promise> { + let files; + const integPath = path.join(this.directory, dirname); + try { + files = await fs.readdir(integPath); + } catch (err: any) { + return { ok: false, error: err }; + } + const versions: string[] = []; + + for (const file of files) { + // TODO handle nested repositories -- assumes integrations are 1 level deep + if (path.extname(file) === '.json' && file.startsWith(`${path.basename(integPath)}-`)) { + const version = file.substring(path.basename(integPath).length + 1, file.length - 5); + if (!version.match(/^\d+(\.\d+)*$/)) { + continue; + } + versions.push(version); + } + } + + versions.sort((a, b) => compareVersions(a, b)); + return { ok: true, value: versions }; } - async isDirectory(dirname: string): Promise { - return (await fs.lstat(this._prepare(dirname))).isDirectory(); + async getDirectoryType(dirname?: string): Promise<'integration' | 'repository' | 'unknown'> { + const isDir = (await fs.lstat(path.join(this.directory, dirname ?? '.'))).isDirectory(); + if (!isDir) { + return 'unknown'; + } + // Sloppily just check for one mandatory integration directory to distinguish. + // Improve if we need to distinguish a repository with an integration named "schemas". + const hasSchemas = ( + await fs.lstat(path.join(this.directory, dirname ?? '.', 'schemas')) + ).isDirectory(); + return hasSchemas ? 'integration' : 'repository'; } join(filename: string): FileSystemCatalogDataAdaptor { diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index 3f5c63677d..2b99635ea9 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -7,32 +7,6 @@ import path from 'path'; import { validateTemplate } from '../validators'; import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; -/** - * Helper function to compare version numbers. - * Assumes that the version numbers are valid, produces undefined behavior otherwise. - * - * @param a Left-hand number - * @param b Right-hand number - * @returns -1 if a > b, 1 if a < b, 0 otherwise. - */ -function compareVersions(a: string, b: string): number { - const aParts = a.split('.').map(Number.parseInt); - const bParts = b.split('.').map(Number.parseInt); - - for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) { - const aValue = i < aParts.length ? aParts[i] : 0; - const bValue = i < bParts.length ? bParts[i] : 0; - - if (aValue > bValue) { - return -1; // a > b - } else if (aValue < bValue) { - return 1; // a < b - } - } - - return 0; // a == b -} - /** * The Integration class represents the data for Integration Templates. * It is backed by the repository file system. @@ -84,22 +58,12 @@ export class Integration { * @returns A string with the latest version, or null if no versions are available. */ async getLatestVersion(): Promise { - const files = await this.reader.readDir(''); - const versions: string[] = []; - - for (const file of files) { - if (path.extname(file) === '.json' && file.startsWith(`${this.name}-`)) { - const version = file.substring(this.name.length + 1, file.length - 5); - if (!version.match(/^\d+(\.\d+)*$/)) { - continue; - } - versions.push(version); - } + const versions = await this.reader.findIntegrationVersions(); + if (!versions.ok) { + console.error(versions.error); + return null; } - - versions.sort((a, b) => compareVersions(a, b)); - - return versions.length > 0 ? versions[0] : null; + return versions.value.length > 0 ? versions.value[0] : null; } /** @@ -109,8 +73,8 @@ export class Integration { * @returns The config if a valid config matching the version is present, otherwise null. */ async getConfig(version?: string): Promise> { - if (!(await this.reader.isDirectory(''))) { - return { ok: false, error: new Error(`${this.directory} is not a valid directory`) }; + if ((await this.reader.getDirectoryType()) !== 'integration') { + return { ok: false, error: new Error(`${this.directory} is not a valid integration`) }; } const maybeVersion: string | null = version ? version : await this.getLatestVersion(); @@ -124,19 +88,8 @@ export class Integration { const configFile = `${this.name}-${maybeVersion}.json`; - try { - const config = await this.reader.readFile(configFile); - const possibleTemplate = JSON.parse(config); - return validateTemplate(possibleTemplate); - } catch (err: any) { - if (err instanceof SyntaxError) { - console.error(`Syntax errors in ${configFile}`, err); - } - if (err instanceof Error && (err as { code?: string }).code === 'ENOENT') { - console.error(`Attempted to retrieve non-existent config ${configFile}`); - } - return { ok: false, error: err }; - } + const config = await this.reader.readFile(configFile); + return validateTemplate(config); } /** @@ -164,14 +117,11 @@ export class Integration { const resultValue: { savedObjects?: object[] } = {}; if (config.assets.savedObjects) { const sobjPath = `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson`; - try { - const ndjson = await this.reader.readFile(sobjPath, 'assets'); - const asJson = '[' + ndjson.trim().replace(/\n/g, ',') + ']'; - const parsed = JSON.parse(asJson); - resultValue.savedObjects = parsed; - } catch (err: any) { - return { ok: false, error: err }; + const assets = await this.reader.readFile(sobjPath, 'assets'); + if (!assets.ok) { + return assets; } + resultValue.savedObjects = assets.value as object[]; } return { ok: true, value: resultValue }; } @@ -199,26 +149,26 @@ export class Integration { const resultValue: { sampleData: object[] | null } = { sampleData: null }; if (config.sampleData) { - try { - const jsonContent = await this.reader.readFile(config.sampleData.path, 'data'); - const parsed = JSON.parse(jsonContent) as object[]; - for (const value of parsed) { - if (!('@timestamp' in value)) { - continue; - } - // Randomly scatter timestamps across last 10 minutes - // Assume for now that the ordering of events isn't important, can change to a sequence if needed - // Also doesn't handle fields like `observedTimestamp` if present - Object.assign(value, { - '@timestamp': new Date( - Date.now() - Math.floor(Math.random() * 1000 * 60 * 10) - ).toISOString(), - }); + const jsonContent = await this.reader.readFile(config.sampleData.path, 'data'); + if (!jsonContent.ok) { + return jsonContent; + } + for (const value of jsonContent.value as object[]) { + if (!('@timestamp' in value)) { + continue; + } + // Randomly scatter timestamps across last 10 minutes + // Assume for now that the ordering of events isn't important, can change to a sequence if needed + // Also doesn't handle fields like `observedTimestamp` if present + const newTime = new Date( + Date.now() - Math.floor(Math.random() * 1000 * 60 * 10) + ).toISOString(); + Object.assign(value, { '@timestamp': newTime }); + if ('observedTimestamp' in value) { + Object.assign(value, { observedTimestamp: newTime }); } - resultValue.sampleData = parsed; - } catch (err: any) { - return { ok: false, error: err }; } + resultValue.sampleData = jsonContent.value as object[]; } return { ok: true, value: resultValue }; } @@ -249,15 +199,13 @@ export class Integration { const resultValue: { mappings: { [key: string]: object } } = { mappings: {}, }; - try { - for (const component of config.components) { - const schemaFile = `${component.name}-${component.version}.mapping.json`; - const rawSchema = await this.reader.readFile(schemaFile, 'schemas'); - const parsedSchema = JSON.parse(rawSchema); - resultValue.mappings[component.name] = parsedSchema; + for (const component of config.components) { + const schemaFile = `${component.name}-${component.version}.mapping.json`; + const schema = await this.reader.readFile(schemaFile, 'schemas'); + if (!schema.ok) { + return schema; } - } catch (err: any) { - return { ok: false, error: err }; + resultValue.mappings[component.name] = schema.value; } return { ok: true, value: resultValue }; } @@ -269,11 +217,6 @@ export class Integration { * @returns A buffer with the static's data if present, otherwise null. */ async getStatic(staticPath: string): Promise> { - try { - const buffer = await this.reader.readFileRaw(staticPath, 'static'); - return { ok: true, value: buffer }; - } catch (err: any) { - return { ok: false, error: err }; - } + return await this.reader.readFileRaw(staticPath, 'static'); } } diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index f3ff15688b..cf8e0312a5 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -17,21 +17,20 @@ export class Repository { } async getIntegrationList(): Promise { - try { - // TODO in the future, we want to support traversing nested directory structures. - const folders = await this.reader.readDir(''); - const integrations = await Promise.all( - folders.map((i) => this.getIntegration(path.basename(i))) - ); - return integrations.filter((x) => x !== null) as Integration[]; - } catch (error) { - console.error(`Error reading integration directories in: ${this.directory}`, error); + // TODO in the future, we want to support traversing nested directory structures. + const folders = await this.reader.findIntegrations(); + if (!folders.ok) { + console.error(`Error reading integration directories in: ${this.directory}`, folders.error); return []; } + const integrations = await Promise.all( + folders.value.map((i) => this.getIntegration(path.basename(i))) + ); + return integrations.filter((x) => x !== null) as Integration[]; } async getIntegration(name: string): Promise { - if (!(await this.reader.isDirectory(name))) { + if ((await this.reader.getDirectoryType(name)) !== 'integration') { console.error(`Requested integration '${name}' does not exist`); return null; } From 309662671c424503d1a28fcb7ae52379d8fe316c Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 13 Sep 2023 16:42:25 -0700 Subject: [PATCH 35/79] Finish migrating functionality to data adaptor Signed-off-by: Simeon Widdis --- .../integrations/repository/__test__/repository.test.ts | 2 +- server/adaptors/integrations/repository/integration.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/adaptors/integrations/repository/__test__/repository.test.ts b/server/adaptors/integrations/repository/__test__/repository.test.ts index 0e08199e9f..c452acd563 100644 --- a/server/adaptors/integrations/repository/__test__/repository.test.ts +++ b/server/adaptors/integrations/repository/__test__/repository.test.ts @@ -38,7 +38,7 @@ describe('Repository', () => { // Mock fs.lstat to return a mix of directories and files jest.spyOn(fs, 'lstat').mockImplementation(async (toLstat) => { - if (toLstat === path.join('path', 'to', 'directory', 'folder1')) { + if (toLstat.toString().startsWith(path.join('path', 'to', 'directory', 'folder1'))) { return { isDirectory: () => true } as Stats; } else { return { isDirectory: () => false } as Stats; diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index 2b99635ea9..d9caa91621 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -89,7 +89,10 @@ export class Integration { const configFile = `${this.name}-${maybeVersion}.json`; const config = await this.reader.readFile(configFile); - return validateTemplate(config); + if (!config.ok) { + return config; + } + return validateTemplate(config.value); } /** From 5498724b8bf384d1066512eb782b42180282cd1d Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 14 Sep 2023 10:47:36 -0700 Subject: [PATCH 36/79] Rename integration types for more clarity Signed-off-by: Simeon Widdis --- .../integrations/__test__/builder.test.ts | 12 +++---- .../__test__/local_repository.test.ts | 12 ++++--- .../integrations/__test__/manager.test.ts | 32 ++++++++++++------- .../integrations/__test__/validators.test.ts | 8 ++--- .../integrations/integrations_builder.ts | 8 ++--- .../integrations/integrations_manager.ts | 11 ++++--- .../repository/__test__/integration.test.ts | 8 ++--- .../repository/__test__/repository.test.ts | 24 +++++++------- .../integrations/repository/integration.ts | 6 ++-- .../integrations/repository/repository.ts | 12 +++---- server/adaptors/integrations/types.ts | 4 +-- server/adaptors/integrations/validators.ts | 4 +-- 12 files changed, 77 insertions(+), 64 deletions(-) diff --git a/server/adaptors/integrations/__test__/builder.test.ts b/server/adaptors/integrations/__test__/builder.test.ts index 84387ca8c6..33aff3497f 100644 --- a/server/adaptors/integrations/__test__/builder.test.ts +++ b/server/adaptors/integrations/__test__/builder.test.ts @@ -5,7 +5,7 @@ import { SavedObjectsClientContract } from '../../../../../../src/core/server'; import { IntegrationInstanceBuilder } from '../integrations_builder'; -import { Integration } from '../repository/integration'; +import { IntegrationReader } from '../repository/integration'; const mockSavedObjectsClient: SavedObjectsClientContract = ({ bulkCreate: jest.fn(), @@ -16,7 +16,7 @@ const mockSavedObjectsClient: SavedObjectsClientContract = ({ update: jest.fn(), } as unknown) as SavedObjectsClientContract; -const sampleIntegration: Integration = ({ +const sampleIntegration: IntegrationReader = ({ deepCheck: jest.fn().mockResolvedValue(true), getAssets: jest.fn().mockResolvedValue({ savedObjects: [ @@ -34,7 +34,7 @@ const sampleIntegration: Integration = ({ name: 'integration-template', type: 'integration-type', }), -} as unknown) as Integration; +} as unknown) as IntegrationReader; describe('IntegrationInstanceBuilder', () => { let builder: IntegrationInstanceBuilder; @@ -93,7 +93,7 @@ describe('IntegrationInstanceBuilder', () => { ], }; - const mockTemplate: Partial = { + const mockTemplate: Partial = { name: 'integration-template', type: 'integration-type', assets: { @@ -298,7 +298,7 @@ describe('IntegrationInstanceBuilder', () => { }; const instance = await builder.buildInstance( - (integration as unknown) as Integration, + (integration as unknown) as IntegrationReader, refs, options ); @@ -326,7 +326,7 @@ describe('IntegrationInstanceBuilder', () => { }; await expect( - builder.buildInstance((integration as unknown) as Integration, refs, options) + builder.buildInstance((integration as unknown) as IntegrationReader, refs, options) ).rejects.toThrowError(); }); }); diff --git a/server/adaptors/integrations/__test__/local_repository.test.ts b/server/adaptors/integrations/__test__/local_repository.test.ts index 162b95414c..f1bfeb9b23 100644 --- a/server/adaptors/integrations/__test__/local_repository.test.ts +++ b/server/adaptors/integrations/__test__/local_repository.test.ts @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Repository } from '../repository/repository'; -import { Integration } from '../repository/integration'; +import { RepositoryReader } from '../repository/repository'; +import { IntegrationReader } from '../repository/integration'; import path from 'path'; import * as fs from 'fs/promises'; @@ -20,15 +20,17 @@ describe('The local repository', () => { return Promise.resolve(null); } // Otherwise, all directories must be integrations - const integ = new Integration(integPath); + const integ = new IntegrationReader(integPath); expect(integ.getConfig()).resolves.toHaveProperty('ok', true); }) ); }); it('Should pass deep validation for all local integrations.', async () => { - const repository: Repository = new Repository(path.join(__dirname, '../__data__/repository')); - const integrations: Integration[] = await repository.getIntegrationList(); + const repository: RepositoryReader = new RepositoryReader( + path.join(__dirname, '../__data__/repository') + ); + const integrations: IntegrationReader[] = await repository.getIntegrationList(); await Promise.all( integrations.map(async (i) => { const result = await i.deepCheck(); diff --git a/server/adaptors/integrations/__test__/manager.test.ts b/server/adaptors/integrations/__test__/manager.test.ts index 7d1972cbf7..75b89e5200 100644 --- a/server/adaptors/integrations/__test__/manager.test.ts +++ b/server/adaptors/integrations/__test__/manager.test.ts @@ -5,14 +5,14 @@ import { IntegrationsManager } from '../integrations_manager'; import { SavedObject, SavedObjectsClientContract } from '../../../../../../src/core/server/types'; -import { Repository } from '../repository/repository'; +import { RepositoryReader } from '../repository/repository'; import { IntegrationInstanceBuilder } from '../integrations_builder'; -import { Integration } from '../repository/integration'; +import { IntegrationReader } from '../repository/integration'; import { SavedObjectsFindResponse } from '../../../../../../src/core/server'; describe('IntegrationsKibanaBackend', () => { let mockSavedObjectsClient: jest.Mocked; - let mockRepository: jest.Mocked; + let mockRepository: jest.Mocked; let backend: IntegrationsManager; beforeEach(() => { @@ -150,7 +150,9 @@ describe('IntegrationsKibanaBackend', () => { const integration = { getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template1' } }), }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getIntegrationTemplates(query); @@ -166,7 +168,7 @@ describe('IntegrationsKibanaBackend', () => { { getConfig: jest.fn().mockResolvedValue({ ok: true, value: { name: 'template2' } }) }, ]; mockRepository.getIntegrationList.mockResolvedValue( - (integrationList as unknown) as Integration[] + (integrationList as unknown) as IntegrationReader[] ); const result = await backend.getIntegrationTemplates(); @@ -226,7 +228,7 @@ describe('IntegrationsKibanaBackend', () => { build: jest.fn().mockResolvedValue({ name, dataset: 'nginx', namespace: 'prod' }), }; const createdInstance = { name, dataset: 'nginx', namespace: 'prod' }; - mockRepository.getIntegration.mockResolvedValue((template as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue((template as unknown) as IntegrationReader); mockSavedObjectsClient.create.mockResolvedValue(({ result: 'created', } as unknown) as SavedObject); @@ -265,7 +267,7 @@ describe('IntegrationsKibanaBackend', () => { build: jest.fn().mockRejectedValue(new Error('Failed to build instance')), }; backend.instanceBuilder = (instanceBuilder as unknown) as IntegrationInstanceBuilder; - mockRepository.getIntegration.mockResolvedValue((template as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue((template as unknown) as IntegrationReader); await expect( backend.loadIntegrationInstance(templateName, name, 'datasource') @@ -281,7 +283,9 @@ describe('IntegrationsKibanaBackend', () => { const integration = { getStatic: jest.fn().mockResolvedValue({ ok: true, value: assetData }), }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getStatic(templateName, staticPath); @@ -325,7 +329,9 @@ describe('IntegrationsKibanaBackend', () => { const integration = { getSchemas: jest.fn().mockResolvedValue({ ok: true, value: schemaData }), }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getSchemas(templateName); @@ -361,7 +367,9 @@ describe('IntegrationsKibanaBackend', () => { const integration = { getAssets: jest.fn().mockResolvedValue({ ok: true, value: assetData }), }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getAssets(templateName); @@ -397,7 +405,9 @@ describe('IntegrationsKibanaBackend', () => { const integration = { getSampleData: jest.fn().mockResolvedValue({ ok: true, value: sampleData }), }; - mockRepository.getIntegration.mockResolvedValue((integration as unknown) as Integration); + mockRepository.getIntegration.mockResolvedValue( + (integration as unknown) as IntegrationReader + ); const result = await backend.getSampleData(templateName); diff --git a/server/adaptors/integrations/__test__/validators.test.ts b/server/adaptors/integrations/__test__/validators.test.ts index 3c6e133f5c..6c09b595ba 100644 --- a/server/adaptors/integrations/__test__/validators.test.ts +++ b/server/adaptors/integrations/__test__/validators.test.ts @@ -5,7 +5,7 @@ import { validateTemplate, validateInstance } from '../validators'; -const validTemplate: IntegrationTemplate = { +const validTemplate: IntegrationConfig = { name: 'test', version: '1.0.0', license: 'Apache-2.0', @@ -29,7 +29,7 @@ const validInstance: IntegrationInstance = { describe('validateTemplate', () => { it('Returns a success value for a valid Integration Template', () => { - const result: Result = validateTemplate(validTemplate); + const result: Result = validateTemplate(validTemplate); expect(result.ok).toBe(true); expect((result as any).value).toBe(validTemplate); }); @@ -38,7 +38,7 @@ describe('validateTemplate', () => { const sample: any = structuredClone(validTemplate); sample.license = undefined; - const result: Result = validateTemplate(sample); + const result: Result = validateTemplate(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); @@ -48,7 +48,7 @@ describe('validateTemplate', () => { const sample: any = structuredClone(validTemplate); sample.components[0].name = 'not-logs'; - const result: Result = validateTemplate(sample); + const result: Result = validateTemplate(sample); expect(result.ok).toBe(false); expect((result as any).error).toBeInstanceOf(Error); diff --git a/server/adaptors/integrations/integrations_builder.ts b/server/adaptors/integrations/integrations_builder.ts index 960912e121..7a8026ceac 100644 --- a/server/adaptors/integrations/integrations_builder.ts +++ b/server/adaptors/integrations/integrations_builder.ts @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from 'uuid'; import { uuidRx } from 'public/components/custom_panels/redux/panel_slice'; import { SavedObjectsClientContract } from '../../../../../src/core/server'; -import { Integration } from './repository/integration'; +import { IntegrationReader } from './repository/integration'; import { SavedObjectsBulkCreateObject } from '../../../../../src/core/public'; interface BuilderOptions { @@ -21,7 +21,7 @@ export class IntegrationInstanceBuilder { this.client = client; } - build(integration: Integration, options: BuilderOptions): Promise { + build(integration: IntegrationReader, options: BuilderOptions): Promise { const instance = integration .deepCheck() .then((result) => { @@ -92,11 +92,11 @@ export class IntegrationInstanceBuilder { } async buildInstance( - integration: Integration, + integration: IntegrationReader, refs: AssetReference[], options: BuilderOptions ): Promise { - const config: Result = await integration.getConfig(); + const config: Result = await integration.getConfig(); if (!config.ok) { return Promise.reject( new Error('Attempted to create instance with invalid template', config.error) diff --git a/server/adaptors/integrations/integrations_manager.ts b/server/adaptors/integrations/integrations_manager.ts index 857e223b76..d365e48eef 100644 --- a/server/adaptors/integrations/integrations_manager.ts +++ b/server/adaptors/integrations/integrations_manager.ts @@ -8,16 +8,17 @@ import { addRequestToMetric } from '../../common/metrics/metrics_helper'; import { IntegrationsAdaptor } from './integrations_adaptor'; import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types'; import { IntegrationInstanceBuilder } from './integrations_builder'; -import { Repository } from './repository/repository'; +import { RepositoryReader } from './repository/repository'; export class IntegrationsManager implements IntegrationsAdaptor { client: SavedObjectsClientContract; instanceBuilder: IntegrationInstanceBuilder; - repository: Repository; + repository: RepositoryReader; - constructor(client: SavedObjectsClientContract, repository?: Repository) { + constructor(client: SavedObjectsClientContract, repository?: RepositoryReader) { this.client = client; - this.repository = repository ?? new Repository(path.join(__dirname, '__data__/repository')); + this.repository = + repository ?? new RepositoryReader(path.join(__dirname, '__data__/repository')); this.instanceBuilder = new IntegrationInstanceBuilder(this.client); } @@ -57,7 +58,7 @@ export class IntegrationsManager implements IntegrationsAdaptor { _getAllIntegrationTemplates = async (): Promise => { const integrationList = await this.repository.getIntegrationList(); const configResults = await Promise.all(integrationList.map((x) => x.getConfig())); - const configs = configResults.filter((cfg) => cfg.ok) as Array<{ value: IntegrationTemplate }>; + const configs = configResults.filter((cfg) => cfg.ok) as Array<{ value: IntegrationConfig }>; return Promise.resolve({ hits: configs.map((cfg) => cfg.value) }); }; diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 7898f485b0..7ffbb176bf 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -4,7 +4,7 @@ */ import * as fs from 'fs/promises'; -import { Integration } from '../integration'; +import { IntegrationReader } from '../integration'; import { Dirent, Stats } from 'fs'; import * as path from 'path'; import { FileSystemCatalogDataAdaptor } from '../fs_data_adaptor'; @@ -12,8 +12,8 @@ import { FileSystemCatalogDataAdaptor } from '../fs_data_adaptor'; jest.mock('fs/promises'); describe('Integration', () => { - let integration: Integration; - const sampleIntegration: IntegrationTemplate = { + let integration: IntegrationReader; + const sampleIntegration: IntegrationConfig = { name: 'sample', version: '2.0.0', license: 'Apache-2.0', @@ -33,7 +33,7 @@ describe('Integration', () => { }; beforeEach(() => { - integration = new Integration('./sample'); + integration = new IntegrationReader('./sample'); jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); }); diff --git a/server/adaptors/integrations/repository/__test__/repository.test.ts b/server/adaptors/integrations/repository/__test__/repository.test.ts index c452acd563..d66fc5e863 100644 --- a/server/adaptors/integrations/repository/__test__/repository.test.ts +++ b/server/adaptors/integrations/repository/__test__/repository.test.ts @@ -4,18 +4,18 @@ */ import * as fs from 'fs/promises'; -import { Repository } from '../repository'; -import { Integration } from '../integration'; +import { RepositoryReader } from '../repository'; +import { IntegrationReader } from '../integration'; import { Dirent, Stats } from 'fs'; import path from 'path'; jest.mock('fs/promises'); describe('Repository', () => { - let repository: Repository; + let repository: RepositoryReader; beforeEach(() => { - repository = new Repository('path/to/directory'); + repository = new RepositoryReader('path/to/directory'); }); describe('getIntegrationList', () => { @@ -23,14 +23,14 @@ describe('Repository', () => { jest.spyOn(fs, 'readdir').mockResolvedValue((['folder1', 'folder2'] as unknown) as Dirent[]); jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); jest - .spyOn(Integration.prototype, 'getConfig') + .spyOn(IntegrationReader.prototype, 'getConfig') .mockResolvedValue({ ok: true, value: {} as any }); const integrations = await repository.getIntegrationList(); expect(integrations).toHaveLength(2); - expect(integrations[0]).toBeInstanceOf(Integration); - expect(integrations[1]).toBeInstanceOf(Integration); + expect(integrations[0]).toBeInstanceOf(IntegrationReader); + expect(integrations[1]).toBeInstanceOf(IntegrationReader); }); it('should filter out null values from the integration list', async () => { @@ -46,13 +46,13 @@ describe('Repository', () => { }); jest - .spyOn(Integration.prototype, 'getConfig') + .spyOn(IntegrationReader.prototype, 'getConfig') .mockResolvedValue({ ok: true, value: {} as any }); const integrations = await repository.getIntegrationList(); expect(integrations).toHaveLength(1); - expect(integrations[0]).toBeInstanceOf(Integration); + expect(integrations[0]).toBeInstanceOf(IntegrationReader); }); it('should handle errors and return an empty array', async () => { @@ -68,17 +68,17 @@ describe('Repository', () => { it('should return an Integration instance if it exists and passes the check', async () => { jest.spyOn(fs, 'lstat').mockResolvedValue({ isDirectory: () => true } as Stats); jest - .spyOn(Integration.prototype, 'getConfig') + .spyOn(IntegrationReader.prototype, 'getConfig') .mockResolvedValue({ ok: true, value: {} as any }); const integration = await repository.getIntegration('integrationName'); - expect(integration).toBeInstanceOf(Integration); + expect(integration).toBeInstanceOf(IntegrationReader); }); it('should return null if the integration does not exist or fails checks', async () => { jest - .spyOn(Integration.prototype, 'getConfig') + .spyOn(IntegrationReader.prototype, 'getConfig') .mockResolvedValue({ ok: false, error: new Error() }); const integration = await repository.getIntegration('invalidIntegration'); diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index d9caa91621..fca1aef5ce 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -12,7 +12,7 @@ import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; * It is backed by the repository file system. * It includes accessor methods for integration configs, as well as helpers for nested components. */ -export class Integration { +export class IntegrationReader { reader: CatalogDataAdaptor; directory: string; name: string; @@ -28,7 +28,7 @@ export class Integration { * * @returns a Result indicating whether the integration is valid. */ - async deepCheck(): Promise> { + async deepCheck(): Promise> { const configResult = await this.getConfig(); if (!configResult.ok) { return configResult; @@ -72,7 +72,7 @@ export class Integration { * @param version The version of the config to retrieve. * @returns The config if a valid config matching the version is present, otherwise null. */ - async getConfig(version?: string): Promise> { + async getConfig(version?: string): Promise> { if ((await this.reader.getDirectoryType()) !== 'integration') { return { ok: false, error: new Error(`${this.directory} is not a valid integration`) }; } diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index cf8e0312a5..08200d474f 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -4,10 +4,10 @@ */ import * as path from 'path'; -import { Integration } from './integration'; +import { IntegrationReader } from './integration'; import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; -export class Repository { +export class RepositoryReader { reader: CatalogDataAdaptor; directory: string; @@ -16,7 +16,7 @@ export class Repository { this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); } - async getIntegrationList(): Promise { + async getIntegrationList(): Promise { // TODO in the future, we want to support traversing nested directory structures. const folders = await this.reader.findIntegrations(); if (!folders.ok) { @@ -26,15 +26,15 @@ export class Repository { const integrations = await Promise.all( folders.value.map((i) => this.getIntegration(path.basename(i))) ); - return integrations.filter((x) => x !== null) as Integration[]; + return integrations.filter((x) => x !== null) as IntegrationReader[]; } - async getIntegration(name: string): Promise { + async getIntegration(name: string): Promise { if ((await this.reader.getDirectoryType(name)) !== 'integration') { console.error(`Requested integration '${name}' does not exist`); return null; } - const integ = new Integration(name, this.reader.join(name)); + const integ = new IntegrationReader(name, this.reader.join(name)); const checkResult = await integ.getConfig(); if (!checkResult.ok) { console.error(`Integration '${name}' is invalid:`, checkResult.error); diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index d84dc3c5a4..c74829d302 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -5,7 +5,7 @@ type Result = { ok: true; value: T } | { ok: false; error: E }; -interface IntegrationTemplate { +interface IntegrationConfig { name: string; version: string; displayName?: string; @@ -48,7 +48,7 @@ interface DisplayAsset { } interface IntegrationTemplateSearchResult { - hits: IntegrationTemplate[]; + hits: IntegrationConfig[]; } interface IntegrationTemplateQuery { diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index b9c7f33f65..7486a38eda 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -17,7 +17,7 @@ const staticAsset: JSONSchemaType = { additionalProperties: false, }; -const templateSchema: JSONSchemaType = { +const templateSchema: JSONSchemaType = { type: 'object', properties: { name: { type: 'string' }, @@ -120,7 +120,7 @@ const instanceValidator = ajv.compile(instanceSchema); * If validation succeeds, returns an object with 'ok' set to true and the validated data. * If validation fails, returns an object with 'ok' set to false and an Error object describing the validation error. */ -export const validateTemplate = (data: unknown): Result => { +export const validateTemplate = (data: unknown): Result => { if (!templateValidator(data)) { return { ok: false, error: new Error(ajv.errorsText(templateValidator.errors)) }; } From b465b9f00dc1849c2338769280ab56b55b333bd2 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 15 Sep 2023 14:18:08 -0700 Subject: [PATCH 37/79] Refactor component usage Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 452 ++++++------------ 1 file changed, 155 insertions(+), 297 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 436ac5af52..b440132414 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -8,20 +8,19 @@ import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/st import React, { useState } from 'react'; interface IntegrationConfig { - instance_name: string; - datasource_name: string; - datasource_description: string; - datasource_filetype: string; - datasourcee_location: string; - connection_name: string; - asset_accel: string; - query_accel: string; + instanceName: string; + createNewDataSource: boolean; + dataSourceName: string; + dataSourceDescription: string; + dataSourceFileType: string; + dataSourceLocation: string; + connectionName: string; } const STEPS: EuiContainedStepProps[] = [ { title: 'Name Integration', children: }, { title: 'Select index or data source for integration', children: }, - { title: 'Select integration assets', children: }, + // { title: 'Select integration assets', children: }, ]; const ALLOWED_FILE_TYPES: Eui.EuiSelectOption[] = [ @@ -62,7 +61,7 @@ const integrationDataTableData = [ }, ]; -const getSteps = (activeStep: number): EuiContainedStepProps[] => { +const getSetupStepStatus = (activeStep: number): EuiContainedStepProps[] => { return STEPS.map((step, idx) => { let status: string = ''; if (idx < activeStep) { @@ -75,7 +74,13 @@ const getSteps = (activeStep: number): EuiContainedStepProps[] => { }); }; -function SetupIntegrationMetadata() { +function SetupIntegrationMetadata({ + name, + setName, +}: { + name: string; + setName: (name: string) => void; +}) { return ( @@ -85,22 +90,35 @@ function SetupIntegrationMetadata() { label="Name" helpText="The name will be used to label the newly added integration" > - + setName(evt.target.value)} /> ); } +function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Element { + return ( + + +

Data Table

+ + + + + + Close + + + + ); +} + function SetupIntegrationNewTable() { return ( - - -

{STEPS[1].title}

- - -

No problem, we can help. Tell us about your data.

- - +
@@ -113,308 +131,143 @@ function SetupIntegrationNewTable() { - +
); } -function IntegrationDataModal( - isDataModalVisible: boolean, - setDataModalVisible: React.Dispatch> -): React.JSX.Element | null { - let dataModal = null; - if (isDataModalVisible) { - dataModal = ( - setDataModalVisible(false)}> - -

Data Table

- - - - - setDataModalVisible(false)} size="s"> - Close - - - - ); - } - return dataModal; -} - -function SetupIntegrationExistingTable( - isDataModalVisible: boolean, - setDataModalVisible: React.Dispatch> -) { +function SetupIntegrationExistingTable({ + showDataModal, + setShowDataModal, +}: { + showDataModal: boolean; + setShowDataModal: (visible: boolean) => void; +}) { + const dataModal = showDataModal ? ( + setShowDataModal(false)} /> + ) : null; return ( - - -

{STEPS[1].title}

- +
- setDataModalVisible(true)}>View table - {IntegrationDataModal(isDataModalVisible, setDataModalVisible)} - + setShowDataModal(true)}>View table + {dataModal} +
); } -function SetupIntegrationAccelerationStandard( - integConfig: IntegrationConfig, - setConfig: React.Dispatch> -) { - return ( - - - - None{': '} - - Set up indices, but don't install any assets. - - - ), - }, - { - id: 'queries', - label: ( - - Minimal{': '} - - Set up indices and include provided saved queries. - - - ), - }, - { - id: 'visualizations', - label: ( - - Complete{': '} - - Indices, queries, and visualizations for the data. - - - ), - }, - { - id: 'all', - label: ( - - Everything{': '} - - Includes additional assets such as detectors or geospatial. - - - ), - }, - ]} - idSelected={integConfig.asset_accel} - onChange={(id) => setConfig(Object.assign({}, integConfig, { asset_accel: id }))} - /> - - - - - None{': '} - - No acceleration. Cheap, but slow. - - - ), - }, - { - id: 'basic', - label: ( - - Basic{': '} - - Minimal optimizations balancing performance and cost. - - - ), - }, - { - id: 'advanced', - label: ( - - Advanced{': '} - - More intensive optimization for better performance. - - - ), - }, - { - id: 'ultra', - label: ( - - Ultra{': '} - - Ideal for performance-critical indices. - - - ), - }, - ]} - idSelected={integConfig.query_accel} - onChange={(id) => setConfig(Object.assign({}, integConfig, { query_accel: id }))} - /> - - - ); -} +function SetupIntegrationDataSource({ + config, + updateConfig, + showDataModal, + setShowDataModal, + tableDetected, + setTableDetected, +}: { + config: IntegrationConfig; + updateConfig: (updates: Partial) => void; + showDataModal: boolean; + setShowDataModal: (show: boolean) => void; + tableDetected: boolean; + setTableDetected: (detected: boolean) => void; +}) { + let tableForm; + if (tableDetected && !config.createNewDataSource) { + tableForm = ( + setShowDataModal(x)} + /> + ); + } else { + tableForm = ; + } -function SetupIntegrationAccelerationAdvanced( - integConfig: IntegrationConfig, - setConfig: React.Dispatch> -) { - return ( - {}, - }, - { - name: 'configure', - description: 'Configure Asset', - type: 'icon', - icon: 'indexSettings', - color: 'primary', - onClick: () => {}, - }, - ], - }, - ]} - items={[ - { - name: '[NGINX Core Logs 1.0] Overview', - type: 'dashboard', - acceleration: 'Enhanced', - }, - { - name: 'ss4o_logs-*-*', - type: 'index-pattern', - acceleration: 'Status', - }, - { - name: 'Top Paths', - type: 'visualization', - acceleration: 'Query', - }, - ]} - hasActions={true} - /> - ); -} + let tablesNotFoundMessage = null; + if (!tableDetected) { + tablesNotFoundMessage = ( + <> + +

No problem, we can help. Tell us about your data.

+ + + + ); + } -function SetupIntegrationAcceleration( - integConfig: IntegrationConfig, - setConfig: React.Dispatch>, - isStandard: boolean, - setIsStandard: React.Dispatch> -) { return ( - - -

{STEPS[2].title}

- - - setIsStandard(true)}> - Standard - - setIsStandard(false)}> - Advanced - - - - {isStandard - ? SetupIntegrationAccelerationStandard(integConfig, setConfig) - : SetupIntegrationAccelerationAdvanced(integConfig, setConfig)} - +
+ setTableDetected(event.target.checked)} + /> + + + +

{STEPS[1].title}

+ + + {tablesNotFoundMessage} + updateConfig({ createNewDataSource: evt.target.checked })} + disabled={!tableDetected} + /> + + {tableForm} + +
); } -function SetupIntegrationStep(activeStep: number) { +function SetupIntegrationStep({ activeStep }: { activeStep: number }) { const [integConfig, setConfig] = useState({ - instance_name: 'NginX Access 2.0', - datasource_name: 'ss4o_logs-nginx-*-*', - datasource_description: 'Integration for viewing Nginx logs in S3.', - datasource_filetype: 'parquet', - datasourcee_location: 'ss4o_logs-nginx-*-*', - connection_name: 'S3 connection name', - asset_accel: 'visualizations', - query_accel: 'basic', - }); + instanceName: '', + createNewDataSource: false, + dataSourceName: '', + dataSourceDescription: '', + dataSourceFileType: '', + dataSourceLocation: '', + connectionName: '', + } as IntegrationConfig); const [isDataModalVisible, setDataModalVisible] = useState(false); const [tableDetected, setTableDetected] = useState(false); - const [isStandard, setIsStandard] = useState(true); + + const updateConfig = (updates: Partial) => + setConfig(Object.assign({}, integConfig, updates)); switch (activeStep) { case 0: - return SetupIntegrationMetadata(); + return ( + updateConfig({ instanceName: name })} + /> + ); case 1: - let tableForm; - if (tableDetected) { - tableForm = SetupIntegrationExistingTable(isDataModalVisible, setDataModalVisible); - } else { - tableForm = SetupIntegrationNewTable(); - } return ( -
- {tableForm} - - setTableDetected(event.target.checked)} - /> -
+ setDataModalVisible(show)} + tableDetected={tableDetected} + setTableDetected={(detected: boolean) => setTableDetected(detected)} + /> ); - case 2: - return SetupIntegrationAcceleration(integConfig, setConfig, isStandard, setIsStandard); default: - return Something went wrong...; + return ( + + Attempted to access integration setup step that doesn't exist. This is a bug. + + ); } } -function SetupBottomBar(step: number, setStep: React.Dispatch>) { +function SetupBottomBar({ step, setStep }: { step: number; setStep: (step: number) => void }) { return ( @@ -438,13 +291,13 @@ function SetupBottomBar(step: number, setStep: React.Dispatch {step > 0 ? ( - setStep(Math.max(step - 1, 0))}> + setStep(step - 1)}> Back ) : null} - setStep(Math.min(step + 1, 2))}> + setStep(step + 1)}> {step === STEPS.length - 1 ? 'Save' : 'Next'} @@ -461,11 +314,16 @@ export function SetupIntegrationStepsPage() { - + + + + - {SetupIntegrationStep(step)} - {SetupBottomBar(step, setStep)} + setStep(Math.min(Math.max(x, 0), STEPS.length - 1))} + /> ); From 2af52f56b740b06c79dc5f9a222ddac288840bbe Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 15 Sep 2023 14:42:40 -0700 Subject: [PATCH 38/79] Connect forms to config state Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 127 +++++++++++++----- 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index b440132414..791fad23b4 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -9,12 +9,12 @@ import React, { useState } from 'react'; interface IntegrationConfig { instanceName: string; - createNewDataSource: boolean; + useExisting: boolean; dataSourceName: string; dataSourceDescription: string; dataSourceFileType: string; dataSourceLocation: string; - connectionName: string; + existingDataSourceName: string; } const STEPS: EuiContainedStepProps[] = [ @@ -116,29 +116,52 @@ function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Eleme ); } -function SetupIntegrationNewTable() { +function SetupIntegrationNewTable({ + config, + updateConfig, +}: { + config: IntegrationConfig; + updateConfig: (updates: Partial) => void; +}) { return (
- + updateConfig({ dataSourceName: evt.target.value })} + /> - + updateConfig({ dataSourceDescription: evt.target.value })} + /> - + updateConfig({ dataSourceDescription: evt.target.value })} + /> - + updateConfig({ dataSourceLocation: evt.target.value })} + />
); } function SetupIntegrationExistingTable({ + config, + updateConfig, showDataModal, setShowDataModal, }: { + config: IntegrationConfig; + updateConfig: (updates: Partial) => void; showDataModal: boolean; setShowDataModal: (visible: boolean) => void; }) { @@ -147,8 +170,15 @@ function SetupIntegrationExistingTable({ ) : null; return (
- - + + updateConfig({ existingDataSourceName: evt.target.value })} + /> setShowDataModal(true)}>View table @@ -173,15 +203,17 @@ function SetupIntegrationDataSource({ setTableDetected: (detected: boolean) => void; }) { let tableForm; - if (tableDetected && !config.createNewDataSource) { + if (tableDetected && config.useExisting) { tableForm = ( setShowDataModal(x)} /> ); } else { - tableForm = ; + tableForm = ; } let tablesNotFoundMessage = null; @@ -211,9 +243,9 @@ function SetupIntegrationDataSource({ {tablesNotFoundMessage} updateConfig({ createNewDataSource: evt.target.checked })} + label="Use existing Data Source" + checked={config.useExisting && tableDetected} + onChange={(evt) => updateConfig({ useExisting: evt.target.checked })} disabled={!tableDetected} /> @@ -223,34 +255,30 @@ function SetupIntegrationDataSource({ ); } -function SetupIntegrationStep({ activeStep }: { activeStep: number }) { - const [integConfig, setConfig] = useState({ - instanceName: '', - createNewDataSource: false, - dataSourceName: '', - dataSourceDescription: '', - dataSourceFileType: '', - dataSourceLocation: '', - connectionName: '', - } as IntegrationConfig); +function SetupIntegrationStep({ + activeStep, + config, + updateConfig, +}: { + activeStep: number; + config: IntegrationConfig; + updateConfig: (updates: Partial) => void; +}) { const [isDataModalVisible, setDataModalVisible] = useState(false); const [tableDetected, setTableDetected] = useState(false); - const updateConfig = (updates: Partial) => - setConfig(Object.assign({}, integConfig, updates)); - switch (activeStep) { case 0: return ( updateConfig({ instanceName: name })} /> ); case 1: return ( setDataModalVisible(show)} @@ -267,7 +295,15 @@ function SetupIntegrationStep({ activeStep }: { activeStep: number }) { } } -function SetupBottomBar({ step, setStep }: { step: number; setStep: (step: number) => void }) { +function SetupBottomBar({ + step, + setStep, + config, +}: { + step: number; + setStep: (step: number) => void; + config: IntegrationConfig; +}) { return ( @@ -297,7 +333,17 @@ function SetupBottomBar({ step, setStep }: { step: number; setStep: (step: numbe ) : null} - setStep(step + 1)}> + { + if (step < STEPS.length - 1) { + setStep(step + 1); + } else { + console.log(config); + } + }} + > {step === STEPS.length - 1 ? 'Save' : 'Next'} @@ -307,8 +353,20 @@ function SetupBottomBar({ step, setStep }: { step: number; setStep: (step: numbe } export function SetupIntegrationStepsPage() { + const [integConfig, setConfig] = useState({ + instanceName: '', + useExisting: true, + dataSourceName: '', + dataSourceDescription: '', + dataSourceFileType: 'parquet', + dataSourceLocation: '', + existingDataSourceName: '', + } as IntegrationConfig); const [step, setStep] = useState(0); + const updateConfig = (updates: Partial) => + setConfig(Object.assign({}, integConfig, updates)); + return ( @@ -317,12 +375,17 @@ export function SetupIntegrationStepsPage() { - + setStep(Math.min(Math.max(x, 0), STEPS.length - 1))} + config={integConfig} /> From 1687fbccd46de43842c7ddce3cccf976086cab26 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 15 Sep 2023 14:55:51 -0700 Subject: [PATCH 39/79] Fix filetype selector Signed-off-by: Simeon Widdis --- public/components/integrations/components/setup_integration.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 791fad23b4..02a68f447e 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -141,7 +141,7 @@ function SetupIntegrationNewTable({ updateConfig({ dataSourceDescription: evt.target.value })} + onChange={(evt) => updateConfig({ dataSourceFileType: evt.target.value })} /> From 1049b13cbe28f3e8b2079d700b294f7f081046f9 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 18 Sep 2023 10:38:05 -0700 Subject: [PATCH 40/79] Remove hardcoded name in path Signed-off-by: Simeon Widdis --- public/components/integrations/components/integration.tsx | 2 +- public/components/integrations/components/setup_integration.tsx | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index e4404b7e30..b7fba8c3ca 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -279,7 +279,7 @@ export function Integration(props: AvailableIntegrationProps) { {IntegrationOverview({ integration, showFlyout: () => { - window.location.hash = '#/available/nginx/setup'; + window.location.hash = `#/available/${integration.name}/setup`; }, setUpSample: () => { addIntegrationRequest(true, integrationTemplateId); diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 02a68f447e..d533559015 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -20,7 +20,6 @@ interface IntegrationConfig { const STEPS: EuiContainedStepProps[] = [ { title: 'Name Integration', children: }, { title: 'Select index or data source for integration', children: }, - // { title: 'Select integration assets', children: }, ]; const ALLOWED_FILE_TYPES: Eui.EuiSelectOption[] = [ From edd6bbe3d26bd9217f55ad47720050542c1718b6 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 18 Sep 2023 10:44:19 -0700 Subject: [PATCH 41/79] Write one snapshot test Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 628 ++++++++++++++++++ .../__tests__/setup_integration.test.tsx | 22 + 2 files changed, 650 insertions(+) create mode 100644 public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap create mode 100644 public/components/integrations/components/__tests__/setup_integration.test.tsx diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap new file mode 100644 index 0000000000..6b1ed1e30c --- /dev/null +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -0,0 +1,628 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Integration Setup Page Test Renders integration setup page as expected 1`] = ` + + +
+ +
+ +
+ +
+ , + "status": "", + "title": "Name Integration", + }, + Object { + "children": , + "status": "disabled", + "title": "Select index or data source for integration", + }, + ] + } + > +
+ +
+
+ + + + + Step 1 + + + + + + +

+ Name Integration +

+ +
+
+ +
+ +
+
+ + +
+
+ + + + + Step 2 is disabled + + + + + + +

+ Select index or data source for integration +

+ +
+
+ +
+ +
+
+ +
+ +
+ + +
+ + + +
+ +

+ Name Integration +

+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + + +
+ The name will be used to label the newly added integration +
+ +
+
+ +
+ + + +
+ +
+ + + + + +
+

+ Page level controls +

+
+
+ +
+
+
+
+
+ +
+
+
+

+ There is a new region landmark with page level controls at the end of the document. +

+
+ } + > + +
+ +

+ Page level controls +

+ + +
+ +
+ + + + + +
+ + +
+ +
+ +
+ + +
+ + + + + +
+ +
+ +
+ + +

+ + There is a new region landmark with page level controls at the end of the document. + +

+ + + + + +
+ +
+ + +`; diff --git a/public/components/integrations/components/__tests__/setup_integration.test.tsx b/public/components/integrations/components/__tests__/setup_integration.test.tsx new file mode 100644 index 0000000000..253211d41c --- /dev/null +++ b/public/components/integrations/components/__tests__/setup_integration.test.tsx @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { configure, mount } from 'enzyme'; +import Adapter from 'enzyme-adapter-react-16'; +import React from 'react'; +import { waitFor } from '@testing-library/react'; +import { SetupIntegrationStepsPage } from '../setup_integration'; + +describe('Integration Setup Page Test', () => { + configure({ adapter: new Adapter() }); + + it('Renders integration setup page as expected', async () => { + const wrapper = mount(); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); +}); From 1c95d417a3b4da28ec0553e91a72036e32289e74 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 18 Sep 2023 10:53:43 -0700 Subject: [PATCH 42/79] Add more tests Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 1394 +++++++++++++++++ .../__tests__/setup_integration.test.tsx | 70 +- .../components/setup_integration.tsx | 14 +- 3 files changed, 1470 insertions(+), 8 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 6b1ed1e30c..dc3fcf6f67 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -626,3 +626,1397 @@ exports[`Integration Setup Page Test Renders integration setup page as expected `; + +exports[`Integration Setup Page Test Renders the data source form as expected 1`] = ` + +
+ +
+ + + (debug) Table detected + +
+ + +
+ + +
+ +

+ Select index or data source for integration +

+ + +
+ + +
+
+ + + No tables were found + +
+ +
+ +
+

+ No problem, we can help. Tell us about your data. +

+
+ +
+ +
+ + +
+ + +
+ + + Use existing Data Source + +
+ + +
+ + +
+ +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+ + + + + +
+ +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ +
+ +
+ +
+ +`; + +exports[`Integration Setup Page Test Renders the existing table form as expected 1`] = ` + +
+ +
+
+ + + +
+
+ + +
+
+ + + + +
+ + + + + +
+ +
+
+ + + +
+ Manage data associated with this data source +
+ +
+
+ + +
+ + + + +
+ +`; + +exports[`Integration Setup Page Test Renders the metadata form as expected 1`] = ` + + +
+ +

+ Name Integration +

+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + + +
+ The name will be used to label the newly added integration +
+ +
+
+ +
+ + +`; + +exports[`Integration Setup Page Test Renders the new table form as expected 1`] = ` + +
+ +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+ + + + + +
+ +
+
+ + +
+
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+
+ + +
+
+ +
+ +`; diff --git a/public/components/integrations/components/__tests__/setup_integration.test.tsx b/public/components/integrations/components/__tests__/setup_integration.test.tsx index 253211d41c..63e0806b2b 100644 --- a/public/components/integrations/components/__tests__/setup_integration.test.tsx +++ b/public/components/integrations/components/__tests__/setup_integration.test.tsx @@ -7,7 +7,23 @@ import { configure, mount } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import React from 'react'; import { waitFor } from '@testing-library/react'; -import { SetupIntegrationStepsPage } from '../setup_integration'; +import { + SetupIntegrationDataSource, + SetupIntegrationExistingTable, + SetupIntegrationMetadata, + SetupIntegrationNewTable, + SetupIntegrationStepsPage, +} from '../setup_integration'; + +const TEST_CONFIG = { + instanceName: 'Test Instance Name', + useExisting: true, + dataSourceName: 'Test Datasource Name', + dataSourceDescription: 'Test Datasource Description', + dataSourceFileType: 'json', + dataSourceLocation: 'ss4o_logs-test-new-location', + existingDataSourceName: 'ss4o_logs-test-existing-location', +}; describe('Integration Setup Page Test', () => { configure({ adapter: new Adapter() }); @@ -19,4 +35,56 @@ describe('Integration Setup Page Test', () => { expect(wrapper).toMatchSnapshot(); }); }); + + it('Renders the metadata form as expected', async () => { + const wrapper = mount( + {}} /> + ); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); + + it('Renders the data source form as expected', async () => { + const wrapper = mount( + {}} + showDataModal={false} + setShowDataModal={() => {}} + tableDetected={false} + setTableDetected={() => {}} + /> + ); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); + + it('Renders the new table form as expected', async () => { + const wrapper = mount( + {}} /> + ); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); + + it('Renders the existing table form as expected', async () => { + const wrapper = mount( + {}} + showDataModal={false} + setShowDataModal={() => {}} + /> + ); + + await waitFor(() => { + expect(wrapper).toMatchSnapshot(); + }); + }); }); diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index d533559015..9e3052abfb 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -73,7 +73,7 @@ const getSetupStepStatus = (activeStep: number): EuiContainedStepProps[] => { }); }; -function SetupIntegrationMetadata({ +export function SetupIntegrationMetadata({ name, setName, }: { @@ -95,7 +95,7 @@ function SetupIntegrationMetadata({ ); } -function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Element { +export function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Element { return ( @@ -115,7 +115,7 @@ function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Eleme ); } -function SetupIntegrationNewTable({ +export function SetupIntegrationNewTable({ config, updateConfig, }: { @@ -153,7 +153,7 @@ function SetupIntegrationNewTable({ ); } -function SetupIntegrationExistingTable({ +export function SetupIntegrationExistingTable({ config, updateConfig, showDataModal, @@ -186,7 +186,7 @@ function SetupIntegrationExistingTable({ ); } -function SetupIntegrationDataSource({ +export function SetupIntegrationDataSource({ config, updateConfig, showDataModal, @@ -254,7 +254,7 @@ function SetupIntegrationDataSource({ ); } -function SetupIntegrationStep({ +export function SetupIntegrationStep({ activeStep, config, updateConfig, @@ -294,7 +294,7 @@ function SetupIntegrationStep({ } } -function SetupBottomBar({ +export function SetupBottomBar({ step, setStep, config, From 176003890349973a501cc790fd54e67de49f2fce Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 18 Sep 2023 10:54:18 -0700 Subject: [PATCH 43/79] Fix test naming Signed-off-by: Simeon Widdis --- .../components/__tests__/setup_integration.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/components/integrations/components/__tests__/setup_integration.test.tsx b/public/components/integrations/components/__tests__/setup_integration.test.tsx index 63e0806b2b..61001ba7c5 100644 --- a/public/components/integrations/components/__tests__/setup_integration.test.tsx +++ b/public/components/integrations/components/__tests__/setup_integration.test.tsx @@ -25,7 +25,7 @@ const TEST_CONFIG = { existingDataSourceName: 'ss4o_logs-test-existing-location', }; -describe('Integration Setup Page Test', () => { +describe('Integration Setup Page', () => { configure({ adapter: new Adapter() }); it('Renders integration setup page as expected', async () => { From d4b27fed6c7f4c42dbdb3d696ce4e6f10807b6ab Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 18 Sep 2023 10:56:50 -0700 Subject: [PATCH 44/79] Update obsolete snapshots Signed-off-by: Simeon Widdis --- .../__snapshots__/setup_integration.test.tsx.snap | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index dc3fcf6f67..1efafd0e03 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Integration Setup Page Test Renders integration setup page as expected 1`] = ` +exports[`Integration Setup Page Renders integration setup page as expected 1`] = `
`; -exports[`Integration Setup Page Test Renders the data source form as expected 1`] = ` +exports[`Integration Setup Page Renders the data source form as expected 1`] = ` `; -exports[`Integration Setup Page Test Renders the existing table form as expected 1`] = ` +exports[`Integration Setup Page Renders the existing table form as expected 1`] = ` `; -exports[`Integration Setup Page Test Renders the metadata form as expected 1`] = ` +exports[`Integration Setup Page Renders the metadata form as expected 1`] = ` `; -exports[`Integration Setup Page Test Renders the new table form as expected 1`] = ` +exports[`Integration Setup Page Renders the new table form as expected 1`] = ` Date: Mon, 18 Sep 2023 13:36:17 -0700 Subject: [PATCH 45/79] Move integration creation helpers to own file Signed-off-by: Simeon Widdis --- .../added_integration_flyout.test.tsx | 329 +---------------- .../create_integration_helpers.test.ts | 333 ++++++++++++++++++ .../components/add_integration_flyout.tsx | 171 +-------- .../components/create_integration_helpers.ts | 173 +++++++++ 4 files changed, 509 insertions(+), 497 deletions(-) create mode 100644 public/components/integrations/components/__tests__/create_integration_helpers.test.ts create mode 100644 public/components/integrations/components/create_integration_helpers.ts diff --git a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx index ae3b609f87..9099e59a14 100644 --- a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx +++ b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx @@ -6,17 +6,7 @@ import { configure, mount } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import { waitFor } from '@testing-library/react'; -import { - AddIntegrationFlyout, - checkDataSourceName, - doTypeValidation, - doNestedPropertyValidation, - doPropertyValidation, - fetchDataSourceMappings, - fetchIntegrationMappings, - doExistingDataSourceValidation, -} from '../add_integration_flyout'; -import * as add_integration_flyout from '../add_integration_flyout'; +import { AddIntegrationFlyout } from '../add_integration_flyout'; import React from 'react'; import { HttpSetup } from '../../../../../../../src/core/public'; @@ -44,320 +34,3 @@ describe('Add Integration Flyout Test', () => { }); }); }); - -describe('doTypeValidation', () => { - it('should return true if required type is not specified', () => { - const toCheck = { type: 'string' }; - const required = {}; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return true if types match', () => { - const toCheck = { type: 'string' }; - const required = { type: 'string' }; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return true if object has properties', () => { - const toCheck = { properties: { prop1: { type: 'string' } } }; - const required = { type: 'object' }; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return false if types do not match', () => { - const toCheck = { type: 'string' }; - const required = { type: 'number' }; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(false); - }); -}); - -describe('doNestedPropertyValidation', () => { - it('should return true if type validation passes and no properties are required', () => { - const toCheck = { type: 'string' }; - const required = { type: 'string' }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return false if type validation fails', () => { - const toCheck = { type: 'string' }; - const required = { type: 'number' }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(false); - }); - - it('should return false if a required property is missing', () => { - const toCheck = { type: 'object', properties: { prop1: { type: 'string' } } }; - const required = { - type: 'object', - properties: { prop1: { type: 'string' }, prop2: { type: 'number' } }, - }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(false); - }); - - it('should return true if all required properties pass validation', () => { - const toCheck = { - type: 'object', - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }; - const required = { - type: 'object', - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); -}); - -describe('doPropertyValidation', () => { - it('should return true if all properties pass validation', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result.ok).toBe(true); - }); - - it('should return false if a property fails validation', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'boolean' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result.ok).toBe(false); - }); - - it('should return false if a required nested property is missing', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result.ok).toBe(false); - }); -}); - -describe('checkDataSourceName', () => { - it('Filters out invalid index names', () => { - const result = checkDataSourceName('ss4o_logs-no-exclams!', 'logs'); - - expect(result.ok).toBe(false); - }); - - it('Filters out incorrectly typed indices', () => { - const result = checkDataSourceName('ss4o_metrics-test-test', 'logs'); - - expect(result.ok).toBe(false); - }); - - it('Accepts correct indices', () => { - const result = checkDataSourceName('ss4o_logs-test-test', 'logs'); - - expect(result.ok).toBe(true); - }); -}); - -describe('fetchDataSourceMappings', () => { - it('Retrieves mappings', async () => { - const mockHttp = { - post: jest.fn().mockResolvedValue({ - source1: { mappings: { properties: { test: true } } }, - source2: { mappings: { properties: { test: true } } }, - }), - } as Partial; - - const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); - - await expect(result).resolves.toMatchObject({ - source1: { properties: { test: true } }, - source2: { properties: { test: true } }, - }); - }); - - it('Catches errors', async () => { - const mockHttp = { - post: jest.fn().mockRejectedValue(new Error('Mock error')), - } as Partial; - - const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); - - await expect(result).resolves.toBeNull(); - }); -}); - -describe('fetchIntegrationMappings', () => { - it('Returns schema mappings', async () => { - const mockHttp = { - get: jest.fn().mockResolvedValue({ data: { mappings: { test: true } }, statusCode: 200 }), - } as Partial; - - const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); - - await expect(result).resolves.toStrictEqual({ test: true }); - }); - - it('Returns null if response fails', async () => { - const mockHttp = { - get: jest.fn().mockResolvedValue({ statusCode: 404 }), - } as Partial; - - const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); - - await expect(result).resolves.toBeNull(); - }); - - it('Catches request error', async () => { - const mockHttp = { - get: jest.fn().mockRejectedValue(new Error('mock error')), - } as Partial; - - const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); - - await expect(result).resolves.toBeNull(); - }); -}); - -describe('doExistingDataSourceValidation', () => { - it('Catches and returns checkDataSourceName errors', async () => { - const mockHttp = {} as Partial; - jest - .spyOn(add_integration_flyout, 'checkDataSourceName') - .mockReturnValue({ ok: false, errors: ['mock'] }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Catches data stream fetch errors', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest.spyOn(add_integration_flyout, 'fetchDataSourceMappings').mockResolvedValue(null); - jest - .spyOn(add_integration_flyout, 'fetchIntegrationMappings') - .mockResolvedValue({ test: { template: { mappings: {} } } }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Catches integration fetch errors', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest - .spyOn(add_integration_flyout, 'fetchDataSourceMappings') - .mockResolvedValue({ test: { properties: {} } }); - jest.spyOn(add_integration_flyout, 'fetchIntegrationMappings').mockResolvedValue(null); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Catches type validation issues', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest - .spyOn(add_integration_flyout, 'fetchDataSourceMappings') - .mockResolvedValue({ test: { properties: {} } }); - jest - .spyOn(add_integration_flyout, 'fetchIntegrationMappings') - .mockResolvedValue({ test: { template: { mappings: {} } } }); - jest - .spyOn(add_integration_flyout, 'doPropertyValidation') - .mockReturnValue({ ok: false, errors: ['mock'] }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Returns no errors if everything passes', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest - .spyOn(add_integration_flyout, 'fetchDataSourceMappings') - .mockResolvedValue({ test: { properties: {} } }); - jest - .spyOn(add_integration_flyout, 'fetchIntegrationMappings') - .mockResolvedValue({ test: { template: { mappings: {} } } }); - jest.spyOn(add_integration_flyout, 'doPropertyValidation').mockReturnValue({ ok: true }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', true); - }); -}); diff --git a/public/components/integrations/components/__tests__/create_integration_helpers.test.ts b/public/components/integrations/components/__tests__/create_integration_helpers.test.ts new file mode 100644 index 0000000000..71ccc99bf6 --- /dev/null +++ b/public/components/integrations/components/__tests__/create_integration_helpers.test.ts @@ -0,0 +1,333 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + checkDataSourceName, + doTypeValidation, + doNestedPropertyValidation, + doPropertyValidation, + fetchDataSourceMappings, + fetchIntegrationMappings, + doExistingDataSourceValidation, +} from '../create_integration_helpers'; +import * as create_integration_helpers from '../create_integration_helpers'; +import { HttpSetup } from '../../../../../../../src/core/public'; + +describe('doTypeValidation', () => { + it('should return true if required type is not specified', () => { + const toCheck = { type: 'string' }; + const required = {}; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return true if types match', () => { + const toCheck = { type: 'string' }; + const required = { type: 'string' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return true if object has properties', () => { + const toCheck = { properties: { prop1: { type: 'string' } } }; + const required = { type: 'object' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return false if types do not match', () => { + const toCheck = { type: 'string' }; + const required = { type: 'number' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); +}); + +describe('doNestedPropertyValidation', () => { + it('should return true if type validation passes and no properties are required', () => { + const toCheck = { type: 'string' }; + const required = { type: 'string' }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return false if type validation fails', () => { + const toCheck = { type: 'string' }; + const required = { type: 'number' }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); + + it('should return false if a required property is missing', () => { + const toCheck = { type: 'object', properties: { prop1: { type: 'string' } } }; + const required = { + type: 'object', + properties: { prop1: { type: 'string' }, prop2: { type: 'number' } }, + }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); + + it('should return true if all required properties pass validation', () => { + const toCheck = { + type: 'object', + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }; + const required = { + type: 'object', + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); +}); + +describe('doPropertyValidation', () => { + it('should return true if all properties pass validation', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(true); + }); + + it('should return false if a property fails validation', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'boolean' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(false); + }); + + it('should return false if a required nested property is missing', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(false); + }); +}); + +describe('checkDataSourceName', () => { + it('Filters out invalid index names', () => { + const result = checkDataSourceName('ss4o_logs-no-exclams!', 'logs'); + + expect(result.ok).toBe(false); + }); + + it('Filters out incorrectly typed indices', () => { + const result = checkDataSourceName('ss4o_metrics-test-test', 'logs'); + + expect(result.ok).toBe(false); + }); + + it('Accepts correct indices', () => { + const result = checkDataSourceName('ss4o_logs-test-test', 'logs'); + + expect(result.ok).toBe(true); + }); +}); + +describe('fetchDataSourceMappings', () => { + it('Retrieves mappings', async () => { + const mockHttp = { + post: jest.fn().mockResolvedValue({ + source1: { mappings: { properties: { test: true } } }, + source2: { mappings: { properties: { test: true } } }, + }), + } as Partial; + + const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); + + await expect(result).resolves.toMatchObject({ + source1: { properties: { test: true } }, + source2: { properties: { test: true } }, + }); + }); + + it('Catches errors', async () => { + const mockHttp = { + post: jest.fn().mockRejectedValue(new Error('Mock error')), + } as Partial; + + const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); +}); + +describe('fetchIntegrationMappings', () => { + it('Returns schema mappings', async () => { + const mockHttp = { + get: jest.fn().mockResolvedValue({ data: { mappings: { test: true } }, statusCode: 200 }), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toStrictEqual({ test: true }); + }); + + it('Returns null if response fails', async () => { + const mockHttp = { + get: jest.fn().mockResolvedValue({ statusCode: 404 }), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); + + it('Catches request error', async () => { + const mockHttp = { + get: jest.fn().mockRejectedValue(new Error('mock error')), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); +}); + +describe('doExistingDataSourceValidation', () => { + it('Catches and returns checkDataSourceName errors', async () => { + const mockHttp = {} as Partial; + jest + .spyOn(create_integration_helpers, 'checkDataSourceName') + .mockReturnValue({ ok: false, errors: ['mock'] }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Catches data stream fetch errors', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest.spyOn(create_integration_helpers, 'fetchDataSourceMappings').mockResolvedValue(null); + jest + .spyOn(create_integration_helpers, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Catches integration fetch errors', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(create_integration_helpers, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest.spyOn(create_integration_helpers, 'fetchIntegrationMappings').mockResolvedValue(null); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Catches type validation issues', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(create_integration_helpers, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest + .spyOn(create_integration_helpers, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + jest + .spyOn(create_integration_helpers, 'doPropertyValidation') + .mockReturnValue({ ok: false, errors: ['mock'] }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Returns no errors if everything passes', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(create_integration_helpers, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest + .spyOn(create_integration_helpers, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + jest.spyOn(create_integration_helpers, 'doPropertyValidation').mockReturnValue({ ok: true }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', true); + }); +}); diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index 0e1e792610..426690adcf 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -20,8 +20,9 @@ import { EuiTitle, } from '@elastic/eui'; import React, { useState } from 'react'; -import { HttpSetup, HttpStart } from '../../../../../../src/core/public'; +import { HttpStart } from '../../../../../../src/core/public'; import { useToast } from '../../../../public/components/common/toast'; +import { doExistingDataSourceValidation } from './create_integration_helpers'; interface IntegrationFlyoutProps { onClose: () => void; @@ -31,174 +32,6 @@ interface IntegrationFlyoutProps { http: HttpStart; } -type ValidationResult = { ok: true } | { ok: false; errors: string[] }; - -export const doTypeValidation = ( - toCheck: { type?: string; properties?: object }, - required: { type?: string; properties?: object } -): ValidationResult => { - if (!required.type) { - return { ok: true }; - } - if (required.type === 'object') { - if (Boolean(toCheck.properties)) { - return { ok: true }; - } - return { ok: false, errors: ["'object' type must have properties."] }; - } - if (required.type !== toCheck.type) { - return { ok: false, errors: [`Type mismatch: '${required.type}' and '${toCheck.type}'`] }; - } - return { ok: true }; -}; - -export const doNestedPropertyValidation = ( - toCheck: { type?: string; properties?: { [key: string]: object } }, - required: { type?: string; properties?: { [key: string]: object } } -): ValidationResult => { - const typeCheck = doTypeValidation(toCheck, required); - if (!typeCheck.ok) { - return typeCheck; - } - for (const property of Object.keys(required.properties ?? {})) { - if (!Object.hasOwn(toCheck.properties ?? {}, property)) { - return { ok: false, errors: [`Missing field '${property}'`] }; - } - // Both are safely non-null after above checks. - const nested = doNestedPropertyValidation( - toCheck.properties![property], - required.properties![property] - ); - if (!nested.ok) { - return nested; - } - } - return { ok: true }; -}; - -export const doPropertyValidation = ( - rootType: string, - dataSourceProps: { [key: string]: { properties?: any } }, - requiredMappings: { [key: string]: { template: { mappings: { properties?: any } } } } -): ValidationResult => { - // Check root object type (without dependencies) - for (const [key, value] of Object.entries( - requiredMappings[rootType].template.mappings.properties - )) { - if ( - !dataSourceProps[key] || - !doNestedPropertyValidation(dataSourceProps[key], value as any).ok - ) { - return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; - } - } - // Check nested dependencies - for (const [key, value] of Object.entries(requiredMappings)) { - if (key === rootType) { - continue; - } - if ( - !dataSourceProps[key] || - !doNestedPropertyValidation(dataSourceProps[key], value.template.mappings.properties).ok - ) { - return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; - } - } - return { ok: true }; -}; - -// Returns true if the data stream is a legal name. -// Appends any additional validation errors to the provided errors array. -export const checkDataSourceName = ( - targetDataSource: string, - integrationType: string -): ValidationResult => { - let errors: string[] = []; - if (!/^[a-z\d\.][a-z\d\._\-\*]*$/.test(targetDataSource)) { - errors = errors.concat('This is not a valid index name.'); - return { ok: false, errors }; - } - const nameValidity: boolean = new RegExp(`^ss4o_${integrationType}-[^\\-]+-[^\\-]+`).test( - targetDataSource - ); - if (!nameValidity) { - errors = errors.concat('This index does not match the suggested naming convention.'); - return { ok: false, errors }; - } - return { ok: true }; -}; - -export const fetchDataSourceMappings = async ( - targetDataSource: string, - http: HttpSetup -): Promise<{ [key: string]: { properties: any } } | null> => { - return http - .post('/api/console/proxy', { - query: { - path: `${targetDataSource}/_mapping`, - method: 'GET', - }, - }) - .then((response) => { - // Un-nest properties by a level for caller convenience - Object.keys(response).forEach((key) => { - response[key].properties = response[key].mappings.properties; - }); - return response; - }) - .catch((err: any) => { - console.error(err); - return null; - }); -}; - -export const fetchIntegrationMappings = async ( - targetName: string, - http: HttpSetup -): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { - return http - .get(`/api/integrations/repository/${targetName}/schema`) - .then((response) => { - if (response.statusCode && response.statusCode !== 200) { - throw new Error('Failed to retrieve Integration schema', { cause: response }); - } - return response.data.mappings; - }) - .catch((err: any) => { - console.error(err); - return null; - }); -}; - -export const doExistingDataSourceValidation = async ( - targetDataSource: string, - integrationName: string, - integrationType: string, - http: HttpSetup -): Promise => { - const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); - if (!dataSourceNameCheck.ok) { - return dataSourceNameCheck; - } - const [dataSourceMappings, integrationMappings] = await Promise.all([ - fetchDataSourceMappings(targetDataSource, http), - fetchIntegrationMappings(integrationName, http), - ]); - if (!dataSourceMappings) { - return { ok: false, errors: ['Provided data stream could not be retrieved'] }; - } - if (!integrationMappings) { - return { ok: false, errors: ['Failed to retrieve integration schema information'] }; - } - const validationResult = Object.values(dataSourceMappings).every( - (value) => doPropertyValidation(integrationType, value.properties, integrationMappings).ok - ); - if (!validationResult) { - return { ok: false, errors: ['The provided index does not match the schema'] }; - } - return { ok: true }; -}; - export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { const { onClose, onCreate, integrationName, integrationType, http } = props; diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts new file mode 100644 index 0000000000..23ee7fbf4a --- /dev/null +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -0,0 +1,173 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { HttpSetup } from '../../../../../../src/core/public'; + +type ValidationResult = { ok: true } | { ok: false; errors: string[] }; + +export const doTypeValidation = ( + toCheck: { type?: string; properties?: object }, + required: { type?: string; properties?: object } +): ValidationResult => { + if (!required.type) { + return { ok: true }; + } + if (required.type === 'object') { + if (Boolean(toCheck.properties)) { + return { ok: true }; + } + return { ok: false, errors: ["'object' type must have properties."] }; + } + if (required.type !== toCheck.type) { + return { ok: false, errors: [`Type mismatch: '${required.type}' and '${toCheck.type}'`] }; + } + return { ok: true }; +}; + +export const doNestedPropertyValidation = ( + toCheck: { type?: string; properties?: { [key: string]: object } }, + required: { type?: string; properties?: { [key: string]: object } } +): ValidationResult => { + const typeCheck = doTypeValidation(toCheck, required); + if (!typeCheck.ok) { + return typeCheck; + } + for (const property of Object.keys(required.properties ?? {})) { + if (!Object.hasOwn(toCheck.properties ?? {}, property)) { + return { ok: false, errors: [`Missing field '${property}'`] }; + } + // Both are safely non-null after above checks. + const nested = doNestedPropertyValidation( + toCheck.properties![property], + required.properties![property] + ); + if (!nested.ok) { + return nested; + } + } + return { ok: true }; +}; + +export const doPropertyValidation = ( + rootType: string, + dataSourceProps: { [key: string]: { properties?: any } }, + requiredMappings: { [key: string]: { template: { mappings: { properties?: any } } } } +): ValidationResult => { + // Check root object type (without dependencies) + for (const [key, value] of Object.entries( + requiredMappings[rootType].template.mappings.properties + )) { + if ( + !dataSourceProps[key] || + !doNestedPropertyValidation(dataSourceProps[key], value as any).ok + ) { + return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; + } + } + // Check nested dependencies + for (const [key, value] of Object.entries(requiredMappings)) { + if (key === rootType) { + continue; + } + if ( + !dataSourceProps[key] || + !doNestedPropertyValidation(dataSourceProps[key], value.template.mappings.properties).ok + ) { + return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; + } + } + return { ok: true }; +}; + +// Returns true if the data stream is a legal name. +// Appends any additional validation errors to the provided errors array. +export const checkDataSourceName = ( + targetDataSource: string, + integrationType: string +): ValidationResult => { + let errors: string[] = []; + if (!/^[a-z\d\.][a-z\d\._\-\*]*$/.test(targetDataSource)) { + errors = errors.concat('This is not a valid index name.'); + return { ok: false, errors }; + } + const nameValidity: boolean = new RegExp(`^ss4o_${integrationType}-[^\\-]+-[^\\-]+`).test( + targetDataSource + ); + if (!nameValidity) { + errors = errors.concat('This index does not match the suggested naming convention.'); + return { ok: false, errors }; + } + return { ok: true }; +}; + +export const fetchDataSourceMappings = async ( + targetDataSource: string, + http: HttpSetup +): Promise<{ [key: string]: { properties: any } } | null> => { + return http + .post('/api/console/proxy', { + query: { + path: `${targetDataSource}/_mapping`, + method: 'GET', + }, + }) + .then((response) => { + // Un-nest properties by a level for caller convenience + Object.keys(response).forEach((key) => { + response[key].properties = response[key].mappings.properties; + }); + return response; + }) + .catch((err: any) => { + console.error(err); + return null; + }); +}; + +export const fetchIntegrationMappings = async ( + targetName: string, + http: HttpSetup +): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { + return http + .get(`/api/integrations/repository/${targetName}/schema`) + .then((response) => { + if (response.statusCode && response.statusCode !== 200) { + throw new Error('Failed to retrieve Integration schema', { cause: response }); + } + return response.data.mappings; + }) + .catch((err: any) => { + console.error(err); + return null; + }); +}; + +export const doExistingDataSourceValidation = async ( + targetDataSource: string, + integrationName: string, + integrationType: string, + http: HttpSetup +): Promise => { + const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); + if (!dataSourceNameCheck.ok) { + return dataSourceNameCheck; + } + const [dataSourceMappings, integrationMappings] = await Promise.all([ + fetchDataSourceMappings(targetDataSource, http), + fetchIntegrationMappings(integrationName, http), + ]); + if (!dataSourceMappings) { + return { ok: false, errors: ['Provided data stream could not be retrieved'] }; + } + if (!integrationMappings) { + return { ok: false, errors: ['Failed to retrieve integration schema information'] }; + } + const validationResult = Object.values(dataSourceMappings).every( + (value) => doPropertyValidation(integrationType, value.properties, integrationMappings).ok + ); + if (!validationResult) { + return { ok: false, errors: ['The provided index does not match the schema'] }; + } + return { ok: true }; +}; From 36d65c63285b6bbc8614a2b540e9484825a569f3 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 19 Sep 2023 11:09:29 -0700 Subject: [PATCH 46/79] Break out integration creation methods Signed-off-by: Simeon Widdis --- .../integrations/components/integration.tsx | 314 ++++++++++-------- 1 file changed, 177 insertions(+), 137 deletions(-) diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index b7fba8c3ca..786bddd948 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -21,97 +21,192 @@ import { IntegrationAssets } from './integration_assets_panel'; import { AvailableIntegrationProps } from './integration_types'; import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { IntegrationScreenshots } from './integration_screenshots_panel'; -import { AddIntegrationFlyout } from './add_integration_flyout'; import { useToast } from '../../../../public/components/common/toast'; +import { coreRefs } from '../../../framework/core_refs'; -export function Integration(props: AvailableIntegrationProps) { - const { http, integrationTemplateId, chrome } = props; +// Toast doesn't export, so we need to redeclare locally. +type Color = 'success' | 'primary' | 'warning' | 'danger' | undefined; - const { setToast } = useToast(); - const [integration, setIntegration] = useState({} as { name: string; type: string }); +interface Integration { + name: string; + type: string; +} - const [integrationMapping, setMapping] = useState(null); - const [integrationAssets, setAssets] = useState([]); - const [loading, setLoading] = useState(false); +const createComponentMapping = async ( + componentName: string, + payload: { + template: { mappings: { _meta: { version: string } } }; + composed_of: string[]; + index_patterns: string[]; + } +): Promise<{ [key: string]: { properties: any } } | null> => { + const http = coreRefs.http!; + const version = payload.template.mappings._meta.version; + return http.post('/api/console/proxy', { + body: JSON.stringify(payload), + query: { + path: `_component_template/ss4o_${componentName}-${version}-template`, + method: 'POST', + }, + }); +}; - const createComponentMapping = async ( - componentName: string, - payload: { - template: { mappings: { _meta: { version: string } } }; - composed_of: string[]; - index_patterns: string[]; +const createIndexMapping = async ( + componentName: string, + payload: { + template: { mappings: { _meta: { version: string } } }; + composed_of: string[]; + index_patterns: string[]; + }, + dataSourceName: string, + integration: Integration +): Promise<{ [key: string]: { properties: any } } | null> => { + const http = coreRefs.http!; + const version = payload.template.mappings._meta.version; + payload.index_patterns = [dataSourceName]; + return http.post('/api/console/proxy', { + body: JSON.stringify(payload), + query: { + path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, + method: 'POST', + }, + }); +}; + +const createDataSourceMappings = async ( + targetDataSource: string, + integrationTemplateId: string, + integration: Integration, + setToast: (title: string, color?: Color, text?: string | undefined) => void +): Promise => { + const http = coreRefs.http!; + const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); + let error: string | null = null; + const mappings = data.data.mappings; + mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( + (componentName: string) => { + const version = mappings[componentName].template.mappings._meta.version; + return `ss4o_${componentName}-${version}-template`; } - ): Promise<{ [key: string]: { properties: any } } | null> => { - const version = payload.template.mappings._meta.version; - return http.post('/api/console/proxy', { - body: JSON.stringify(payload), + ); + + try { + // Create component mappings before the index mapping + // The assumption is that index mapping relies on component mappings for creation + await Promise.all( + Object.entries(mappings).map(([key, mapping]) => { + if (key === integration.type) { + return Promise.resolve(); + } + return createComponentMapping(key, mapping as any); + }) + ); + // In order to see our changes, we need to manually provoke a refresh + await http.post('/api/console/proxy', { query: { - path: `_component_template/ss4o_${componentName}-${version}-template`, - method: 'POST', + path: '_refresh', + method: 'GET', }, }); - }; + await createIndexMapping( + integration.type, + mappings[integration.type], + targetDataSource, + integration + ); + } catch (err: any) { + error = err.message; + } - const createIndexMapping = async ( - componentName: string, - payload: { - template: { mappings: { _meta: { version: string } } }; - composed_of: string[]; - index_patterns: string[]; - }, - dataSourceName: string - ): Promise<{ [key: string]: { properties: any } } | null> => { - const version = payload.template.mappings._meta.version; - payload.index_patterns = [dataSourceName]; - return http.post('/api/console/proxy', { - body: JSON.stringify(payload), + if (error !== null) { + setToast('Failure creating index template', 'danger', error); + } else { + setToast(`Successfully created index template`); + } +}; + +async function addIntegrationRequest( + addSample: boolean, + templateName: string, + integrationTemplateId: string, + integration: Integration, + setLoading: React.Dispatch>, + setToast: (title: string, color?: Color, text?: string | undefined) => void, + name?: string, + dataSource?: string +) { + const http = coreRefs.http!; + setLoading(true); + if (addSample) { + createDataSourceMappings( + `ss4o_${integration.type}-${integrationTemplateId}-*-sample`, + integrationTemplateId, + integration, + setToast + ); + name = `${integrationTemplateId}-sample`; + dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; + } + + const response: boolean = await http + .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { + body: JSON.stringify({ name, dataSource }), + }) + .then((_res) => { + setToast(`${name} integration successfully added!`, 'success'); + window.location.hash = `#/installed/${_res.data?.id}`; + return true; + }) + .catch((_err) => { + setToast( + 'Failed to load integration. Check Added Integrations table for more details', + 'danger' + ); + return false; + }); + if (!addSample || !response) { + setLoading(false); + return; + } + const data: { sampleData: unknown[] } = await http + .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) + .then((res) => res.data) + .catch((err) => { + console.error(err); + setToast('The sample data could not be retrieved', 'danger'); + return { sampleData: [] }; + }); + const requestBody = + data.sampleData + .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) + .join('\n') + '\n'; + http + .post('/api/console/proxy', { + body: requestBody, query: { - path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, + path: `${dataSource}/_bulk?refresh=wait_for`, method: 'POST', }, + }) + .catch((err) => { + console.error(err); + setToast('Failed to load sample data', 'danger'); + }) + .finally(() => { + setLoading(false); }); - }; +} - const createDataSourceMappings = async (targetDataSource: string): Promise => { - const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); - let error: string | null = null; - const mappings = data.data.mappings; - mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( - (componentName: string) => { - const version = mappings[componentName].template.mappings._meta.version; - return `ss4o_${componentName}-${version}-template`; - } - ); +export function Integration(props: AvailableIntegrationProps) { + const http = coreRefs.http!; + const { integrationTemplateId, chrome } = props; - try { - // Create component mappings before the index mapping - // The assumption is that index mapping relies on component mappings for creation - await Promise.all( - Object.entries(mappings).map(([key, mapping]) => { - if (key === integration.type) { - return Promise.resolve(); - } - return createComponentMapping(key, mapping as any); - }) - ); - // In order to see our changes, we need to manually provoke a refresh - await http.post('/api/console/proxy', { - query: { - path: '_refresh', - method: 'GET', - }, - }); - await createIndexMapping(integration.type, mappings[integration.type], targetDataSource); - } catch (err: any) { - error = err.message; - } + const { setToast } = useToast(); + const [integration, setIntegration] = useState({} as Integration); - if (error !== null) { - setToast('Failure creating index template', 'danger', error); - } else { - setToast(`Successfully created index template`); - } - }; + const [integrationMapping, setMapping] = useState(null); + const [integrationAssets, setAssets] = useState([]); + const [loading, setLoading] = useState(false); useEffect(() => { chrome.setBreadcrumbs([ @@ -170,68 +265,6 @@ export function Integration(props: AvailableIntegrationProps) { }); }, [integration]); - async function addIntegrationRequest( - addSample: boolean, - templateName: string, - name?: string, - dataSource?: string - ) { - setLoading(true); - if (addSample) { - createDataSourceMappings(`ss4o_${integration.type}-${integrationTemplateId}-*-sample`); - name = `${integrationTemplateId}-sample`; - dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; - } - - const response: boolean = await http - .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { - body: JSON.stringify({ name, dataSource }), - }) - .then((_res) => { - setToast(`${name} integration successfully added!`, 'success'); - window.location.hash = `#/installed/${_res.data?.id}`; - return true; - }) - .catch((_err) => { - setToast( - 'Failed to load integration. Check Added Integrations table for more details', - 'danger' - ); - return false; - }); - if (!addSample || !response) { - setLoading(false); - return; - } - const data: { sampleData: unknown[] } = await http - .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) - .then((res) => res.data) - .catch((err) => { - console.error(err); - setToast('The sample data could not be retrieved', 'danger'); - return { sampleData: [] }; - }); - const requestBody = - data.sampleData - .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) - .join('\n') + '\n'; - http - .post('/api/console/proxy', { - body: requestBody, - query: { - path: `${dataSource}/_bulk?refresh=wait_for`, - method: 'POST', - }, - }) - .catch((err) => { - console.error(err); - setToast('Failed to load sample data', 'danger'); - }) - .finally(() => { - setLoading(false); - }); - } - const tabs = [ { id: 'assets', @@ -282,7 +315,14 @@ export function Integration(props: AvailableIntegrationProps) { window.location.hash = `#/available/${integration.name}/setup`; }, setUpSample: () => { - addIntegrationRequest(true, integrationTemplateId); + addIntegrationRequest( + true, + integration.name, + integrationTemplateId, + integration, + setLoading, + setToast + ); }, loading, })} From fe160079ee792407b84a73ebf7c66d2686b4f2c3 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 22 Sep 2023 11:43:05 -0700 Subject: [PATCH 47/79] Isolate more create_integration helpers Signed-off-by: Simeon Widdis --- .../components/create_integration_helpers.ts | 175 ++++++++++++++++++ .../integrations/components/integration.tsx | 174 +---------------- 2 files changed, 176 insertions(+), 173 deletions(-) diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index 23ee7fbf4a..4c4c2558e7 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -3,9 +3,19 @@ * SPDX-License-Identifier: Apache-2.0 */ import { HttpSetup } from '../../../../../../src/core/public'; +import { coreRefs } from '../../../framework/core_refs'; +import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; type ValidationResult = { ok: true } | { ok: false; errors: string[] }; +// Toast doesn't export, so we need to redeclare locally. +type Color = 'success' | 'primary' | 'warning' | 'danger' | undefined; + +export interface Integration { + name: string; + type: string; +} + export const doTypeValidation = ( toCheck: { type?: string; properties?: object }, required: { type?: string; properties?: object } @@ -171,3 +181,168 @@ export const doExistingDataSourceValidation = async ( } return { ok: true }; }; + +const createComponentMapping = async ( + componentName: string, + payload: { + template: { mappings: { _meta: { version: string } } }; + composed_of: string[]; + index_patterns: string[]; + } +): Promise<{ [key: string]: { properties: any } } | null> => { + const http = coreRefs.http!; + const version = payload.template.mappings._meta.version; + return http.post('/api/console/proxy', { + body: JSON.stringify(payload), + query: { + path: `_component_template/ss4o_${componentName}-${version}-template`, + method: 'POST', + }, + }); +}; + +const createIndexMapping = async ( + componentName: string, + payload: { + template: { mappings: { _meta: { version: string } } }; + composed_of: string[]; + index_patterns: string[]; + }, + dataSourceName: string, + integration: Integration +): Promise<{ [key: string]: { properties: any } } | null> => { + const http = coreRefs.http!; + const version = payload.template.mappings._meta.version; + payload.index_patterns = [dataSourceName]; + return http.post('/api/console/proxy', { + body: JSON.stringify(payload), + query: { + path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, + method: 'POST', + }, + }); +}; + +const createDataSourceMappings = async ( + targetDataSource: string, + integrationTemplateId: string, + integration: Integration, + setToast: (title: string, color?: Color, text?: string | undefined) => void +): Promise => { + const http = coreRefs.http!; + const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); + let error: string | null = null; + const mappings = data.data.mappings; + mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( + (componentName: string) => { + const version = mappings[componentName].template.mappings._meta.version; + return `ss4o_${componentName}-${version}-template`; + } + ); + + try { + // Create component mappings before the index mapping + // The assumption is that index mapping relies on component mappings for creation + await Promise.all( + Object.entries(mappings).map(([key, mapping]) => { + if (key === integration.type) { + return Promise.resolve(); + } + return createComponentMapping(key, mapping as any); + }) + ); + // In order to see our changes, we need to manually provoke a refresh + await http.post('/api/console/proxy', { + query: { + path: '_refresh', + method: 'GET', + }, + }); + await createIndexMapping( + integration.type, + mappings[integration.type], + targetDataSource, + integration + ); + } catch (err: any) { + error = err.message; + } + + if (error !== null) { + setToast('Failure creating index template', 'danger', error); + } else { + setToast(`Successfully created index template`); + } +}; + +export async function addIntegrationRequest( + addSample: boolean, + templateName: string, + integrationTemplateId: string, + integration: Integration, + setLoading: React.Dispatch>, + setToast: (title: string, color?: Color, text?: string | undefined) => void, + name?: string, + dataSource?: string +) { + const http = coreRefs.http!; + setLoading(true); + if (addSample) { + createDataSourceMappings( + `ss4o_${integration.type}-${integrationTemplateId}-*-sample`, + integrationTemplateId, + integration, + setToast + ); + name = `${integrationTemplateId}-sample`; + dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; + } + + const response: boolean = await http + .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { + body: JSON.stringify({ name, dataSource }), + }) + .then((_res) => { + setToast(`${name} integration successfully added!`, 'success'); + window.location.hash = `#/installed/${_res.data?.id}`; + return true; + }) + .catch((_err) => { + setToast( + 'Failed to load integration. Check Added Integrations table for more details', + 'danger' + ); + return false; + }); + if (!addSample || !response) { + setLoading(false); + return; + } + const data: { sampleData: unknown[] } = await http + .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) + .then((res) => res.data) + .catch((err) => { + console.error(err); + setToast('The sample data could not be retrieved', 'danger'); + return { sampleData: [] }; + }); + const requestBody = + data.sampleData + .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) + .join('\n') + '\n'; + http + .post('/api/console/proxy', { + body: requestBody, + query: { + path: `${dataSource}/_bulk?refresh=wait_for`, + method: 'POST', + }, + }) + .catch((err) => { + console.error(err); + setToast('Failed to load sample data', 'danger'); + }) + .finally(() => { + setLoading(false); + }); +} diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index 786bddd948..70e56b9f6d 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -23,179 +23,7 @@ import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { IntegrationScreenshots } from './integration_screenshots_panel'; import { useToast } from '../../../../public/components/common/toast'; import { coreRefs } from '../../../framework/core_refs'; - -// Toast doesn't export, so we need to redeclare locally. -type Color = 'success' | 'primary' | 'warning' | 'danger' | undefined; - -interface Integration { - name: string; - type: string; -} - -const createComponentMapping = async ( - componentName: string, - payload: { - template: { mappings: { _meta: { version: string } } }; - composed_of: string[]; - index_patterns: string[]; - } -): Promise<{ [key: string]: { properties: any } } | null> => { - const http = coreRefs.http!; - const version = payload.template.mappings._meta.version; - return http.post('/api/console/proxy', { - body: JSON.stringify(payload), - query: { - path: `_component_template/ss4o_${componentName}-${version}-template`, - method: 'POST', - }, - }); -}; - -const createIndexMapping = async ( - componentName: string, - payload: { - template: { mappings: { _meta: { version: string } } }; - composed_of: string[]; - index_patterns: string[]; - }, - dataSourceName: string, - integration: Integration -): Promise<{ [key: string]: { properties: any } } | null> => { - const http = coreRefs.http!; - const version = payload.template.mappings._meta.version; - payload.index_patterns = [dataSourceName]; - return http.post('/api/console/proxy', { - body: JSON.stringify(payload), - query: { - path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, - method: 'POST', - }, - }); -}; - -const createDataSourceMappings = async ( - targetDataSource: string, - integrationTemplateId: string, - integration: Integration, - setToast: (title: string, color?: Color, text?: string | undefined) => void -): Promise => { - const http = coreRefs.http!; - const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); - let error: string | null = null; - const mappings = data.data.mappings; - mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( - (componentName: string) => { - const version = mappings[componentName].template.mappings._meta.version; - return `ss4o_${componentName}-${version}-template`; - } - ); - - try { - // Create component mappings before the index mapping - // The assumption is that index mapping relies on component mappings for creation - await Promise.all( - Object.entries(mappings).map(([key, mapping]) => { - if (key === integration.type) { - return Promise.resolve(); - } - return createComponentMapping(key, mapping as any); - }) - ); - // In order to see our changes, we need to manually provoke a refresh - await http.post('/api/console/proxy', { - query: { - path: '_refresh', - method: 'GET', - }, - }); - await createIndexMapping( - integration.type, - mappings[integration.type], - targetDataSource, - integration - ); - } catch (err: any) { - error = err.message; - } - - if (error !== null) { - setToast('Failure creating index template', 'danger', error); - } else { - setToast(`Successfully created index template`); - } -}; - -async function addIntegrationRequest( - addSample: boolean, - templateName: string, - integrationTemplateId: string, - integration: Integration, - setLoading: React.Dispatch>, - setToast: (title: string, color?: Color, text?: string | undefined) => void, - name?: string, - dataSource?: string -) { - const http = coreRefs.http!; - setLoading(true); - if (addSample) { - createDataSourceMappings( - `ss4o_${integration.type}-${integrationTemplateId}-*-sample`, - integrationTemplateId, - integration, - setToast - ); - name = `${integrationTemplateId}-sample`; - dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; - } - - const response: boolean = await http - .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { - body: JSON.stringify({ name, dataSource }), - }) - .then((_res) => { - setToast(`${name} integration successfully added!`, 'success'); - window.location.hash = `#/installed/${_res.data?.id}`; - return true; - }) - .catch((_err) => { - setToast( - 'Failed to load integration. Check Added Integrations table for more details', - 'danger' - ); - return false; - }); - if (!addSample || !response) { - setLoading(false); - return; - } - const data: { sampleData: unknown[] } = await http - .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) - .then((res) => res.data) - .catch((err) => { - console.error(err); - setToast('The sample data could not be retrieved', 'danger'); - return { sampleData: [] }; - }); - const requestBody = - data.sampleData - .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) - .join('\n') + '\n'; - http - .post('/api/console/proxy', { - body: requestBody, - query: { - path: `${dataSource}/_bulk?refresh=wait_for`, - method: 'POST', - }, - }) - .catch((err) => { - console.error(err); - setToast('Failed to load sample data', 'danger'); - }) - .finally(() => { - setLoading(false); - }); -} +import { Integration, addIntegrationRequest } from './create_integration_helpers'; export function Integration(props: AvailableIntegrationProps) { const http = coreRefs.http!; From dfadcb597a3f530c1c0d0ffc5a9bdb82dcbc3820 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 22 Sep 2023 15:05:06 -0700 Subject: [PATCH 48/79] Simplify setup form Signed-off-by: Simeon Widdis --- .../components/create_integration_helpers.ts | 8 +- .../integrations/components/integration.tsx | 4 +- .../components/setup_integration.tsx | 413 ++++++------------ public/components/integrations/home.tsx | 10 +- 4 files changed, 135 insertions(+), 300 deletions(-) diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index 4c4c2558e7..ffbf7bf70e 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -11,7 +11,7 @@ type ValidationResult = { ok: true } | { ok: false; errors: string[] }; // Toast doesn't export, so we need to redeclare locally. type Color = 'success' | 'primary' | 'warning' | 'danger' | undefined; -export interface Integration { +export interface IntegrationTemplate { name: string; type: string; } @@ -209,7 +209,7 @@ const createIndexMapping = async ( index_patterns: string[]; }, dataSourceName: string, - integration: Integration + integration: IntegrationTemplate ): Promise<{ [key: string]: { properties: any } } | null> => { const http = coreRefs.http!; const version = payload.template.mappings._meta.version; @@ -226,7 +226,7 @@ const createIndexMapping = async ( const createDataSourceMappings = async ( targetDataSource: string, integrationTemplateId: string, - integration: Integration, + integration: IntegrationTemplate, setToast: (title: string, color?: Color, text?: string | undefined) => void ): Promise => { const http = coreRefs.http!; @@ -279,7 +279,7 @@ export async function addIntegrationRequest( addSample: boolean, templateName: string, integrationTemplateId: string, - integration: Integration, + integration: IntegrationTemplate, setLoading: React.Dispatch>, setToast: (title: string, color?: Color, text?: string | undefined) => void, name?: string, diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index 70e56b9f6d..723847fea2 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -23,14 +23,14 @@ import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { IntegrationScreenshots } from './integration_screenshots_panel'; import { useToast } from '../../../../public/components/common/toast'; import { coreRefs } from '../../../framework/core_refs'; -import { Integration, addIntegrationRequest } from './create_integration_helpers'; +import { IntegrationTemplate, addIntegrationRequest } from './create_integration_helpers'; export function Integration(props: AvailableIntegrationProps) { const http = coreRefs.http!; const { integrationTemplateId, chrome } = props; const { setToast } = useToast(); - const [integration, setIntegration] = useState({} as Integration); + const [integration, setIntegration] = useState({} as IntegrationTemplate); const [integrationMapping, setMapping] = useState(null); const [integrationAssets, setAssets] = useState([]); diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 9e3052abfb..8a56b1e8f9 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -4,96 +4,101 @@ */ import * as Eui from '@elastic/eui'; -import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; import React, { useState } from 'react'; interface IntegrationConfig { - instanceName: string; - useExisting: boolean; - dataSourceName: string; - dataSourceDescription: string; - dataSourceFileType: string; - dataSourceLocation: string; - existingDataSourceName: string; + displayName: string; + connectionType: string; + connectionDataSource: string; } -const STEPS: EuiContainedStepProps[] = [ - { title: 'Name Integration', children: }, - { title: 'Select index or data source for integration', children: }, -]; - -const ALLOWED_FILE_TYPES: Eui.EuiSelectOption[] = [ - { value: 'parquet', text: 'parquet' }, - { value: 'json', text: 'json' }, -]; +interface IntegrationConfigProps { + config: IntegrationConfig; + updateConfig: (updates: Partial) => void; +} const INTEGRATION_DATA_TABLE_COLUMNS = [ { field: 'field', - name: 'Field Name', + name: 'Field', + }, + { + field: 'sourceType', + name: 'Source Data Type', + }, + { + field: 'destType', + name: 'Destination Data Type', + }, + { + field: 'group', + name: 'Mapping Group', }, +]; + +// TODO support localization +const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map = new Map([ + ['s3', 'table'], + ['index', 'index'], +]); + +const integrationConnectionSelectorItems = [ { - field: 'type', - name: 'Field Type', + value: 's3', + text: 'S3 Connection', + dataSourceName: ['table', 'tables'], }, { - field: 'isTimestamp', - name: 'Timestamp', + value: 'index', + text: 'OpenSearch Index', + dataSourceName: ['index', 'indexes'], }, ]; const integrationDataTableData = [ { - field: 'spanId', - type: 'string', - isTimestamp: false, + field: 'domain.bytes', + sourceType: 'string', + destType: 'integer', + group: 'communication', + }, + { + field: 'http.url', + sourceType: 'string', + destType: 'keyword', + group: 'http', }, { - field: 'severity.number', - type: 'long', - isTimestamp: false, + field: 'destination.address', + sourceType: 'string', + destType: 'keyword', + group: 'communication', }, { - field: '@timestamp', - type: 'date', - isTimestamp: true, + field: 'netflow.error_rate', + sourceType: 'string', + destType: 'string', + group: 'http', + }, + { + field: 'http.route', + sourceType: 'string', + destType: 'string', + group: 'http', + }, + { + field: 'destination.packets', + sourceType: 'integer', + destType: 'integer', + group: 'communication', }, ]; -const getSetupStepStatus = (activeStep: number): EuiContainedStepProps[] => { - return STEPS.map((step, idx) => { - let status: string = ''; - if (idx < activeStep) { - status = 'complete'; - } - if (idx > activeStep) { - status = 'disabled'; - } - return Object.assign({}, step, { status }); - }); -}; - -export function SetupIntegrationMetadata({ - name, - setName, -}: { - name: string; - setName: (name: string) => void; -}) { - return ( - - -

{STEPS[0].title}

- - - setName(evt.target.value)} /> - - - ); -} +const availableIndices = [ + { value: 'ss4o_logs-nginx-prod', text: 'ss4o_logs-nginx-prod' }, + { value: 'ss4o_logs-nginx-test', text: 'ss4o_logs-nginx-test' }, + { value: 'ss4o_logs-apache-prod', text: 'ss4o_logs-apache-prod' }, +]; export function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Element { return ( @@ -115,201 +120,64 @@ export function IntegrationDataModal({ close }: { close: () => void }): React.JS ); } -export function SetupIntegrationNewTable({ - config, - updateConfig, -}: { - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; -}) { +export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfigProps) { + const connectionType = + INTEGRATION_CONNECTION_DATA_SOURCE_TYPES.get(config.connectionType) ?? 'index'; + const indefiniteArticle = 'aeiou'.includes(connectionType.charAt(0)) ? 'an' : 'a'; + const capitalizedConnectionType = + connectionType.charAt(0).toUpperCase() + connectionType.slice(1); + return ( -
- - updateConfig({ dataSourceName: evt.target.value })} - /> - - + + +

Set Up Integration

+ + + +

Integration Details

+ + + updateConfig({ dataSourceDescription: evt.target.value })} + value={config.displayName} + onChange={(event) => updateConfig({ displayName: event.target.value })} /> - + + +

Integration Connection

+ + + updateConfig({ dataSourceFileType: evt.target.value })} + options={integrationConnectionSelectorItems} + value={config.connectionType} + onChange={(event) => updateConfig({ connectionType: event.target.value })} /> - - updateConfig({ dataSourceLocation: evt.target.value })} - /> - -
- ); -} - -export function SetupIntegrationExistingTable({ - config, - updateConfig, - showDataModal, - setShowDataModal, -}: { - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; - showDataModal: boolean; - setShowDataModal: (visible: boolean) => void; -}) { - const dataModal = showDataModal ? ( - setShowDataModal(false)} /> - ) : null; - return ( -
- + updateConfig({ existingDataSourceName: evt.target.value })} + options={availableIndices} + value={config.connectionDataSource} + onChange={(event) => updateConfig({ connectionDataSource: event.target.value })} /> - - setShowDataModal(true)}>View table - {dataModal} -
- ); -} - -export function SetupIntegrationDataSource({ - config, - updateConfig, - showDataModal, - setShowDataModal, - tableDetected, - setTableDetected, -}: { - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; - showDataModal: boolean; - setShowDataModal: (show: boolean) => void; - tableDetected: boolean; - setTableDetected: (detected: boolean) => void; -}) { - let tableForm; - if (tableDetected && config.useExisting) { - tableForm = ( - setShowDataModal(x)} - /> - ); - } else { - tableForm = ; - } - - let tablesNotFoundMessage = null; - if (!tableDetected) { - tablesNotFoundMessage = ( - <> - -

No problem, we can help. Tell us about your data.

- - - - ); - } - - return ( -
- setTableDetected(event.target.checked)} - /> - - - -

{STEPS[1].title}

- - - {tablesNotFoundMessage} - updateConfig({ useExisting: evt.target.checked })} - disabled={!tableDetected} - /> - - {tableForm} - -
+ Validate + ); } -export function SetupIntegrationStep({ - activeStep, - config, - updateConfig, -}: { - activeStep: number; - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; -}) { - const [isDataModalVisible, setDataModalVisible] = useState(false); - const [tableDetected, setTableDetected] = useState(false); - - switch (activeStep) { - case 0: - return ( - updateConfig({ instanceName: name })} - /> - ); - case 1: - return ( - setDataModalVisible(show)} - tableDetected={tableDetected} - setTableDetected={(detected: boolean) => setTableDetected(detected)} - /> - ); - default: - return ( - - Attempted to access integration setup step that doesn't exist. This is a bug. - - ); - } -} - -export function SetupBottomBar({ - step, - setStep, - config, -}: { - step: number; - setStep: (step: number) => void; - config: IntegrationConfig; -}) { +export function SetupBottomBar({ config }: { config: IntegrationConfig }) { return ( { // TODO evil hack because props aren't set up let hash = window.location.hash; @@ -318,32 +186,17 @@ export function SetupBottomBar({ window.location.hash = hash; }} > - Cancel + Discard - - - - {step > 0 ? ( - - setStep(step - 1)}> - Back - - - ) : null} { - if (step < STEPS.length - 1) { - setStep(step + 1); - } else { - console.log(config); - } - }} + iconType="arrowRight" + iconSide="right" + onClick={() => console.log(config)} > - {step === STEPS.length - 1 ? 'Save' : 'Next'} + Add Integration @@ -351,17 +204,12 @@ export function SetupBottomBar({ ); } -export function SetupIntegrationStepsPage() { +export function SetupIntegrationPage() { const [integConfig, setConfig] = useState({ - instanceName: '', - useExisting: true, - dataSourceName: '', - dataSourceDescription: '', - dataSourceFileType: 'parquet', - dataSourceLocation: '', - existingDataSourceName: '', + displayName: 'Test', + connectionType: 'index', + connectionDataSource: 'ss4o_logs-nginx-prod', } as IntegrationConfig); - const [step, setStep] = useState(0); const updateConfig = (updates: Partial) => setConfig(Object.assign({}, integConfig, updates)); @@ -369,23 +217,12 @@ export function SetupIntegrationStepsPage() { return ( - - - - - - - - - setStep(Math.min(Math.max(x, 0), STEPS.length - 1))} - config={integConfig} - /> + + + + + + ); diff --git a/public/components/integrations/home.tsx b/public/components/integrations/home.tsx index 67ec7e74f1..6e7da8669e 100644 --- a/public/components/integrations/home.tsx +++ b/public/components/integrations/home.tsx @@ -3,17 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useState } from 'react'; +import React from 'react'; import { HashRouter, Route, RouteComponentProps, Switch } from 'react-router-dom'; -import { EuiGlobalToastList } from '@elastic/eui'; -import { Toast } from '@elastic/eui/src/components/toast/global_toast_list'; -import { Integration } from './components/integration'; import { TraceAnalyticsCoreDeps } from '../trace_analytics/home'; import { ChromeBreadcrumb } from '../../../../../src/core/public'; import { AvailableIntegrationOverviewPage } from './components/available_integration_overview_page'; import { AddedIntegrationOverviewPage } from './components/added_integration_overview_page'; import { AddedIntegration } from './components/added_integration'; -import { SetupIntegrationStepsPage } from './components/setup_integration'; +import { SetupIntegrationPage } from './components/setup_integration'; +import { Integration } from './components/integration'; export type AppAnalyticsCoreDeps = TraceAnalyticsCoreDeps; @@ -63,7 +61,7 @@ export const Home = (props: HomeProps) => { /> )} /> - } /> + } />
From af1471d58e988fea324e45ad51ab49e8002a6a2a Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 22 Sep 2023 15:27:09 -0700 Subject: [PATCH 49/79] Add data source picker items Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 8a56b1e8f9..485475a25e 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -4,7 +4,7 @@ */ import * as Eui from '@elastic/eui'; -import React, { useState } from 'react'; +import React, { useState, useEffect } from 'react'; interface IntegrationConfig { displayName: string; @@ -100,6 +100,28 @@ const availableIndices = [ { value: 'ss4o_logs-apache-prod', text: 'ss4o_logs-apache-prod' }, ]; +const fetchAvailableDataSources = async (type: string): Promise => { + // Artificial delay for UI testing + const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + await sleep(500); + // TODO fetch actual items instead of hardcoded values + if (type === 'index') { + return [ + { value: 'ss4o_logs-nginx-prod', text: 'ss4o_logs-nginx-prod' }, + { value: 'ss4o_logs-nginx-test', text: 'ss4o_logs-nginx-test' }, + { value: 'ss4o_logs-apache-prod', text: 'ss4o_logs-apache-prod' }, + ]; + } else if (type === 's3') { + return [ + { value: 'table_1', text: 'table_1' }, + { value: 'table_2', text: 'table_2' }, + ]; + } else { + console.error(`Unknown connection type: ${type}`); + return []; + } +}; + export function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Element { return ( @@ -127,6 +149,18 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig const capitalizedConnectionType = connectionType.charAt(0).toUpperCase() + connectionType.slice(1); + const [availableDataSources, setAvailableDataSources] = useState([] as Eui.EuiSelectOption[]); + useEffect(() => { + const updateDataSources = async () => { + const data = await fetchAvailableDataSources(config.connectionType); + + setAvailableDataSources(data); + }; + + setAvailableDataSources([]); + updateDataSources(); + }, [config.connectionType]); + return ( @@ -160,7 +194,7 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig helpText={`Select ${indefiniteArticle} ${connectionType} to pull the data from.`} > updateConfig({ connectionDataSource: event.target.value })} /> From 9c9d8001f2ee2260654fbfdf4cea4181b26f75e3 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 22 Sep 2023 16:49:33 -0700 Subject: [PATCH 50/79] Add better selector logic Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 67 +++++++++++-------- public/components/integrations/home.tsx | 8 ++- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 485475a25e..4abe2c2848 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -94,28 +94,19 @@ const integrationDataTableData = [ }, ]; -const availableIndices = [ - { value: 'ss4o_logs-nginx-prod', text: 'ss4o_logs-nginx-prod' }, - { value: 'ss4o_logs-nginx-test', text: 'ss4o_logs-nginx-test' }, - { value: 'ss4o_logs-apache-prod', text: 'ss4o_logs-apache-prod' }, -]; - -const fetchAvailableDataSources = async (type: string): Promise => { +const suggestDataSources = async (type: string): Promise => { // Artificial delay for UI testing const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); - await sleep(500); + await sleep(Math.random() * 2000); // TODO fetch actual items instead of hardcoded values if (type === 'index') { return [ - { value: 'ss4o_logs-nginx-prod', text: 'ss4o_logs-nginx-prod' }, - { value: 'ss4o_logs-nginx-test', text: 'ss4o_logs-nginx-test' }, - { value: 'ss4o_logs-apache-prod', text: 'ss4o_logs-apache-prod' }, + { label: 'ss4o_logs-nginx-prod' }, + { label: 'ss4o_logs-nginx-test' }, + { label: 'ss4o_logs-apache-prod' }, ]; } else if (type === 's3') { - return [ - { value: 'table_1', text: 'table_1' }, - { value: 'table_2', text: 'table_2' }, - ]; + return [{ label: 'table_1' }, { label: 'table_2' }]; } else { console.error(`Unknown connection type: ${type}`); return []; @@ -149,15 +140,18 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig const capitalizedConnectionType = connectionType.charAt(0).toUpperCase() + connectionType.slice(1); - const [availableDataSources, setAvailableDataSources] = useState([] as Eui.EuiSelectOption[]); + const [dataSourceSuggestions, setDataSourceSuggestions] = useState( + [] as Eui.EuiSelectableOption[] + ); + const [isSuggestionsLoading, setIsSuggestionsLoading] = useState(true); useEffect(() => { const updateDataSources = async () => { - const data = await fetchAvailableDataSources(config.connectionType); - - setAvailableDataSources(data); + const data = await suggestDataSources(config.connectionType); + setDataSourceSuggestions(data); + setIsSuggestionsLoading(false); }; - setAvailableDataSources([]); + setIsSuggestionsLoading(true); updateDataSources(); }, [config.connectionType]); @@ -193,11 +187,28 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig label={capitalizedConnectionType} helpText={`Select ${indefiniteArticle} ${connectionType} to pull the data from.`} > - updateConfig({ connectionDataSource: event.target.value })} - /> + { + setDataSourceSuggestions(suggestions); + for (const suggestion of suggestions) { + if (suggestion.checked) { + updateConfig({ connectionDataSource: suggestion.label }); + return; + } + } + }} + singleSelection + searchable + > + {(list, search) => ( + <> + {search} + {list} + + )} + Validate @@ -238,11 +249,11 @@ export function SetupBottomBar({ config }: { config: IntegrationConfig }) { ); } -export function SetupIntegrationPage() { +export function SetupIntegrationPage({ integration }: { integration: string }) { const [integConfig, setConfig] = useState({ - displayName: 'Test', + displayName: `${integration} Integration`, connectionType: 'index', - connectionDataSource: 'ss4o_logs-nginx-prod', + connectionDataSource: '', } as IntegrationConfig); const updateConfig = (updates: Partial) => diff --git a/public/components/integrations/home.tsx b/public/components/integrations/home.tsx index 6e7da8669e..7a509c9d23 100644 --- a/public/components/integrations/home.tsx +++ b/public/components/integrations/home.tsx @@ -61,7 +61,13 @@ export const Home = (props: HomeProps) => { /> )} /> - } /> + ( + + )} + />
From 69e4978417647a59526b6a1ac23bcdd2442ebdde Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 25 Sep 2023 12:06:46 -0700 Subject: [PATCH 51/79] Add queries for data sources Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 4abe2c2848..d3dbffbb63 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -5,6 +5,8 @@ import * as Eui from '@elastic/eui'; import React, { useState, useEffect } from 'react'; +import { string } from 'joi'; +import { coreRefs } from '../../../framework/core_refs'; interface IntegrationConfig { displayName: string; @@ -95,20 +97,42 @@ const integrationDataTableData = [ ]; const suggestDataSources = async (type: string): Promise => { - // Artificial delay for UI testing - const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); - await sleep(Math.random() * 2000); - // TODO fetch actual items instead of hardcoded values - if (type === 'index') { - return [ - { label: 'ss4o_logs-nginx-prod' }, - { label: 'ss4o_logs-nginx-test' }, - { label: 'ss4o_logs-apache-prod' }, - ]; - } else if (type === 's3') { - return [{ label: 'table_1' }, { label: 'table_2' }]; - } else { - console.error(`Unknown connection type: ${type}`); + const http = coreRefs.http!; + try { + if (type === 'index') { + const result = await http.post('/api/console/proxy', { + body: '{}', + query: { + path: '_data_stream/ss4o_*', + method: 'GET', + }, + }); + return ( + result.data_streams?.map((item: { name: string }) => { + return { label: item.name }; + }) ?? [] + ); + } else if (type === 's3') { + const result = (await http.post('/api/console/proxy', { + body: '{}', + query: { + path: '_plugins/_query/_datasources', + method: 'GET', + }, + })) as Array<{ name: string; connector: string }>; + return ( + result + ?.filter((item) => item.connector === 'S3GLUE') + .map((item) => { + return { label: item.name }; + }) ?? [] + ); + } else { + console.error(`Unknown connection type: ${type}`); + return []; + } + } catch (err: any) { + console.error(err.message); return []; } }; From 038eefbedaa0ab0a9a988c873e194a7e4f884abd Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 25 Sep 2023 15:25:31 -0700 Subject: [PATCH 52/79] Switch from selector to combobox Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index d3dbffbb63..c6cbc8c51d 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -96,7 +96,7 @@ const integrationDataTableData = [ }, ]; -const suggestDataSources = async (type: string): Promise => { +const suggestDataSources = async (type: string): Promise> => { const http = coreRefs.http!; try { if (type === 'index') { @@ -165,7 +165,7 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig connectionType.charAt(0).toUpperCase() + connectionType.slice(1); const [dataSourceSuggestions, setDataSourceSuggestions] = useState( - [] as Eui.EuiSelectableOption[] + [] as Array<{ label: string }> ); const [isSuggestionsLoading, setIsSuggestionsLoading] = useState(true); useEffect(() => { @@ -204,35 +204,28 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig updateConfig({ connectionType: event.target.value })} + onChange={(event) => + updateConfig({ connectionType: event.target.value, connectionDataSource: '' }) + } /> - { - setDataSourceSuggestions(suggestions); - for (const suggestion of suggestions) { - if (suggestion.checked) { - updateConfig({ connectionDataSource: suggestion.label }); - return; - } + onChange={(selected) => { + if (selected.length === 0) { + updateConfig({ connectionDataSource: '' }); + } else { + updateConfig({ connectionDataSource: selected[0].label }); } }} - singleSelection - searchable - > - {(list, search) => ( - <> - {search} - {list} - - )} - + selectedOptions={[{ label: config.connectionDataSource }]} + singleSelection={{ asPlainText: true }} + /> Validate From 16745d1c87a542b3197484bc7e0f730804b14daa Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Mon, 25 Sep 2023 15:32:03 -0700 Subject: [PATCH 53/79] Update snapshots Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 2721 ++++++++--------- .../__tests__/setup_integration.test.tsx | 70 +- .../components/setup_integration.tsx | 2 +- 3 files changed, 1250 insertions(+), 1543 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 1efafd0e03..2d1b995512 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -1,7 +1,9 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`Integration Setup Page Renders integration setup page as expected 1`] = ` - +
- -
+ - -
+ - , - "status": "", - "title": "Name Integration", - }, - Object { - "children": , - "status": "disabled", - "title": "Select index or data source for integration", - }, - ] - } +
-
- -
-
- - - - - Step 1 - - - - - - -

- Name Integration -

- -
-
- -
- -
-
- - -
-
- - - - - Step 2 is disabled - - - - - - -

- Select index or data source for integration -

- -
-
- -
- -
-
- -
- -
- - -
- -
- Name Integration + Set Up Integration + +
+ + +

+ Integration Details +

+ + +
+
- Name + Display Name
@@ -230,12 +100,11 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiFormRow__fieldWrapper" > @@ -264,43 +132,619 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
+
+
+ + +
+ + +

+ Integration Connection +

+ + +
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+ + + + + +
+ +
+
+ + + +
+ Select a data source to connect to. +
+ +
+
+ + +
+
+ + + +
+
+ +
+ + +
+
+
+ + + + +
+ +
+
+ +
+ +
+ + + + + + + + + + + +
+ +
+
+ + +
+
- The name will be used to label the newly added integration + Select an index to pull the data from.
+ + + + +
- - -
- -
- + +
+ +
+ + @@ -323,7 +767,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = class="euiFlexItem euiFlexItem--flexGrowZero" >
-
-
-
@@ -362,7 +799,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = type="button" > @@ -429,13 +866,13 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiFlexItem euiFlexItem--flexGrowZero" >
- -
- -
- -
- @@ -524,7 +950,8 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = > - Next + Add Integration @@ -624,939 +1052,142 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
- + `; -exports[`Integration Setup Page Renders the data source form as expected 1`] = ` - -
- +
-
- - + + +
+ + +

- (debug) Table detected - -

- - -
- - -
+ + +
+ + - -

- Select index or data source for integration -

- - -
- -
-
- - - No tables were found - -
- -
- -
-

- No problem, we can help. Tell us about your data. -

-
- -
- + Display Name + +
- - -
- -
- - - Use existing Data Source - -
- - -
- - -
- -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
-
- - - -
-
- - -
-
- - - - -
- - - - - -
- -
-
- - -
-
- - -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- -
- -
- -
- -`; - -exports[`Integration Setup Page Renders the existing table form as expected 1`] = ` - -
- -
-
- - - -
-
- - -
- - - - -
- - - - - -
- -
-
- - - -
- Manage data associated with this data source -
- -
-
- - -
- - - - -
- -`; - -exports[`Integration Setup Page Renders the metadata form as expected 1`] = ` - - -
+ + + + +
+
+ + +
+
+ + +
+ -

- Name Integration -

+ Integration Connection + + +
+
- Name + Data Source
-
- + onMouseUp={[Function]} + value="index" + > + + + - + +
+ + + + + +
+
- +
- The name will be used to label the newly added integration + Select a data source to connect to.
-
- - -`; - -exports[`Integration Setup Page Renders the new table form as expected 1`] = ` - -
- -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
-
- - - -
-
- + +
+
-
-
- - - -
- - -
- -
+ +
- - -
-
- - -
-
- - - +
+ Select an index to pull the data from. +
+ +
-
+ + - - -
-
- - - - -
-
- - -
-
- -
- + Validate + + + + + + +
+ + `; diff --git a/public/components/integrations/components/__tests__/setup_integration.test.tsx b/public/components/integrations/components/__tests__/setup_integration.test.tsx index 61001ba7c5..b88d32ec30 100644 --- a/public/components/integrations/components/__tests__/setup_integration.test.tsx +++ b/public/components/integrations/components/__tests__/setup_integration.test.tsx @@ -8,80 +8,30 @@ import Adapter from 'enzyme-adapter-react-16'; import React from 'react'; import { waitFor } from '@testing-library/react'; import { - SetupIntegrationDataSource, - SetupIntegrationExistingTable, - SetupIntegrationMetadata, - SetupIntegrationNewTable, - SetupIntegrationStepsPage, + IntegrationConfig, + SetupIntegrationPage, + SetupIntegrationForm, } from '../setup_integration'; -const TEST_CONFIG = { - instanceName: 'Test Instance Name', - useExisting: true, - dataSourceName: 'Test Datasource Name', - dataSourceDescription: 'Test Datasource Description', - dataSourceFileType: 'json', - dataSourceLocation: 'ss4o_logs-test-new-location', - existingDataSourceName: 'ss4o_logs-test-existing-location', +const TEST_CONFIG: IntegrationConfig = { + displayName: 'Test Instance Name', + connectionType: 'index', + connectionDataSource: 'ss4o_logs-nginx-test', }; describe('Integration Setup Page', () => { configure({ adapter: new Adapter() }); it('Renders integration setup page as expected', async () => { - const wrapper = mount(); + const wrapper = mount(); await waitFor(() => { expect(wrapper).toMatchSnapshot(); }); }); - it('Renders the metadata form as expected', async () => { - const wrapper = mount( - {}} /> - ); - - await waitFor(() => { - expect(wrapper).toMatchSnapshot(); - }); - }); - - it('Renders the data source form as expected', async () => { - const wrapper = mount( - {}} - showDataModal={false} - setShowDataModal={() => {}} - tableDetected={false} - setTableDetected={() => {}} - /> - ); - - await waitFor(() => { - expect(wrapper).toMatchSnapshot(); - }); - }); - - it('Renders the new table form as expected', async () => { - const wrapper = mount( - {}} /> - ); - - await waitFor(() => { - expect(wrapper).toMatchSnapshot(); - }); - }); - - it('Renders the existing table form as expected', async () => { - const wrapper = mount( - {}} - showDataModal={false} - setShowDataModal={() => {}} - /> - ); + it('Renders the form as expected', async () => { + const wrapper = mount( {}} />); await waitFor(() => { expect(wrapper).toMatchSnapshot(); diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index c6cbc8c51d..e66e6e032c 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -8,7 +8,7 @@ import React, { useState, useEffect } from 'react'; import { string } from 'joi'; import { coreRefs } from '../../../framework/core_refs'; -interface IntegrationConfig { +export interface IntegrationConfig { displayName: string; connectionType: string; connectionDataSource: string; From 1892b8f6ce630c834cb97132b7c42018e3c664fe Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 26 Sep 2023 11:02:31 -0700 Subject: [PATCH 54/79] Connect validation button to data source validation method Signed-off-by: Simeon Widdis --- .../components/add_integration_flyout.tsx | 3 +- .../components/create_integration_helpers.ts | 8 +++- .../components/setup_integration.tsx | 42 ++++++++++++++++--- public/components/integrations/home.tsx | 7 +++- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index 426690adcf..a06d4bdda5 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -85,8 +85,7 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { const validationResult = await doExistingDataSourceValidation( dataSource, integrationName, - integrationType, - http + integrationType ); if (validationResult.ok) { setToast('Index name or wildcard pattern is valid', 'success'); diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index ffbf7bf70e..0e9a612677 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -65,6 +65,10 @@ export const doPropertyValidation = ( requiredMappings: { [key: string]: { template: { mappings: { properties?: any } } } } ): ValidationResult => { // Check root object type (without dependencies) + if (!Object.hasOwn(requiredMappings, rootType)) { + // This is a configuration error for the integration. + return { ok: false, errors: ['Required mapping for integration has no root type.'] }; + } for (const [key, value] of Object.entries( requiredMappings[rootType].template.mappings.properties )) { @@ -156,9 +160,9 @@ export const fetchIntegrationMappings = async ( export const doExistingDataSourceValidation = async ( targetDataSource: string, integrationName: string, - integrationType: string, - http: HttpSetup + integrationType: string ): Promise => { + const http = coreRefs.http!; const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); if (!dataSourceNameCheck.ok) { return dataSourceNameCheck; diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index e66e6e032c..5f1da89db5 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -5,8 +5,8 @@ import * as Eui from '@elastic/eui'; import React, { useState, useEffect } from 'react'; -import { string } from 'joi'; import { coreRefs } from '../../../framework/core_refs'; +import { doExistingDataSourceValidation } from './create_integration_helpers'; export interface IntegrationConfig { displayName: string; @@ -17,6 +17,10 @@ export interface IntegrationConfig { interface IntegrationConfigProps { config: IntegrationConfig; updateConfig: (updates: Partial) => void; + integration: { + name: string; + type: string; + }; } const INTEGRATION_DATA_TABLE_COLUMNS = [ @@ -157,7 +161,11 @@ export function IntegrationDataModal({ close }: { close: () => void }): React.JS ); } -export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfigProps) { +export function SetupIntegrationForm({ + config, + updateConfig, + integration, +}: IntegrationConfigProps) { const connectionType = INTEGRATION_CONNECTION_DATA_SOURCE_TYPES.get(config.connectionType) ?? 'index'; const indefiniteArticle = 'aeiou'.includes(connectionType.charAt(0)) ? 'an' : 'a'; @@ -227,7 +235,18 @@ export function SetupIntegrationForm({ config, updateConfig }: IntegrationConfig singleSelection={{ asPlainText: true }} /> - Validate + { + const validationResult = await doExistingDataSourceValidation( + config.connectionDataSource, + integration.name, + integration.type + ); + console.log(validationResult); + }} + > + Validate + ); } @@ -266,9 +285,16 @@ export function SetupBottomBar({ config }: { config: IntegrationConfig }) { ); } -export function SetupIntegrationPage({ integration }: { integration: string }) { +export function SetupIntegrationPage({ + integration, +}: { + integration: { + name: string; + type: string; + }; +}) { const [integConfig, setConfig] = useState({ - displayName: `${integration} Integration`, + displayName: `${integration.name} Integration`, connectionType: 'index', connectionDataSource: '', } as IntegrationConfig); @@ -281,7 +307,11 @@ export function SetupIntegrationPage({ integration }: { integration: string }) { - + diff --git a/public/components/integrations/home.tsx b/public/components/integrations/home.tsx index 7a509c9d23..7a6b3db5df 100644 --- a/public/components/integrations/home.tsx +++ b/public/components/integrations/home.tsx @@ -65,7 +65,12 @@ export const Home = (props: HomeProps) => { exact path={'/available/:id/setup'} render={(routerProps) => ( - + )} /> From ebdb008dfbdc3bfcbd12b22004e207a08f24987e Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 26 Sep 2023 16:56:37 -0700 Subject: [PATCH 55/79] Reimplement add integration button Signed-off-by: Simeon Widdis --- .../components/create_integration_helpers.ts | 6 --- .../integrations/components/integration.tsx | 7 ++-- .../components/setup_integration.tsx | 38 ++++++++++++++++--- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index 0e9a612677..15b74b3025 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -284,13 +284,11 @@ export async function addIntegrationRequest( templateName: string, integrationTemplateId: string, integration: IntegrationTemplate, - setLoading: React.Dispatch>, setToast: (title: string, color?: Color, text?: string | undefined) => void, name?: string, dataSource?: string ) { const http = coreRefs.http!; - setLoading(true); if (addSample) { createDataSourceMappings( `ss4o_${integration.type}-${integrationTemplateId}-*-sample`, @@ -319,7 +317,6 @@ export async function addIntegrationRequest( return false; }); if (!addSample || !response) { - setLoading(false); return; } const data: { sampleData: unknown[] } = await http @@ -345,8 +342,5 @@ export async function addIntegrationRequest( .catch((err) => { console.error(err); setToast('Failed to load sample data', 'danger'); - }) - .finally(() => { - setLoading(false); }); } diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index 723847fea2..c005e98d0b 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -142,15 +142,16 @@ export function Integration(props: AvailableIntegrationProps) { showFlyout: () => { window.location.hash = `#/available/${integration.name}/setup`; }, - setUpSample: () => { - addIntegrationRequest( + setUpSample: async () => { + setLoading(true); + await addIntegrationRequest( true, integration.name, integrationTemplateId, integration, - setLoading, setToast ); + setLoading(false); }, loading, })} diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 5f1da89db5..39610c7579 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -6,7 +6,12 @@ import * as Eui from '@elastic/eui'; import React, { useState, useEffect } from 'react'; import { coreRefs } from '../../../framework/core_refs'; -import { doExistingDataSourceValidation } from './create_integration_helpers'; +import { + addIntegrationRequest, + doExistingDataSourceValidation, +} from './create_integration_helpers'; +import { useToast } from '../../../../public/components/common/toast'; +import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; export interface IntegrationConfig { displayName: string; @@ -161,6 +166,12 @@ export function IntegrationDataModal({ close }: { close: () => void }): React.JS ); } +const findTemplate = async (integrationTemplateId: string) => { + const http = coreRefs.http!; + const result = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}`); + return result; +}; + export function SetupIntegrationForm({ config, updateConfig, @@ -251,7 +262,10 @@ export function SetupIntegrationForm({ ); } -export function SetupBottomBar({ config }: { config: IntegrationConfig }) { +export function SetupBottomBar({ config, integration }: IntegrationConfigProps) { + const { setToast } = useToast(); + const [loading, setLoading] = useState(false); + return ( @@ -275,7 +289,21 @@ export function SetupBottomBar({ config }: { config: IntegrationConfig }) { fill iconType="arrowRight" iconSide="right" - onClick={() => console.log(config)} + isLoading={loading} + onClick={async () => { + setLoading(true); + const template = await findTemplate(integration.name); + await addIntegrationRequest( + false, + integration.name, + config.displayName, + template, + setToast, + config.displayName, + config.connectionDataSource + ); + setLoading(false); + }} > Add Integration @@ -306,7 +334,7 @@ export function SetupIntegrationPage({ - + - + ); From 6230a6581b20ab83e7f5224bdfe4bab787b5f8ba Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 26 Sep 2023 17:00:32 -0700 Subject: [PATCH 56/79] Temporarily remove validate button Signed-off-by: Simeon Widdis --- .../integrations/components/setup_integration.tsx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 39610c7579..973e14f927 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -246,7 +246,8 @@ export function SetupIntegrationForm({ singleSelection={{ asPlainText: true }} /> - { const validationResult = await doExistingDataSourceValidation( config.connectionDataSource, @@ -257,12 +258,18 @@ export function SetupIntegrationForm({ }} > Validate - + */} ); } -export function SetupBottomBar({ config, integration }: IntegrationConfigProps) { +export function SetupBottomBar({ + config, + integration, +}: { + config: IntegrationConfig; + integration: { name: string; type: string }; +}) { const { setToast } = useToast(); const [loading, setLoading] = useState(false); From bbb137b9d6f90d282fdd1594bee0b6539893eb1a Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 28 Sep 2023 13:53:23 -0700 Subject: [PATCH 57/79] Add queries to integrations config Signed-off-by: Simeon Widdis --- .../aws_elb/assets/elb_query-1.0.0.sql | 1 + .../repository/aws_elb/aws_elb-1.0.0.json | 9 ++++- .../integrations/repository/integration.ts | 34 ++++++++++++++++++- server/adaptors/integrations/types.ts | 19 +++++++---- server/adaptors/integrations/validators.ts | 13 +++++++ 5 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 server/adaptors/integrations/__data__/repository/aws_elb/assets/elb_query-1.0.0.sql diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/assets/elb_query-1.0.0.sql b/server/adaptors/integrations/__data__/repository/aws_elb/assets/elb_query-1.0.0.sql new file mode 100644 index 0000000000..eb0513bbfd --- /dev/null +++ b/server/adaptors/integrations/__data__/repository/aws_elb/assets/elb_query-1.0.0.sql @@ -0,0 +1 @@ +SELECT * FROM ${TABLE} LIMIT 100; diff --git a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json index 3a8a58d291..c981c31381 100644 --- a/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json +++ b/server/adaptors/integrations/__data__/repository/aws_elb/aws_elb-1.0.0.json @@ -50,7 +50,14 @@ "savedObjects": { "name": "aws_elb", "version": "1.0.0" - } + }, + "queries": [ + { + "name": "elb_query", + "version": "1.0.0", + "language": "sql" + } + ] }, "sampleData": { "path": "sample.json" diff --git a/server/adaptors/integrations/repository/integration.ts b/server/adaptors/integrations/repository/integration.ts index fca1aef5ce..83e5779aca 100644 --- a/server/adaptors/integrations/repository/integration.ts +++ b/server/adaptors/integrations/repository/integration.ts @@ -109,6 +109,10 @@ export class IntegrationReader { ): Promise< Result<{ savedObjects?: object[]; + queries?: Array<{ + query: string; + language: string; + }>; }> > { const configResult = await this.getConfig(version); @@ -117,7 +121,10 @@ export class IntegrationReader { } const config = configResult.value; - const resultValue: { savedObjects?: object[] } = {}; + const resultValue: { + savedObjects?: object[]; + queries?: Array<{ query: string; language: string }>; + } = {}; if (config.assets.savedObjects) { const sobjPath = `${config.assets.savedObjects.name}-${config.assets.savedObjects.version}.ndjson`; const assets = await this.reader.readFile(sobjPath, 'assets'); @@ -126,6 +133,31 @@ export class IntegrationReader { } resultValue.savedObjects = assets.value as object[]; } + if (config.assets.queries) { + resultValue.queries = []; + const queries = await Promise.all( + config.assets.queries.map(async (item) => { + const queryPath = `${item.name}-${item.version}.${item.language}`; + const query = await this.reader.readFileRaw(queryPath, 'assets'); + if (!query.ok) { + return query; + } + return { + ok: true as const, + value: { + language: item.language, + query: query.value.toString('utf8'), + }, + }; + }) + ); + for (const query of queries) { + if (!query.ok) { + return query; + } + resultValue.queries.push(query.value); + } + } return { ok: true, value: resultValue }; } diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index c74829d302..fd5729afcc 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -5,6 +5,18 @@ type Result = { ok: true; value: T } | { ok: false; error: E }; +interface IntegrationAssets { + savedObjects?: { + name: string; + version: string; + }; + queries?: Array<{ + name: string; + version: string; + language: string; + }>; +} + interface IntegrationConfig { name: string; version: string; @@ -22,12 +34,7 @@ interface IntegrationConfig { darkModeGallery?: StaticAsset[]; }; components: IntegrationComponent[]; - assets: { - savedObjects?: { - name: string; - version: string; - }; - }; + assets: IntegrationAssets; sampleData?: { path: string; }; diff --git a/server/adaptors/integrations/validators.ts b/server/adaptors/integrations/validators.ts index 7486a38eda..2a4a717a90 100644 --- a/server/adaptors/integrations/validators.ts +++ b/server/adaptors/integrations/validators.ts @@ -64,6 +64,19 @@ const templateSchema: JSONSchemaType = { nullable: true, additionalProperties: false, }, + queries: { + type: 'array', + items: { + type: 'object', + properties: { + name: { type: 'string' }, + version: { type: 'string' }, + language: { type: 'string' }, + }, + required: ['name', 'version', 'language'], + }, + nullable: true, + }, }, additionalProperties: false, }, From 764d0283295dc3ecd03eb27e444f7044e0b1560f Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 29 Sep 2023 10:06:08 -0700 Subject: [PATCH 58/79] Simplify dynamic table term selection Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 79 ++++++------------- 1 file changed, 26 insertions(+), 53 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 973e14f927..944443b30f 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -28,41 +28,41 @@ interface IntegrationConfigProps { }; } -const INTEGRATION_DATA_TABLE_COLUMNS = [ - { - field: 'field', - name: 'Field', - }, - { - field: 'sourceType', - name: 'Source Data Type', - }, - { - field: 'destType', - name: 'Destination Data Type', - }, - { - field: 'group', - name: 'Mapping Group', - }, -]; - // TODO support localization -const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map = new Map([ - ['s3', 'table'], - ['index', 'index'], +const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map< + string, + { + title: string; + lower: string; + help: string; + } +> = new Map([ + [ + 's3', + { + title: 'Table', + lower: 'table', + help: 'Select a table to pull the data from.', + }, + ], + [ + 'index', + { + title: 'Index', + lower: 'index', + help: 'Select an index to pull the data from.', + }, + ], ]); const integrationConnectionSelectorItems = [ { value: 's3', text: 'S3 Connection', - dataSourceName: ['table', 'tables'], }, { value: 'index', text: 'OpenSearch Index', - dataSourceName: ['index', 'indexes'], }, ]; @@ -146,26 +146,6 @@ const suggestDataSources = async (type: string): Promise void }): React.JSX.Element { - return ( - - -

Data Table

- - - - - - Close - - - - ); -} - const findTemplate = async (integrationTemplateId: string) => { const http = coreRefs.http!; const result = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}`); @@ -177,11 +157,7 @@ export function SetupIntegrationForm({ updateConfig, integration, }: IntegrationConfigProps) { - const connectionType = - INTEGRATION_CONNECTION_DATA_SOURCE_TYPES.get(config.connectionType) ?? 'index'; - const indefiniteArticle = 'aeiou'.includes(connectionType.charAt(0)) ? 'an' : 'a'; - const capitalizedConnectionType = - connectionType.charAt(0).toUpperCase() + connectionType.slice(1); + const connectionType = INTEGRATION_CONNECTION_DATA_SOURCE_TYPES.get(config.connectionType)!; const [dataSourceSuggestions, setDataSourceSuggestions] = useState( [] as Array<{ label: string }> @@ -228,10 +204,7 @@ export function SetupIntegrationForm({ } /> - + Date: Fri, 29 Sep 2023 10:10:37 -0700 Subject: [PATCH 59/79] Remove unused validate code Signed-off-by: Simeon Widdis --- .../integrations/components/setup_integration.tsx | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 944443b30f..e775203b58 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -219,19 +219,6 @@ export function SetupIntegrationForm({ singleSelection={{ asPlainText: true }} /> - {/* TODO temporarily removing validate button until error states are ready. */} - {/* { - const validationResult = await doExistingDataSourceValidation( - config.connectionDataSource, - integration.name, - integration.type - ); - console.log(validationResult); - }} - > - Validate - */} ); } From aaeddefd774dd62f3233aa6990ea77a7653df2ca Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 29 Sep 2023 10:14:24 -0700 Subject: [PATCH 60/79] Undo wildcard import Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 100 ++++++++++-------- 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index e775203b58..edfdfbb6c5 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -3,7 +3,23 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as Eui from '@elastic/eui'; +import { + EuiBottomBar, + EuiButton, + EuiComboBox, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiPage, + EuiPageBody, + EuiPageContent, + EuiPageContentBody, + EuiSelect, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; import React, { useState, useEffect } from 'react'; import { coreRefs } from '../../../framework/core_refs'; import { @@ -175,37 +191,37 @@ export function SetupIntegrationForm({ }, [config.connectionType]); return ( - - + +

Set Up Integration

- - - + + +

Integration Details

- - - - + + + updateConfig({ displayName: event.target.value })} /> - - - + + +

Integration Connection

- - - - + + + updateConfig({ connectionType: event.target.value, connectionDataSource: '' }) } /> - - - + + { @@ -218,8 +234,8 @@ export function SetupIntegrationForm({ selectedOptions={[{ label: config.connectionDataSource }]} singleSelection={{ asPlainText: true }} /> - - + + ); } @@ -234,10 +250,10 @@ export function SetupBottomBar({ const [loading, setLoading] = useState(false); return ( - - - - + + + { @@ -249,10 +265,10 @@ export function SetupBottomBar({ }} > Discard - - - - + + + Add Integration - - - - + + + + ); } @@ -298,19 +314,19 @@ export function SetupIntegrationPage({ setConfig(Object.assign({}, integConfig, updates)); return ( - - - - + + + + - - + + - - + + ); } From 3ae20be9ada3778f9e033b01dcbfc47f56d66dd0 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 29 Sep 2023 10:21:19 -0700 Subject: [PATCH 61/79] Switch from proxy to dataconnections endpoint Signed-off-by: Simeon Widdis --- .../integrations/components/setup_integration.tsx | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index edfdfbb6c5..30a3069e57 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -138,13 +138,10 @@ const suggestDataSources = async (type: string): Promise; + const result = (await http.get('/api/dataconnections')) as Array<{ + name: string; + connector: string; + }>; return ( result ?.filter((item) => item.connector === 'S3GLUE') From 0158cc329add002d3469073ca24f2d8b979d8b24 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 29 Sep 2023 10:22:50 -0700 Subject: [PATCH 62/79] Remove unused table fields Signed-off-by: Simeon Widdis --- .../components/setup_integration.tsx | 44 +------------------ 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 30a3069e57..5327406bfa 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -22,10 +22,7 @@ import { } from '@elastic/eui'; import React, { useState, useEffect } from 'react'; import { coreRefs } from '../../../framework/core_refs'; -import { - addIntegrationRequest, - doExistingDataSourceValidation, -} from './create_integration_helpers'; +import { addIntegrationRequest } from './create_integration_helpers'; import { useToast } from '../../../../public/components/common/toast'; import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; @@ -82,45 +79,6 @@ const integrationConnectionSelectorItems = [ }, ]; -const integrationDataTableData = [ - { - field: 'domain.bytes', - sourceType: 'string', - destType: 'integer', - group: 'communication', - }, - { - field: 'http.url', - sourceType: 'string', - destType: 'keyword', - group: 'http', - }, - { - field: 'destination.address', - sourceType: 'string', - destType: 'keyword', - group: 'communication', - }, - { - field: 'netflow.error_rate', - sourceType: 'string', - destType: 'string', - group: 'http', - }, - { - field: 'http.route', - sourceType: 'string', - destType: 'string', - group: 'http', - }, - { - field: 'destination.packets', - sourceType: 'integer', - destType: 'integer', - group: 'communication', - }, -]; - const suggestDataSources = async (type: string): Promise> => { const http = coreRefs.http!; try { From 3cb6c57870e6b93f91476a9e93602204b74078a3 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 29 Sep 2023 10:26:45 -0700 Subject: [PATCH 63/79] Switch dataconnections base to const Signed-off-by: Simeon Widdis --- .../components/integrations/components/setup_integration.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 5327406bfa..e86e5db6a2 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -25,6 +25,7 @@ import { coreRefs } from '../../../framework/core_refs'; import { addIntegrationRequest } from './create_integration_helpers'; import { useToast } from '../../../../public/components/common/toast'; import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; +import { DATACONNECTIONS_BASE } from '../../../../common/constants/shared'; export interface IntegrationConfig { displayName: string; @@ -96,7 +97,7 @@ const suggestDataSources = async (type: string): Promise; From dd1d763f8a55c815799f6f650b0d8b1cda61b8bc Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Fri, 29 Sep 2023 10:29:14 -0700 Subject: [PATCH 64/79] Add console proxy to route constants Signed-off-by: Simeon Widdis --- common/constants/shared.ts | 1 + .../components/create_integration_helpers.ts | 12 ++++++------ .../integrations/components/setup_integration.tsx | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/common/constants/shared.ts b/common/constants/shared.ts index 2187f28ce2..b19ae966de 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -19,6 +19,7 @@ export const EVENT_ANALYTICS = '/event_analytics'; export const SAVED_OBJECTS = '/saved_objects'; export const SAVED_QUERY = '/query'; export const SAVED_VISUALIZATION = '/vis'; +export const CONSOLE_PROXY = '/api/console/proxy'; // Server route export const PPL_ENDPOINT = '/_plugins/_ppl'; diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts index 15b74b3025..79e6825b37 100644 --- a/public/components/integrations/components/create_integration_helpers.ts +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -4,7 +4,7 @@ */ import { HttpSetup } from '../../../../../../src/core/public'; import { coreRefs } from '../../../framework/core_refs'; -import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; +import { CONSOLE_PROXY, INTEGRATIONS_BASE } from '../../../../common/constants/shared'; type ValidationResult = { ok: true } | { ok: false; errors: string[] }; @@ -120,7 +120,7 @@ export const fetchDataSourceMappings = async ( http: HttpSetup ): Promise<{ [key: string]: { properties: any } } | null> => { return http - .post('/api/console/proxy', { + .post(CONSOLE_PROXY, { query: { path: `${targetDataSource}/_mapping`, method: 'GET', @@ -196,7 +196,7 @@ const createComponentMapping = async ( ): Promise<{ [key: string]: { properties: any } } | null> => { const http = coreRefs.http!; const version = payload.template.mappings._meta.version; - return http.post('/api/console/proxy', { + return http.post(CONSOLE_PROXY, { body: JSON.stringify(payload), query: { path: `_component_template/ss4o_${componentName}-${version}-template`, @@ -218,7 +218,7 @@ const createIndexMapping = async ( const http = coreRefs.http!; const version = payload.template.mappings._meta.version; payload.index_patterns = [dataSourceName]; - return http.post('/api/console/proxy', { + return http.post(CONSOLE_PROXY, { body: JSON.stringify(payload), query: { path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, @@ -256,7 +256,7 @@ const createDataSourceMappings = async ( }) ); // In order to see our changes, we need to manually provoke a refresh - await http.post('/api/console/proxy', { + await http.post(CONSOLE_PROXY, { query: { path: '_refresh', method: 'GET', @@ -332,7 +332,7 @@ export async function addIntegrationRequest( .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) .join('\n') + '\n'; http - .post('/api/console/proxy', { + .post(CONSOLE_PROXY, { body: requestBody, query: { path: `${dataSource}/_bulk?refresh=wait_for`, diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index e86e5db6a2..45e45f0a28 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -24,7 +24,7 @@ import React, { useState, useEffect } from 'react'; import { coreRefs } from '../../../framework/core_refs'; import { addIntegrationRequest } from './create_integration_helpers'; import { useToast } from '../../../../public/components/common/toast'; -import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; +import { CONSOLE_PROXY, INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { DATACONNECTIONS_BASE } from '../../../../common/constants/shared'; export interface IntegrationConfig { @@ -84,7 +84,7 @@ const suggestDataSources = async (type: string): Promise Date: Fri, 29 Sep 2023 10:31:07 -0700 Subject: [PATCH 65/79] Update snapshots Signed-off-by: Simeon Widdis --- .../setup_integration.test.tsx.snap | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 1786f79721..051d231ad2 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -204,18 +204,10 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = options={ Array [ Object { - "dataSourceName": Array [ - "table", - "tables", - ], "text": "S3 Connection", "value": "s3", }, Object { - "dataSourceName": Array [ - "index", - "indexes", - ], "text": "OpenSearch Index", "value": "index", }, @@ -253,24 +245,12 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = value="index" >