From 2b1e2d9cef60870ddd41525d3765b6b31e94ed42 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Tue, 3 Oct 2023 12:59:53 -0700 Subject: [PATCH] Update UI for integrations setup (#1052) * Stub catalog reader interface Signed-off-by: Simeon Widdis * Add basic catalog functionality to new catalog reader Signed-off-by: Simeon Widdis * Refactor validation logic with a deeper interface Signed-off-by: Simeon Widdis * Refactor validation logic with a deeper interface Signed-off-by: Simeon Widdis * Remove redundant test. This test is unneeded after 12c4bcf Signed-off-by: Simeon Widdis * Add tests for new validators Signed-off-by: Simeon Widdis * Make better failure mode for invalid objects Signed-off-by: Simeon Widdis * Generalize Result type Signed-off-by: Simeon Widdis * Convert backend to use catalog reader (unstable) Signed-off-by: Simeon Widdis * Repair tests for integrations class (unstable) Signed-off-by: Simeon Widdis * Refactor repository for new integration interface Signed-off-by: Simeon Widdis * Fix outer repository and backend tests Signed-off-by: Simeon Widdis * Add tests for sample data Signed-off-by: Simeon Widdis * Add CatalogReader JavaDocs Signed-off-by: Simeon Widdis * Repair integrations builder Signed-off-by: Simeon Widdis * Remove extra commented test Signed-off-by: Simeon Widdis * Remove unnecessary log statement Signed-off-by: Simeon Widdis * Repair getSchemas behavior to return correct type Let it be known at on this day, with this commit, I have truly grokked why we don't use `any` in typescript. Signed-off-by: Simeon Widdis * Add tests for getSchemas Signed-off-by: Simeon Widdis * Add tests for asset and sample data backend methods Signed-off-by: Simeon Widdis * Break flyout validation methods out of constructing method Signed-off-by: Simeon Widdis * Add tests for extracted flyout methods Signed-off-by: Simeon Widdis * Switch validation method to use ValidationResult Signed-off-by: Simeon Widdis * Swap out flyout for hello-world setup page Signed-off-by: Simeon Widdis * Add basic step incrementing Signed-off-by: Simeon Widdis * Add basic field skeleton for each step Signed-off-by: Simeon Widdis * Add a cancel button Signed-off-by: Simeon Widdis * Add config type to developing form Signed-off-by: Simeon Widdis * Flatten integration config Signed-off-by: Simeon Widdis * Add sample data table modal Signed-off-by: Simeon Widdis * Add toggle for standard and advanced asset config Signed-off-by: Simeon Widdis * Simplify imports Signed-off-by: Simeon Widdis * Refactor major class names Signed-off-by: Simeon Widdis * (WIP) begin refactoring functionality into adaptor Signed-off-by: Simeon Widdis * Finish migrating functionality to data adaptor Signed-off-by: Simeon Widdis * Rename integration types for more clarity Signed-off-by: Simeon Widdis * Refactor component usage Signed-off-by: Simeon Widdis * Connect forms to config state Signed-off-by: Simeon Widdis * Fix filetype selector Signed-off-by: Simeon Widdis * Remove hardcoded name in path Signed-off-by: Simeon Widdis * Write one snapshot test Signed-off-by: Simeon Widdis * Add more tests Signed-off-by: Simeon Widdis * Fix test naming Signed-off-by: Simeon Widdis * Update obsolete snapshots Signed-off-by: Simeon Widdis * Move integration creation helpers to own file Signed-off-by: Simeon Widdis * Break out integration creation methods Signed-off-by: Simeon Widdis * Isolate more create_integration helpers Signed-off-by: Simeon Widdis * Simplify setup form Signed-off-by: Simeon Widdis * Add data source picker items Signed-off-by: Simeon Widdis * Add better selector logic Signed-off-by: Simeon Widdis * Add queries for data sources Signed-off-by: Simeon Widdis * Switch from selector to combobox Signed-off-by: Simeon Widdis * Update snapshots Signed-off-by: Simeon Widdis * Connect validation button to data source validation method Signed-off-by: Simeon Widdis * Reimplement add integration button Signed-off-by: Simeon Widdis * Temporarily remove validate button Signed-off-by: Simeon Widdis * Simplify dynamic table term selection Signed-off-by: Simeon Widdis * Remove unused validate code Signed-off-by: Simeon Widdis * Undo wildcard import Signed-off-by: Simeon Widdis * Switch from proxy to dataconnections endpoint Signed-off-by: Simeon Widdis * Remove unused table fields Signed-off-by: Simeon Widdis * Switch dataconnections base to const Signed-off-by: Simeon Widdis * Add console proxy to route constants Signed-off-by: Simeon Widdis * Update snapshots Signed-off-by: Simeon Widdis * Move color to constants Signed-off-by: Simeon Widdis * Move index name validation to constants and improve matching Signed-off-by: Simeon Widdis * Move test constants to test constants Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- common/constants/integrations.ts | 4 + common/constants/shared.ts | 1 + .../setup_integration.test.tsx.snap | 2681 ++++++++--------- .../added_integration_flyout.test.tsx | 329 +- .../create_integration_helpers.test.ts | 333 ++ .../__tests__/setup_integration.test.tsx | 68 +- .../components/add_integration_flyout.tsx | 174 +- .../components/create_integration_helpers.ts | 344 +++ .../integrations/components/integration.tsx | 161 +- .../components/setup_integration.tsx | 556 ++-- public/components/integrations/home.tsx | 21 +- .../repository/__test__/integration.test.ts | 46 +- test/constants.ts | 28 + 13 files changed, 2194 insertions(+), 2552 deletions(-) create mode 100644 public/components/integrations/components/__tests__/create_integration_helpers.test.ts create mode 100644 public/components/integrations/components/create_integration_helpers.ts diff --git a/common/constants/integrations.ts b/common/constants/integrations.ts index 9df90f0047..2d31c3c0f5 100644 --- a/common/constants/integrations.ts +++ b/common/constants/integrations.ts @@ -5,3 +5,7 @@ export const OPENSEARCH_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/integrations/index'; export const ASSET_FILTER_OPTIONS = ['index-pattern', 'search', 'visualization', 'dashboard']; +export const VALID_INDEX_NAME = /^[a-z\d\.][a-z\d\._\-\*]*$/; + +// Upstream doesn't export this, so we need to redeclare it for our use. +export type Color = 'success' | 'primary' | 'warning' | 'danger' | undefined; diff --git a/common/constants/shared.ts b/common/constants/shared.ts index 2187f28ce2..b19ae966de 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -19,6 +19,7 @@ export const EVENT_ANALYTICS = '/event_analytics'; export const SAVED_OBJECTS = '/saved_objects'; export const SAVED_QUERY = '/query'; export const SAVED_VISUALIZATION = '/vis'; +export const CONSOLE_PROXY = '/api/console/proxy'; // Server route export const PPL_ENDPOINT = '/_plugins/_ppl'; diff --git a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap index 1efafd0e03..516b053c95 100644 --- a/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap +++ b/public/components/integrations/components/__tests__/__snapshots__/setup_integration.test.tsx.snap @@ -1,7 +1,28 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`Integration Setup Page Renders integration setup page as expected 1`] = ` - +
- -
+ - -
- + +
+ , - "status": "", - "title": "Name Integration", - }, + "connectionDataSource": "", + "connectionType": "index", + "displayName": "sample Integration", + } + } + integration={ Object { - "children": , - "status": "disabled", - "title": "Select index or data source for integration", - }, - ] - } - > -
- -
-
- - - - - Step 1 - - - - - - -

- Name Integration -

- -
-
- -
- -
-
- - -
-
- - - - - Step 2 is disabled - - - - - - -

- Select index or data source for integration -

- -
-
- -
- -
-
- -
- -
- - -
- -
- Name Integration + Set Up Integration + +
+ + +

+ Integration Details +

+ + +
+
- Name + Display Name
@@ -230,12 +137,11 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiFormRow__fieldWrapper" > @@ -264,16 +169,538 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
+
+
+ + +
+ + +

+ Integration Connection +

+ + +
+ + +
+
+ + + +
+
+ + +
+
+ + + + +
+ + + + + +
+ +
+
+ + + +
+ Select a data source to connect to. +
+ +
+
+ + +
+
+ + + +
+
+ +
+ + +
+
+
+ + + + +
+ +
+
+ +
+ +
+ + + + + + + + + + + +
+ +
+
+ + +
+
- The name will be used to label the newly added integration + Select an index to pull the data from.
@@ -281,26 +708,40 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
- - -
- -
- + +
+ +
+ + @@ -323,7 +764,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = class="euiFlexItem euiFlexItem--flexGrowZero" >
-
-
-
@@ -362,7 +796,7 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = type="button" > @@ -429,13 +863,13 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = className="euiFlexItem euiFlexItem--flexGrowZero" >
- -
- -
- -
- @@ -524,7 +947,9 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = > @@ -550,8 +977,9 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] = > - Next + Add Integration @@ -624,939 +1052,162 @@ exports[`Integration Setup Page Renders integration setup page as expected 1`] =
- + `; -exports[`Integration Setup Page Renders the data source form as expected 1`] = ` - -
- +
-
- - + + +
+ + +

- (debug) Table detected - -

- - -
- - -
+ + +
+ + - -

- Select index or data source for integration -

- - -
- -
-
- - - No tables were found - -
- -
- -
-

- No problem, we can help. Tell us about your data. -

-
- -
- + Display Name + +
- - -
- -
- - - Use existing Data Source - -
- - -
- - -
- -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
-
- - - -
-
- - -
-
- - - - -
- - - - - -
- -
-
- - -
-
- - -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- -
- -
- -
- -`; - -exports[`Integration Setup Page Renders the existing table form as expected 1`] = ` - -
- -
-
- - - -
-
- - -
- - - - -
- - - - - -
- + + + + +
-
- - - -
- Manage data associated with this data source -
- + + +
-
- - -
- - - - -
- -`; - -exports[`Integration Setup Page Renders the metadata form as expected 1`] = ` - - -
+ + +
+ -

- Name Integration -

+ Integration Connection + + +
+
- Name + Data Source
-
- + onMouseUp={[Function]} + value="index" + > + + + - + +
+ + + + + +
+
- +
- The name will be used to label the newly added integration + Select a data source to connect to.
-
- - -`; - -exports[`Integration Setup Page Renders the new table form as expected 1`] = ` - -
- -
-
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
- - - -
-
- - -
-
- - - - -
-
- - -
-
- - -
-
- - - -
-
- + +
+
-
-
- - - -
- - -
- -
+ +
- - -
-
- - -
-
- - - -
-
- - +
-
- - - - -
+ Select an index to pull the data from.
- - + +
-
- -
- + +
+ + `; diff --git a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx index ae3b609f87..9099e59a14 100644 --- a/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx +++ b/public/components/integrations/components/__tests__/added_integration_flyout.test.tsx @@ -6,17 +6,7 @@ import { configure, mount } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import { waitFor } from '@testing-library/react'; -import { - AddIntegrationFlyout, - checkDataSourceName, - doTypeValidation, - doNestedPropertyValidation, - doPropertyValidation, - fetchDataSourceMappings, - fetchIntegrationMappings, - doExistingDataSourceValidation, -} from '../add_integration_flyout'; -import * as add_integration_flyout from '../add_integration_flyout'; +import { AddIntegrationFlyout } from '../add_integration_flyout'; import React from 'react'; import { HttpSetup } from '../../../../../../../src/core/public'; @@ -44,320 +34,3 @@ describe('Add Integration Flyout Test', () => { }); }); }); - -describe('doTypeValidation', () => { - it('should return true if required type is not specified', () => { - const toCheck = { type: 'string' }; - const required = {}; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return true if types match', () => { - const toCheck = { type: 'string' }; - const required = { type: 'string' }; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return true if object has properties', () => { - const toCheck = { properties: { prop1: { type: 'string' } } }; - const required = { type: 'object' }; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return false if types do not match', () => { - const toCheck = { type: 'string' }; - const required = { type: 'number' }; - - const result = doTypeValidation(toCheck, required); - - expect(result.ok).toBe(false); - }); -}); - -describe('doNestedPropertyValidation', () => { - it('should return true if type validation passes and no properties are required', () => { - const toCheck = { type: 'string' }; - const required = { type: 'string' }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); - - it('should return false if type validation fails', () => { - const toCheck = { type: 'string' }; - const required = { type: 'number' }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(false); - }); - - it('should return false if a required property is missing', () => { - const toCheck = { type: 'object', properties: { prop1: { type: 'string' } } }; - const required = { - type: 'object', - properties: { prop1: { type: 'string' }, prop2: { type: 'number' } }, - }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(false); - }); - - it('should return true if all required properties pass validation', () => { - const toCheck = { - type: 'object', - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }; - const required = { - type: 'object', - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }; - - const result = doNestedPropertyValidation(toCheck, required); - - expect(result.ok).toBe(true); - }); -}); - -describe('doPropertyValidation', () => { - it('should return true if all properties pass validation', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result.ok).toBe(true); - }); - - it('should return false if a property fails validation', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'boolean' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result.ok).toBe(false); - }); - - it('should return false if a required nested property is missing', () => { - const rootType = 'root'; - const dataSourceProps = { - prop1: { type: 'string' }, - }; - const requiredMappings = { - root: { - template: { - mappings: { - properties: { - prop1: { type: 'string' }, - prop2: { type: 'number' }, - }, - }, - }, - }, - }; - - const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); - - expect(result.ok).toBe(false); - }); -}); - -describe('checkDataSourceName', () => { - it('Filters out invalid index names', () => { - const result = checkDataSourceName('ss4o_logs-no-exclams!', 'logs'); - - expect(result.ok).toBe(false); - }); - - it('Filters out incorrectly typed indices', () => { - const result = checkDataSourceName('ss4o_metrics-test-test', 'logs'); - - expect(result.ok).toBe(false); - }); - - it('Accepts correct indices', () => { - const result = checkDataSourceName('ss4o_logs-test-test', 'logs'); - - expect(result.ok).toBe(true); - }); -}); - -describe('fetchDataSourceMappings', () => { - it('Retrieves mappings', async () => { - const mockHttp = { - post: jest.fn().mockResolvedValue({ - source1: { mappings: { properties: { test: true } } }, - source2: { mappings: { properties: { test: true } } }, - }), - } as Partial; - - const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); - - await expect(result).resolves.toMatchObject({ - source1: { properties: { test: true } }, - source2: { properties: { test: true } }, - }); - }); - - it('Catches errors', async () => { - const mockHttp = { - post: jest.fn().mockRejectedValue(new Error('Mock error')), - } as Partial; - - const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); - - await expect(result).resolves.toBeNull(); - }); -}); - -describe('fetchIntegrationMappings', () => { - it('Returns schema mappings', async () => { - const mockHttp = { - get: jest.fn().mockResolvedValue({ data: { mappings: { test: true } }, statusCode: 200 }), - } as Partial; - - const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); - - await expect(result).resolves.toStrictEqual({ test: true }); - }); - - it('Returns null if response fails', async () => { - const mockHttp = { - get: jest.fn().mockResolvedValue({ statusCode: 404 }), - } as Partial; - - const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); - - await expect(result).resolves.toBeNull(); - }); - - it('Catches request error', async () => { - const mockHttp = { - get: jest.fn().mockRejectedValue(new Error('mock error')), - } as Partial; - - const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); - - await expect(result).resolves.toBeNull(); - }); -}); - -describe('doExistingDataSourceValidation', () => { - it('Catches and returns checkDataSourceName errors', async () => { - const mockHttp = {} as Partial; - jest - .spyOn(add_integration_flyout, 'checkDataSourceName') - .mockReturnValue({ ok: false, errors: ['mock'] }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Catches data stream fetch errors', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest.spyOn(add_integration_flyout, 'fetchDataSourceMappings').mockResolvedValue(null); - jest - .spyOn(add_integration_flyout, 'fetchIntegrationMappings') - .mockResolvedValue({ test: { template: { mappings: {} } } }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Catches integration fetch errors', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest - .spyOn(add_integration_flyout, 'fetchDataSourceMappings') - .mockResolvedValue({ test: { properties: {} } }); - jest.spyOn(add_integration_flyout, 'fetchIntegrationMappings').mockResolvedValue(null); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Catches type validation issues', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest - .spyOn(add_integration_flyout, 'fetchDataSourceMappings') - .mockResolvedValue({ test: { properties: {} } }); - jest - .spyOn(add_integration_flyout, 'fetchIntegrationMappings') - .mockResolvedValue({ test: { template: { mappings: {} } } }); - jest - .spyOn(add_integration_flyout, 'doPropertyValidation') - .mockReturnValue({ ok: false, errors: ['mock'] }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', false); - }); - - it('Returns no errors if everything passes', async () => { - const mockHttp = {} as Partial; - jest.spyOn(add_integration_flyout, 'checkDataSourceName').mockReturnValue({ ok: true }); - jest - .spyOn(add_integration_flyout, 'fetchDataSourceMappings') - .mockResolvedValue({ test: { properties: {} } }); - jest - .spyOn(add_integration_flyout, 'fetchIntegrationMappings') - .mockResolvedValue({ test: { template: { mappings: {} } } }); - jest.spyOn(add_integration_flyout, 'doPropertyValidation').mockReturnValue({ ok: true }); - - const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); - - await expect(result).resolves.toHaveProperty('ok', true); - }); -}); diff --git a/public/components/integrations/components/__tests__/create_integration_helpers.test.ts b/public/components/integrations/components/__tests__/create_integration_helpers.test.ts new file mode 100644 index 0000000000..71ccc99bf6 --- /dev/null +++ b/public/components/integrations/components/__tests__/create_integration_helpers.test.ts @@ -0,0 +1,333 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + checkDataSourceName, + doTypeValidation, + doNestedPropertyValidation, + doPropertyValidation, + fetchDataSourceMappings, + fetchIntegrationMappings, + doExistingDataSourceValidation, +} from '../create_integration_helpers'; +import * as create_integration_helpers from '../create_integration_helpers'; +import { HttpSetup } from '../../../../../../../src/core/public'; + +describe('doTypeValidation', () => { + it('should return true if required type is not specified', () => { + const toCheck = { type: 'string' }; + const required = {}; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return true if types match', () => { + const toCheck = { type: 'string' }; + const required = { type: 'string' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return true if object has properties', () => { + const toCheck = { properties: { prop1: { type: 'string' } } }; + const required = { type: 'object' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return false if types do not match', () => { + const toCheck = { type: 'string' }; + const required = { type: 'number' }; + + const result = doTypeValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); +}); + +describe('doNestedPropertyValidation', () => { + it('should return true if type validation passes and no properties are required', () => { + const toCheck = { type: 'string' }; + const required = { type: 'string' }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); + + it('should return false if type validation fails', () => { + const toCheck = { type: 'string' }; + const required = { type: 'number' }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); + + it('should return false if a required property is missing', () => { + const toCheck = { type: 'object', properties: { prop1: { type: 'string' } } }; + const required = { + type: 'object', + properties: { prop1: { type: 'string' }, prop2: { type: 'number' } }, + }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(false); + }); + + it('should return true if all required properties pass validation', () => { + const toCheck = { + type: 'object', + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }; + const required = { + type: 'object', + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }; + + const result = doNestedPropertyValidation(toCheck, required); + + expect(result.ok).toBe(true); + }); +}); + +describe('doPropertyValidation', () => { + it('should return true if all properties pass validation', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(true); + }); + + it('should return false if a property fails validation', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'boolean' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(false); + }); + + it('should return false if a required nested property is missing', () => { + const rootType = 'root'; + const dataSourceProps = { + prop1: { type: 'string' }, + }; + const requiredMappings = { + root: { + template: { + mappings: { + properties: { + prop1: { type: 'string' }, + prop2: { type: 'number' }, + }, + }, + }, + }, + }; + + const result = doPropertyValidation(rootType, dataSourceProps as any, requiredMappings); + + expect(result.ok).toBe(false); + }); +}); + +describe('checkDataSourceName', () => { + it('Filters out invalid index names', () => { + const result = checkDataSourceName('ss4o_logs-no-exclams!', 'logs'); + + expect(result.ok).toBe(false); + }); + + it('Filters out incorrectly typed indices', () => { + const result = checkDataSourceName('ss4o_metrics-test-test', 'logs'); + + expect(result.ok).toBe(false); + }); + + it('Accepts correct indices', () => { + const result = checkDataSourceName('ss4o_logs-test-test', 'logs'); + + expect(result.ok).toBe(true); + }); +}); + +describe('fetchDataSourceMappings', () => { + it('Retrieves mappings', async () => { + const mockHttp = { + post: jest.fn().mockResolvedValue({ + source1: { mappings: { properties: { test: true } } }, + source2: { mappings: { properties: { test: true } } }, + }), + } as Partial; + + const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); + + await expect(result).resolves.toMatchObject({ + source1: { properties: { test: true } }, + source2: { properties: { test: true } }, + }); + }); + + it('Catches errors', async () => { + const mockHttp = { + post: jest.fn().mockRejectedValue(new Error('Mock error')), + } as Partial; + + const result = fetchDataSourceMappings('sample', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); +}); + +describe('fetchIntegrationMappings', () => { + it('Returns schema mappings', async () => { + const mockHttp = { + get: jest.fn().mockResolvedValue({ data: { mappings: { test: true } }, statusCode: 200 }), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toStrictEqual({ test: true }); + }); + + it('Returns null if response fails', async () => { + const mockHttp = { + get: jest.fn().mockResolvedValue({ statusCode: 404 }), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); + + it('Catches request error', async () => { + const mockHttp = { + get: jest.fn().mockRejectedValue(new Error('mock error')), + } as Partial; + + const result = fetchIntegrationMappings('target', mockHttp as HttpSetup); + + await expect(result).resolves.toBeNull(); + }); +}); + +describe('doExistingDataSourceValidation', () => { + it('Catches and returns checkDataSourceName errors', async () => { + const mockHttp = {} as Partial; + jest + .spyOn(create_integration_helpers, 'checkDataSourceName') + .mockReturnValue({ ok: false, errors: ['mock'] }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Catches data stream fetch errors', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest.spyOn(create_integration_helpers, 'fetchDataSourceMappings').mockResolvedValue(null); + jest + .spyOn(create_integration_helpers, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Catches integration fetch errors', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(create_integration_helpers, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest.spyOn(create_integration_helpers, 'fetchIntegrationMappings').mockResolvedValue(null); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Catches type validation issues', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(create_integration_helpers, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest + .spyOn(create_integration_helpers, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + jest + .spyOn(create_integration_helpers, 'doPropertyValidation') + .mockReturnValue({ ok: false, errors: ['mock'] }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', false); + }); + + it('Returns no errors if everything passes', async () => { + const mockHttp = {} as Partial; + jest.spyOn(create_integration_helpers, 'checkDataSourceName').mockReturnValue({ ok: true }); + jest + .spyOn(create_integration_helpers, 'fetchDataSourceMappings') + .mockResolvedValue({ test: { properties: {} } }); + jest + .spyOn(create_integration_helpers, 'fetchIntegrationMappings') + .mockResolvedValue({ test: { template: { mappings: {} } } }); + jest.spyOn(create_integration_helpers, 'doPropertyValidation').mockReturnValue({ ok: true }); + + const result = doExistingDataSourceValidation('target', 'name', 'type', mockHttp as HttpSetup); + + await expect(result).resolves.toHaveProperty('ok', true); + }); +}); diff --git a/public/components/integrations/components/__tests__/setup_integration.test.tsx b/public/components/integrations/components/__tests__/setup_integration.test.tsx index 61001ba7c5..4d0431b5e1 100644 --- a/public/components/integrations/components/__tests__/setup_integration.test.tsx +++ b/public/components/integrations/components/__tests__/setup_integration.test.tsx @@ -7,79 +7,29 @@ import { configure, mount } from 'enzyme'; import Adapter from 'enzyme-adapter-react-16'; import React from 'react'; import { waitFor } from '@testing-library/react'; +import { SetupIntegrationPage, SetupIntegrationForm } from '../setup_integration'; import { - SetupIntegrationDataSource, - SetupIntegrationExistingTable, - SetupIntegrationMetadata, - SetupIntegrationNewTable, - SetupIntegrationStepsPage, -} from '../setup_integration'; - -const TEST_CONFIG = { - instanceName: 'Test Instance Name', - useExisting: true, - dataSourceName: 'Test Datasource Name', - dataSourceDescription: 'Test Datasource Description', - dataSourceFileType: 'json', - dataSourceLocation: 'ss4o_logs-test-new-location', - existingDataSourceName: 'ss4o_logs-test-existing-location', -}; + TEST_INTEGRATION_CONFIG, + TEST_INTEGRATION_SETUP_INPUTS, +} from '../../../../../test/constants'; describe('Integration Setup Page', () => { configure({ adapter: new Adapter() }); it('Renders integration setup page as expected', async () => { - const wrapper = mount(); - - await waitFor(() => { - expect(wrapper).toMatchSnapshot(); - }); - }); - - it('Renders the metadata form as expected', async () => { - const wrapper = mount( - {}} /> - ); - - await waitFor(() => { - expect(wrapper).toMatchSnapshot(); - }); - }); - - it('Renders the data source form as expected', async () => { - const wrapper = mount( - {}} - showDataModal={false} - setShowDataModal={() => {}} - tableDetected={false} - setTableDetected={() => {}} - /> - ); - - await waitFor(() => { - expect(wrapper).toMatchSnapshot(); - }); - }); - - it('Renders the new table form as expected', async () => { - const wrapper = mount( - {}} /> - ); + const wrapper = mount(); await waitFor(() => { expect(wrapper).toMatchSnapshot(); }); }); - it('Renders the existing table form as expected', async () => { + it('Renders the form as expected', async () => { const wrapper = mount( - {}} - showDataModal={false} - setShowDataModal={() => {}} + integration={TEST_INTEGRATION_CONFIG} /> ); diff --git a/public/components/integrations/components/add_integration_flyout.tsx b/public/components/integrations/components/add_integration_flyout.tsx index 0e1e792610..a06d4bdda5 100644 --- a/public/components/integrations/components/add_integration_flyout.tsx +++ b/public/components/integrations/components/add_integration_flyout.tsx @@ -20,8 +20,9 @@ import { EuiTitle, } from '@elastic/eui'; import React, { useState } from 'react'; -import { HttpSetup, HttpStart } from '../../../../../../src/core/public'; +import { HttpStart } from '../../../../../../src/core/public'; import { useToast } from '../../../../public/components/common/toast'; +import { doExistingDataSourceValidation } from './create_integration_helpers'; interface IntegrationFlyoutProps { onClose: () => void; @@ -31,174 +32,6 @@ interface IntegrationFlyoutProps { http: HttpStart; } -type ValidationResult = { ok: true } | { ok: false; errors: string[] }; - -export const doTypeValidation = ( - toCheck: { type?: string; properties?: object }, - required: { type?: string; properties?: object } -): ValidationResult => { - if (!required.type) { - return { ok: true }; - } - if (required.type === 'object') { - if (Boolean(toCheck.properties)) { - return { ok: true }; - } - return { ok: false, errors: ["'object' type must have properties."] }; - } - if (required.type !== toCheck.type) { - return { ok: false, errors: [`Type mismatch: '${required.type}' and '${toCheck.type}'`] }; - } - return { ok: true }; -}; - -export const doNestedPropertyValidation = ( - toCheck: { type?: string; properties?: { [key: string]: object } }, - required: { type?: string; properties?: { [key: string]: object } } -): ValidationResult => { - const typeCheck = doTypeValidation(toCheck, required); - if (!typeCheck.ok) { - return typeCheck; - } - for (const property of Object.keys(required.properties ?? {})) { - if (!Object.hasOwn(toCheck.properties ?? {}, property)) { - return { ok: false, errors: [`Missing field '${property}'`] }; - } - // Both are safely non-null after above checks. - const nested = doNestedPropertyValidation( - toCheck.properties![property], - required.properties![property] - ); - if (!nested.ok) { - return nested; - } - } - return { ok: true }; -}; - -export const doPropertyValidation = ( - rootType: string, - dataSourceProps: { [key: string]: { properties?: any } }, - requiredMappings: { [key: string]: { template: { mappings: { properties?: any } } } } -): ValidationResult => { - // Check root object type (without dependencies) - for (const [key, value] of Object.entries( - requiredMappings[rootType].template.mappings.properties - )) { - if ( - !dataSourceProps[key] || - !doNestedPropertyValidation(dataSourceProps[key], value as any).ok - ) { - return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; - } - } - // Check nested dependencies - for (const [key, value] of Object.entries(requiredMappings)) { - if (key === rootType) { - continue; - } - if ( - !dataSourceProps[key] || - !doNestedPropertyValidation(dataSourceProps[key], value.template.mappings.properties).ok - ) { - return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; - } - } - return { ok: true }; -}; - -// Returns true if the data stream is a legal name. -// Appends any additional validation errors to the provided errors array. -export const checkDataSourceName = ( - targetDataSource: string, - integrationType: string -): ValidationResult => { - let errors: string[] = []; - if (!/^[a-z\d\.][a-z\d\._\-\*]*$/.test(targetDataSource)) { - errors = errors.concat('This is not a valid index name.'); - return { ok: false, errors }; - } - const nameValidity: boolean = new RegExp(`^ss4o_${integrationType}-[^\\-]+-[^\\-]+`).test( - targetDataSource - ); - if (!nameValidity) { - errors = errors.concat('This index does not match the suggested naming convention.'); - return { ok: false, errors }; - } - return { ok: true }; -}; - -export const fetchDataSourceMappings = async ( - targetDataSource: string, - http: HttpSetup -): Promise<{ [key: string]: { properties: any } } | null> => { - return http - .post('/api/console/proxy', { - query: { - path: `${targetDataSource}/_mapping`, - method: 'GET', - }, - }) - .then((response) => { - // Un-nest properties by a level for caller convenience - Object.keys(response).forEach((key) => { - response[key].properties = response[key].mappings.properties; - }); - return response; - }) - .catch((err: any) => { - console.error(err); - return null; - }); -}; - -export const fetchIntegrationMappings = async ( - targetName: string, - http: HttpSetup -): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { - return http - .get(`/api/integrations/repository/${targetName}/schema`) - .then((response) => { - if (response.statusCode && response.statusCode !== 200) { - throw new Error('Failed to retrieve Integration schema', { cause: response }); - } - return response.data.mappings; - }) - .catch((err: any) => { - console.error(err); - return null; - }); -}; - -export const doExistingDataSourceValidation = async ( - targetDataSource: string, - integrationName: string, - integrationType: string, - http: HttpSetup -): Promise => { - const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); - if (!dataSourceNameCheck.ok) { - return dataSourceNameCheck; - } - const [dataSourceMappings, integrationMappings] = await Promise.all([ - fetchDataSourceMappings(targetDataSource, http), - fetchIntegrationMappings(integrationName, http), - ]); - if (!dataSourceMappings) { - return { ok: false, errors: ['Provided data stream could not be retrieved'] }; - } - if (!integrationMappings) { - return { ok: false, errors: ['Failed to retrieve integration schema information'] }; - } - const validationResult = Object.values(dataSourceMappings).every( - (value) => doPropertyValidation(integrationType, value.properties, integrationMappings).ok - ); - if (!validationResult) { - return { ok: false, errors: ['The provided index does not match the schema'] }; - } - return { ok: true }; -}; - export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { const { onClose, onCreate, integrationName, integrationType, http } = props; @@ -252,8 +85,7 @@ export function AddIntegrationFlyout(props: IntegrationFlyoutProps) { const validationResult = await doExistingDataSourceValidation( dataSource, integrationName, - integrationType, - http + integrationType ); if (validationResult.ok) { setToast('Index name or wildcard pattern is valid', 'success'); diff --git a/public/components/integrations/components/create_integration_helpers.ts b/public/components/integrations/components/create_integration_helpers.ts new file mode 100644 index 0000000000..fe822ca64e --- /dev/null +++ b/public/components/integrations/components/create_integration_helpers.ts @@ -0,0 +1,344 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { Color, VALID_INDEX_NAME } from '../../../../common/constants/integrations'; +import { HttpSetup } from '../../../../../../src/core/public'; +import { coreRefs } from '../../../framework/core_refs'; +import { CONSOLE_PROXY, INTEGRATIONS_BASE } from '../../../../common/constants/shared'; + +type ValidationResult = { ok: true } | { ok: false; errors: string[] }; + +export interface IntegrationTemplate { + name: string; + type: string; +} + +export const doTypeValidation = ( + toCheck: { type?: string; properties?: object }, + required: { type?: string; properties?: object } +): ValidationResult => { + if (!required.type) { + return { ok: true }; + } + if (required.type === 'object') { + if (Boolean(toCheck.properties)) { + return { ok: true }; + } + return { ok: false, errors: ["'object' type must have properties."] }; + } + if (required.type !== toCheck.type) { + return { ok: false, errors: [`Type mismatch: '${required.type}' and '${toCheck.type}'`] }; + } + return { ok: true }; +}; + +export const doNestedPropertyValidation = ( + toCheck: { type?: string; properties?: { [key: string]: object } }, + required: { type?: string; properties?: { [key: string]: object } } +): ValidationResult => { + const typeCheck = doTypeValidation(toCheck, required); + if (!typeCheck.ok) { + return typeCheck; + } + for (const property of Object.keys(required.properties ?? {})) { + if (!Object.hasOwn(toCheck.properties ?? {}, property)) { + return { ok: false, errors: [`Missing field '${property}'`] }; + } + // Both are safely non-null after above checks. + const nested = doNestedPropertyValidation( + toCheck.properties![property], + required.properties![property] + ); + if (!nested.ok) { + return nested; + } + } + return { ok: true }; +}; + +export const doPropertyValidation = ( + rootType: string, + dataSourceProps: { [key: string]: { properties?: any } }, + requiredMappings: { [key: string]: { template: { mappings: { properties?: any } } } } +): ValidationResult => { + // Check root object type (without dependencies) + if (!Object.hasOwn(requiredMappings, rootType)) { + // This is a configuration error for the integration. + return { ok: false, errors: ['Required mapping for integration has no root type.'] }; + } + for (const [key, value] of Object.entries( + requiredMappings[rootType].template.mappings.properties + )) { + if ( + !dataSourceProps[key] || + !doNestedPropertyValidation(dataSourceProps[key], value as any).ok + ) { + return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; + } + } + // Check nested dependencies + for (const [key, value] of Object.entries(requiredMappings)) { + if (key === rootType) { + continue; + } + if ( + !dataSourceProps[key] || + !doNestedPropertyValidation(dataSourceProps[key], value.template.mappings.properties).ok + ) { + return { ok: false, errors: [`Data source is invalid at key '${key}'`] }; + } + } + return { ok: true }; +}; + +// Returns true if the data stream is a legal name. +// Appends any additional validation errors to the provided errors array. +export const checkDataSourceName = ( + targetDataSource: string, + integrationType: string +): ValidationResult => { + let errors: string[] = []; + if (!VALID_INDEX_NAME.test(targetDataSource)) { + errors = errors.concat('This is not a valid index name.'); + return { ok: false, errors }; + } + const nameValidity: boolean = new RegExp(`^ss4?o_${integrationType}-[^\\-]+-.+`).test( + targetDataSource + ); + if (!nameValidity) { + errors = errors.concat('This index does not match the suggested naming convention.'); + return { ok: false, errors }; + } + return { ok: true }; +}; + +export const fetchDataSourceMappings = async ( + targetDataSource: string, + http: HttpSetup +): Promise<{ [key: string]: { properties: any } } | null> => { + return http + .post(CONSOLE_PROXY, { + query: { + path: `${targetDataSource}/_mapping`, + method: 'GET', + }, + }) + .then((response) => { + // Un-nest properties by a level for caller convenience + Object.keys(response).forEach((key) => { + response[key].properties = response[key].mappings.properties; + }); + return response; + }) + .catch((err: any) => { + console.error(err); + return null; + }); +}; + +export const fetchIntegrationMappings = async ( + targetName: string, + http: HttpSetup +): Promise<{ [key: string]: { template: { mappings: { properties?: any } } } } | null> => { + return http + .get(`/api/integrations/repository/${targetName}/schema`) + .then((response) => { + if (response.statusCode && response.statusCode !== 200) { + throw new Error('Failed to retrieve Integration schema', { cause: response }); + } + return response.data.mappings; + }) + .catch((err: any) => { + console.error(err); + return null; + }); +}; + +export const doExistingDataSourceValidation = async ( + targetDataSource: string, + integrationName: string, + integrationType: string +): Promise => { + const http = coreRefs.http!; + const dataSourceNameCheck = checkDataSourceName(targetDataSource, integrationType); + if (!dataSourceNameCheck.ok) { + return dataSourceNameCheck; + } + const [dataSourceMappings, integrationMappings] = await Promise.all([ + fetchDataSourceMappings(targetDataSource, http), + fetchIntegrationMappings(integrationName, http), + ]); + if (!dataSourceMappings) { + return { ok: false, errors: ['Provided data stream could not be retrieved'] }; + } + if (!integrationMappings) { + return { ok: false, errors: ['Failed to retrieve integration schema information'] }; + } + const validationResult = Object.values(dataSourceMappings).every( + (value) => doPropertyValidation(integrationType, value.properties, integrationMappings).ok + ); + if (!validationResult) { + return { ok: false, errors: ['The provided index does not match the schema'] }; + } + return { ok: true }; +}; + +const createComponentMapping = async ( + componentName: string, + payload: { + template: { mappings: { _meta: { version: string } } }; + composed_of: string[]; + index_patterns: string[]; + } +): Promise<{ [key: string]: { properties: any } } | null> => { + const http = coreRefs.http!; + const version = payload.template.mappings._meta.version; + return http.post(CONSOLE_PROXY, { + body: JSON.stringify(payload), + query: { + path: `_component_template/ss4o_${componentName}-${version}-template`, + method: 'POST', + }, + }); +}; + +const createIndexMapping = async ( + componentName: string, + payload: { + template: { mappings: { _meta: { version: string } } }; + composed_of: string[]; + index_patterns: string[]; + }, + dataSourceName: string, + integration: IntegrationTemplate +): Promise<{ [key: string]: { properties: any } } | null> => { + const http = coreRefs.http!; + const version = payload.template.mappings._meta.version; + payload.index_patterns = [dataSourceName]; + return http.post(CONSOLE_PROXY, { + body: JSON.stringify(payload), + query: { + path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, + method: 'POST', + }, + }); +}; + +const createDataSourceMappings = async ( + targetDataSource: string, + integrationTemplateId: string, + integration: IntegrationTemplate, + setToast: (title: string, color?: Color, text?: string | undefined) => void +): Promise => { + const http = coreRefs.http!; + const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); + let error: string | null = null; + const mappings = data.data.mappings; + mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( + (componentName: string) => { + const version = mappings[componentName].template.mappings._meta.version; + return `ss4o_${componentName}-${version}-template`; + } + ); + + try { + // Create component mappings before the index mapping + // The assumption is that index mapping relies on component mappings for creation + await Promise.all( + Object.entries(mappings).map(([key, mapping]) => { + if (key === integration.type) { + return Promise.resolve(); + } + return createComponentMapping(key, mapping as any); + }) + ); + // In order to see our changes, we need to manually provoke a refresh + await http.post(CONSOLE_PROXY, { + query: { + path: '_refresh', + method: 'GET', + }, + }); + await createIndexMapping( + integration.type, + mappings[integration.type], + targetDataSource, + integration + ); + } catch (err: any) { + error = err.message; + } + + if (error !== null) { + setToast('Failure creating index template', 'danger', error); + } else { + setToast(`Successfully created index template`); + } +}; + +export async function addIntegrationRequest( + addSample: boolean, + templateName: string, + integrationTemplateId: string, + integration: IntegrationTemplate, + setToast: (title: string, color?: Color, text?: string | undefined) => void, + name?: string, + dataSource?: string +) { + const http = coreRefs.http!; + if (addSample) { + createDataSourceMappings( + `ss4o_${integration.type}-${integrationTemplateId}-*-sample`, + integrationTemplateId, + integration, + setToast + ); + name = `${integrationTemplateId}-sample`; + dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; + } + + const response: boolean = await http + .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { + body: JSON.stringify({ name, dataSource }), + }) + .then((_res) => { + setToast(`${name} integration successfully added!`, 'success'); + window.location.hash = `#/installed/${_res.data?.id}`; + return true; + }) + .catch((_err) => { + setToast( + 'Failed to load integration. Check Added Integrations table for more details', + 'danger' + ); + return false; + }); + if (!addSample || !response) { + return; + } + const data: { sampleData: unknown[] } = await http + .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) + .then((res) => res.data) + .catch((err) => { + console.error(err); + setToast('The sample data could not be retrieved', 'danger'); + return { sampleData: [] }; + }); + const requestBody = + data.sampleData + .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) + .join('\n') + '\n'; + http + .post(CONSOLE_PROXY, { + body: requestBody, + query: { + path: `${dataSource}/_bulk?refresh=wait_for`, + method: 'POST', + }, + }) + .catch((err) => { + console.error(err); + setToast('Failed to load sample data', 'danger'); + }); +} diff --git a/public/components/integrations/components/integration.tsx b/public/components/integrations/components/integration.tsx index b7fba8c3ca..c005e98d0b 100644 --- a/public/components/integrations/components/integration.tsx +++ b/public/components/integrations/components/integration.tsx @@ -21,98 +21,21 @@ import { IntegrationAssets } from './integration_assets_panel'; import { AvailableIntegrationProps } from './integration_types'; import { INTEGRATIONS_BASE } from '../../../../common/constants/shared'; import { IntegrationScreenshots } from './integration_screenshots_panel'; -import { AddIntegrationFlyout } from './add_integration_flyout'; import { useToast } from '../../../../public/components/common/toast'; +import { coreRefs } from '../../../framework/core_refs'; +import { IntegrationTemplate, addIntegrationRequest } from './create_integration_helpers'; export function Integration(props: AvailableIntegrationProps) { - const { http, integrationTemplateId, chrome } = props; + const http = coreRefs.http!; + const { integrationTemplateId, chrome } = props; const { setToast } = useToast(); - const [integration, setIntegration] = useState({} as { name: string; type: string }); + const [integration, setIntegration] = useState({} as IntegrationTemplate); const [integrationMapping, setMapping] = useState(null); const [integrationAssets, setAssets] = useState([]); const [loading, setLoading] = useState(false); - const createComponentMapping = async ( - componentName: string, - payload: { - template: { mappings: { _meta: { version: string } } }; - composed_of: string[]; - index_patterns: string[]; - } - ): Promise<{ [key: string]: { properties: any } } | null> => { - const version = payload.template.mappings._meta.version; - return http.post('/api/console/proxy', { - body: JSON.stringify(payload), - query: { - path: `_component_template/ss4o_${componentName}-${version}-template`, - method: 'POST', - }, - }); - }; - - const createIndexMapping = async ( - componentName: string, - payload: { - template: { mappings: { _meta: { version: string } } }; - composed_of: string[]; - index_patterns: string[]; - }, - dataSourceName: string - ): Promise<{ [key: string]: { properties: any } } | null> => { - const version = payload.template.mappings._meta.version; - payload.index_patterns = [dataSourceName]; - return http.post('/api/console/proxy', { - body: JSON.stringify(payload), - query: { - path: `_index_template/ss4o_${componentName}-${integration.name}-${version}-sample`, - method: 'POST', - }, - }); - }; - - const createDataSourceMappings = async (targetDataSource: string): Promise => { - const data = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}/schema`); - let error: string | null = null; - const mappings = data.data.mappings; - mappings[integration.type].composed_of = mappings[integration.type].composed_of.map( - (componentName: string) => { - const version = mappings[componentName].template.mappings._meta.version; - return `ss4o_${componentName}-${version}-template`; - } - ); - - try { - // Create component mappings before the index mapping - // The assumption is that index mapping relies on component mappings for creation - await Promise.all( - Object.entries(mappings).map(([key, mapping]) => { - if (key === integration.type) { - return Promise.resolve(); - } - return createComponentMapping(key, mapping as any); - }) - ); - // In order to see our changes, we need to manually provoke a refresh - await http.post('/api/console/proxy', { - query: { - path: '_refresh', - method: 'GET', - }, - }); - await createIndexMapping(integration.type, mappings[integration.type], targetDataSource); - } catch (err: any) { - error = err.message; - } - - if (error !== null) { - setToast('Failure creating index template', 'danger', error); - } else { - setToast(`Successfully created index template`); - } - }; - useEffect(() => { chrome.setBreadcrumbs([ { @@ -170,68 +93,6 @@ export function Integration(props: AvailableIntegrationProps) { }); }, [integration]); - async function addIntegrationRequest( - addSample: boolean, - templateName: string, - name?: string, - dataSource?: string - ) { - setLoading(true); - if (addSample) { - createDataSourceMappings(`ss4o_${integration.type}-${integrationTemplateId}-*-sample`); - name = `${integrationTemplateId}-sample`; - dataSource = `ss4o_${integration.type}-${integrationTemplateId}-sample-sample`; - } - - const response: boolean = await http - .post(`${INTEGRATIONS_BASE}/store/${templateName}`, { - body: JSON.stringify({ name, dataSource }), - }) - .then((_res) => { - setToast(`${name} integration successfully added!`, 'success'); - window.location.hash = `#/installed/${_res.data?.id}`; - return true; - }) - .catch((_err) => { - setToast( - 'Failed to load integration. Check Added Integrations table for more details', - 'danger' - ); - return false; - }); - if (!addSample || !response) { - setLoading(false); - return; - } - const data: { sampleData: unknown[] } = await http - .get(`${INTEGRATIONS_BASE}/repository/${templateName}/data`) - .then((res) => res.data) - .catch((err) => { - console.error(err); - setToast('The sample data could not be retrieved', 'danger'); - return { sampleData: [] }; - }); - const requestBody = - data.sampleData - .map((record) => `{"create": { "_index": "${dataSource}" } }\n${JSON.stringify(record)}`) - .join('\n') + '\n'; - http - .post('/api/console/proxy', { - body: requestBody, - query: { - path: `${dataSource}/_bulk?refresh=wait_for`, - method: 'POST', - }, - }) - .catch((err) => { - console.error(err); - setToast('Failed to load sample data', 'danger'); - }) - .finally(() => { - setLoading(false); - }); - } - const tabs = [ { id: 'assets', @@ -281,8 +142,16 @@ export function Integration(props: AvailableIntegrationProps) { showFlyout: () => { window.location.hash = `#/available/${integration.name}/setup`; }, - setUpSample: () => { - addIntegrationRequest(true, integrationTemplateId); + setUpSample: async () => { + setLoading(true); + await addIntegrationRequest( + true, + integration.name, + integrationTemplateId, + integration, + setToast + ); + setLoading(false); }, loading, })} diff --git a/public/components/integrations/components/setup_integration.tsx b/public/components/integrations/components/setup_integration.tsx index 9e3052abfb..9fd7101575 100644 --- a/public/components/integrations/components/setup_integration.tsx +++ b/public/components/integrations/components/setup_integration.tsx @@ -3,313 +3,215 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as Eui from '@elastic/eui'; -import { EuiContainedStepProps } from '@opensearch-project/oui/src/components/steps/steps'; -import React, { useState } from 'react'; +import { + EuiBottomBar, + EuiButton, + EuiComboBox, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiPage, + EuiPageBody, + EuiPageContent, + EuiPageContentBody, + EuiSelect, + EuiSpacer, + EuiTitle, +} from '@elastic/eui'; +import React, { useState, useEffect } from 'react'; +import { coreRefs } from '../../../framework/core_refs'; +import { addIntegrationRequest } from './create_integration_helpers'; +import { useToast } from '../../../../public/components/common/toast'; +import { CONSOLE_PROXY, INTEGRATIONS_BASE } from '../../../../common/constants/shared'; +import { DATACONNECTIONS_BASE } from '../../../../common/constants/shared'; -interface IntegrationConfig { - instanceName: string; - useExisting: boolean; - dataSourceName: string; - dataSourceDescription: string; - dataSourceFileType: string; - dataSourceLocation: string; - existingDataSourceName: string; +export interface IntegrationSetupInputs { + displayName: string; + connectionType: string; + connectionDataSource: string; } -const STEPS: EuiContainedStepProps[] = [ - { title: 'Name Integration', children: }, - { title: 'Select index or data source for integration', children: }, -]; - -const ALLOWED_FILE_TYPES: Eui.EuiSelectOption[] = [ - { value: 'parquet', text: 'parquet' }, - { value: 'json', text: 'json' }, -]; +interface IntegrationConfigProps { + config: IntegrationSetupInputs; + updateConfig: (updates: Partial) => void; + integration: { + name: string; + type: string; + }; +} -const INTEGRATION_DATA_TABLE_COLUMNS = [ - { - field: 'field', - name: 'Field Name', - }, - { - field: 'type', - name: 'Field Type', - }, +// TODO support localization +const INTEGRATION_CONNECTION_DATA_SOURCE_TYPES: Map< + string, { - field: 'isTimestamp', - name: 'Timestamp', - }, -]; + title: string; + lower: string; + help: string; + } +> = new Map([ + [ + 's3', + { + title: 'Table', + lower: 'table', + help: 'Select a table to pull the data from.', + }, + ], + [ + 'index', + { + title: 'Index', + lower: 'index', + help: 'Select an index to pull the data from.', + }, + ], +]); -const integrationDataTableData = [ - { - field: 'spanId', - type: 'string', - isTimestamp: false, - }, +const integrationConnectionSelectorItems = [ { - field: 'severity.number', - type: 'long', - isTimestamp: false, + value: 's3', + text: 'S3 Connection', }, { - field: '@timestamp', - type: 'date', - isTimestamp: true, + value: 'index', + text: 'OpenSearch Index', }, ]; -const getSetupStepStatus = (activeStep: number): EuiContainedStepProps[] => { - return STEPS.map((step, idx) => { - let status: string = ''; - if (idx < activeStep) { - status = 'complete'; - } - if (idx > activeStep) { - status = 'disabled'; +const suggestDataSources = async (type: string): Promise> => { + const http = coreRefs.http!; + try { + if (type === 'index') { + const result = await http.post(CONSOLE_PROXY, { + body: '{}', + query: { + path: '_data_stream/ss4o_*', + method: 'GET', + }, + }); + return ( + result.data_streams?.map((item: { name: string }) => { + return { label: item.name }; + }) ?? [] + ); + } else if (type === 's3') { + const result = (await http.get(DATACONNECTIONS_BASE)) as Array<{ + name: string; + connector: string; + }>; + return ( + result + ?.filter((item) => item.connector === 'S3GLUE') + .map((item) => { + return { label: item.name }; + }) ?? [] + ); + } else { + console.error(`Unknown connection type: ${type}`); + return []; } - return Object.assign({}, step, { status }); - }); + } catch (err: any) { + console.error(err.message); + return []; + } }; -export function SetupIntegrationMetadata({ - name, - setName, -}: { - name: string; - setName: (name: string) => void; -}) { - return ( - - -

{STEPS[0].title}

- - - setName(evt.target.value)} /> - - - ); -} - -export function IntegrationDataModal({ close }: { close: () => void }): React.JSX.Element { - return ( - - -

Data Table

- - - - - - Close - - - - ); -} +const findTemplate = async (integrationTemplateId: string) => { + const http = coreRefs.http!; + const result = await http.get(`${INTEGRATIONS_BASE}/repository/${integrationTemplateId}`); + return result; +}; -export function SetupIntegrationNewTable({ +export function SetupIntegrationForm({ config, updateConfig, -}: { - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; -}) { - return ( -
- - updateConfig({ dataSourceName: evt.target.value })} - /> - - - updateConfig({ dataSourceDescription: evt.target.value })} - /> - - - updateConfig({ dataSourceFileType: evt.target.value })} - /> - - - updateConfig({ dataSourceLocation: evt.target.value })} - /> - -
- ); -} + integration, +}: IntegrationConfigProps) { + const connectionType = INTEGRATION_CONNECTION_DATA_SOURCE_TYPES.get(config.connectionType)!; -export function SetupIntegrationExistingTable({ - config, - updateConfig, - showDataModal, - setShowDataModal, -}: { - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; - showDataModal: boolean; - setShowDataModal: (visible: boolean) => void; -}) { - const dataModal = showDataModal ? ( - setShowDataModal(false)} /> - ) : null; - return ( -
- - updateConfig({ existingDataSourceName: evt.target.value })} - /> - - - setShowDataModal(true)}>View table - {dataModal} -
+ const [dataSourceSuggestions, setDataSourceSuggestions] = useState( + [] as Array<{ label: string }> ); -} + const [isSuggestionsLoading, setIsSuggestionsLoading] = useState(true); + useEffect(() => { + const updateDataSources = async () => { + const data = await suggestDataSources(config.connectionType); + setDataSourceSuggestions(data); + setIsSuggestionsLoading(false); + }; -export function SetupIntegrationDataSource({ - config, - updateConfig, - showDataModal, - setShowDataModal, - tableDetected, - setTableDetected, -}: { - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; - showDataModal: boolean; - setShowDataModal: (show: boolean) => void; - tableDetected: boolean; - setTableDetected: (detected: boolean) => void; -}) { - let tableForm; - if (tableDetected && config.useExisting) { - tableForm = ( - setShowDataModal(x)} - /> - ); - } else { - tableForm = ; - } - - let tablesNotFoundMessage = null; - if (!tableDetected) { - tablesNotFoundMessage = ( - <> - -

No problem, we can help. Tell us about your data.

- - - - ); - } + setIsSuggestionsLoading(true); + updateDataSources(); + }, [config.connectionType]); return ( -
- setTableDetected(event.target.checked)} - /> - - - -

{STEPS[1].title}

- - - {tablesNotFoundMessage} - updateConfig({ useExisting: evt.target.checked })} - disabled={!tableDetected} + + +

Set Up Integration

+ + + +

Integration Details

+ + + + updateConfig({ displayName: event.target.value })} /> - - {tableForm} - -
- ); -} - -export function SetupIntegrationStep({ - activeStep, - config, - updateConfig, -}: { - activeStep: number; - config: IntegrationConfig; - updateConfig: (updates: Partial) => void; -}) { - const [isDataModalVisible, setDataModalVisible] = useState(false); - const [tableDetected, setTableDetected] = useState(false); - - switch (activeStep) { - case 0: - return ( - updateConfig({ instanceName: name })} + + + +

Integration Connection

+ + + + + updateConfig({ connectionType: event.target.value, connectionDataSource: '' }) + } /> - ); - case 1: - return ( - setDataModalVisible(show)} - tableDetected={tableDetected} - setTableDetected={(detected: boolean) => setTableDetected(detected)} + + + { + if (selected.length === 0) { + updateConfig({ connectionDataSource: '' }); + } else { + updateConfig({ connectionDataSource: selected[0].label }); + } + }} + selectedOptions={[{ label: config.connectionDataSource }]} + singleSelection={{ asPlainText: true }} /> - ); - default: - return ( - - Attempted to access integration setup step that doesn't exist. This is a bug. - - ); - } + + + ); } export function SetupBottomBar({ - step, - setStep, config, + integration, }: { - step: number; - setStep: (step: number) => void; - config: IntegrationConfig; + config: IntegrationSetupInputs; + integration: { name: string; type: string }; }) { + const { setToast } = useToast(); + const [loading, setLoading] = useState(false); + return ( - - - - + + + { // TODO evil hack because props aren't set up let hash = window.location.hash; @@ -318,75 +220,69 @@ export function SetupBottomBar({ window.location.hash = hash; }} > - Cancel - - - - - - {step > 0 ? ( - - setStep(step - 1)}> - Back - - - ) : null} - - + + + { - if (step < STEPS.length - 1) { - setStep(step + 1); - } else { - console.log(config); - } + iconType="arrowRight" + iconSide="right" + isLoading={loading} + onClick={async () => { + setLoading(true); + const template = await findTemplate(integration.name); + await addIntegrationRequest( + false, + integration.name, + config.displayName, + template, + setToast, + config.displayName, + config.connectionDataSource + ); + setLoading(false); }} > - {step === STEPS.length - 1 ? 'Save' : 'Next'} - - - - + Add Integration + + + + ); } -export function SetupIntegrationStepsPage() { +export function SetupIntegrationPage({ + integration, +}: { + integration: { + name: string; + type: string; + }; +}) { const [integConfig, setConfig] = useState({ - instanceName: '', - useExisting: true, - dataSourceName: '', - dataSourceDescription: '', - dataSourceFileType: 'parquet', - dataSourceLocation: '', - existingDataSourceName: '', - } as IntegrationConfig); - const [step, setStep] = useState(0); + displayName: `${integration.name} Integration`, + connectionType: 'index', + connectionDataSource: '', + } as IntegrationSetupInputs); - const updateConfig = (updates: Partial) => + const updateConfig = (updates: Partial) => setConfig(Object.assign({}, integConfig, updates)); return ( - - - - - - - - + + + + - - - setStep(Math.min(Math.max(x, 0), STEPS.length - 1))} - config={integConfig} - /> - - + + + + + ); } diff --git a/public/components/integrations/home.tsx b/public/components/integrations/home.tsx index 67ec7e74f1..7a6b3db5df 100644 --- a/public/components/integrations/home.tsx +++ b/public/components/integrations/home.tsx @@ -3,17 +3,15 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useState } from 'react'; +import React from 'react'; import { HashRouter, Route, RouteComponentProps, Switch } from 'react-router-dom'; -import { EuiGlobalToastList } from '@elastic/eui'; -import { Toast } from '@elastic/eui/src/components/toast/global_toast_list'; -import { Integration } from './components/integration'; import { TraceAnalyticsCoreDeps } from '../trace_analytics/home'; import { ChromeBreadcrumb } from '../../../../../src/core/public'; import { AvailableIntegrationOverviewPage } from './components/available_integration_overview_page'; import { AddedIntegrationOverviewPage } from './components/added_integration_overview_page'; import { AddedIntegration } from './components/added_integration'; -import { SetupIntegrationStepsPage } from './components/setup_integration'; +import { SetupIntegrationPage } from './components/setup_integration'; +import { Integration } from './components/integration'; export type AppAnalyticsCoreDeps = TraceAnalyticsCoreDeps; @@ -63,7 +61,18 @@ export const Home = (props: HomeProps) => { /> )} /> - } /> + ( + + )} + />
diff --git a/server/adaptors/integrations/repository/__test__/integration.test.ts b/server/adaptors/integrations/repository/__test__/integration.test.ts index 7ffbb176bf..ec77acac1a 100644 --- a/server/adaptors/integrations/repository/__test__/integration.test.ts +++ b/server/adaptors/integrations/repository/__test__/integration.test.ts @@ -7,30 +7,12 @@ import * as fs from 'fs/promises'; import { IntegrationReader } from '../integration'; import { Dirent, Stats } from 'fs'; import * as path from 'path'; -import { FileSystemCatalogDataAdaptor } from '../fs_data_adaptor'; +import { TEST_INTEGRATION_CONFIG } from '../../../../../test/constants'; jest.mock('fs/promises'); describe('Integration', () => { let integration: IntegrationReader; - const sampleIntegration: IntegrationConfig = { - name: 'sample', - version: '2.0.0', - license: 'Apache-2.0', - type: 'logs', - components: [ - { - name: 'logs', - version: '1.0.0', - }, - ], - assets: { - savedObjects: { - name: 'sample', - version: '1.0.1', - }, - }, - }; beforeEach(() => { integration = new IntegrationReader('./sample'); @@ -79,19 +61,19 @@ describe('Integration', () => { }); it('should return the parsed config template if it is valid', async () => { - jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(sampleIntegration)); + jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(TEST_INTEGRATION_CONFIG)); jest.spyOn(fs, 'lstat').mockResolvedValueOnce({ isDirectory: () => true } as Stats); - const result = await integration.getConfig(sampleIntegration.version); + const result = await integration.getConfig(TEST_INTEGRATION_CONFIG.version); - expect(result).toEqual({ ok: true, value: sampleIntegration }); + expect(result).toEqual({ ok: true, value: TEST_INTEGRATION_CONFIG }); }); it('should return an error if the config template is invalid', async () => { - const invalidTemplate = { ...sampleIntegration, version: 2 }; + const invalidTemplate = { ...TEST_INTEGRATION_CONFIG, version: 2 }; jest.spyOn(fs, 'readFile').mockResolvedValue(JSON.stringify(invalidTemplate)); - const result = await integration.getConfig(sampleIntegration.version); + const result = await integration.getConfig(TEST_INTEGRATION_CONFIG.version); expect(result.ok).toBe(false); }); @@ -99,7 +81,7 @@ describe('Integration', () => { it('should return an error if the config file has syntax errors', async () => { jest.spyOn(fs, 'readFile').mockResolvedValue('Invalid JSON'); - const result = await integration.getConfig(sampleIntegration.version); + const result = await integration.getConfig(TEST_INTEGRATION_CONFIG.version); expect(result.ok).toBe(false); }); @@ -114,7 +96,7 @@ describe('Integration', () => { return Promise.reject(error); }); - const result = await integration.getConfig(sampleIntegration.version); + const result = await integration.getConfig(TEST_INTEGRATION_CONFIG.version); expect(readFileMock).toHaveBeenCalled(); expect(result.ok).toBe(false); @@ -123,10 +105,12 @@ describe('Integration', () => { describe('getAssets', () => { it('should return linked saved object assets when available', async () => { - integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleIntegration }); + integration.getConfig = jest + .fn() + .mockResolvedValue({ ok: true, value: TEST_INTEGRATION_CONFIG }); jest.spyOn(fs, 'readFile').mockResolvedValue('{"name":"asset1"}\n{"name":"asset2"}'); - const result = await integration.getAssets(sampleIntegration.version); + const result = await integration.getAssets(TEST_INTEGRATION_CONFIG.version); expect(result.ok).toBe(true); expect((result as any).value.savedObjects).toStrictEqual([ @@ -142,10 +126,12 @@ describe('Integration', () => { }); it('should return an error if the saved object assets are invalid', async () => { - integration.getConfig = jest.fn().mockResolvedValue({ ok: true, value: sampleIntegration }); + integration.getConfig = jest + .fn() + .mockResolvedValue({ ok: true, value: TEST_INTEGRATION_CONFIG }); jest.spyOn(fs, 'readFile').mockResolvedValue('{"unclosed":'); - const result = await integration.getAssets(sampleIntegration.version); + const result = await integration.getAssets(TEST_INTEGRATION_CONFIG.version); expect(result.ok).toBe(false); }); diff --git a/test/constants.ts b/test/constants.ts index f332a905be..dcbb6cdb3c 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { IntegrationSetupInputs } from 'public/components/integrations/components/setup_integration'; + export const TEST_SPAN_RESPONSE = { took: 1, timed_out: false, @@ -558,6 +560,32 @@ export const TEST_SERVICE_MAP = { }, }; +export const TEST_INTEGRATION_SETUP_INPUTS: IntegrationSetupInputs = { + displayName: 'Test Instance Name', + connectionType: 'index', + connectionDataSource: 'ss4o_logs-nginx-test', +}; + +// TODO fill in the rest of the fields +export const TEST_INTEGRATION_CONFIG: IntegrationConfig = { + name: 'sample', + version: '2.0.0', + license: 'Apache-2.0', + type: 'logs', + components: [ + { + name: 'logs', + version: '1.0.0', + }, + ], + assets: { + savedObjects: { + name: 'sample', + version: '1.0.1', + }, + }, +}; + export const mockSavedObjectActions = ({ get = [], getBulk = [] }) => { return { get: jest.fn().mockResolvedValue({ observabilityObjectList: get }),