diff --git a/.cypress/integration/ad/dashboard/ad_dashboard.spec.ts b/.cypress/integration/ad/dashboard/ad_dashboard.spec.ts index 6161e947..b9f3a8f9 100644 --- a/.cypress/integration/ad/dashboard/ad_dashboard.spec.ts +++ b/.cypress/integration/ad/dashboard/ad_dashboard.spec.ts @@ -49,7 +49,7 @@ context('AD Dashboard', () => { }); cy.mockSearchIndexOnAction('search_index_response.json', () => { - cy.get('a[data-test-subj="add_detector"]').click({ + cy.get('a[data-test-subj="createDetectorButton"]').click({ force: true, }); }); diff --git a/public/components/ContentPanel/ContentPanel.tsx b/public/components/ContentPanel/ContentPanel.tsx index f2764265..10c9ad9a 100644 --- a/public/components/ContentPanel/ContentPanel.tsx +++ b/public/components/ContentPanel/ContentPanel.tsx @@ -31,6 +31,7 @@ type ContentPanelProps = { title: string | React.ReactNode | React.ReactNode[]; titleSize?: EuiTitleSize; subTitle?: React.ReactNode | React.ReactNode[]; + badgeLabel?: string; bodyStyles?: React.CSSProperties; panelStyles?: React.CSSProperties; horizontalRuleClassName?: string; @@ -45,6 +46,7 @@ const ContentPanel = (props: ContentPanelProps) => ( { + return ( + + + + Try a sample detector + + + + + Create detector + + + + ); +}; diff --git a/public/pages/Dashboard/Components/EmptyDashboard/EmptyDashboard.tsx b/public/pages/Dashboard/Components/EmptyDashboard/EmptyDashboard.tsx index a95085ba..0b9ee175 100644 --- a/public/pages/Dashboard/Components/EmptyDashboard/EmptyDashboard.tsx +++ b/public/pages/Dashboard/Components/EmptyDashboard/EmptyDashboard.tsx @@ -13,9 +13,9 @@ * permissions and limitations under the License. */ -import { EuiButton, EuiEmptyPrompt, EuiLink, EuiIcon } from '@elastic/eui'; +import { EuiEmptyPrompt, EuiLink, EuiIcon } from '@elastic/eui'; import React, { Component, Fragment } from 'react'; -import { APP_PATH, PLUGIN_NAME } from '../../../../utils/constants'; +import { CreateDetectorButtons } from '../../../../components/CreateDetectorButtons/CreateDetectorButtons'; export class EmptyDashboard extends Component<{}, {}> { render() { @@ -27,7 +27,7 @@ export class EmptyDashboard extends Component<{}, {}> {

Create detector first to detect anomalies in your data.

Dashboard will generate insights on the anomalies across all of - your detectors + your detectors.

Read about{' '} @@ -41,15 +41,7 @@ export class EmptyDashboard extends Component<{}, {}> {

} - actions={ - - Create detector - - } + actions={} /> ); } diff --git a/public/pages/Dashboard/Components/EmptyDashboard/__tests__/__snapshots__/EmptyDashboard.test.tsx.snap b/public/pages/Dashboard/Components/EmptyDashboard/__tests__/__snapshots__/EmptyDashboard.test.tsx.snap index f22dd09b..0a58dcf2 100644 --- a/public/pages/Dashboard/Components/EmptyDashboard/__tests__/__snapshots__/EmptyDashboard.test.tsx.snap +++ b/public/pages/Dashboard/Components/EmptyDashboard/__tests__/__snapshots__/EmptyDashboard.test.tsx.snap @@ -22,7 +22,7 @@ exports[` spec Empty results renders component with empty messa Create detector first to detect anomalies in your data.

- Dashboard will generate insights on the anomalies across all of your detectors + Dashboard will generate insights on the anomalies across all of your detectors.

Read about @@ -55,21 +55,51 @@ exports[` spec Empty results renders component with empty messa

- - - - Create detector - - - + + + Try a sample detector + + + +
+
+ + + + Create detector + + + +
+ `; diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx index 47fd898e..7e42ebb9 100644 --- a/public/pages/DetectorResults/containers/AnomalyResults.tsx +++ b/public/pages/DetectorResults/containers/AnomalyResults.tsx @@ -63,6 +63,11 @@ import { FEATURE_DATA_CHECK_WINDOW_OFFSET, } from '../../utils/anomalyResultUtils'; import { getDetectorResults } from '../../../redux/reducers/anomalyResults'; +import { + detectorIsSample, + getAssociatedIndex, +} from '../../SampleData/utils/helpers'; +import { SampleIndexDetailsCallout } from '../../SampleData/components/SampleIndexDetailsCallout/SampleIndexDetailsCallout'; interface AnomalyResultsProps extends RouteComponentProps { detectorId: string; @@ -112,6 +117,15 @@ export function AnomalyResults(props: AnomalyResultsProps) { } }, [detector]); + useEffect(() => { + if (detector && detectorIsSample(detector)) { + setIsSampleDetector(true); + setSampleIndexName(getAssociatedIndex(detector)); + } else { + setIsSampleDetector(false); + } + }, [detector]); + const monitors = useSelector((state: AppState) => state.alerting.monitors); const monitor = get(monitors, `${detectorId}.0`); @@ -119,6 +133,10 @@ export function AnomalyResults(props: AnomalyResultsProps) { MISSING_FEATURE_DATA_SEVERITY >(); + const [isSampleDetector, setIsSampleDetector] = useState(false); + + const [sampleIndexName, setSampleIndexName] = useState(''); + const [featureNamesAtHighSev, setFeatureNamesAtHighSev] = useState( [] as string[] ); @@ -367,6 +385,15 @@ export function AnomalyResults(props: AnomalyResultsProps) { isDetectorInitializing || isDetectorFailed ? ( + {isSampleDetector ? ( + + {' '} + {' '} + {' '} + + ) : null} {isDetectorUpdated || isDetectorMissingData || isInitializingNormally || diff --git a/public/pages/DetectorsList/components/EmptyMessage/EmptyMessage.tsx b/public/pages/DetectorsList/components/EmptyMessage/EmptyMessage.tsx index 1438440f..4f95d65a 100644 --- a/public/pages/DetectorsList/components/EmptyMessage/EmptyMessage.tsx +++ b/public/pages/DetectorsList/components/EmptyMessage/EmptyMessage.tsx @@ -15,7 +15,7 @@ import { EuiButton, EuiEmptyPrompt, EuiText } from '@elastic/eui'; import React from 'react'; -import { APP_PATH, PLUGIN_NAME } from '../../../../utils/constants'; +import { CreateDetectorButtons } from '../../../../components/CreateDetectorButtons/CreateDetectorButtons'; const filterText = 'There are no detectors matching your applied filters. Reset your filters to view all detectors.'; @@ -45,13 +45,7 @@ export const EmptyDetectorMessage = (props: EmptyDetectorProps) => ( Reset filters ) : ( - - Create detector - + ) } /> diff --git a/public/pages/DetectorsList/components/EmptyMessage/__tests__/__snapshots__/EmptyMessage.test.tsx.snap b/public/pages/DetectorsList/components/EmptyMessage/__tests__/__snapshots__/EmptyMessage.test.tsx.snap index 9a2f9809..9223cf78 100644 --- a/public/pages/DetectorsList/components/EmptyMessage/__tests__/__snapshots__/EmptyMessage.test.tsx.snap +++ b/public/pages/DetectorsList/components/EmptyMessage/__tests__/__snapshots__/EmptyMessage.test.tsx.snap @@ -26,22 +26,52 @@ exports[` spec Empty results renders component with empt
- - - - Create detector - - - + + + Try a sample detector + + + +
+
+ + + + Create detector + + + +
+ `; diff --git a/public/pages/SampleData/components/SampleDataBox/SampleDataBox.tsx b/public/pages/SampleData/components/SampleDataBox/SampleDataBox.tsx new file mode 100644 index 00000000..13c7dc1f --- /dev/null +++ b/public/pages/SampleData/components/SampleDataBox/SampleDataBox.tsx @@ -0,0 +1,118 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import ContentPanel from '../../../../components/ContentPanel/ContentPanel'; +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiTitle, + EuiLink, +} from '@elastic/eui'; +import { PLUGIN_NAME } from '../../../../utils/constants'; + +interface SampleDataBoxProps { + title: string; + icon: any; + description: string; + loadDataButtonDescription: string; + onOpenFlyout(): void; + onLoadData(): void; + isLoadingData: boolean; + isDataLoaded: boolean; + detectorId: string; +} + +export const SampleDataBox = (props: SampleDataBoxProps) => { + return ( +
+ + {props.icon} + +

+ {props.title} +

+ + + Info + +
+ } + titleSize="s" + badgeLabel={props.isDataLoaded ? 'INSTALLED' : undefined} + > + + +

+ {props.description} +

+ + + + { + props.onLoadData(); + }} + > + {props.isLoadingData + ? 'Creating detector' + : props.isDataLoaded + ? 'Detector created' + : props.loadDataButtonDescription} + + + + {props.isDataLoaded ? ( + + View detector and sample data + + ) : null} + + + + + + ); +}; diff --git a/public/pages/SampleData/components/SampleDataBox/__tests__/SampleDataBox.test.tsx b/public/pages/SampleData/components/SampleDataBox/__tests__/SampleDataBox.test.tsx new file mode 100644 index 00000000..a265f507 --- /dev/null +++ b/public/pages/SampleData/components/SampleDataBox/__tests__/SampleDataBox.test.tsx @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import { EuiIcon } from '@elastic/eui'; +import { SampleDataBox } from '../SampleDataBox'; + +const defaultProps = { + title: 'Sample title', + icon: , + description: 'Sample description', + loadDataButtonDescription: 'Sample button description', + onLoadData: jest.fn(), + isLoadingData: false, + isDataLoaded: false, + detectorId: 'sample-detector-id', +}; + +describe(' spec', () => { + describe('Data not loaded', () => { + test('renders component', () => { + const { container, getByText } = render( + + ); + expect(container.firstChild).toMatchSnapshot(); + getByText('Sample title'); + getByText('Sample description'); + getByText('Sample button description'); + }); + }); + describe('Data is loading', () => { + test('renders component', () => { + const { container, getByText } = render( + + ); + expect(container.firstChild).toMatchSnapshot(); + getByText('Sample title'); + getByText('Sample description'); + getByText('Creating detector'); + }); + }); + describe('Data is loaded', () => { + test('renders component', () => { + const { container, getByText } = render( + + ); + expect(container.firstChild).toMatchSnapshot(); + getByText('Sample title'); + getByText('Sample description'); + getByText('Detector created'); + }); + }); +}); diff --git a/public/pages/SampleData/components/SampleDataBox/__tests__/__snapshots__/SampleDataBox.test.tsx.snap b/public/pages/SampleData/components/SampleDataBox/__tests__/__snapshots__/SampleDataBox.test.tsx.snap new file mode 100644 index 00000000..4ffe4943 --- /dev/null +++ b/public/pages/SampleData/components/SampleDataBox/__tests__/__snapshots__/SampleDataBox.test.tsx.snap @@ -0,0 +1,393 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[` spec Data is loaded renders component 1`] = ` +
+
+ + + INSTALLED + + +
+
+
+
+
+ +

+ Sample title +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Sample description +

+
+
+
+ +
+
+ + View detector and sample data + +
+
+
+
+
+
+`; + +exports[` spec Data is loading renders component 1`] = ` +
+
+
+
+
+
+
+ +

+ Sample title +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Sample description +

+
+
+
+ +
+
+
+
+
+
+
+`; + +exports[` spec Data not loaded renders component 1`] = ` +
+
+
+
+
+
+
+ +

+ Sample title +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Sample description +

+
+
+
+ +
+
+
+
+
+
+
+`; diff --git a/public/pages/SampleData/components/SampleDataCallout/SampleDataCallout.tsx b/public/pages/SampleData/components/SampleDataCallout/SampleDataCallout.tsx new file mode 100644 index 00000000..27b11d4d --- /dev/null +++ b/public/pages/SampleData/components/SampleDataCallout/SampleDataCallout.tsx @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { EuiCallOut, EuiLink } from '@elastic/eui'; +import { APP_PATH, PLUGIN_NAME } from '../../../../utils/constants'; + +export const SampleDataCallout = () => { + return ( + +

+ Read the{' '} + + documentation + {' '} + or create a{' '} + + sample detector + {' '} + to get started. +

+ + ); +}; diff --git a/public/pages/SampleData/components/SampleDataCallout/__tests__/SampleDataCallout.test.tsx b/public/pages/SampleData/components/SampleDataCallout/__tests__/SampleDataCallout.test.tsx new file mode 100644 index 00000000..d28ba781 --- /dev/null +++ b/public/pages/SampleData/components/SampleDataCallout/__tests__/SampleDataCallout.test.tsx @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its /* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { render } from '@testing-library/react'; +import { SampleDataCallout } from '../SampleDataCallout'; + +describe(' spec', () => { + describe('Data not loaded', () => { + test('renders component', () => { + const { container, getByText } = render(); + expect(container.firstChild).toMatchSnapshot(); + getByText('Looking to get more familiar with anomaly detection?'); + }); + }); +}); diff --git a/public/pages/SampleData/components/SampleDataCallout/__tests__/__snapshots__/SampleDataCallout.test.tsx.snap b/public/pages/SampleData/components/SampleDataCallout/__tests__/__snapshots__/SampleDataCallout.test.tsx.snap new file mode 100644 index 00000000..7b77c4cf --- /dev/null +++ b/public/pages/SampleData/components/SampleDataCallout/__tests__/__snapshots__/SampleDataCallout.test.tsx.snap @@ -0,0 +1,55 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[` spec Data not loaded renders component 1`] = ` +
+
+ + + Looking to get more familiar with anomaly detection? + +
+
+

+ Read the + + + documentation + + + or create a + + + sample detector + + + to get started. +

+
+
+`; diff --git a/public/pages/SampleData/components/SampleDetailsFlyout/SampleDetailsFlyout.tsx b/public/pages/SampleData/components/SampleDetailsFlyout/SampleDetailsFlyout.tsx new file mode 100644 index 00000000..e6a1cdf5 --- /dev/null +++ b/public/pages/SampleData/components/SampleDetailsFlyout/SampleDetailsFlyout.tsx @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { get } from 'lodash'; +import { + EuiAccordion, + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutHeader, + EuiText, + EuiTitle, + EuiSpacer, +} from '@elastic/eui'; +import { + getFieldsAndTypesGrid, + getFeaturesAndAggsAndFieldsGrid, +} from '../../utils/helpers'; +import { SAMPLE_DATA } from '../../utils/constants'; +import { EuiHorizontalRule } from '@elastic/eui'; + +interface SampleDetailsFlyoutProps { + title: string; + sampleData: SAMPLE_DATA; + interval: number; + onClose(): void; +} + +export const SampleDetailsFlyout = (props: SampleDetailsFlyoutProps) => { + const fieldValues = Object.keys(props.sampleData.fieldMappings); + const fieldTypes = fieldValues.map((field) => + get(props.sampleData.fieldMappings, `${field}.type`) + ); + const featureNames = Object.keys( + get(props.sampleData.detectorConfig, 'uiMetadata.features') + ); + const featureAggs = featureNames.map((feature) => + get( + props.sampleData.detectorConfig, + `uiMetadata.features.${feature}.aggregationBy` + ) + ); + const featureFields = featureNames.map((feature) => + get( + props.sampleData.detectorConfig, + `uiMetadata.features.${feature}.aggregationOf` + ) + ); + const detectorInterval = get( + props.sampleData.detectorConfig, + 'detection_interval.period.interval' + ); + + return ( + + + +

{props.title}

+ + + + +

Detector details

+ + } + initialIsOpen={true} + paddingSize="m" + > + + Name: + {props.sampleData.detectorName} +

+ Detection interval: + Every {detectorInterval} minutes +

+ Feature details: + + + {getFeaturesAndAggsAndFieldsGrid( + featureNames, + featureAggs, + featureFields + )} + + + +

Index details

+ + } + initialIsOpen={false} + paddingSize="m" + > + + Name: + {props.sampleData.indexName} +

+ Log frequency: Every {props.interval} minute(s) +

+ Log duration: 3 weeks +

+ Field details: + + + {getFieldsAndTypesGrid(fieldValues, fieldTypes)} + + + + ); +}; diff --git a/public/pages/SampleData/components/SampleIndexDetailsCallout/SampleIndexDetailsCallout.tsx b/public/pages/SampleData/components/SampleIndexDetailsCallout/SampleIndexDetailsCallout.tsx new file mode 100644 index 00000000..b1efbfd6 --- /dev/null +++ b/public/pages/SampleData/components/SampleIndexDetailsCallout/SampleIndexDetailsCallout.tsx @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { EuiCallOut, EuiLink } from '@elastic/eui'; +import { KIBANA_NAME, KIBANA_PATH } from '../../../../utils/constants'; + +interface SampleIndexDetailsCalloutProps { + indexName: string; +} + +export const SampleIndexDetailsCallout = ( + props: SampleIndexDetailsCalloutProps +) => { + return ( + +

+ Check out the{' '} + + Kibana Discover app + + {''} to view the raw data for sample index '{props.indexName}'. +

+ + ); +}; diff --git a/public/pages/SampleData/containers/SampleData/SampleData.tsx b/public/pages/SampleData/containers/SampleData/SampleData.tsx new file mode 100644 index 00000000..2ee4f83e --- /dev/null +++ b/public/pages/SampleData/containers/SampleData/SampleData.tsx @@ -0,0 +1,306 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { + EuiSpacer, + EuiPageHeader, + EuiTitle, + EuiText, + EuiFlexItem, + EuiFlexGroup, +} from '@elastic/eui'; +import React, { Fragment, useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +//@ts-ignore +import chrome from 'ui/chrome'; +//@ts-ignore +import { toastNotifications } from 'ui/notify'; +import { BREADCRUMBS, SAMPLE_TYPE } from '../../../../utils/constants'; +import { + GET_SAMPLE_DETECTORS_QUERY_PARAMS, + GET_SAMPLE_INDICES_QUERY, +} from '../../../utils/constants'; +import { AppState } from '../../../../redux/reducers'; +import { getDetectorList } from '../../../../redux/reducers/ad'; +import { createSampleData } from '../../../../redux/reducers/sampleData'; + +import { + getIndices, + createIndex, +} from '../../../../redux/reducers/elasticsearch'; +import { createDetector, startDetector } from '../../../../redux/reducers/ad'; +import { + sampleHttpResponses, + sampleEcommerce, + sampleHostHealth, +} from '../../utils/constants'; +import { + containsSampleIndex, + containsSampleDetector, + getDetectorId, +} from '../../utils/helpers'; +import { SampleDataBox } from '../../components/SampleDataBox/SampleDataBox'; +import { SampleDetailsFlyout } from '../../components/SampleDetailsFlyout/SampleDetailsFlyout'; + +export const SampleData = () => { + const dispatch = useDispatch(); + const visibleIndices = useSelector( + (state: AppState) => state.elasticsearch.indices + ); + const allDetectors = Object.values( + useSelector((state: AppState) => state.ad.detectorList) + ); + + const [isLoadingHttpData, setIsLoadingHttpData] = useState(false); + const [isLoadingEcommerceData, setIsLoadingEcommerceData] = useState( + false + ); + const [isLoadingHostHealthData, setIsLoadingHostHealthData] = useState< + boolean + >(false); + const [ + showHttpResponseDetailsFlyout, + setShowHttpResponseDetailsFlyout, + ] = useState(false); + const [showEcommerceDetailsFlyout, setShowEcommerceDetailsFlyout] = useState< + boolean + >(false); + const [ + showHostHealthDetailsFlyout, + setShowHostHealthDetailsFlyout, + ] = useState(false); + + const getAllSampleDetectors = async () => { + await dispatch(getDetectorList(GET_SAMPLE_DETECTORS_QUERY_PARAMS)).catch( + (error: any) => { + console.error('Error getting all detectors: ', error); + } + ); + }; + + const getAllSampleIndices = async () => { + await dispatch(getIndices(GET_SAMPLE_INDICES_QUERY)).catch((error: any) => { + console.error('Error getting all indices: ', error); + }); + }; + + // Set breadcrumbs on page initialization + useEffect(() => { + chrome.breadcrumbs.set([ + BREADCRUMBS.ANOMALY_DETECTOR, + BREADCRUMBS.SAMPLE_DETECTORS, + ]); + }, []); + + // Getting all initial sample detectors & indices + useEffect(() => { + getAllSampleDetectors(); + getAllSampleIndices(); + }, []); + + // Create and populate sample index, create and start sample detector + const handleLoadData = async ( + sampleType: SAMPLE_TYPE, + indexConfig: any, + detectorConfig: any, + setLoadingState: (isLoading: boolean) => void + ) => { + setLoadingState(true); + let errorDuringAction = false; + let errorMessage = ''; + + // Create the index (if it doesn't exist yet) + if (!containsSampleIndex(visibleIndices, sampleType)) { + await dispatch(createIndex(indexConfig)).catch((error: any) => { + errorDuringAction = true; + errorMessage = 'Error creating sample index.'; + console.error('Error creating sample index: ', error); + }); + } + + // Get the sample data from the server and bulk insert + if (!errorDuringAction) { + await dispatch(createSampleData(sampleType)).catch((error: any) => { + errorDuringAction = true; + errorMessage = error; + console.error('Error bulk inserting data: ', error); + }); + } + + // Create the detector + if (!errorDuringAction) { + await dispatch(createDetector(detectorConfig)) + .then(function (response: any) { + const detectorId = response.data.response.id; + // Start the detector + dispatch(startDetector(detectorId)).catch((error: any) => { + errorDuringAction = true; + errorMessage = error.data.message; + console.error('Error starting sample detector: ', error); + }); + }) + .catch((error: any) => { + errorDuringAction = true; + errorMessage = error; + console.error('Error creating sample detector: ', error); + }); + } + + getAllSampleDetectors(); + getAllSampleIndices(); + setLoadingState(false); + if (!errorDuringAction) { + toastNotifications.addSuccess('Successfully loaded sample detector'); + } else { + toastNotifications.addDanger( + `Unable to load all sample data, please try again. ${errorMessage}` + ); + } + }; + + return ( + + + +

Sample detectors

+ + + + Create a detector with streaming sample data to get a deeper + understanding of how anomaly detection works. You can create and + initialize a detector with configured settings for your selected sample + index. + + + + + { + setShowHttpResponseDetailsFlyout(true); + setShowEcommerceDetailsFlyout(false); + setShowHostHealthDetailsFlyout(false); + }} + onLoadData={() => { + handleLoadData( + SAMPLE_TYPE.HTTP_RESPONSES, + sampleHttpResponses.indexConfig, + sampleHttpResponses.detectorConfig, + setIsLoadingHttpData + ); + }} + isLoadingData={isLoadingHttpData} + isDataLoaded={containsSampleDetector( + allDetectors, + SAMPLE_TYPE.HTTP_RESPONSES + )} + detectorId={getDetectorId( + allDetectors, + sampleHttpResponses.detectorName + )} + /> + + + { + setShowHttpResponseDetailsFlyout(false); + setShowEcommerceDetailsFlyout(true); + setShowHostHealthDetailsFlyout(false); + }} + onLoadData={() => { + handleLoadData( + SAMPLE_TYPE.ECOMMERCE, + sampleEcommerce.indexConfig, + sampleEcommerce.detectorConfig, + setIsLoadingEcommerceData + ); + }} + isLoadingData={isLoadingEcommerceData} + isDataLoaded={containsSampleDetector( + allDetectors, + SAMPLE_TYPE.ECOMMERCE + )} + detectorId={getDetectorId( + allDetectors, + sampleEcommerce.detectorName + )} + /> + + + { + setShowHttpResponseDetailsFlyout(false); + setShowEcommerceDetailsFlyout(false); + setShowHostHealthDetailsFlyout(true); + }} + onLoadData={() => { + handleLoadData( + SAMPLE_TYPE.HOST_HEALTH, + sampleHostHealth.indexConfig, + sampleHostHealth.detectorConfig, + setIsLoadingHostHealthData + ); + }} + isLoadingData={isLoadingHostHealthData} + isDataLoaded={containsSampleDetector( + allDetectors, + SAMPLE_TYPE.HOST_HEALTH + )} + detectorId={getDetectorId( + allDetectors, + sampleHostHealth.detectorName + )} + /> + + + + {showHttpResponseDetailsFlyout ? ( + setShowHttpResponseDetailsFlyout(false)} + /> + ) : null} + {showEcommerceDetailsFlyout ? ( + setShowEcommerceDetailsFlyout(false)} + /> + ) : null} + {showHostHealthDetailsFlyout ? ( + setShowHostHealthDetailsFlyout(false)} + /> + ) : null} + + ); +}; diff --git a/public/pages/SampleData/containers/SampleData/__tests__/SampleData.test.tsx b/public/pages/SampleData/containers/SampleData/__tests__/SampleData.test.tsx new file mode 100644 index 00000000..76da8ef4 --- /dev/null +++ b/public/pages/SampleData/containers/SampleData/__tests__/SampleData.test.tsx @@ -0,0 +1,128 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { render, wait } from '@testing-library/react'; +import { SampleData } from '../SampleData'; +import { Provider } from 'react-redux'; +import { + MemoryRouter as Router, + Redirect, + Route, + Switch, +} from 'react-router-dom'; +import { httpClientMock } from '../../../../../../test/mocks'; +import configureStore from '../../../../../redux/configureStore'; +import { + Detectors, + initialDetectorsState, +} from '../../../../../redux/reducers/ad'; +import { sampleHttpResponses } from '../../../utils/constants'; + +const renderWithRouter = ( + initialAdState: Detectors = initialDetectorsState +) => ({ + ...render( + + + + } /> + + + + + ), +}); + +describe(' spec', () => { + jest.clearAllMocks(); + describe('No sample detectors created', () => { + test('renders component', async () => { + httpClientMock.get = jest.fn().mockResolvedValue({ + data: { ok: true, response: { detectorList: [], totalDetectors: 0 } }, + }); + const { container, getByText, queryByText } = renderWithRouter(); + expect(container).toMatchSnapshot(); + getByText('Sample detectors'); + getByText('Monitor HTTP responses'); + getByText('Monitor eCommerce orders'); + getByText('Monitor host health'); + expect(queryByText('INSTALLED')).toBeNull(); + expect(queryByText('Detector created')).toBeNull(); + expect(queryByText('View detector and sample data')).toBeNull(); + }); + }); + + describe('Some detectors created', () => { + jest.clearAllMocks(); + test('renders component with sample detector', async () => { + httpClientMock.get = jest.fn().mockResolvedValue({ + data: { + ok: true, + response: { + detectorList: [ + { + id: 'sample-detector-id', + name: sampleHttpResponses.detectorName, + indices: sampleHttpResponses.indexName, + totalAnomalies: 0, + lastActiveAnomaly: 0, + }, + ], + totalDetectors: 1, + }, + }, + }); + const { container, getByText, getAllByText } = renderWithRouter(); + await wait(); + expect(container).toMatchSnapshot(); + getByText('Sample detectors'); + getByText('Monitor HTTP responses'); + getByText('Monitor eCommerce orders'); + getByText('Monitor host health'); + expect(getAllByText('Detector created')).toHaveLength(1); + expect(getAllByText('View detector and sample data')).toHaveLength(1); + expect(getAllByText('INSTALLED')).toHaveLength(1); + }); + test('renders component with non-sample detector', async () => { + httpClientMock.get = jest.fn().mockResolvedValue({ + data: { + ok: true, + response: { + detectorList: [ + { + id: 'non-sample-detector-id', + name: 'non-sample-detector', + indices: 'non-sample-index', + totalAnomalies: 0, + lastActiveAnomaly: 0, + }, + ], + totalDetectors: 1, + }, + }, + }); + const { container, getByText, queryByText } = renderWithRouter(); + await wait(); + expect(container).toMatchSnapshot(); + getByText('Sample detectors'); + getByText('Monitor HTTP responses'); + getByText('Monitor eCommerce orders'); + getByText('Monitor host health'); + expect(queryByText('INSTALLED')).toBeNull(); + expect(queryByText('Detector created')).toBeNull(); + }); + }); +}); diff --git a/public/pages/SampleData/containers/SampleData/__tests__/__snapshots__/SampleData.test.tsx.snap b/public/pages/SampleData/containers/SampleData/__tests__/__snapshots__/SampleData.test.tsx.snap new file mode 100644 index 00000000..2f7e9bd9 --- /dev/null +++ b/public/pages/SampleData/containers/SampleData/__tests__/__snapshots__/SampleData.test.tsx.snap @@ -0,0 +1,1247 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[` spec No sample detectors created renders component 1`] = ` +
+
+

+ Sample detectors +

+
+
+ Create a detector with streaming sample data to get a deeper understanding of how anomaly detection works. You can create and initialize a detector with configured settings for your selected sample index. +
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor HTTP responses +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect high numbers of error response codes in an index containing HTTP response data. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor eCommerce orders +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect any unual increase or decrease of orders in an index containing online order data. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor host health +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect increases in CPU and memory utilization in an index containing various health metrics from a host. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec Some detectors created renders component with non-sample detector 1`] = ` +
+
+

+ Sample detectors +

+
+
+ Create a detector with streaming sample data to get a deeper understanding of how anomaly detection works. You can create and initialize a detector with configured settings for your selected sample index. +
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor HTTP responses +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect high numbers of error response codes in an index containing HTTP response data. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor eCommerce orders +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect any unual increase or decrease of orders in an index containing online order data. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor host health +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect increases in CPU and memory utilization in an index containing various health metrics from a host. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+`; + +exports[` spec Some detectors created renders component with sample detector 1`] = ` +
+
+

+ Sample detectors +

+
+
+ Create a detector with streaming sample data to get a deeper understanding of how anomaly detection works. You can create and initialize a detector with configured settings for your selected sample index. +
+
+
+
+
+
+ + + INSTALLED + + +
+
+
+
+
+ +

+ Monitor HTTP responses +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect high numbers of error response codes in an index containing HTTP response data. +

+
+
+
+ +
+
+ + View detector and sample data + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor eCommerce orders +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect any unual increase or decrease of orders in an index containing online order data. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

+ Monitor host health +

+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ Detect increases in CPU and memory utilization in an index containing various health metrics from a host. +

+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+`; diff --git a/public/pages/SampleData/index.ts b/public/pages/SampleData/index.ts new file mode 100644 index 00000000..cd70b1ef --- /dev/null +++ b/public/pages/SampleData/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +export { SampleData } from './containers/SampleData/SampleData'; diff --git a/public/pages/SampleData/utils/constants.tsx b/public/pages/SampleData/utils/constants.tsx new file mode 100644 index 00000000..734d4e10 --- /dev/null +++ b/public/pages/SampleData/utils/constants.tsx @@ -0,0 +1,360 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +//@ts-ignore +import moment from 'moment'; +import React from 'react'; +import { EuiIcon } from '@elastic/eui'; + +// same as default Kibana sample data +export const indexSettings = { + index: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + }, +}; + +export interface SAMPLE_DATA { + indexName: string; + detectorName: string; + description: string; + icon: any; + fieldMappings: {}; + indexConfig: {}; + detectorConfig: {}; +} + +/* + *** SAMPLE HTTP RESPONSES CONSTANTS *** + */ +const httpResponsesIndexName = 'opendistro-sample-http-responses'; +const httpResponsesDetectorName = 'opendistro-sample-http-responses-detector'; +const httpFieldMappings = { + timestamp: { + type: 'date', + }, + status_code: { + type: 'integer', + }, + http_1xx: { + type: 'integer', + }, + http_2xx: { + type: 'integer', + }, + http_3xx: { + type: 'integer', + }, + http_4xx: { + type: 'integer', + }, + http_5xx: { + type: 'integer', + }, +}; +export const sampleHttpResponses = { + indexName: httpResponsesIndexName, + detectorName: httpResponsesDetectorName, + description: + 'Detect high numbers of error response codes in an index containing HTTP response data.', + icon: , + fieldMappings: httpFieldMappings, + indexConfig: { + index: httpResponsesIndexName, + body: { + settings: indexSettings, + mappings: { + properties: httpFieldMappings, + }, + }, + }, + detectorConfig: { + name: httpResponsesDetectorName, + description: + 'A sample detector to detect anomalies with HTTP response code logs.', + time_field: 'timestamp', + indices: [httpResponsesIndexName], + featureAttributes: [ + { + feature_name: 'sum_http_4xx', + feature_enabled: true, + importance: 1, + aggregationQuery: { sum_http_4xx: { sum: { field: 'http_4xx' } } }, + }, + { + feature_name: 'sum_http_5xx', + feature_enabled: true, + importance: 2, + aggregationQuery: { sum_http_5xx: { sum: { field: 'http_5xx' } } }, + }, + ], + uiMetadata: { + features: { + sum_http_4xx: { + featureType: 'simple_aggs', + aggregationBy: 'sum', + aggregationOf: 'http_4xx', + }, + sum_http_5xx: { + featureType: 'simple_aggs', + aggregationBy: 'sum', + aggregationOf: 'http_5xx', + }, + }, + filters: [], + }, + detection_interval: { + period: { + interval: 10, + unit: 'Minutes', + }, + }, + window_delay: { + period: { + interval: 1, + unit: 'Minutes', + }, + }, + }, +} as SAMPLE_DATA; + +/* + *** ECOMMERCE CONSTANTS *** + */ +const ecommerceIndexName = 'opendistro-sample-ecommerce'; +const ecommerceDetectorName = 'opendistro-sample-ecommerce-detector'; +const ecommerceFieldMappings = { + timestamp: { + type: 'date', + }, + order_id: { + type: 'integer', + }, + items_purchased_success: { + type: 'integer', + }, + items_purchased_failure: { + type: 'integer', + }, + total_revenue_usd: { + type: 'integer', + }, +}; +export const sampleEcommerce = { + indexName: ecommerceIndexName, + detectorName: ecommerceDetectorName, + description: + 'Detect any unual increase or decrease of orders in an index containing online order data.', + icon: , + fieldMappings: ecommerceFieldMappings, + indexConfig: { + index: ecommerceIndexName, + body: { + settings: indexSettings, + mappings: { + properties: ecommerceFieldMappings, + }, + }, + }, + detectorConfig: { + name: ecommerceDetectorName, + description: 'A sample detector to detect anomalies with ecommerce logs.', + time_field: 'timestamp', + indices: [ecommerceIndexName], + featureAttributes: [ + { + feature_name: 'sum_failed_items', + feature_enabled: true, + importance: 1, + aggregationQuery: { + sum_failed_items: { sum: { field: 'items_purchased_failure' } }, + }, + }, + { + feature_name: 'avg_total_revenue', + feature_enabled: true, + importance: 2, + aggregationQuery: { + avg_total_revenue: { avg: { field: 'total_revenue_usd' } }, + }, + }, + { + feature_name: 'max_total_revenue', + feature_enabled: true, + importance: 3, + aggregationQuery: { + max_total_revenue: { max: { field: 'total_revenue_usd' } }, + }, + }, + { + feature_name: 'min_total_revenue', + feature_enabled: true, + importance: 4, + aggregationQuery: { + min_total_revenue: { min: { field: 'total_revenue_usd' } }, + }, + }, + ], + uiMetadata: { + features: { + sum_failed_items: { + featureType: 'simple_aggs', + aggregationBy: 'sum', + aggregationOf: 'items_purchased_failure', + }, + avg_total_revenue: { + featureType: 'simple_aggs', + aggregationBy: 'avg', + aggregationOf: 'total_revenue_usd', + }, + max_total_revenue: { + featureType: 'simple_aggs', + aggregationBy: 'max', + aggregationOf: 'total_revenue_usd', + }, + min_total_revenue: { + featureType: 'simple_aggs', + aggregationBy: 'min', + aggregationOf: 'total_revenue_usd', + }, + }, + filters: [], + }, + detection_interval: { + period: { + interval: 10, + unit: 'Minutes', + }, + }, + window_delay: { + period: { + interval: 1, + unit: 'Minutes', + }, + }, + }, +} as SAMPLE_DATA; + +/* + *** HOST HEALTH CONSTANTS *** + */ +const hostHealthIndexName = 'opendistro-sample-host-health'; +const hostHealthDetectorName = 'opendistro-sample-host-health-detector'; +const hostHealthFieldMappings = { + timestamp: { + type: 'date', + }, + cpu_usage_percentage: { + type: 'integer', + }, + memory_usage_percentage: { + type: 'integer', + }, +}; +export const sampleHostHealth = { + indexName: hostHealthIndexName, + detectorName: hostHealthDetectorName, + description: + 'Detect increases in CPU and memory utilization in an index containing various health metrics from a host.', + icon: , + fieldMappings: hostHealthFieldMappings, + indexConfig: { + index: hostHealthIndexName, + body: { + settings: indexSettings, + mappings: { + properties: hostHealthFieldMappings, + }, + }, + }, + detectorConfig: { + name: hostHealthDetectorName, + description: + 'A sample detector to detect anomalies with logs related to the health of a host.', + time_field: 'timestamp', + indices: [hostHealthIndexName], + featureAttributes: [ + { + feature_name: 'max_cpu_usage', + feature_enabled: true, + importance: 1, + aggregationQuery: { + sum_cpu_usage: { max: { field: 'cpu_usage_percentage' } }, + }, + }, + { + feature_name: 'max_memory_usage', + feature_enabled: true, + importance: 2, + aggregationQuery: { + sum_memory_usage: { max: { field: 'memory_usage_percentage' } }, + }, + }, + { + feature_name: 'avg_cpu_usage', + feature_enabled: true, + importance: 3, + aggregationQuery: { + avg_cpu_usage: { avg: { field: 'cpu_usage_percentage' } }, + }, + }, + { + feature_name: 'avg_memory_usage', + feature_enabled: true, + importance: 4, + aggregationQuery: { + avg_memory_usage: { avg: { field: 'memory_usage_percentage' } }, + }, + }, + ], + uiMetadata: { + features: { + max_cpu_usage: { + featureType: 'simple_aggs', + aggregationBy: 'max', + aggregationOf: 'cpu_usage_percentage', + }, + max_memory_usage: { + featureType: 'simple_aggs', + aggregationBy: 'max', + aggregationOf: 'memory_usage_percentage', + }, + avg_cpu_usage: { + featureType: 'simple_aggs', + aggregationBy: 'avg', + aggregationOf: 'cpu_usage_percentage', + }, + avg_memory_usage: { + featureType: 'simple_aggs', + aggregationBy: 'avg', + aggregationOf: 'memory_usage_percentage', + }, + }, + filters: [], + }, + detection_interval: { + period: { + interval: 10, + unit: 'Minutes', + }, + }, + window_delay: { + period: { + interval: 1, + unit: 'Minutes', + }, + }, + }, +} as SAMPLE_DATA; diff --git a/public/pages/SampleData/utils/helpers.tsx b/public/pages/SampleData/utils/helpers.tsx new file mode 100644 index 00000000..34085881 --- /dev/null +++ b/public/pages/SampleData/utils/helpers.tsx @@ -0,0 +1,233 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import React from 'react'; +import { EuiDataGrid } from '@elastic/eui'; +import { CatIndex } from '../../../../server/models/types'; +import { Detector, DetectorListItem } from '../../../models/interfaces'; +import { SAMPLE_TYPE, ANOMALY_DETECTORS_INDEX } from '../../../utils/constants'; +import { + sampleHttpResponses, + sampleEcommerce, + sampleHostHealth, +} from '../utils/constants'; + +export const containsDetectorsIndex = (indices: CatIndex[]) => { + return indices.map((index) => index.index).includes(ANOMALY_DETECTORS_INDEX); +}; + +export const containsSampleIndex = ( + indices: CatIndex[], + sampleType: SAMPLE_TYPE +) => { + let indexName = ''; + switch (sampleType) { + case SAMPLE_TYPE.HTTP_RESPONSES: { + indexName = sampleHttpResponses.indexName; + break; + } + case SAMPLE_TYPE.ECOMMERCE: { + indexName = sampleEcommerce.indexName; + break; + } + case SAMPLE_TYPE.HOST_HEALTH: { + indexName = sampleHostHealth.indexName; + break; + } + } + return indices.map((index) => index.index).includes(indexName); +}; + +export const containsSampleDetector = ( + detectors: DetectorListItem[], + sampleType: SAMPLE_TYPE +) => { + let detectorName = ''; + switch (sampleType) { + case SAMPLE_TYPE.HTTP_RESPONSES: { + detectorName = sampleHttpResponses.detectorName; + break; + } + case SAMPLE_TYPE.ECOMMERCE: { + detectorName = sampleEcommerce.detectorName; + break; + } + case SAMPLE_TYPE.HOST_HEALTH: { + detectorName = sampleHostHealth.detectorName; + break; + } + } + return detectors.map((detector) => detector.name).includes(detectorName); +}; + +export const detectorIsSample = (detector: Detector) => { + return ( + detector.name === sampleHttpResponses.detectorName || + detector.name === sampleEcommerce.detectorName || + detector.name === sampleHostHealth.detectorName + ); +}; + +export const getAssociatedIndex = (detector: Detector) => { + if (detector.name === sampleHttpResponses.detectorName) { + return sampleHttpResponses.indexName; + } + if (detector.name === sampleEcommerce.detectorName) { + return sampleEcommerce.indexName; + } + if (detector.name === sampleHostHealth.detectorName) { + return sampleHostHealth.indexName; + } + console.error( + 'Error getting associated sample index for detector ', + detector.name + ); + return ''; +}; + +export const getDetectorId = ( + detectors: DetectorListItem[], + detectorName: string +) => { + let detectorId = ''; + detectors.some((detector) => { + if (detector.name === detectorName) { + detectorId = detector.id; + return true; + } + return false; + }); + return detectorId; +}; + +const getFieldsAndTypesData = (fields: string[], types: string[]) => { + let data = []; + for (let i = 0; i < fields.length; i++) { + const field = fields[i]; + const type = types[i]; + data.push({ + Field: field, + Type: type, + }); + } + return data; +}; + +const getFeaturesAndAggsAndFieldsData = ( + features: string[], + aggs: string[], + fields: string[] +) => { + let data = []; + for (let i = 0; i < features.length; i++) { + const feature = features[i]; + const agg = aggs[i]; + const field = fields[i]; + data.push({ + Feature: feature, + Aggregation: agg, + 'Index field': field, + }); + } + return data; +}; + +export const getFieldsAndTypesGrid = (fields: string[], types: string[]) => { + const gridData = getFieldsAndTypesData(fields, types); + return ( + {}, + }} + rowCount={gridData.length} + renderCellValue={({ rowIndex, columnId }) => + //@ts-ignore + gridData[rowIndex][columnId] + } + gridStyle={{ + border: 'horizontal', + header: 'shade', + rowHover: 'highlight', + stripes: true, + }} + toolbarVisibility={false} + /> + ); +}; + +export const getFeaturesAndAggsAndFieldsGrid = ( + features: string[], + aggs: string[], + fields: string[] +) => { + const gridData = getFeaturesAndAggsAndFieldsData(features, aggs, fields); + return ( + {}, + }} + rowCount={gridData.length} + renderCellValue={({ rowIndex, columnId }) => + //@ts-ignore + gridData[rowIndex][columnId] + } + gridStyle={{ + border: 'horizontal', + header: 'shade', + rowHover: 'highlight', + stripes: true, + }} + toolbarVisibility={false} + /> + ); +}; diff --git a/public/pages/createDetector/containers/CreateDetector.tsx b/public/pages/createDetector/containers/CreateDetector.tsx index 749ed5d8..30203d95 100644 --- a/public/pages/createDetector/containers/CreateDetector.tsx +++ b/public/pages/createDetector/containers/CreateDetector.tsx @@ -27,8 +27,8 @@ import { } from '@elastic/eui'; import { Formik } from 'formik'; import { get, isEmpty } from 'lodash'; -import React, { Fragment, useEffect } from 'react'; -import { useDispatch } from 'react-redux'; +import React, { Fragment, useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; import { RouteComponentProps } from 'react-router'; import { Dispatch } from 'redux'; //@ts-ignore @@ -41,6 +41,8 @@ import { searchDetector, updateDetector, } from '../../../redux/reducers/ad'; +import { getIndices } from '../../../redux/reducers/elasticsearch'; +import { AppState } from '../../../redux/reducers'; import { BREADCRUMBS, MAX_DETECTORS } from '../../../utils/constants'; import { getErrorMessage, validateName } from '../../../utils/utils'; import { DetectorInfo } from '../components/DetectorInfo'; @@ -52,6 +54,9 @@ import { formikToDetector } from './utils/formikToDetector'; import { Detector } from '../../../models/interfaces'; import { Settings } from '../components/Settings/Settings'; import { useHideSideNavBar } from '../../main/hooks/useHideSideNavBar'; +import { CatIndex } from '../../../../server/models/types'; +import { SampleDataCallout } from '../../SampleData/components/SampleDataCallout/SampleDataCallout'; +import { containsDetectorsIndex } from '../../SampleData/utils/helpers'; interface CreateRouterProps { detectorId?: string; @@ -68,6 +73,25 @@ export function CreateDetector(props: CreateADProps) { //In case user is refreshing Edit detector page, we'll lose existing detector state //This will ensure to fetch the detector based on id from URL const { detector, hasError } = useFetchDetectorInfo(detectorId); + const [sampleCalloutVisible, setSampleCalloutVisible] = useState( + false + ); + const visibleIndices = useSelector( + (state: AppState) => state.elasticsearch.indices + ) as CatIndex[]; + + // Getting all initial indices + useEffect(() => { + const getInitialIndices = async () => { + await dispatch(getIndices('')); + }; + getInitialIndices(); + }, []); + + // Check if the sample data callout should be visible based on detector index + useEffect(() => { + setSampleCalloutVisible(!containsDetectorsIndex(visibleIndices)); + }, [visibleIndices]); //Set breadcrumbs based on Create / Update useEffect(() => { @@ -196,9 +220,23 @@ export function CreateDetector(props: CreateADProps) { } }; + const handleHideSampleCallout = () => { + setSampleCalloutVisible(false); + }; + return ( + {sampleCalloutVisible ? ( + + + + + + + + + ) : null} @@ -211,7 +249,7 @@ export function CreateDetector(props: CreateADProps) { initialValues={detectorToFormik(detector)} onSubmit={handleSubmit} > - {formikProps => ( + {(formikProps) => ( diff --git a/public/pages/createDetector/containers/__tests__/__snapshots__/CreateDetector.test.tsx.snap b/public/pages/createDetector/containers/__tests__/__snapshots__/CreateDetector.test.tsx.snap index 5f9243d9..7290b36f 100644 --- a/public/pages/createDetector/containers/__tests__/__snapshots__/CreateDetector.test.tsx.snap +++ b/public/pages/createDetector/containers/__tests__/__snapshots__/CreateDetector.test.tsx.snap @@ -7,6 +7,72 @@ exports[` spec create detector renders the component 1`] = `
+
+
+
+
+ + + Looking to get more familiar with anomaly detection? + +
+
+

+ Read the + + + documentation + + + or create a + + + sample detector + + + to get started. +

+
+
+
+
+
+
+
diff --git a/public/pages/main/Main.tsx b/public/pages/main/Main.tsx index 4308e519..22e71a41 100644 --- a/public/pages/main/Main.tsx +++ b/public/pages/main/Main.tsx @@ -18,6 +18,7 @@ import React from 'react'; import { AppState } from '../../redux/reducers'; import { CreateDetector } from '../createDetector'; import { DetectorList } from '../DetectorsList'; +import { SampleData } from '../SampleData'; import { ListRouterParams } from '../DetectorsList/containers/List/List'; // @ts-ignore import { EuiSideNav, EuiPage, EuiPageBody, EuiPageSideBar } from '@elastic/eui'; @@ -31,11 +32,13 @@ enum Navigation { AnomalyDetection = 'Anomaly detection', Dashboard = 'Dashboard', Detectors = 'Detectors', + SampleDetectors = 'Sample detectors', } enum Pathname { Dashboard = '/dashboard', Detectors = '/detectors', + SampleDetectors = '/sample-detectors', } interface MainProps extends RouteComponentProps {} @@ -62,6 +65,12 @@ export function Main(props: MainProps) { href: `#${Pathname.Detectors}`, isSelected: props.location.pathname === Pathname.Detectors, }, + { + name: Navigation.SampleDetectors, + id: 3, + href: `#${Pathname.SampleDetectors}`, + isSelected: props.location.pathname === Pathname.SampleDetectors, + }, ], }, ]; @@ -84,6 +93,11 @@ export function Main(props: MainProps) { )} /> + } + /> { this.createItem('Detectors', 2, { href: `#${APP_PATH.LIST_DETECTORS}`, }), + this.createItem('Sample Data', 3, { + href: `#${APP_PATH.SAMPLE_DATA}`, + }), ], }, ]; diff --git a/public/pages/utils/constants.ts b/public/pages/utils/constants.ts index 50ab4100..90ca6fbe 100644 --- a/public/pages/utils/constants.ts +++ b/public/pages/utils/constants.ts @@ -63,3 +63,14 @@ export const GET_ALL_DETECTORS_QUERY_PARAMS = { sortDirection: SORT_DIRECTION.ASC, sortField: 'name', }; + +export const GET_SAMPLE_DETECTORS_QUERY_PARAMS = { + from: 0, + search: 'opendistro-sample', + indices: '', + size: MAX_DETECTORS, + sortDirection: SORT_DIRECTION.ASC, + sortField: 'name', +}; + +export const GET_SAMPLE_INDICES_QUERY = 'opendistro-sample-*'; diff --git a/public/redux/reducers/elasticsearch.ts b/public/redux/reducers/elasticsearch.ts index db4c1fcb..a05d40da 100644 --- a/public/redux/reducers/elasticsearch.ts +++ b/public/redux/reducers/elasticsearch.ts @@ -30,6 +30,9 @@ const GET_INDICES = 'elasticsearch/GET_INDICES'; const GET_ALIASES = 'elasticsearch/GET_ALIASES'; const GET_MAPPINGS = 'elasticsearch/GET_MAPPINGS'; const SEARCH_ES = 'elasticsearch/SEARCH_ES'; +const CREATE_INDEX = 'elasticsearch/CREATE_INDEX'; +const BULK = 'elasticsearch/BULK'; +const DELETE_INDEX = 'elasticsearch/DELETE_INDEX'; export type Mappings = { [key: string]: any; @@ -170,7 +173,75 @@ const reducer = handleActions( ...state, requesting: false, errorMessage: get(action, 'error.data.error', action.error), - dataTypes: {} + dataTypes: {}, + }), + }, + [CREATE_INDEX]: { + REQUEST: (state: ElasticsearchState): ElasticsearchState => { + return { ...state, requesting: true, errorMessage: '' }; + }, + SUCCESS: ( + state: ElasticsearchState, + action: APIResponseAction + ): ElasticsearchState => { + return { + ...state, + requesting: false, + indices: action.result.data.response.indices, + }; + }, + FAILURE: ( + state: ElasticsearchState, + action: APIErrorAction + ): ElasticsearchState => ({ + ...state, + requesting: false, + errorMessage: action.error, + }), + }, + [BULK]: { + REQUEST: (state: ElasticsearchState): ElasticsearchState => { + return { ...state, requesting: true, errorMessage: '' }; + }, + SUCCESS: ( + state: ElasticsearchState, + action: APIResponseAction + ): ElasticsearchState => { + return { + ...state, + requesting: false, + }; + }, + FAILURE: ( + state: ElasticsearchState, + action: APIErrorAction + ): ElasticsearchState => ({ + ...state, + requesting: false, + errorMessage: action.error, + }), + }, + [DELETE_INDEX]: { + REQUEST: (state: ElasticsearchState): ElasticsearchState => { + return { ...state, requesting: true, errorMessage: '' }; + }, + SUCCESS: ( + state: ElasticsearchState, + action: APIResponseAction + ): ElasticsearchState => { + return { + ...state, + requesting: false, + indices: action.result.data.response.indices, + }; + }, + FAILURE: ( + state: ElasticsearchState, + action: APIErrorAction + ): ElasticsearchState => ({ + ...state, + requesting: false, + errorMessage: action.error, }), }, }, @@ -204,6 +275,24 @@ export const searchES = (requestData: any): APIAction => ({ client.post(`..${AD_NODE_API._SEARCH}`, requestData), }); +export const createIndex = (indexConfig: any): APIAction => ({ + type: CREATE_INDEX, + request: (client: IHttpService) => + client.put(`..${AD_NODE_API.CREATE_INDEX}`, { indexConfig }), +}); + +export const bulk = (body: any): APIAction => ({ + type: BULK, + request: (client: IHttpService) => + client.post(`..${AD_NODE_API.BULK}`, { body }), +}); + +export const deleteIndex = (index: string): APIAction => ({ + type: DELETE_INDEX, + request: (client: IHttpService) => + client.post(`..${AD_NODE_API.DELETE_INDEX}`, { index }), +}); + export const getPrioritizedIndices = (searchKey: string): ThunkAction => async ( dispatch, getState diff --git a/public/redux/reducers/sampleData.ts b/public/redux/reducers/sampleData.ts new file mode 100644 index 00000000..7cfb6879 --- /dev/null +++ b/public/redux/reducers/sampleData.ts @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { + APIAction, + APIResponseAction, + IHttpService, +} from '../middleware/types'; +import handleActions from '../utils/handleActions'; +import { AD_NODE_API } from '../../../utils/constants'; +import { SAMPLE_TYPE } from '../../utils/constants'; + +const CREATE_SAMPLE_DATA = 'ad/CREATE_SAMPLE_DATA'; + +export interface SampleDataState { + requesting: boolean; + errorMessage: string; +} +export const initialState: SampleDataState = { + requesting: false, + errorMessage: '', +}; + +const reducer = handleActions( + { + [CREATE_SAMPLE_DATA]: { + REQUEST: (state: SampleDataState): SampleDataState => ({ + ...state, + requesting: true, + errorMessage: '', + }), + SUCCESS: ( + state: SampleDataState, + action: APIResponseAction + ): SampleDataState => ({ + ...state, + requesting: false, + }), + FAILURE: ( + state: SampleDataState, + action: APIResponseAction + ): SampleDataState => ({ + ...state, + requesting: false, + errorMessage: action.error.data.error, + }), + }, + }, + initialState +); + +export const createSampleData = (sampleDataType: SAMPLE_TYPE): APIAction => ({ + type: CREATE_SAMPLE_DATA, + request: (client: IHttpService) => + client.post(`..${AD_NODE_API.CREATE_SAMPLE_DATA}`, { + type: sampleDataType, + }), +}); + +export default reducer; diff --git a/public/utils/constants.ts b/public/utils/constants.ts index b2ae663b..3cfbdb2e 100644 --- a/public/utils/constants.ts +++ b/public/utils/constants.ts @@ -24,6 +24,7 @@ export enum DATA_TYPES { export const BREADCRUMBS = Object.freeze({ ANOMALY_DETECTOR: { text: 'Anomaly detection', href: '#/' }, DETECTORS: { text: 'Detectors', href: '#/detectors' }, + SAMPLE_DETECTORS: { text: 'Sample detectors', href: '#/sample-detectors' }, CREATE_DETECTOR: { text: 'Create detector' }, EDIT_DETECTOR: { text: 'Edit detector' }, DASHBOARD: { text: 'Dashboard', href: '#/' }, @@ -33,15 +34,25 @@ export const BREADCRUMBS = Object.freeze({ export const APP_PATH = { DASHBOARD: '/dashboard', LIST_DETECTORS: '/detectors', + SAMPLE_DETECTORS: '/sample-detectors', CREATE_DETECTOR: '/create-ad/', EDIT_DETECTOR: '/detectors/:detectorId/edit', EDIT_FEATURES: '/detectors/:detectorId/features/', DETECTOR_DETAIL: '/detectors/:detectorId/', }; + +export const KIBANA_PATH = { + DISCOVER: '/discover', +}; + export const PLUGIN_NAME = 'opendistro-anomaly-detection-kibana'; export const ALERTING_PLUGIN_NAME = 'opendistro-alerting'; +export const KIBANA_NAME = 'kibana'; + +export const ANOMALY_DETECTORS_INDEX = '.opendistro-anomaly-detectors'; + export const ANOMALY_RESULT_INDEX = '.opendistro-anomaly-results'; export const MAX_DETECTORS = 1000; @@ -57,6 +68,12 @@ export enum DETECTOR_STATE { UNEXPECTED_FAILURE = 'Unexpected failure', } +export enum SAMPLE_TYPE { + HTTP_RESPONSES = 'http-responses', + HOST_HEALTH = 'host-health', + ECOMMERCE = 'ecommerce', +} + export const MAX_FEATURE_NUM = 5; export const MAX_FEATURE_NAME_SIZE = 64; diff --git a/server/plugin.ts b/server/plugin.ts index 1b4b3060..735093b4 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -21,6 +21,7 @@ import { default as createRouter, Router } from './router'; import registerADRoutes from './routes/ad'; import registerAlertingRoutes from './routes/alerting'; import registerElasticsearchRoute from './routes/elasticsearch'; +import registerSampleDataRoutes from './routes/sampleData'; interface CoreSetup { elasticsearch: Legacy.Plugins.elasticsearch.Plugin; @@ -46,6 +47,7 @@ export class ADPlugin { registerElasticsearchRoute(apiRouter); registerADRoutes(apiRouter); registerAlertingRoutes(apiRouter); + registerSampleDataRoutes(apiRouter); } public start() {} diff --git a/server/routes/ad.ts b/server/routes/ad.ts index 34ef5e92..f9ad1fe7 100644 --- a/server/routes/ad.ts +++ b/server/routes/ad.ts @@ -100,7 +100,9 @@ const previewDetector = async ( try { const { detectorId } = req.params; //@ts-ignore - const requestBody = JSON.stringify(convertPreviewInputKeysToSnakeCase(req.payload)); + const requestBody = JSON.stringify( + convertPreviewInputKeysToSnakeCase(req.payload) + ); const response = await callWithRequest(req, 'ad.previewDetector', { detectorId, body: requestBody, @@ -333,7 +335,7 @@ const getDetectors = async ( query_string: { fields: ['name', 'description'], default_operator: 'AND', - query: `*${search.trim().split(' ').join('* *')}*`, + query: `*${search.trim().split('-').join('* *')}*`, }, }); } diff --git a/server/routes/elasticsearch.ts b/server/routes/elasticsearch.ts index 6fa7fb34..7586b318 100644 --- a/server/routes/elasticsearch.ts +++ b/server/routes/elasticsearch.ts @@ -27,12 +27,16 @@ import { ServerResponse, } from '../models/types'; import { Router } from '../router'; +import { isIndexNotFoundError } from './utils/adHelpers'; -export default function(apiRouter: Router) { +export default function (apiRouter: Router) { apiRouter.get('/_indices', getIndices); apiRouter.get('/_aliases', getAliases); apiRouter.get('/_mappings', getMapping); apiRouter.post('/_search', executeSearch); + apiRouter.put('/create_index', createIndex); + apiRouter.post('/bulk', bulk); + apiRouter.post('/delete_index', deleteIndex); } type SearchParams = { @@ -133,6 +137,90 @@ const getAliases = async ( } }; +const createIndex = async ( + req: Request, + h: ResponseToolkit, + callWithRequest: CallClusterWithRequest +): Promise> => { + //@ts-ignore + const index = req.payload.indexConfig.index; + //@ts-ignore + const body = req.payload.indexConfig.body; + try { + await callWithRequest(req, 'indices.create', { + index: index, + body: body, + }); + } catch (err) { + console.log('Anomaly detector - Unable to create index', err); + return { ok: false, error: err.message }; + } + try { + const response: CatIndex[] = await callWithRequest(req, 'cat.indices', { + index, + format: 'json', + h: 'health,index', + }); + return { ok: true, response: { indices: response } }; + } catch (err) { + console.log('Anomaly detector - Unable to get indices', err); + return { ok: false, error: err.message }; + } +}; + +const bulk = async ( + req: Request, + h: ResponseToolkit, + callWithRequest: CallClusterWithRequest +): Promise> => { + //@ts-ignore + const body = req.payload.body; + try { + const response: any = await callWithRequest(req, 'bulk', { + body: body, + }); + //@ts-ignore + return { ok: true, response: { response } }; + } catch (err) { + console.log('Anomaly detector - Unable to perform bulk action', err); + return { ok: false, error: err.message }; + } +}; + +const deleteIndex = async ( + req: Request, + h: ResponseToolkit, + callWithRequest: CallClusterWithRequest +): Promise> => { + //@ts-ignore + const index = req.payload.index; + try { + await callWithRequest(req, 'indices.delete', { + index: index, + }); + } catch (err) { + console.log( + 'Anomaly detector - Unable to perform delete index action', + err + ); + // Ignore the error if it's an index_not_found_exception + if (!isIndexNotFoundError(err)) { + return { ok: false, error: err.message }; + } + } + try { + const response: CatIndex[] = await callWithRequest(req, 'cat.indices', { + index, + format: 'json', + h: 'health,index', + }); + return { ok: true, response: { indices: response } }; + } catch (err) { + console.log('Anomaly detector - Unable to get indices', err); + return { ok: false, error: err.message }; + } +}; + const getMapping = async ( req: Request, h: ResponseToolkit, diff --git a/server/routes/sampleData.ts b/server/routes/sampleData.ts new file mode 100644 index 00000000..07948b83 --- /dev/null +++ b/server/routes/sampleData.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +//@ts-ignore +import moment from 'moment'; +import { Request, ResponseToolkit } from 'hapi'; +import path from 'path'; +//@ts-ignore +import { CallClusterWithRequest } from 'src/legacy/core_plugins/elasticsearch'; +import { ServerResponse } from '../models/types'; +import { Router } from '../router'; +import { SAMPLE_TYPE } from '../../public/utils/constants'; +import { loadSampleData } from '../sampleData/utils/helpers'; + +export default function (apiRouter: Router) { + apiRouter.post('/create_sample_data', createSampleData); +} + +// Get the zip file stored in server, unzip it, and bulk insert it +const createSampleData = async ( + req: Request, + h: ResponseToolkit, + callWithRequest: CallClusterWithRequest +): Promise> => { + //@ts-ignore + const type = req.payload.type as SAMPLE_TYPE; + try { + let filePath = ''; + let indexName = ''; + + switch (type) { + case SAMPLE_TYPE.HTTP_RESPONSES: { + filePath = path.resolve( + __dirname, + '../sampleData/rawData/httpResponses.json.gz' + ); + indexName = 'opendistro-sample-http-responses'; + break; + } + case SAMPLE_TYPE.ECOMMERCE: { + filePath = path.resolve( + __dirname, + '../sampleData/rawData/ecommerce.json.gz' + ); + indexName = 'opendistro-sample-ecommerce'; + break; + } + case SAMPLE_TYPE.HOST_HEALTH: { + filePath = path.resolve( + __dirname, + '../sampleData/rawData/hostHealth.json.gz' + ); + indexName = 'opendistro-sample-host-health'; + break; + } + } + + await loadSampleData(filePath, indexName, req, callWithRequest); + + //@ts-ignore + return { ok: true }; + } catch (err) { + console.log('Anomaly detector - Unable to load the sample data', err); + return { ok: false, error: err.message }; + } +}; diff --git a/server/sampleData/rawData/ecommerce.json.gz b/server/sampleData/rawData/ecommerce.json.gz new file mode 100644 index 00000000..e7044715 Binary files /dev/null and b/server/sampleData/rawData/ecommerce.json.gz differ diff --git a/server/sampleData/rawData/hostHealth.json.gz b/server/sampleData/rawData/hostHealth.json.gz new file mode 100644 index 00000000..7952e697 Binary files /dev/null and b/server/sampleData/rawData/hostHealth.json.gz differ diff --git a/server/sampleData/rawData/httpResponses.json.gz b/server/sampleData/rawData/httpResponses.json.gz new file mode 100644 index 00000000..1eba7bee Binary files /dev/null and b/server/sampleData/rawData/httpResponses.json.gz differ diff --git a/server/sampleData/utils/helpers.ts b/server/sampleData/utils/helpers.ts new file mode 100644 index 00000000..4dd8c249 --- /dev/null +++ b/server/sampleData/utils/helpers.ts @@ -0,0 +1,142 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +//@ts-ignore +import moment from 'moment'; +import readline from 'readline'; +import { Request } from 'hapi'; + +import fs from 'fs'; +import { createUnzip } from 'zlib'; +//@ts-ignore +import { CallClusterWithRequest } from 'src/legacy/core_plugins/elasticsearch'; + +const BULK_INSERT_SIZE = 500; + +export const loadSampleData = ( + filePath: string, + indexName: string, + req: Request, + callWithRequest: CallClusterWithRequest +) => { + return new Promise((resolve, reject) => { + let count: number = 0; + let docs: any[] = []; + let isPaused: boolean = false; + let offset = 0; + const startTime = moment(new Date().getTime()).subtract({ days: 7 }); + + // Create the read stream for the file. Set a smaller buffer size here to prevent it from + // getting too large, to prevent inserting too many docs at once into the index. + const readStream = fs.createReadStream(filePath, { + highWaterMark: 1024 * 4, + }); + const lineStream = readline.createInterface({ + input: readStream.pipe(createUnzip()), + }); + + // This is only ran when the end of lineStream closes normally. It is used to + // bulk insert the final batch of lines that are < BULK_INSERT_SIZE + const onClose = async () => { + if (docs.length > 0) { + try { + await bulkInsert(docs); + } catch (err) { + reject(err); + return; + } + } + resolve(count); + }; + lineStream.on('close', onClose); + lineStream.on('pause', async () => { + isPaused = true; + }); + lineStream.on('resume', async () => { + isPaused = false; + }); + lineStream.on('line', async (doc) => { + // for the initial doc, get the timestamp to properly set an offset + if (count === 0) { + const initialTime = moment(JSON.parse(doc).timestamp); + offset = startTime.diff(initialTime); + } + count++; + docs.push(doc); + + // If not currently paused: pause the stream to prevent concurrent bulk inserts + // on the cluster which could cause performance issues. + // Also, empty the current docs[] before performing the bulk insert to prevent + // buffered docs from being dropped. + if (docs.length >= BULK_INSERT_SIZE && !isPaused) { + lineStream.pause(); + + // save the docs to insert, and empty out current docs list + const docsToInsert = docs.slice(); + docs = []; + try { + await bulkInsert(docsToInsert); + lineStream.resume(); + } catch (err) { + lineStream.removeListener('close', onClose); + lineStream.close(); + reject(err); + } + } + }); + + const bulkInsert = async (docs: any[]) => { + try { + const bulkBody = prepareBody(docs, offset); + const resp = await callWithRequest(req, 'bulk', { + body: bulkBody, + }); + if (resp.errors) { + console.log('Error while bulk inserting. ', resp.errors); + return Promise.reject( + new Error('Error while bulk inserting. Please try again.') + ); + } + } catch (err) { + console.log('Error while bulk inserting. ', err); + return Promise.reject( + new Error('Error while bulk inserting. Please try again.') + ); + } + }; + + const prepareBody = (docs: string[], offset: number) => { + const bulkBody = [] as any[]; + let docIdCount = count - docs.length; + docs.forEach((doc: string) => { + bulkBody.push(getDocDetails(docIdCount)); + bulkBody.push(updateTimestamp(doc, offset)); + docIdCount++; + }); + return bulkBody; + }; + + const updateTimestamp = (doc: any, offset: number) => { + let docAsJSON = JSON.parse(doc); + const updatedTimestamp = docAsJSON.timestamp + offset; + docAsJSON.timestamp = updatedTimestamp; + return docAsJSON; + }; + + const getDocDetails = (docIdCount: number) => { + return `{ "index": { "_index": "${indexName}", "_id": ${docIdCount} } }`; + }; + }); +}; diff --git a/utils/constants.ts b/utils/constants.ts index 965263fe..e2d9b727 100644 --- a/utils/constants.ts +++ b/utils/constants.ts @@ -21,6 +21,10 @@ export const AD_NODE_API = Object.freeze({ _ALIASES: `${BASE_NODE_API_PATH}/_aliases`, _MAPPINGS: `${BASE_NODE_API_PATH}/_mappings`, DETECTOR: `${BASE_NODE_API_PATH}/detectors`, + CREATE_INDEX: `${BASE_NODE_API_PATH}/create_index`, + BULK: `${BASE_NODE_API_PATH}/bulk`, + DELETE_INDEX: `${BASE_NODE_API_PATH}/delete_index`, + CREATE_SAMPLE_DATA: `${BASE_NODE_API_PATH}/create_sample_data`, }); export const ALERTING_NODE_API = Object.freeze({ _SEARCH: `${BASE_NODE_API_PATH}/monitors/_search`,