From c0732106af97e0065e5671797dface64b5ec85e1 Mon Sep 17 00:00:00 2001 From: Mykola Harmash Date: Tue, 10 Dec 2024 14:07:11 +0100 Subject: [PATCH] [Observability Onboarding] Show existing data callout in Firehose flow (#203072) Closes https://github.com/elastic/kibana/issues/190795 Adds the logic to display a message to the user in case there is already an existing Firehose data in their cluster and to show the identified AWS services in the "Visualize Data" step right away without waiting for the window to loose focus first. ![CleanShot 2024-12-05 at 11 50 59@2x](https://github.com/user-attachments/assets/00653bf0-f711-4029-9011-a34a160b4b9b) ## How to test 1. Open the Firehose flow 2. Make sure there is no callout and the third step is not active 3. Go to Kibana dev console and ingest some dummy data (see examples bellow) 4. Refresh the page with the Firehose flow 5. make sure there is a callout and the third steps shows the identified AWS service ``` POST logs-aws.apigateway_logs-default/_doc { "@timestamp": "2024-11-25T13:32:01.000Z", "some": 111, "aws.kinesis.name": "Elastic-CloudwatchLogs" } POST metrics-aws.apigateway_metrics-default/_doc { "@timestamp": "2024-11-25T13:31:01.000Z", "agent": { "type": "firehose" }, "aws": { "cloudwatch": { "namespace": "AWS/ApiGateway" }, "exporter": { "arn": "arn:aws:cloudwatch:us-west-2:975050175126:metric-stream/Elastic-CloudwatchLogsAndMetricsToFirehose-CloudWatchMetricStream-Nhb4NhzPdL4J" } }, "cloud": { "account": { "id": "975050175126" }, "provider": "aws", "region": "us-west-2" } } ``` (cherry picked from commit 6cb14302a1a3fd157d1e1947493a0d0d5f4980f6) --- .../firehose/existing_data_callout.tsx | 36 +++++++++++++++++++ .../quickstart_flows/firehose/index.tsx | 25 +++++++++---- .../firehose/use_populated_aws_index_list.ts | 25 +++++++++++++ .../firehose/visualize_data.tsx | 29 +++++---------- .../observability/onboarding/firehose.ts | 34 ++++++++++++++++++ 5 files changed, 122 insertions(+), 27 deletions(-) create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/existing_data_callout.tsx create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/existing_data_callout.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/existing_data_callout.tsx new file mode 100644 index 0000000000000..2be3a663d8020 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/existing_data_callout.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiCallOut, useIsWithinMaxBreakpoint } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +export function ExistingDataCallout() { + const isMobile = useIsWithinMaxBreakpoint('s'); + + return ( +
+ +

+ {i18n.translate( + 'xpack.observability_onboarding.firehose.existingDataCallout.description', + { + defaultMessage: `If the Amazon Firehose Data stream(s) associated with this workflow are still active, you will encounter errors during onboarding. Navigate to Step 3 below in order to explore your services.`, + } + )} +

+ +
+ ); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx index 381630702d737..01e2dd02c3c47 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx @@ -31,6 +31,8 @@ import { useFirehoseFlow } from './use_firehose_flow'; import { VisualizeData } from './visualize_data'; import { ObservabilityOnboardingAppServices } from '../../..'; import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger'; +import { ExistingDataCallout } from './existing_data_callout'; +import { usePopulatedAWSIndexList } from './use_populated_aws_index_list'; const OPTIONS = [ { @@ -61,6 +63,9 @@ export function FirehosePanel() { }, } = useKibana(); const { data, status, error, refetch } = useFirehoseFlow(); + const { data: populatedAWSIndexList } = usePopulatedAWSIndexList(); + + const hasExistingData = Array.isArray(populatedAWSIndexList) && populatedAWSIndexList.length > 0; const telemetryEventContext: OnboardingFlowEventContext = useMemo( () => ({ @@ -72,12 +77,13 @@ export function FirehosePanel() { [cloudServiceProvider, selectedOptionId] ); - const isMonitoringData = useWindowBlurDataMonitoringTrigger({ - isActive: status === FETCH_STATUS.SUCCESS, - onboardingFlowType: 'firehose', - onboardingId: data?.onboardingId, - telemetryEventContext, - }); + const isMonitoringData = + useWindowBlurDataMonitoringTrigger({ + isActive: status === FETCH_STATUS.SUCCESS, + onboardingFlowType: 'firehose', + onboardingId: data?.onboardingId, + telemetryEventContext, + }) || hasExistingData; const onOptionChange = useCallback((id: string) => { setSelectedOptionId(id as CreateStackOption); @@ -193,6 +199,7 @@ export function FirehosePanel() { ), }, @@ -200,6 +207,12 @@ export function FirehosePanel() { return ( + {hasExistingData && ( + <> + + + + )} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts new file mode 100644 index 0000000000000..070a5aed4e2db --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useFetcher } from '../../../hooks/use_fetcher'; +import { + FIREHOSE_CLOUDFORMATION_STACK_NAME, + FIREHOSE_LOGS_STREAM_NAME, +} from '../../../../common/aws_firehose'; + +export function usePopulatedAWSIndexList() { + return useFetcher((callApi) => { + return callApi('GET /internal/observability_onboarding/firehose/has-data', { + params: { + query: { + logsStreamName: FIREHOSE_LOGS_STREAM_NAME, + stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME, + }, + }, + }); + }, []); +} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx index 45a2089c2d1c4..30023f20bf6cc 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx @@ -13,11 +13,7 @@ import { unionBy } from 'lodash'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import { FormattedMessage } from '@kbn/i18n-react'; import { ObservabilityOnboardingAppServices } from '../../..'; -import { - FIREHOSE_CLOUDFORMATION_STACK_NAME, - FIREHOSE_LOGS_STREAM_NAME, -} from '../../../../common/aws_firehose'; -import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { AccordionWithIcon } from '../shared/accordion_with_icon'; import { GetStartedPanel } from '../shared/get_started_panel'; import { @@ -28,34 +24,25 @@ import { AutoRefreshCallout } from './auto_refresh_callout'; import { ProgressCallout } from './progress_callout'; import { HAS_DATA_FETCH_INTERVAL } from './utils'; import { CreateStackOption } from './types'; +import { usePopulatedAWSIndexList } from './use_populated_aws_index_list'; const REQUEST_PENDING_STATUS_LIST = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED]; interface Props { onboardingId: string; selectedCreateStackOption: CreateStackOption; + hasExistingData: boolean; } -export function VisualizeData({ onboardingId, selectedCreateStackOption }: Props) { +export function VisualizeData({ onboardingId, selectedCreateStackOption, hasExistingData }: Props) { const accordionId = useGeneratedHtmlId({ prefix: 'accordion' }); const [orderedVisibleAWSServiceList, setOrderedVisibleAWSServiceList] = useState< AWSServiceGetStartedConfig[] >([]); - const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState(true); - const { - data: populatedAWSIndexList, - status, - refetch, - } = useFetcher((callApi) => { - return callApi('GET /internal/observability_onboarding/firehose/has-data', { - params: { - query: { - logsStreamName: FIREHOSE_LOGS_STREAM_NAME, - stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME, - }, - }, - }); - }, []); + const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState( + !hasExistingData + ); + const { data: populatedAWSIndexList, status, refetch } = usePopulatedAWSIndexList(); const { services: { notifications, diff --git a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts index 22c700d16be2d..163ff88c8ee5d 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts @@ -83,5 +83,39 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { // Checking that an AWS service item is visible after data is detected await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`); }); + + it('shows the existing data callout and detected AWS services when data was ingested previously', async () => { + const DATASET = 'aws.vpcflow'; + const AWS_SERVICE_ID = 'vpc-flow'; + await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton'); + const copiedCommand = await browser.getClipboardValue(); + const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? []; + + await testSubjects.missingOrFail('observabilityOnboardingFirehosePanelExistingDataCallout'); + + expect(logsStreamName).toBeDefined(); + + // Simulate Firehose stream ingesting log files + const to = new Date().toISOString(); + const count = 1; + await synthtrace.index( + timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return log.create().dataset(DATASET).timestamp(timestamp).defaults({ + 'aws.kinesis.name': logsStreamName, + }); + }) + ); + + await browser.refresh(); + + // Checking that the existing data callout is visible after data is detected + await testSubjects.isDisplayed('observabilityOnboardingFirehosePanelExistingDataCallout'); + + // Checking that an AWS service item is visible after data is detected + await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`); + }); }); }