[Observability Onboarding] Show existing data callout in Firehose flow (

elastic#203072)

Closes elastic#190795

Adds the logic to display a message to the user in case there is already
an existing Firehose data in their cluster and to show the identified
AWS services in the "Visualize Data" step right away without waiting for
the window to loose focus first.

![CleanShot 2024-12-05 at 11 50
59@2x](https://github.com/user-attachments/assets/00653bf0-f711-4029-9011-a34a160b4b9b)

## How to test

1. Open the Firehose flow
2. Make sure there is no callout and the third step is not active
3. Go to Kibana dev console and ingest some dummy data (see examples
bellow)
4. Refresh the page with the Firehose flow
5. make sure there is a callout and the third steps shows the identified
AWS service

```
POST logs-aws.apigateway_logs-default/_doc
{
  "@timestamp": "2024-11-25T13:32:01.000Z",
  "some": 111,
  "aws.kinesis.name": "Elastic-CloudwatchLogs"
}

POST metrics-aws.apigateway_metrics-default/_doc
{
    "@timestamp": "2024-11-25T13:31:01.000Z",
    "agent": {
      "type": "firehose"
    },
    "aws": {
      "cloudwatch": {
        "namespace": "AWS/ApiGateway"
      },
      "exporter": {
        "arn": "arn:aws:cloudwatch:us-west-2:975050175126:metric-stream/Elastic-CloudwatchLogsAndMetricsToFirehose-CloudWatchMetricStream-Nhb4NhzPdL4J"
      }
    },
    "cloud": {
      "account": {
        "id": "975050175126"
      },
      "provider": "aws",
      "region": "us-west-2"
    }
}
```

(cherry picked from commit 6cb1430)

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/existing_data_callout.tsx

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiCallOut, useIsWithinMaxBreakpoint } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+export function ExistingDataCallout() {
+  const isMobile = useIsWithinMaxBreakpoint('s');
+
+  return (
+    <div css={{ maxWidth: isMobile ? '100%' : '80%' }}>
+      <EuiCallOut
+        title={i18n.translate('xpack.observability_onboarding.firehose.existingDataCallout.title', {
+          defaultMessage: 'This workflow has been used before.',
+        })}
+        iconType="iInCircle"
+        color="warning"
+        data-test-subj="observabilityOnboardingFirehosePanelExistingDataCallout"
+      >
+        <p>
+          {i18n.translate(
+            'xpack.observability_onboarding.firehose.existingDataCallout.description',
+            {
+              defaultMessage: `If the Amazon Firehose Data stream(s) associated with this workflow are still active, you will encounter errors during onboarding. Navigate to Step 3 below in order to explore your services.`,
+            }
+          )}
+        </p>
+      </EuiCallOut>
+    </div>
+  );
+}

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx

@@ -31,6 +31,8 @@ import { useFirehoseFlow } from './use_firehose_flow';
 import { VisualizeData } from './visualize_data';
 import { ObservabilityOnboardingAppServices } from '../../..';
 import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger';
+import { ExistingDataCallout } from './existing_data_callout';
+import { usePopulatedAWSIndexList } from './use_populated_aws_index_list';
 
 const OPTIONS = [
   {
@@ -61,6 +63,9 @@ export function FirehosePanel() {
     },
   } = useKibana<ObservabilityOnboardingAppServices>();
   const { data, status, error, refetch } = useFirehoseFlow();
+  const { data: populatedAWSIndexList } = usePopulatedAWSIndexList();
+
+  const hasExistingData = Array.isArray(populatedAWSIndexList) && populatedAWSIndexList.length > 0;
 
   const telemetryEventContext: OnboardingFlowEventContext = useMemo(
     () => ({
@@ -72,12 +77,13 @@ export function FirehosePanel() {
     [cloudServiceProvider, selectedOptionId]
   );
 
-  const isMonitoringData = useWindowBlurDataMonitoringTrigger({
-    isActive: status === FETCH_STATUS.SUCCESS,
-    onboardingFlowType: 'firehose',
-    onboardingId: data?.onboardingId,
-    telemetryEventContext,
-  });
+  const isMonitoringData =
+    useWindowBlurDataMonitoringTrigger({
+      isActive: status === FETCH_STATUS.SUCCESS,
+      onboardingFlowType: 'firehose',
+      onboardingId: data?.onboardingId,
+      telemetryEventContext,
+    }) || hasExistingData;
 
   const onOptionChange = useCallback((id: string) => {
     setSelectedOptionId(id as CreateStackOption);
@@ -193,13 +199,20 @@ export function FirehosePanel() {
         <VisualizeData
           selectedCreateStackOption={selectedOptionId}
           onboardingId={data.onboardingId}
+          hasExistingData={hasExistingData}
         />
       ),
     },
   ];
 
   return (
     <EuiPanel hasBorder paddingSize="xl">
+      {hasExistingData && (
+        <>
+          <ExistingDataCallout />
+          <EuiSpacer size="xl" />
+        </>
+      )}
       <EuiSteps steps={steps} />
       <FeedbackButtons flow="firehose" />
     </EuiPanel>

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useFetcher } from '../../../hooks/use_fetcher';
+import {
+  FIREHOSE_CLOUDFORMATION_STACK_NAME,
+  FIREHOSE_LOGS_STREAM_NAME,
+} from '../../../../common/aws_firehose';
+
+export function usePopulatedAWSIndexList() {
+  return useFetcher((callApi) => {
+    return callApi('GET /internal/observability_onboarding/firehose/has-data', {
+      params: {
+        query: {
+          logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
+          stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
+        },
+      },
+    });
+  }, []);
+}

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx

@@ -13,11 +13,7 @@ import { unionBy } from 'lodash';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { ObservabilityOnboardingAppServices } from '../../..';
-import {
-  FIREHOSE_CLOUDFORMATION_STACK_NAME,
-  FIREHOSE_LOGS_STREAM_NAME,
-} from '../../../../common/aws_firehose';
-import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
+import { FETCH_STATUS } from '../../../hooks/use_fetcher';
 import { AccordionWithIcon } from '../shared/accordion_with_icon';
 import { GetStartedPanel } from '../shared/get_started_panel';
 import {
@@ -28,34 +24,25 @@ import { AutoRefreshCallout } from './auto_refresh_callout';
 import { ProgressCallout } from './progress_callout';
 import { HAS_DATA_FETCH_INTERVAL } from './utils';
 import { CreateStackOption } from './types';
+import { usePopulatedAWSIndexList } from './use_populated_aws_index_list';
 
 const REQUEST_PENDING_STATUS_LIST = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED];
 
 interface Props {
   onboardingId: string;
   selectedCreateStackOption: CreateStackOption;
+  hasExistingData: boolean;
 }
 
-export function VisualizeData({ onboardingId, selectedCreateStackOption }: Props) {
+export function VisualizeData({ onboardingId, selectedCreateStackOption, hasExistingData }: Props) {
   const accordionId = useGeneratedHtmlId({ prefix: 'accordion' });
   const [orderedVisibleAWSServiceList, setOrderedVisibleAWSServiceList] = useState<
     AWSServiceGetStartedConfig[]
   >([]);
-  const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState<boolean>(true);
-  const {
-    data: populatedAWSIndexList,
-    status,
-    refetch,
-  } = useFetcher((callApi) => {
-    return callApi('GET /internal/observability_onboarding/firehose/has-data', {
-      params: {
-        query: {
-          logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
-          stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
-        },
-      },
-    });
-  }, []);
+  const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState<boolean>(
+    !hasExistingData
+  );
+  const { data: populatedAWSIndexList, status, refetch } = usePopulatedAWSIndexList();
   const {
     services: {
       notifications,

x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts

@@ -83,5 +83,39 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       // Checking that an AWS service item is visible after data is detected
       await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`);
     });
+
+    it('shows the existing data callout and detected AWS services when data was ingested previously', async () => {
+      const DATASET = 'aws.vpcflow';
+      const AWS_SERVICE_ID = 'vpc-flow';
+      await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
+      const copiedCommand = await browser.getClipboardValue();
+      const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
+
+      await testSubjects.missingOrFail('observabilityOnboardingFirehosePanelExistingDataCallout');
+
+      expect(logsStreamName).toBeDefined();
+
+      // Simulate Firehose stream ingesting log files
+      const to = new Date().toISOString();
+      const count = 1;
+      await synthtrace.index(
+        timerange(moment(to).subtract(count, 'minute'), moment(to))
+          .interval('1m')
+          .rate(1)
+          .generator((timestamp) => {
+            return log.create().dataset(DATASET).timestamp(timestamp).defaults({
+              'aws.kinesis.name': logsStreamName,
+            });
+          })
+      );
+
+      await browser.refresh();
+
+      // Checking that the existing data callout is visible after data is detected
+      await testSubjects.isDisplayed('observabilityOnboardingFirehosePanelExistingDataCallout');
+
+      // Checking that an AWS service item is visible after data is detected
+      await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`);
+    });
   });
 }