[Logs onboarding] Generating yaml configuration in server

x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx

@@ -15,16 +15,15 @@ import {
   EuiButtonGroup,
   EuiCodeBlock,
   EuiSteps,
+  EuiSkeletonRectangle,
 } from '@elastic/eui';
 import {
   StepPanel,
   StepPanelContent,
   StepPanelFooter,
 } from '../../../shared/step_panel';
 import { useWizard } from '.';
-import { useFetcher } from '../../../../hooks/use_fetcher';
-// import { useKibana } from '@kbn/kibana-react-plugin/public';
-// import type { CloudSetup } from '@kbn/cloud-plugin/public';
+import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher';
 
 type ElasticAgentPlatform = 'linux-tar' | 'macos' | 'windows';
 export function InstallElasticAgent() {
@@ -41,48 +40,47 @@ export function InstallElasticAgent() {
     goBack();
   }
 
-  const { data: installShipperSetup } = useFetcher((callApi) => {
-    if (CurrentStep !== InstallElasticAgent) {
-      return;
-    }
-    return callApi(
-      'POST /internal/observability_onboarding/custom_logs/install_shipper_setup',
-      {
-        params: {
-          body: {
-            name: wizardState.datasetName,
-            state: {
-              datasetName: wizardState.datasetName,
-              customConfigurations: wizardState.customConfigurations,
-              logFilePaths: wizardState.logFilePaths,
+  const { data: installShipperSetup, status: installShipperSetupStatus } =
+    useFetcher((callApi) => {
+      if (CurrentStep !== InstallElasticAgent) {
+        return;
+      }
+
+      return callApi(
+        'POST /internal/observability_onboarding/custom_logs/install_shipper_setup',
+        {
+          params: {
+            body: {
+              name: wizardState.datasetName,
+              state: {
+                datasetName: wizardState.datasetName,
+                namespace: wizardState.namespace,
+                customConfigurations: wizardState.customConfigurations,
+                logFilePaths: wizardState.logFilePaths,
+              },
             },
           },
-        },
+        }
+      );
+    }, []);
+
+  const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher(
+    (callApi) => {
+      if (installShipperSetup?.apiKeyId) {
+        return callApi(
+          'GET /api/observability_onboarding/elastic_agent/config',
+          {
+            headers: {
+              authorization: `ApiKey ${installShipperSetup?.apiKeyEncoded}`,
+            },
+          }
+        );
       }
-    );
-  }, []);
+    },
+    [installShipperSetup?.apiKeyId, installShipperSetup?.apiKeyEncoded]
+  );
 
   const apiKeyEncoded = installShipperSetup?.apiKeyEncoded;
-  const esHost = installShipperSetup?.esHost;
-
-  const elasticAgentYaml = getElasticAgentYaml({
-    esHost,
-    apiKeyEncoded,
-    logfileId: 'custom-logs-abcdefgh',
-    logfileNamespace: 'default',
-    logfileStreams: [
-      ...wizardState.logFilePaths.map((path) => ({
-        id: `logs-onboarding-${wizardState.datasetName}`,
-        dataset: wizardState.datasetName,
-        path,
-      })),
-      // {
-      //   id: 'logs-onboarding-demo-app',
-      //   dataset: 'demo1',
-      //   path: '/home/oliver/github/logs-onboarding-demo-app/combined.log',
-      // },
-    ],
-  });
 
   return (
     <StepPanel title="Install shipper to collect data">
@@ -101,7 +99,10 @@ export function InstallElasticAgent() {
           steps={[
             {
               title: 'Install the Elastic Agent',
-              status: 'current',
+              status:
+                installShipperSetupStatus === FETCH_STATUS.LOADING
+                  ? 'loading'
+                  : 'current',
               children: (
                 <>
                   <EuiText color="subdued">
@@ -128,20 +129,34 @@ export function InstallElasticAgent() {
                     }
                   />
                   <EuiSpacer size="m" />
-                  <EuiCodeBlock language="bash" isCopyable>
-                    {getInstallShipperCommand({
-                      elasticAgentPlatform,
-                      apiKeyEncoded,
-                      apiEndpoint: installShipperSetup?.apiEndpoint,
-                      scriptDownloadUrl: installShipperSetup?.scriptDownloadUrl,
-                    })}
-                  </EuiCodeBlock>
+                  <EuiSkeletonRectangle
+                    isLoading={
+                      installShipperSetupStatus === FETCH_STATUS.LOADING
+                    }
+                    contentAriaLabel="Command to install elastic agent"
+                    width="100%"
+                    height={80}
+                    borderRadius="s"
+                  >
+                    <EuiCodeBlock language="bash" isCopyable>
+                      {getInstallShipperCommand({
+                        elasticAgentPlatform,
+                        apiKeyEncoded,
+                        apiEndpoint: installShipperSetup?.apiEndpoint,
+                        scriptDownloadUrl:
+                          installShipperSetup?.scriptDownloadUrl,
+                      })}
+                    </EuiCodeBlock>
+                  </EuiSkeletonRectangle>
                 </>
               ),
             },
             {
               title: 'Configure the agent',
-              status: 'incomplete',
+              status:
+                yamlConfigStatus === FETCH_STATUS.LOADING
+                  ? 'loading'
+                  : 'incomplete',
               children: (
                 <>
                   <EuiText color="subdued">
@@ -151,15 +166,23 @@ export function InstallElasticAgent() {
                     </p>
                   </EuiText>
                   <EuiSpacer size="m" />
-                  <EuiCodeBlock language="yaml" isCopyable>
-                    {elasticAgentYaml}
-                  </EuiCodeBlock>
+                  <EuiSkeletonRectangle
+                    isLoading={yamlConfigStatus === FETCH_STATUS.LOADING}
+                    contentAriaLabel="Elastic agent yaml configuration"
+                    width="100%"
+                    height={300}
+                    borderRadius="s"
+                  >
+                    <EuiCodeBlock language="yaml" isCopyable>
+                      {yamlConfig}
+                    </EuiCodeBlock>
+                  </EuiSkeletonRectangle>
                   <EuiSpacer size="m" />
                   <EuiButton
                     iconType="download"
                     color="primary"
                     href={`data:application/yaml;base64,${Buffer.from(
-                      elasticAgentYaml,
+                      yamlConfig,
                       'utf8'
                     ).toString('base64')}`}
                     download="elastic-agent.yml"
@@ -220,42 +243,3 @@ function oneLine(parts: TemplateStringsArray, ...args: string[]) {
   const str = flatten(zip(parts, args)).join('');
   return str.replace(/\s+/g, ' ').trim();
 }
-
-function getElasticAgentYaml({
-  esHost = '$ES_HOST',
-  apiKeyEncoded = '$API_KEY',
-  logfileId,
-  logfileNamespace,
-  logfileStreams,
-}: {
-  esHost: string | undefined;
-  apiKeyEncoded: string | undefined;
-  logfileId: string;
-  logfileNamespace: string;
-  logfileStreams: Array<{ id: string; dataset: string; path: string }>;
-}) {
-  const apiKeyBeats = Buffer.from(apiKeyEncoded, 'base64').toString('utf8');
-  return `
-outputs:
-  default:
-    type: elasticsearch
-    hosts:
-      - '${esHost}'
-    api_key: ${apiKeyBeats}
-
-inputs:
-  - id: ${logfileId}
-    type: logfile
-    data_stream:
-      namespace: ${logfileNamespace}
-    streams:
-${logfileStreams
-  .map(
-    ({ id, dataset, path }) => `      - id: ${id}
-        data_stream:
-          dataset: ${dataset}
-        paths:
-          - ${path}`
-  )
-  .join('\n')}`.trim();
-}

x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts

@@ -68,11 +68,13 @@ const createApiKeyRoute = createObservabilityOnboardingServerRoute({
     });
 
     const savedObjectsClient = coreStart.savedObjects.getScopedClient(request);
-    saveObservabilityOnboardingState({
+
+    await saveObservabilityOnboardingState({
       savedObjectsClient,
       apiKeyId,
       observabilityOnboardingState: { state } as ObservabilityOnboardingState,
     });
+
     return {
       apiKeyId, // key the status off this
       apiKeyEncoded,

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { dump, load } from 'js-yaml';
+
+export const generateYml = ({
+  datasetName = '',
+  namespace = '',
+  customConfigurations,
+  logFilePaths = [],
+  apiKey,
+  esHost,
+  logfileId,
+}: {
+  datasetName?: string;
+  namespace?: string;
+  customConfigurations?: string;
+  logFilePaths?: string[];
+  apiKey: string;
+  esHost: string[];
+  logfileId: string;
+}) => {
+  const customConfigYaml = load(customConfigurations ?? '');
+
+  return dump({
+    ...{
+      outputs: {
+        default: {
+          type: 'elasticsearch',
+          hosts: esHost,
+          api_key: apiKey,
+        },
+      },
+      inputs: [
+        {
+          id: logfileId,
+          type: 'logfile',
+          data_stream: {
+            namespace,
+          },
+          streams: [
+            {
+              id: `logs-onboarding-${datasetName}`,
+              data_stream: {
+                dataset: datasetName,
+              },
+              paths: logFilePaths,
+            },
+          ],
+        },
+      ],
+    },
+    ...customConfigYaml,
+  });
+};

x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import { KibanaRequest } from '@kbn/core-http-server';
+import { HTTPAuthorizationHeader } from '@kbn/security-plugin/server';
+import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route';
+import { findLatestObservabilityOnboardingState } from '../custom_logs/find_latest_observability_onboarding_state';
+import { getESHosts } from '../custom_logs/get_es_hosts';
+import { generateYml } from './generate_yml';
+
+const getAuthenticationAPIKey = (request: KibanaRequest) => {
+  const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request);
+  if (authorizationHeader && authorizationHeader.credentials) {
+    const apiKey = Buffer.from(authorizationHeader.credentials, 'base64')
+      .toString()
+      .split(':');
+    return {
+      apiKeyId: apiKey[0],
+      apiKey: apiKey[1],
+    };
+  }
+  throw new Error('Authorization header is missing');
+};
+
+const generateConfig = createObservabilityOnboardingServerRoute({
+  endpoint: 'GET /api/observability_onboarding/elastic_agent/config',
+  options: { tags: [] },
+  async handler(resources): Promise<string> {
+    const { core, plugins, request } = resources;
+    const { apiKeyId, apiKey } = getAuthenticationAPIKey(request);
+
+    const coreStart = await core.start();
+    const savedObjectsClient =
+      coreStart.savedObjects.createInternalRepository();
+
+    const esHost = getESHosts({
+      cloudSetup: plugins.cloud.setup,
+      esClient: coreStart.elasticsearch.client.asInternalUser as Client,
+    });
+
+    const savedState = await findLatestObservabilityOnboardingState({
+      savedObjectsClient,
+    });
+
+    const yaml = generateYml({
+      datasetName: savedState?.state.datasetName,
+      customConfigurations: savedState?.state.customConfigurations,
+      logFilePaths: savedState?.state.logFilePaths,
+      namespace: savedState?.state.namespace,
+      apiKey: `${apiKeyId}:${apiKey}`,
+      esHost,
+      logfileId: `custom-logs-${Date.now()}`,
+    });
+
+    return yaml;
+  },
+});
+
+export const elasticAgentRouteRepository = {
+  ...generateConfig,
+};

x-pack/plugins/observability_onboarding/server/routes/index.ts

@@ -10,11 +10,13 @@ import type {
 } from '@kbn/server-route-repository';
 import { statusRouteRepository } from './status/route';
 import { customLogsRouteRepository } from './custom_logs/route';
+import { elasticAgentRouteRepository } from './elastic_agent/route';
 
 function getTypedObservabilityOnboardingServerRouteRepository() {
   const repository = {
     ...statusRouteRepository,
     ...customLogsRouteRepository,
+    ...elasticAgentRouteRepository,
   };
 
   return repository;

x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts

@@ -14,7 +14,8 @@ export interface ObservabilityOnboardingState {
   state: {
     datasetName: string;
     customConfigurations: string;
-    logFilePaths: string;
+    logFilePaths: string[];
+    namespace: string;
     progress: Record<string, string>;
   };
 }