From f3948072ffbd6e76b99f61695f690ebf17c52b45 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Thu, 18 May 2023 16:36:15 +0200 Subject: [PATCH 1/2] Generating yaml configuration in server --- .../wizard/install_elastic_agent.tsx | 164 ++++++++---------- .../find_observability_onboarding_state.ts | 8 +- .../server/routes/custom_logs/route.ts | 4 +- .../routes/elastic_agent/generate_yml.ts | 59 +++++++ .../server/routes/elastic_agent/route.ts | 75 ++++++++ .../server/routes/index.ts | 2 + 6 files changed, 217 insertions(+), 95 deletions(-) create mode 100644 x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts create mode 100644 x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts diff --git a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx index 0609f1d825f8c..1371706a80796 100644 --- a/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx +++ b/x-pack/plugins/observability_onboarding/public/components/app/custom_logs/wizard/install_elastic_agent.tsx @@ -15,6 +15,7 @@ import { EuiButtonGroup, EuiCodeBlock, EuiSteps, + EuiSkeletonRectangle, } from '@elastic/eui'; import { StepPanel, @@ -22,9 +23,7 @@ import { StepPanelFooter, } from '../../../shared/step_panel'; import { useWizard } from '.'; -import { useFetcher } from '../../../../hooks/use_fetcher'; -// import { useKibana } from '@kbn/kibana-react-plugin/public'; -// import type { CloudSetup } from '@kbn/cloud-plugin/public'; +import { FETCH_STATUS, useFetcher } from '../../../../hooks/use_fetcher'; type ElasticAgentPlatform = 'linux-tar' | 'macos' | 'windows'; export function InstallElasticAgent() { @@ -41,45 +40,43 @@ export function InstallElasticAgent() { goBack(); } - const { data: installShipperSetup } = useFetcher((callApi) => { - return callApi( - 'POST /internal/observability_onboarding/custom_logs/install_shipper_setup', - { - params: { - body: { - name: wizardState.datasetName, - state: { - datasetName: wizardState.datasetName, - customConfigurations: wizardState.customConfigurations, - logFilePaths: wizardState.logFilePaths, + const { data: installShipperSetup, status: installShipperSetupStatus } = + useFetcher((callApi) => { + return callApi( + 'POST /internal/observability_onboarding/custom_logs/install_shipper_setup', + { + params: { + body: { + name: wizardState.datasetName, + state: { + datasetName: wizardState.datasetName, + namespace: wizardState.namespace, + customConfigurations: wizardState.customConfigurations, + logFilePaths: wizardState.logFilePaths, + }, }, }, - }, + } + ); + }, []); + + const { data: yamlConfig = '', status: yamlConfigStatus } = useFetcher( + (callApi) => { + if (installShipperSetup?.apiKeyId) { + return callApi( + 'GET /api/observability_onboarding/elastic_agent/config', + { + headers: { + authorization: `ApiKey ${installShipperSetup?.apiKeyEncoded}`, + }, + } + ); } - ); - }, []); + }, + [installShipperSetup?.apiKeyId, installShipperSetup?.apiKeyEncoded] + ); const apiKeyEncoded = installShipperSetup?.apiKeyEncoded; - const esHost = installShipperSetup?.esHost; - - const elasticAgentYaml = getElasticAgentYaml({ - esHost, - apiKeyEncoded, - logfileId: 'custom-logs-abcdefgh', - logfileNamespace: 'default', - logfileStreams: [ - ...wizardState.logFilePaths.map((path) => ({ - id: `logs-onboarding-${wizardState.datasetName}`, - dataset: wizardState.datasetName, - path, - })), - // { - // id: 'logs-onboarding-demo-app', - // dataset: 'demo1', - // path: '/home/oliver/github/logs-onboarding-demo-app/combined.log', - // }, - ], - }); return ( @@ -98,7 +95,10 @@ export function InstallElasticAgent() { steps={[ { title: 'Install the Elastic Agent', - status: 'current', + status: + installShipperSetupStatus === FETCH_STATUS.LOADING + ? 'loading' + : 'current', children: ( <> @@ -125,20 +125,35 @@ export function InstallElasticAgent() { } /> - - {getInstallShipperCommand({ - elasticAgentPlatform, - apiKeyEncoded, - statusApiEndpoint: installShipperSetup?.statusApiEndpoint, - scriptDownloadUrl: installShipperSetup?.scriptDownloadUrl, - })} - + + + {getInstallShipperCommand({ + elasticAgentPlatform, + apiKeyEncoded, + statusApiEndpoint: + installShipperSetup?.statusApiEndpoint, + scriptDownloadUrl: + installShipperSetup?.scriptDownloadUrl, + })} + + ), }, { title: 'Configure the agent', - status: 'incomplete', + status: + yamlConfigStatus === FETCH_STATUS.LOADING + ? 'loading' + : 'incomplete', children: ( <> @@ -148,15 +163,23 @@ export function InstallElasticAgent() {

- - {elasticAgentYaml} - + + + {yamlConfig} + + ; -}) { - const apiKeyBeats = Buffer.from(apiKeyEncoded, 'base64').toString('utf8'); - return ` -outputs: - default: - type: elasticsearch - hosts: - - '${esHost}' - api_key: ${apiKeyBeats} - -inputs: - - id: ${logfileId} - type: logfile - data_stream: - namespace: ${logfileNamespace} - streams: -${logfileStreams - .map( - ({ id, dataset, path }) => ` - id: ${id} - data_stream: - dataset: ${dataset} - paths: - - ${path}` - ) - .join('\n')}`.trim(); -} diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_observability_onboarding_state.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_observability_onboarding_state.ts index 1b6880b821fa0..3427f3f992320 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_observability_onboarding_state.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/find_observability_onboarding_state.ts @@ -18,18 +18,18 @@ export async function findObservabilityOnboardingState({ }: { savedObjectsClient: SavedObjectsClientContract; apiKeyId: string; -}): Promise { +}): Promise { const result = await savedObjectsClient.find({ type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE, page: 1, perPage: 1, filter: `${OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE}.attributes.apiKeyId: "${apiKeyId}"`, }); - return result.saved_objects.map( - ({ id, attributes, updated_at: upatedAt }) => ({ + return ( + result.saved_objects.map(({ id, attributes, updated_at: upatedAt }) => ({ id, updatedAt: upatedAt ? Date.parse(upatedAt) : 0, ...attributes, - }) + }))?.[0] ?? {} ); } diff --git a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts index e68561b488d69..f1b24bd5a9397 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/custom_logs/route.ts @@ -61,10 +61,12 @@ const createApiKeyRoute = createObservabilityOnboardingServerRoute({ }); const savedObjectsClient = coreStart.savedObjects.getScopedClient(request); - saveObservabilityOnboardingState({ + + await saveObservabilityOnboardingState({ savedObjectsClient, observabilityOnboardingState: { apiKeyId, state }, }); + return { apiKeyId, // key the status off this apiKeyEncoded, diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts new file mode 100644 index 0000000000000..fb27500639fc1 --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/generate_yml.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { dump, load } from 'js-yaml'; + +export const generateYml = ({ + datasetName = '', + namespace = '', + customConfigurations, + logFilePaths = [], + apiKey, + esHost, + logfileId, +}: { + datasetName?: string; + namespace?: string; + customConfigurations?: string; + logFilePaths?: string[]; + apiKey: string; + esHost: string[]; + logfileId: string; +}) => { + const customConfigYaml = load(customConfigurations ?? ''); + + return dump({ + ...{ + outputs: { + default: { + type: 'elasticsearch', + hosts: esHost, + api_key: apiKey, + }, + }, + inputs: [ + { + id: logfileId, + type: 'logfile', + data_stream: { + namespace, + }, + streams: [ + { + id: `logs-onboarding-${datasetName}`, + data_stream: { + dataset: datasetName, + }, + paths: logFilePaths, + }, + ], + }, + ], + }, + ...customConfigYaml, + }); +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts new file mode 100644 index 0000000000000..335031d9ef32a --- /dev/null +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import { KibanaRequest } from '@kbn/core-http-server'; +import { HTTPAuthorizationHeader } from '@kbn/security-plugin/server'; +import { ObservabilityOnboardingState } from '../../saved_objects/observability_onboarding_status'; +import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; +import { findObservabilityOnboardingState } from '../custom_logs/find_observability_onboarding_state'; +import { getESHosts } from '../custom_logs/get_es_hosts'; +import { generateYml } from './generate_yml'; + +const getAuthenticationAPIKey = (request: KibanaRequest) => { + const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request); + if (authorizationHeader && authorizationHeader.credentials) { + const apiKey = Buffer.from(authorizationHeader.credentials, 'base64') + .toString() + .split(':'); + return { + apiKeyId: apiKey[0], + apiKey: apiKey[1], + }; + } + throw new Error('Authorization header is missing'); +}; + +type SavedState = ObservabilityOnboardingState['state'] & { + datasetName?: string; + customConfigurations?: string; + logFilePaths?: string[]; + namespace?: string; +}; + +const generateConfig = createObservabilityOnboardingServerRoute({ + endpoint: 'GET /api/observability_onboarding/elastic_agent/config', + options: { tags: [] }, + async handler(resources): Promise { + const { core, plugins, request } = resources; + const { apiKeyId, apiKey } = getAuthenticationAPIKey(request); + + const coreStart = await core.start(); + const savedObjectsClient = coreStart.savedObjects.getScopedClient(request); + + const esHost = getESHosts({ + cloudSetup: plugins.cloud.setup, + esClient: coreStart.elasticsearch.client.asInternalUser as Client, + }); + + const { state }: { state: SavedState } = + await findObservabilityOnboardingState({ + savedObjectsClient, + apiKeyId, + }); + + const yaml = generateYml({ + datasetName: state?.datasetName, + customConfigurations: state?.customConfigurations, + logFilePaths: state?.logFilePaths, + namespace: state?.namespace, + apiKey: `${apiKeyId}:${apiKey}`, + esHost, + logfileId: `custom-logs-${Date.now()}`, + }); + + return yaml; + }, +}); + +export const elasticAgentRouteRepository = { + ...generateConfig, +}; diff --git a/x-pack/plugins/observability_onboarding/server/routes/index.ts b/x-pack/plugins/observability_onboarding/server/routes/index.ts index e68bc3e075993..4833f44a2936c 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/index.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/index.ts @@ -10,11 +10,13 @@ import type { } from '@kbn/server-route-repository'; import { statusRouteRepository } from './status/route'; import { customLogsRouteRepository } from './custom_logs/route'; +import { elasticAgentRouteRepository } from './elastic_agent/route'; function getTypedObservabilityOnboardingServerRouteRepository() { const repository = { ...statusRouteRepository, ...customLogsRouteRepository, + ...elasticAgentRouteRepository, }; return repository; From 2dce8801f890014f813dc4acad5d4c9893c96faf Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 22 May 2023 14:51:59 +0200 Subject: [PATCH 2/2] Using internal savedObjects client --- .../server/routes/elastic_agent/route.ts | 29 +++++++------------ .../observability_onboarding_status.ts | 3 +- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts index 335031d9ef32a..da71ec1f7f3c5 100644 --- a/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts +++ b/x-pack/plugins/observability_onboarding/server/routes/elastic_agent/route.ts @@ -8,9 +8,8 @@ import type { Client } from '@elastic/elasticsearch'; import { KibanaRequest } from '@kbn/core-http-server'; import { HTTPAuthorizationHeader } from '@kbn/security-plugin/server'; -import { ObservabilityOnboardingState } from '../../saved_objects/observability_onboarding_status'; import { createObservabilityOnboardingServerRoute } from '../create_observability_onboarding_server_route'; -import { findObservabilityOnboardingState } from '../custom_logs/find_observability_onboarding_state'; +import { findLatestObservabilityOnboardingState } from '../custom_logs/find_latest_observability_onboarding_state'; import { getESHosts } from '../custom_logs/get_es_hosts'; import { generateYml } from './generate_yml'; @@ -28,13 +27,6 @@ const getAuthenticationAPIKey = (request: KibanaRequest) => { throw new Error('Authorization header is missing'); }; -type SavedState = ObservabilityOnboardingState['state'] & { - datasetName?: string; - customConfigurations?: string; - logFilePaths?: string[]; - namespace?: string; -}; - const generateConfig = createObservabilityOnboardingServerRoute({ endpoint: 'GET /api/observability_onboarding/elastic_agent/config', options: { tags: [] }, @@ -43,24 +35,23 @@ const generateConfig = createObservabilityOnboardingServerRoute({ const { apiKeyId, apiKey } = getAuthenticationAPIKey(request); const coreStart = await core.start(); - const savedObjectsClient = coreStart.savedObjects.getScopedClient(request); + const savedObjectsClient = + coreStart.savedObjects.createInternalRepository(); const esHost = getESHosts({ cloudSetup: plugins.cloud.setup, esClient: coreStart.elasticsearch.client.asInternalUser as Client, }); - const { state }: { state: SavedState } = - await findObservabilityOnboardingState({ - savedObjectsClient, - apiKeyId, - }); + const savedState = await findLatestObservabilityOnboardingState({ + savedObjectsClient, + }); const yaml = generateYml({ - datasetName: state?.datasetName, - customConfigurations: state?.customConfigurations, - logFilePaths: state?.logFilePaths, - namespace: state?.namespace, + datasetName: savedState?.state.datasetName, + customConfigurations: savedState?.state.customConfigurations, + logFilePaths: savedState?.state.logFilePaths, + namespace: savedState?.state.namespace, apiKey: `${apiKeyId}:${apiKey}`, esHost, logfileId: `custom-logs-${Date.now()}`, diff --git a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts index 7fa558dc20df3..40267f2318625 100644 --- a/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts +++ b/x-pack/plugins/observability_onboarding/server/saved_objects/observability_onboarding_status.ts @@ -14,7 +14,8 @@ export interface ObservabilityOnboardingState { state: { datasetName: string; customConfigurations: string; - logFilePaths: string; + logFilePaths: string[]; + namespace: string; progress: Record; }; }