From 98dd709a33679fe18851122132e990059023ae0b Mon Sep 17 00:00:00 2001 From: Joseph Crail Date: Mon, 5 Jun 2023 14:35:57 -0700 Subject: [PATCH] [Profiling] Manage indices via Elasticsearch (#157949) ## Summary This PR sets up the profiling indices via Elasticsearch instead of using the single-click installation in Kibana. ### Notable changes * Set Elasticsearch flag (`xpack.profiling.templates.enabled`) to `true` so that indices are bootstrapped in Elasticsearch * Remove component templates and mappings (handled in Elasticsearch plugin now) * Remove six steps from Kibana's single-click installation (handled in Elasticsearch plugin now) * Use new status API in Elasticsearch to check if resources (templates, indices, etc) are ready * Invoke creation of collector and symbolizer package policies in parallel to updating APM policy * Refactor initialization and verification steps so that we can later give the user more granular insight into the status of steps --- x-pack/plugins/profiling/common/setup.ts | 108 ++++++++++ .../plugins/profiling/common/stack_traces.ts | 12 ++ .../fixtures/es_resources_data_false.json | 46 +--- .../fixtures/es_resources_setup_false.json | 46 +--- .../profiling/e2e/cypress_test_runner.ts | 3 + .../server/lib/setup/{mappings => }/README.md | 0 .../profiling/server/lib/setup/apm_package.ts | 46 ++++ .../server/lib/setup/cluster_settings.ts | 61 ++++++ .../server/lib/setup/fleet_policies.ts | 204 ++++++++++++++++++ .../lib/setup/{steps => }/get_apm_policy.ts | 2 +- .../lib/setup/get_setup_instructions.ts | 2 +- .../server/lib/setup/has_profiling_data.ts | 14 +- .../server/lib/setup/security_role.ts | 37 ++++ ...catch_resource_already_exists_exception.ts | 16 -- .../component_template_profiling_events.json | 73 ------- ...ponent_template_profiling_executables.json | 35 --- .../component_template_profiling_ilm.json | 9 - ...ponent_template_profiling_stackframes.json | 45 ---- ...ponent_template_profiling_stacktraces.json | 32 --- .../lib/setup/steps/get_apm_package_step.ts | 44 ---- .../setup/steps/get_cluster_settings_step.ts | 32 --- .../steps/get_component_templates_step.ts | 86 -------- .../steps/get_create_events_data_streams.ts | 63 ------ .../setup/steps/get_create_indices_step.ts | 177 --------------- .../lib/setup/steps/get_fleet_policy_step.ts | 159 -------------- .../server/lib/setup/steps/get_ilm_step.ts | 43 ---- .../setup/steps/get_index_templates_step.ts | 80 ------- .../setup/steps/get_is_cloud_enabled_step.ts | 24 --- .../lib/setup/steps/get_security_step.ts | 49 ----- .../server/lib/setup/steps/ilm_profiling.json | 35 --- .../profiling/server/lib/setup/steps/index.ts | 35 --- .../profiling_returnpads_private.json | 49 ----- .../mappings/profiling_sq_executables.json | 34 --- .../mappings/profiling_sq_leafframes.json | 34 --- .../steps/mappings/profiling_symbols.json | 93 -------- .../mappings/profiling_symbols_private.json | 93 -------- .../profiling/server/lib/setup/types.ts | 8 +- .../plugins/profiling/server/routes/setup.ts | 184 +++++++++------- .../profiling/server/routes/topn.test.ts | 16 +- .../utils/create_profiling_es_client.ts | 50 +++-- x-pack/plugins/profiling/tsconfig.json | 2 +- 41 files changed, 631 insertions(+), 1550 deletions(-) create mode 100644 x-pack/plugins/profiling/common/setup.ts rename x-pack/plugins/profiling/server/lib/setup/{mappings => }/README.md (100%) create mode 100644 x-pack/plugins/profiling/server/lib/setup/apm_package.ts create mode 100644 x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts create mode 100644 x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts rename x-pack/plugins/profiling/server/lib/setup/{steps => }/get_apm_policy.ts (88%) create mode 100644 x-pack/plugins/profiling/server/lib/setup/security_role.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/catch_resource_already_exists_exception.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_events.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_executables.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_ilm.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stackframes.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stacktraces.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_apm_package_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_cluster_settings_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_component_templates_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_create_events_data_streams.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_fleet_policy_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_ilm_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_index_templates_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_is_cloud_enabled_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/get_security_step.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/ilm_profiling.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/index.ts delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json delete mode 100644 x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json diff --git a/x-pack/plugins/profiling/common/setup.ts b/x-pack/plugins/profiling/common/setup.ts new file mode 100644 index 0000000000000..4a05162ab537e --- /dev/null +++ b/x-pack/plugins/profiling/common/setup.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RecursivePartial } from '@kbn/apm-plugin/typings/common'; + +export interface SetupState { + cloud: { + available: boolean; + required: boolean; + }; + data: { + available: boolean; + }; + packages: { + installed: boolean; + }; + permissions: { + configured: boolean; + }; + policies: { + apm: { + installed: boolean; + }; + collector: { + installed: boolean; + }; + symbolizer: { + installed: boolean; + }; + }; + resource_management: { + enabled: boolean; + }; + resources: { + created: boolean; + }; + settings: { + configured: boolean; + }; +} + +export type PartialSetupState = RecursivePartial; + +export function createDefaultSetupState(): SetupState { + return { + cloud: { + available: false, + required: true, + }, + data: { + available: false, + }, + packages: { + installed: false, + }, + permissions: { + configured: false, + }, + policies: { + apm: { + installed: false, + }, + collector: { + installed: false, + }, + symbolizer: { + installed: false, + }, + }, + resource_management: { + enabled: false, + }, + resources: { + created: false, + }, + settings: { + configured: false, + }, + }; +} + +export function areResourcesSetup(state: SetupState): boolean { + return ( + state.resource_management.enabled && + state.resources.created && + state.packages.installed && + state.permissions.configured && + state.policies.apm.installed && + state.policies.collector.installed && + state.policies.symbolizer.installed && + state.settings.configured + ); +} + +function mergeRecursivePartial(base: T, partial: RecursivePartial): T { + return { ...base, ...partial }; +} + +export function mergePartialSetupStates( + base: SetupState, + partials: PartialSetupState[] +): SetupState { + return partials.reduce(mergeRecursivePartial, base); +} diff --git a/x-pack/plugins/profiling/common/stack_traces.ts b/x-pack/plugins/profiling/common/stack_traces.ts index 40c070e2887f1..5384e60639b9d 100644 --- a/x-pack/plugins/profiling/common/stack_traces.ts +++ b/x-pack/plugins/profiling/common/stack_traces.ts @@ -7,6 +7,18 @@ import { ProfilingESField } from './elasticsearch'; +export interface ProfilingStatusResponse { + profiling: { + enabled: boolean; + }; + resource_management: { + enabled: boolean; + }; + resources: { + created: boolean; + }; +} + interface ProfilingEvents { [key: string]: number; } diff --git a/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_data_false.json b/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_data_false.json index c6f988a235f1a..ee5725bcf460d 100644 --- a/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_data_false.json +++ b/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_data_false.json @@ -1,46 +1,4 @@ { "has_setup": true, - "has_data": false, - "steps": [ - { - "name": "is_cloud", - "completed": false - }, - { - "name": "apm_package", - "completed": false - }, - { - "name": "cluster_settings", - "completed": false - }, - { - "name": "ilm", - "completed": false - }, - { - "name": "component_templates", - "completed": false - }, - { - "name": "index_templates", - "completed": false - }, - { - "name": "create_events_data_streams", - "completed": false - }, - { - "name": "create_indices", - "completed": false - }, - { - "name": "security", - "completed": false - }, - { - "name": "fleet_policy", - "completed": false - } - ] -} \ No newline at end of file + "has_data": false +} diff --git a/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_setup_false.json b/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_setup_false.json index c07f2a6af31fd..e86b5b846d908 100644 --- a/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_setup_false.json +++ b/x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_setup_false.json @@ -1,46 +1,4 @@ { "has_setup": false, - "has_data": false, - "steps": [ - { - "name": "is_cloud", - "completed": false - }, - { - "name": "apm_package", - "completed": false - }, - { - "name": "cluster_settings", - "completed": false - }, - { - "name": "ilm", - "completed": false - }, - { - "name": "component_templates", - "completed": false - }, - { - "name": "index_templates", - "completed": false - }, - { - "name": "create_events_data_streams", - "completed": false - }, - { - "name": "create_indices", - "completed": false - }, - { - "name": "security", - "completed": false - }, - { - "name": "fleet_policy", - "completed": false - } - ] -} \ No newline at end of file + "has_data": false +} diff --git a/x-pack/plugins/profiling/e2e/cypress_test_runner.ts b/x-pack/plugins/profiling/e2e/cypress_test_runner.ts index d1ced10770762..3faeac0ab8401 100644 --- a/x-pack/plugins/profiling/e2e/cypress_test_runner.ts +++ b/x-pack/plugins/profiling/e2e/cypress_test_runner.ts @@ -41,6 +41,9 @@ export async function cypressTestRunner({ auth: `${username}:${password}`, }); + // Ensure Fleet setup is complete + await axios.post(`${kibanaUrlWithAuth}/api/fleet/setup`, {}, { headers: { 'kbn-xsrf': true } }); + const profilingResources = await axios.get<{ has_setup: boolean; has_data: boolean }>( `${kibanaUrlWithAuth}/api/profiling/v1/setup/es_resources`, { headers: { 'kbn-xsrf': true } } diff --git a/x-pack/plugins/profiling/server/lib/setup/mappings/README.md b/x-pack/plugins/profiling/server/lib/setup/README.md similarity index 100% rename from x-pack/plugins/profiling/server/lib/setup/mappings/README.md rename to x-pack/plugins/profiling/server/lib/setup/README.md diff --git a/x-pack/plugins/profiling/server/lib/setup/apm_package.ts b/x-pack/plugins/profiling/server/lib/setup/apm_package.ts new file mode 100644 index 0000000000000..dd7b6de8dda51 --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/apm_package.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installPackage, getInstallation } from '@kbn/fleet-plugin/server/services/epm/packages'; +import { + fetchFindLatestPackageOrThrow, + pkgToPkgKey, +} from '@kbn/fleet-plugin/server/services/epm/registry'; +import { ProfilingSetupOptions } from './types'; +import { PartialSetupState } from '../../../common/setup'; + +export async function isApmPackageInstalled({ + soClient, +}: ProfilingSetupOptions): Promise { + const installation = await getInstallation({ + pkgName: 'apm', + savedObjectsClient: soClient, + }); + return { + packages: { + installed: !!installation, + }, + }; +} + +export async function installLatestApmPackage({ + client, + soClient, + spaceId, +}: ProfilingSetupOptions) { + const esClient = client.getEsClient(); + const { name, version } = await fetchFindLatestPackageOrThrow('apm'); + + await installPackage({ + installSource: 'registry', + esClient, + savedObjectsClient: soClient, + pkgkey: pkgToPkgKey({ name, version }), + spaceId, + force: true, + }); +} diff --git a/x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts b/x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts new file mode 100644 index 0000000000000..72d84b329ab2d --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ProfilingSetupOptions } from './types'; +import { PartialSetupState } from '../../../common/setup'; + +const MAX_BUCKETS = 150000; + +export async function validateMaximumBuckets({ + client, +}: ProfilingSetupOptions): Promise { + const settings = await client.getEsClient().cluster.getSettings({}); + const maxBuckets = settings.persistent.search?.max_buckets; + return { + settings: { + configured: maxBuckets === MAX_BUCKETS.toString(), + }, + }; +} + +export async function setMaximumBuckets({ client }: ProfilingSetupOptions) { + await client.getEsClient().cluster.putSettings({ + persistent: { + search: { + max_buckets: MAX_BUCKETS, + }, + }, + }); +} + +export async function validateResourceManagement({ + client, +}: ProfilingSetupOptions): Promise { + const statusResponse = await client.profilingStatus(); + return { + resource_management: { + enabled: statusResponse.resource_management.enabled, + }, + resources: { + created: statusResponse.resources.created, + }, + }; +} + +export async function enableResourceManagement({ client }: ProfilingSetupOptions) { + await client.getEsClient().cluster.putSettings({ + persistent: { + xpack: { + profiling: { + templates: { + enabled: true, + }, + }, + }, + }, + }); +} diff --git a/x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts b/x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts new file mode 100644 index 0000000000000..03faf606ec31d --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/fleet_policies.ts @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { merge } from 'lodash'; +import { ElasticsearchClient } from '@kbn/core/server'; +import { fetchFindLatestPackageOrThrow } from '@kbn/fleet-plugin/server/services/epm/registry'; +import { getApmPolicy } from './get_apm_policy'; +import { ProfilingSetupOptions } from './types'; +import { PartialSetupState } from '../../../common/setup'; + +async function createIngestAPIKey(esClient: ElasticsearchClient) { + const apiKeyResponse = await esClient.security.createApiKey({ + name: 'profiling-manager', + role_descriptors: { + profiling_manager: { + indices: [ + { + names: ['profiling-*', '.profiling-*'], + privileges: [ + 'read', + 'create_doc', + 'create', + 'write', + 'index', + 'create_index', + 'view_index_metadata', + 'manage', + ], + }, + ], + cluster: ['monitor'], + }, + }, + }); + + return atob(apiKeyResponse.encoded); +} + +export async function validateApmPolicy({ + soClient, + packagePolicyClient, +}: ProfilingSetupOptions): Promise { + const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient }); + return { + policies: { + apm: { + installed: !!(apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value?.profiling), + }, + }, + }; +} + +export async function updateApmPolicy({ + client, + soClient, + packagePolicyClient, +}: ProfilingSetupOptions) { + const esClient = client.getEsClient(); + const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient }); + + if (!apmPolicy) { + throw new Error(`Could not find APM policy`); + } + + const apmPolicyApiKey = await createIngestAPIKey(esClient); + + const profilingApmConfig = { + profiling: { + enabled: true, + elasticsearch: { + api_key: apmPolicyApiKey, + }, + metrics: { + elasticsearch: { + hosts: ['https://1b6c02856ea642a6ac14499b01507233.us-east-2.aws.elastic-cloud.com:443'], + api_key: 'woq-IoMBRbbiEbPugtWW:_iBmc1PdSout7sf5FCkEpA', + }, + }, + keyvalue_retention: { + // 60 days + age: '1440h', + // 200 Gib + size_bytes: 200 * 1024 * 1024 * 1024, + execution_interval: '12h', + }, + }, + }; + + const { + id, + revision, + updated_at: updateAt, + updated_by: updateBy, + ...apmPolicyModified + } = apmPolicy; + + apmPolicyModified.inputs = apmPolicy.inputs.map((input) => { + return input.type === 'apm' + ? merge({}, input, { config: { 'apm-server': { value: profilingApmConfig } } }) + : input; + }); + + await packagePolicyClient.update(soClient, esClient, id, apmPolicyModified); +} + +const CLOUD_AGENT_POLICY_ID = 'policy-elastic-agent-on-cloud'; +const COLLECTOR_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-collector'; +const SYMBOLIZER_PACKAGE_POLICY_NAME = 'elastic-universal-profiling-symbolizer'; + +export async function validateCollectorPackagePolicy({ + soClient, + packagePolicyClient, +}: ProfilingSetupOptions): Promise { + const packagePolicies = await packagePolicyClient.list(soClient, {}); + return { + policies: { + collector: { + installed: packagePolicies.items.some((pkg) => pkg.name === COLLECTOR_PACKAGE_POLICY_NAME), + }, + }, + }; +} + +export async function createCollectorPackagePolicy({ + client, + soClient, + packagePolicyClient, +}: ProfilingSetupOptions) { + const packageName = 'profiler_collector'; + const { version } = await fetchFindLatestPackageOrThrow(packageName, { prerelease: true }); + const packagePolicy = { + policy_id: CLOUD_AGENT_POLICY_ID, + enabled: true, + package: { + name: packageName, + title: 'Universal Profiling Collector', + version, + }, + name: COLLECTOR_PACKAGE_POLICY_NAME, + namespace: 'default', + inputs: [ + { + policy_template: 'universal_profiling_collector', + enabled: true, + streams: [], + type: 'pf-elastic-collector', + }, + ], + }; + const esClient = client.getEsClient(); + await packagePolicyClient.create(soClient, esClient, packagePolicy, { + force: true, + }); +} + +export async function validateSymbolizerPackagePolicy({ + soClient, + packagePolicyClient, +}: ProfilingSetupOptions): Promise { + const packagePolicies = await packagePolicyClient.list(soClient, {}); + return { + policies: { + symbolizer: { + installed: packagePolicies.items.some((pkg) => pkg.name === SYMBOLIZER_PACKAGE_POLICY_NAME), + }, + }, + }; +} + +export async function createSymbolizerPackagePolicy({ + client, + soClient, + packagePolicyClient, +}: ProfilingSetupOptions) { + const packageName = 'profiler_symbolizer'; + const { version } = await fetchFindLatestPackageOrThrow(packageName, { prerelease: true }); + const packagePolicy = { + policy_id: CLOUD_AGENT_POLICY_ID, + enabled: true, + package: { + name: packageName, + title: 'Universal Profiling Symbolizer', + version, + }, + name: SYMBOLIZER_PACKAGE_POLICY_NAME, + namespace: 'default', + inputs: [ + { + policy_template: 'universal_profiling_symbolizer', + enabled: true, + streams: [], + type: 'pf-elastic-symbolizer', + }, + ], + }; + const esClient = client.getEsClient(); + await packagePolicyClient.create(soClient, esClient, packagePolicy, { + force: true, + }); +} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_apm_policy.ts b/x-pack/plugins/profiling/server/lib/setup/get_apm_policy.ts similarity index 88% rename from x-pack/plugins/profiling/server/lib/setup/steps/get_apm_policy.ts rename to x-pack/plugins/profiling/server/lib/setup/get_apm_policy.ts index ee7a5913e1a57..ecc23f42ac99c 100644 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_apm_policy.ts +++ b/x-pack/plugins/profiling/server/lib/setup/get_apm_policy.ts @@ -17,5 +17,5 @@ export async function getApmPolicy({ packagePolicyClient: PackagePolicyClient; soClient: SavedObjectsClientContract; }) { - return await packagePolicyClient.get(soClient, ELASTIC_CLOUD_APM_POLICY); + return packagePolicyClient.get(soClient, ELASTIC_CLOUD_APM_POLICY); } diff --git a/x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts b/x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts index 8a9540e1fd583..93aef861bbe41 100644 --- a/x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts +++ b/x-pack/plugins/profiling/server/lib/setup/get_setup_instructions.ts @@ -7,7 +7,7 @@ import { SavedObjectsClientContract } from '@kbn/core/server'; import { PackagePolicyClient } from '@kbn/fleet-plugin/server'; -import { getApmPolicy } from './steps/get_apm_policy'; +import { getApmPolicy } from './get_apm_policy'; export interface SetupDataCollectionInstructions { variables: { diff --git a/x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts b/x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts index d4478ee6c70ed..45b6f2aaf7476 100644 --- a/x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts +++ b/x-pack/plugins/profiling/server/lib/setup/has_profiling_data.ts @@ -5,19 +5,21 @@ * 2.0. */ -import { ProfilingESClient } from '../../utils/create_profiling_es_client'; +import { ProfilingSetupOptions } from './types'; +import { PartialSetupState } from '../../../common/setup'; export async function hasProfilingData({ client, -}: { - client: ProfilingESClient; -}): Promise { +}: ProfilingSetupOptions): Promise { const hasProfilingDataResponse = await client.search('has_any_profiling_data', { index: 'profiling*', size: 0, track_total_hits: 1, terminate_after: 1, }); - - return hasProfilingDataResponse.hits.total.value > 0; + return { + data: { + available: hasProfilingDataResponse.hits.total.value > 0, + }, + }; } diff --git a/x-pack/plugins/profiling/server/lib/setup/security_role.ts b/x-pack/plugins/profiling/server/lib/setup/security_role.ts new file mode 100644 index 0000000000000..a6f65e438ce60 --- /dev/null +++ b/x-pack/plugins/profiling/server/lib/setup/security_role.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ProfilingSetupOptions } from './types'; +import { PartialSetupState } from '../../../common/setup'; + +const PROFILING_READER_ROLE_NAME = 'profiling-reader'; + +export async function validateSecurityRole({ + client, +}: ProfilingSetupOptions): Promise { + const esClient = client.getEsClient(); + const roles = await esClient.security.getRole(); + return { + settings: { + configured: PROFILING_READER_ROLE_NAME in roles, + }, + }; +} + +export async function setSecurityRole({ client }: ProfilingSetupOptions) { + const esClient = client.getEsClient(); + await esClient.security.putRole({ + name: PROFILING_READER_ROLE_NAME, + indices: [ + { + names: ['profiling-*'], + privileges: ['read', 'view_index_metadata'], + }, + ], + cluster: ['monitor'], + }); +} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/catch_resource_already_exists_exception.ts b/x-pack/plugins/profiling/server/lib/setup/steps/catch_resource_already_exists_exception.ts deleted file mode 100644 index eaea6c2708342..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/catch_resource_already_exists_exception.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isResponseError } from '@kbn/es-errors'; - -export function catchResourceAlreadyExistsException(error: any) { - if (isResponseError(error) && error.body?.error?.type === 'resource_already_exists_exception') { - return Promise.resolve(); - } - - return Promise.reject(error); -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_events.json b/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_events.json deleted file mode 100644 index 8a45a18ff60e6..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_events.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "4", - "max_result_window": 150000, - "refresh_interval": "10s", - "sort": { - "field": [ - "service.name", - "@timestamp", - "orchestrator.resource.name", - "container.name", - "process.thread.name", - "host.id" - ] - } - }, - "codec": "best_compression" - }, - "mappings": { - "_source": { - "enabled": false - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "service.name": { - "type": "keyword" - }, - "@timestamp": { - "type": "date", - "format": "epoch_second" - }, - "host.id": { - "type": "keyword" - }, - "Stacktrace.id": { - "type": "keyword", - "index": false - }, - "orchestrator.resource.name": { - "type": "keyword" - }, - "container.name": { - "type": "keyword" - }, - "process.thread.name": { - "type": "keyword" - }, - "Stacktrace.count": { - "type": "short", - "index": false - }, - "agent.version": { - "type": "keyword" - }, - "host.ip": { - "type": "ip" - }, - "host.name": { - "type": "keyword" - }, - "os.kernel": { - "type": "keyword" - }, - "tags": { - "type": "keyword" - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_executables.json b/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_executables.json deleted file mode 100644 index f80cf8a92a26f..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_executables.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "Executable.build.id": { - "type": "keyword", - "index": true - }, - "Executable.file.name": { - "type": "keyword", - "index": true - }, - "@timestamp": { - "type": "date", - "format": "epoch_second" - }, - "Symbolization.lastprocessed": { - "type": "date", - "format": "epoch_second", - "index": false - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_ilm.json b/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_ilm.json deleted file mode 100644 index 36741bc9310a0..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_ilm.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "settings": { - "index": { - "lifecycle": { - "name": "profiling" - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stackframes.json b/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stackframes.json deleted file mode 100644 index 09368fa8671f2..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stackframes.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 16, - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Stackframe.line.number": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Stackframe.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Stackframe.function.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Stackframe.function.offset": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stacktraces.json b/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stacktraces.json deleted file mode 100644 index 19c8f15158901..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/component_template_profiling_stacktraces.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": 16, - "refresh_interval": "10s", - "sort": { - "field": [ - "Stacktrace.frame.ids" - ] - } - } - }, - "mappings": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "Stacktrace.frame.ids": { - "type": "keyword", - "index": false - }, - "Stacktrace.frame.types": { - "type": "keyword", - "index": false - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_apm_package_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_apm_package_step.ts deleted file mode 100644 index ad31b85baef19..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_apm_package_step.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { installPackage, getInstallation } from '@kbn/fleet-plugin/server/services/epm/packages'; -import { - fetchFindLatestPackageOrThrow, - pkgToPkgKey, -} from '@kbn/fleet-plugin/server/services/epm/registry'; -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; - -export function getApmPackageStep({ - client, - soClient, - spaceId, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - return { - name: 'apm_package', - hasCompleted: async () => { - const installation = await getInstallation({ - pkgName: 'apm', - savedObjectsClient: soClient, - }); - - return !!installation; - }, - init: async () => { - const { name, version } = await fetchFindLatestPackageOrThrow('apm'); - - await installPackage({ - installSource: 'registry', - esClient, - savedObjectsClient: soClient, - pkgkey: pkgToPkgKey({ name, version }), - spaceId, - force: true, - }); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_cluster_settings_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_cluster_settings_step.ts deleted file mode 100644 index b281745552eb2..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_cluster_settings_step.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; - -const MAX_BUCKETS = 150000; - -export function getClusterSettingsStep({ - client, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - return { - name: 'cluster_settings', - hasCompleted: async () => { - const settings = await client.getEsClient().cluster.getSettings({}); - - return settings.persistent.search?.max_buckets === MAX_BUCKETS.toString(); - }, - init: async () => { - await client.getEsClient().cluster.putSettings({ - persistent: { - search: { - max_buckets: MAX_BUCKETS, - }, - }, - }); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_component_templates_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_component_templates_step.ts deleted file mode 100644 index 3a37e3e3970d2..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_component_templates_step.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/types'; -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import componentTemplateProfilingIlm from './component_template_profiling_ilm.json'; -import componentTemplateProfilingEvents from './component_template_profiling_events.json'; -import componentTemplateProfilingExecutables from './component_template_profiling_executables.json'; -import componentTemplateProfilingStackframes from './component_template_profiling_stackframes.json'; -import componentTemplateProfilingStacktraces from './component_template_profiling_stacktraces.json'; - -export enum ProfilingComponentTemplateName { - Ilm = 'profiling-ilm', - Events = 'profiling-events', - Executables = 'profiling-executables', - Stackframes = 'profiling-stackframes', - Stacktraces = 'profiling-stacktraces', -} - -export function getComponentTemplatesStep({ - client, - logger, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - - return { - name: 'component_templates', - hasCompleted: async () => { - return Promise.all( - [ - ProfilingComponentTemplateName.Ilm, - ProfilingComponentTemplateName.Events, - ProfilingComponentTemplateName.Executables, - ProfilingComponentTemplateName.Stackframes, - ProfilingComponentTemplateName.Stacktraces, - ].map((componentTemplateName) => - esClient.cluster.getComponentTemplate({ - name: componentTemplateName, - }) - ) - ).then( - () => Promise.resolve(true), - (error) => { - logger.debug('Some component templates could not be fetched'); - logger.debug(error); - return Promise.resolve(false); - } - ); - }, - init: async () => { - await Promise.all([ - esClient.cluster.putComponentTemplate({ - name: ProfilingComponentTemplateName.Ilm, - create: false, - template: componentTemplateProfilingIlm, - }), - esClient.cluster.putComponentTemplate({ - name: ProfilingComponentTemplateName.Events, - create: false, - template: componentTemplateProfilingEvents as IndicesIndexState, - _meta: { - description: 'Mappings for profiling events data stream', - }, - }), - esClient.cluster.putComponentTemplate({ - name: ProfilingComponentTemplateName.Executables, - create: false, - template: componentTemplateProfilingExecutables as IndicesIndexState, - }), - esClient.cluster.putComponentTemplate({ - name: ProfilingComponentTemplateName.Stackframes, - create: false, - template: componentTemplateProfilingStackframes as IndicesIndexState, - }), - esClient.cluster.putComponentTemplate({ - name: ProfilingComponentTemplateName.Stacktraces, - create: false, - template: componentTemplateProfilingStacktraces as IndicesIndexState, - }), - ]); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_create_events_data_streams.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_create_events_data_streams.ts deleted file mode 100644 index 16cdee47f000d..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_create_events_data_streams.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception'; - -function getEventDataStreamNames() { - const subSampledIndicesIdx = Array.from(Array(11).keys(), (item: number) => item + 1); - const subSampledIndexName = (pow: number): string => { - return `profiling-events-5pow${String(pow).padStart(2, '0')}`; - }; - // Generate all the possible index template names - const eventsIndices = ['profiling-events-all'].concat( - subSampledIndicesIdx.map((pow) => subSampledIndexName(pow)) - ); - - return eventsIndices; -} - -export function getCreateEventsDataStreamsStep({ - client, - logger, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - - const dataStreamNames = getEventDataStreamNames(); - - return { - name: 'create_events_data_streams', - hasCompleted: async () => { - const dataStreams = await esClient.indices.getDataStream({ - name: 'profiling-events*', - }); - - const allDataStreams = dataStreams.data_streams.map((dataStream) => dataStream.name); - - const missingDataStreams = dataStreamNames.filter( - (eventIndex) => !allDataStreams.includes(eventIndex) - ); - - if (missingDataStreams.length > 0) { - logger.debug(`Missing event indices: ${missingDataStreams.join(', ')}`); - } - - return missingDataStreams.length === 0; - }, - init: async () => { - await Promise.all( - dataStreamNames.map((dataStreamName) => - esClient.indices - .createDataStream({ - name: dataStreamName, - }) - .catch(catchResourceAlreadyExistsException) - ) - ); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts deleted file mode 100644 index 30d5c49a50c49..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_create_indices_step.ts +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception'; -import profilingReturnpadsPrivateMapping from './mappings/profiling_returnpads_private.json'; -import profilingSymbolsPrivateMapping from './mappings/profiling_symbols_private.json'; -import profilingSymbolsMapping from './mappings/profiling_symbols.json'; -import profilingSQLeafframesMapping from './mappings/profiling_sq_leafframes.json'; -import profilingSQExecutablesMapping from './mappings/profiling_sq_executables.json'; - -const RETURNPADS_PRIVATE_INDEX = 'profiling-returnpads-private'; -const SQ_EXECUTABLES_INDEX = 'profiling-sq-executables'; -const SQ_LEAFFRAMES_INDEX = 'profiling-sq-leafframes'; -const SYMBOLS_INDEX = 'profiling-symbols'; -const SYMBOLS_PRIVATE_INDEX = 'profiling-symbols-private'; -const ILM_LOCK_INDEX = '.profiling-ilm-lock'; - -const getKeyValueIndices = () => { - const kvIndices = ['profiling-stacktraces', 'profiling-stackframes', 'profiling-executables']; - - const pairs: Array<{ index: string; alias: string }> = kvIndices.flatMap((index) => { - return [ - { index: `${index}-000001`, alias: index }, - { index: `${index}-000002`, alias: `${index}-next` }, - ]; - }); - - return pairs; -}; - -export function getCreateIndicesStep({ - client, - logger, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - const keyValueIndices = getKeyValueIndices(); - - return { - name: 'create_indices', - hasCompleted: async () => { - const nonKvIndices = [ - RETURNPADS_PRIVATE_INDEX, - SQ_EXECUTABLES_INDEX, - SQ_LEAFFRAMES_INDEX, - SYMBOLS_INDEX, - SYMBOLS_PRIVATE_INDEX, - ILM_LOCK_INDEX, - ]; - - const results = await Promise.all([ - esClient.cat - .indices({ - index: keyValueIndices - .map(({ index }) => index) - .concat(nonKvIndices) - .map((index) => index + '*') - .join(','), - format: 'json', - }) - .then((response) => { - const allIndices = response.map((index) => index.index!); - - const missingIndices = keyValueIndices - .map(({ index }) => index) - .concat(nonKvIndices) - .filter((index) => !allIndices.includes(index)); - - if (missingIndices.length) { - logger.debug(`Missing indices: ${missingIndices.join(',')}`); - } - - return missingIndices.length === 0; - }) - .catch((error) => { - logger.debug(`Failed fetching indices: ${error}`); - return Promise.resolve(false); - }), - esClient.cat - .aliases({ - name: keyValueIndices.map(({ alias }) => alias + '*').join(','), - format: 'json', - }) - .then((response) => { - const allAliases = response.map((index) => index.alias!); - - const missingAliases = keyValueIndices - .map(({ alias }) => alias) - .filter((alias) => !allAliases.includes(alias)); - - if (missingAliases.length) { - logger.debug(`Missing aliases: ${missingAliases.join(',')}`); - } - - return missingAliases.length === 0; - }) - .catch((error) => { - logger.debug(`Failed fetching aliases: ${error}`); - return Promise.resolve(false); - }), - ]); - - return results.every(Boolean); - }, - init: async () => { - await Promise.all([ - ...keyValueIndices.map(({ index, alias }) => { - return esClient.indices - .create({ - index, - aliases: { - [alias]: { - is_write_index: true, - }, - }, - }) - .catch(catchResourceAlreadyExistsException); - }), - esClient.indices - .create({ - index: SQ_EXECUTABLES_INDEX, - ...profilingSQExecutablesMapping, - }) - .catch(catchResourceAlreadyExistsException), - esClient.indices - .create({ - index: SQ_LEAFFRAMES_INDEX, - ...profilingSQLeafframesMapping, - }) - .catch(catchResourceAlreadyExistsException), - esClient.indices - .create({ - index: SYMBOLS_INDEX, - ...profilingSymbolsMapping, - }) - .catch(catchResourceAlreadyExistsException), - esClient.indices - .create({ - index: SYMBOLS_PRIVATE_INDEX, - ...profilingSymbolsPrivateMapping, - }) - .catch(catchResourceAlreadyExistsException), - esClient.indices - .create({ - index: RETURNPADS_PRIVATE_INDEX, - ...profilingReturnpadsPrivateMapping, - }) - .catch(catchResourceAlreadyExistsException), - esClient.indices - .create({ - index: ILM_LOCK_INDEX, - settings: { - index: { - hidden: true, - }, - }, - mappings: { - properties: { - '@timestamp': { - type: 'date', - format: 'epoch_second', - }, - phase: { - type: 'keyword', - }, - }, - }, - }) - .catch(catchResourceAlreadyExistsException), - ]); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_fleet_policy_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_fleet_policy_step.ts deleted file mode 100644 index cffe2bab3bedc..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_fleet_policy_step.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchClient } from '@kbn/core/server'; -import { merge, omit } from 'lodash'; -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import { getApmPolicy } from './get_apm_policy'; - -async function createIngestAPIKey(esClient: ElasticsearchClient) { - const apiKeyResponse = await esClient.security.createApiKey({ - name: 'profiling-manager', - role_descriptors: { - profiling_manager: { - indices: [ - { - names: ['profiling-*', '.profiling-*'], - privileges: [ - 'read', - 'create_doc', - 'create', - 'write', - 'index', - 'create_index', - 'view_index_metadata', - 'manage', - ], - }, - ], - cluster: ['monitor'], - }, - }, - }); - - return atob(apiKeyResponse.encoded); -} - -export function getFleetPolicyStep({ - client, - soClient, - logger, - packagePolicyClient, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - return { - name: 'fleet_policy', - hasCompleted: async () => { - try { - const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient }); - - return apmPolicy && apmPolicy?.inputs[0].config?.['apm-server'].value.profiling; - } catch (error) { - logger.debug('Could not fetch fleet policy'); - logger.debug(error); - return false; - } - }, - init: async () => { - const apmPolicyApiKey = await createIngestAPIKey(client.getEsClient()); - - const profilingApmConfig = { - profiling: { - enabled: true, - elasticsearch: { - api_key: apmPolicyApiKey, - }, - metrics: { - elasticsearch: { - hosts: [ - 'https://1b6c02856ea642a6ac14499b01507233.us-east-2.aws.elastic-cloud.com:443', - ], - api_key: 'woq-IoMBRbbiEbPugtWW:_iBmc1PdSout7sf5FCkEpA', - }, - }, - keyvalue_retention: { - // 60 days - age: '1440h', - // 200 Gib - size_bytes: 200 * 1024 * 1024 * 1024, - execution_interval: '12h', - }, - }, - }; - - const apmPolicy = await getApmPolicy({ packagePolicyClient, soClient }); - - if (!apmPolicy) { - throw new Error(`Could not find APM policy`); - } - - const modifiedPolicyInputs = apmPolicy.inputs.map((input) => { - return input.type === 'apm' - ? merge({}, input, { config: { 'apm-server': { value: profilingApmConfig } } }) - : input; - }); - - await packagePolicyClient.update(soClient, esClient, apmPolicy.id, { - ...omit(apmPolicy, 'id', 'revision', 'updated_at', 'updated_by'), - inputs: modifiedPolicyInputs, - }); - - // We add here the creation of the new package_policy for symbolizer. - // We create the new policy and bind it to the Cloud default agent policy; - // to do so, force s required to be set to true. - const cloudAgentPolicyId = 'policy-elastic-agent-on-cloud'; - await packagePolicyClient.create( - soClient, - esClient, - { - policy_id: cloudAgentPolicyId, - enabled: true, - package: { - name: 'profiler_symbolizer', - title: 'Universal Profiling Symbolizer', - version: '8.8.0-preview', - }, - name: 'elastic-universal-profiling-symbolizer', - namespace: 'default', - inputs: [ - { - policy_template: 'universal_profiling_symbolizer', - enabled: true, - streams: [], - type: 'pf-elastic-symbolizer', - }, - ], - }, - { force: true } - ); - await packagePolicyClient.create( - soClient, - esClient, - { - policy_id: cloudAgentPolicyId, - enabled: true, - package: { - name: 'profiler_collector', - title: 'Universal Profiling Collector', - version: '8.9.0-preview', - }, - name: 'elastic-universal-profiling-collector', - namespace: 'default', - inputs: [ - { - policy_template: 'universal_profiling_collector', - enabled: true, - streams: [], - type: 'pf-elastic-collector', - }, - ], - }, - { force: true } - ); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_ilm_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_ilm_step.ts deleted file mode 100644 index 04d57b3c11d15..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_ilm_step.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import { catchResourceAlreadyExistsException } from './catch_resource_already_exists_exception'; -import ilmProfiling from './ilm_profiling.json'; - -const LIFECYCLE_POLICY_NAME = 'profiling'; - -export function getIlmStep({ - client, - logger, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - - return { - name: 'ilm', - hasCompleted: () => { - return esClient.ilm.getLifecycle({ name: LIFECYCLE_POLICY_NAME }).then( - () => { - return Promise.resolve(true); - }, - (error) => { - logger.debug('ILM policy not installed'); - logger.debug(error); - return Promise.resolve(false); - } - ); - }, - init: async () => { - await esClient.ilm - .putLifecycle({ - name: LIFECYCLE_POLICY_NAME, - policy: ilmProfiling, - }) - .catch(catchResourceAlreadyExistsException); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_index_templates_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_index_templates_step.ts deleted file mode 100644 index 2e6fd0bec1f56..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_index_templates_step.ts +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import { ProfilingComponentTemplateName } from './get_component_templates_step'; - -enum ProfilingIndexTemplate { - Events = 'profiling-events', - Executables = 'profiling-executables', - Stacktraces = 'profiling-stacktraces', - Stackframes = 'profiling-stackframes', -} - -export function getIndexTemplatesStep({ - client, - logger, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - - return { - name: 'index_templates', - hasCompleted: async () => { - return Promise.all( - [ - ProfilingIndexTemplate.Events, - ProfilingIndexTemplate.Executables, - ProfilingIndexTemplate.Stacktraces, - ProfilingIndexTemplate.Stackframes, - ].map((indexTemplateName) => - esClient.indices.getIndexTemplate({ - name: indexTemplateName, - }) - ) - ).then( - () => Promise.resolve(true), - (error) => { - logger.debug('Some index templates could not be fetched'); - logger.debug(error); - return Promise.resolve(false); - } - ); - }, - init: async () => { - await Promise.all([ - esClient.indices.putIndexTemplate({ - name: ProfilingIndexTemplate.Events, - create: false, - index_patterns: [ProfilingIndexTemplate.Events + '*'], - data_stream: { - hidden: false, - }, - composed_of: [ProfilingComponentTemplateName.Events, ProfilingComponentTemplateName.Ilm], - priority: 100, - _meta: { - description: `Index template for ${ProfilingIndexTemplate.Events}`, - }, - }), - ...[ - ProfilingIndexTemplate.Executables, - ProfilingIndexTemplate.Stacktraces, - ProfilingIndexTemplate.Stackframes, - ].map((indexTemplateName) => { - return esClient.indices.putIndexTemplate({ - name: indexTemplateName, - // Don't fail if the index template already exists, simply overwrite the format - create: false, - index_patterns: [indexTemplateName + '*'], - composed_of: [indexTemplateName], - _meta: { - description: `Index template for ${indexTemplateName}`, - }, - }); - }), - ]); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_is_cloud_enabled_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_is_cloud_enabled_step.ts deleted file mode 100644 index 920317eb47974..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_is_cloud_enabled_step.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; - -export function getIsCloudEnabledStep({ - isCloudEnabled, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - return { - name: 'is_cloud', - hasCompleted: async () => { - return isCloudEnabled; - }, - init: async () => { - if (!isCloudEnabled) { - throw new Error(`Universal Profiling is only available on Elastic Cloud.`); - } - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/get_security_step.ts b/x-pack/plugins/profiling/server/lib/setup/steps/get_security_step.ts deleted file mode 100644 index c3ae1a41488a6..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/get_security_step.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; - -const PROFILING_READER_ROLE_NAME = 'profiling-reader'; - -export function getSecurityStep({ - client, - logger, -}: ProfilingSetupStepFactoryOptions): ProfilingSetupStep { - const esClient = client.getEsClient(); - - return { - name: 'security', - hasCompleted: () => { - return esClient.security - .getRole({ - name: PROFILING_READER_ROLE_NAME, - }) - .then( - () => { - return Promise.resolve(true); - }, - (error) => { - logger.debug('Could not fetch profiling-reader role'); - logger.debug(error); - return Promise.resolve(false); - } - ); - }, - init: async () => { - await esClient.security.putRole({ - name: PROFILING_READER_ROLE_NAME, - indices: [ - { - names: ['profiling-*'], - privileges: ['read', 'view_index_metadata'], - }, - ], - cluster: ['monitor'], - }); - }, - }; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/ilm_profiling.json b/x-pack/plugins/profiling/server/lib/setup/steps/ilm_profiling.json deleted file mode 100644 index 0d39bf08b0a02..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/ilm_profiling.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "phases": { - "hot": { - "min_age": "0ms", - "actions": { - "rollover": { - "max_primary_shard_size": "50gb", - "max_age": "7d" - }, - "set_priority": { - "priority": 100 - } - } - }, - "warm": { - "min_age": "30d", - "actions": { - "set_priority": { - "priority": 50 - }, - "shrink": { - "number_of_shards": 2 - } - } - }, - "delete": { - "min_age": "60d", - "actions": { - "delete": { - "delete_searchable_snapshot": true - } - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/index.ts b/x-pack/plugins/profiling/server/lib/setup/steps/index.ts deleted file mode 100644 index b0c5bbf0a50e7..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getClusterSettingsStep } from './get_cluster_settings_step'; -import { ProfilingSetupStep, ProfilingSetupStepFactoryOptions } from '../types'; -import { getComponentTemplatesStep } from './get_component_templates_step'; -import { getIlmStep } from './get_ilm_step'; -import { getIndexTemplatesStep } from './get_index_templates_step'; -import { getFleetPolicyStep } from './get_fleet_policy_step'; -import { getSecurityStep } from './get_security_step'; -import { getApmPackageStep } from './get_apm_package_step'; -import { getCreateEventsDataStreamsStep } from './get_create_events_data_streams'; -import { getCreateIndicesStep } from './get_create_indices_step'; -import { getIsCloudEnabledStep } from './get_is_cloud_enabled_step'; - -export function getProfilingSetupSteps( - options: ProfilingSetupStepFactoryOptions -): ProfilingSetupStep[] { - return [ - getIsCloudEnabledStep(options), - getApmPackageStep(options), - getClusterSettingsStep(options), - getIlmStep(options), - getComponentTemplatesStep(options), - getIndexTemplatesStep(options), - getCreateEventsDataStreamsStep(options), - getCreateIndicesStep(options), - getSecurityStep(options), - getFleetPolicyStep(options), - ]; -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json deleted file mode 100644 index 533b196ff9ce1..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_returnpads_private.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbfile.created": { - "type": "date", - "doc_values": false, - "index": true, - "store": false - }, - "Symbfile.file.id": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbfile.part": { - "type": "short", - "index": false, - "doc_values": false, - "store": false - }, - "Symbfile.parts": { - "type": "short", - "index": false, - "doc_values": false, - "store": false - }, - "Symbfile.data": { - "type": "binary", - "doc_values": false, - "store": false - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json deleted file mode 100644 index 8e5e9935c6ecb..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_executables.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "Executable.file.id": { - "type": "keyword", - "index": false - }, - "Time.created": { - "type": "date", - "index": true - }, - "Symbolization.time.next": { - "type": "date", - "index": true - }, - "Symbolization.retries": { - "type": "short", - "index": true - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json deleted file mode 100644 index ed778a1309bc5..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_sq_leafframes.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "settings": { - "index": { - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true - }, - "Stacktrace.frame.id": { - "type": "keyword", - "index": false - }, - "Time.created": { - "type": "date", - "index": true - }, - "Symbolization.time.next": { - "type": "date", - "index": true - }, - "Symbolization.retries": { - "type": "short", - "index": true - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json deleted file mode 100644 index 7736a6202a9ad..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.function.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.function.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.depth": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.base": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.length": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.offsets": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.lines": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.id": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.exec.pcrange": { - "type": "ip_range", - "index": true, - "doc_values": false, - "store": false - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json b/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json deleted file mode 100644 index 7736a6202a9ad..0000000000000 --- a/x-pack/plugins/profiling/server/lib/setup/steps/mappings/profiling_symbols_private.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "settings": { - "index": { - "number_of_shards": "16", - "refresh_interval": "10s" - } - }, - "mappings": { - "_source": { - "enabled": true - }, - "properties": { - "ecs.version": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.function.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.file.name": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.call.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.function.line": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.depth": { - "type": "integer", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.base": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.length": { - "type": "unsigned_long", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.offsets": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.linetable.lines": { - "type": "keyword", - "index": false, - "doc_values": false, - "store": false - }, - "Symbol.file.id": { - "type": "keyword", - "index": true, - "doc_values": false, - "store": false - }, - "Symbol.exec.pcrange": { - "type": "ip_range", - "index": true, - "doc_values": false, - "store": false - } - } - } -} diff --git a/x-pack/plugins/profiling/server/lib/setup/types.ts b/x-pack/plugins/profiling/server/lib/setup/types.ts index 65a0ae684052b..b590567605a2b 100644 --- a/x-pack/plugins/profiling/server/lib/setup/types.ts +++ b/x-pack/plugins/profiling/server/lib/setup/types.ts @@ -10,13 +10,7 @@ import { PackagePolicyClient } from '@kbn/fleet-plugin/server'; import { Logger } from '@kbn/logging'; import { ProfilingESClient } from '../../utils/create_profiling_es_client'; -export interface ProfilingSetupStep { - name: string; - init: () => Promise; - hasCompleted: () => Promise; -} - -export interface ProfilingSetupStepFactoryOptions { +export interface ProfilingSetupOptions { client: ProfilingESClient; soClient: SavedObjectsClientContract; packagePolicyClient: PackagePolicyClient; diff --git a/x-pack/plugins/profiling/server/routes/setup.ts b/x-pack/plugins/profiling/server/routes/setup.ts index 4f97497b6ab96..c8df37ce76140 100644 --- a/x-pack/plugins/profiling/server/routes/setup.ts +++ b/x-pack/plugins/profiling/server/routes/setup.ts @@ -4,29 +4,35 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { eachSeries } from 'async'; -import { Logger } from '@kbn/logging'; + import { RouteRegisterParameters } from '.'; -import { getRoutePaths } from '../../common'; +import { getClient } from './compat'; +import { installLatestApmPackage, isApmPackageInstalled } from '../lib/setup/apm_package'; +import { + enableResourceManagement, + setMaximumBuckets, + validateMaximumBuckets, + validateResourceManagement, +} from '../lib/setup/cluster_settings'; +import { + createCollectorPackagePolicy, + createSymbolizerPackagePolicy, + updateApmPolicy, + validateApmPolicy, + validateCollectorPackagePolicy, + validateSymbolizerPackagePolicy, +} from '../lib/setup/fleet_policies'; import { getSetupInstructions } from '../lib/setup/get_setup_instructions'; -import { getProfilingSetupSteps } from '../lib/setup/steps'; -import { handleRouteHandlerError } from '../utils/handle_route_error_handler'; import { hasProfilingData } from '../lib/setup/has_profiling_data'; -import { getClient } from './compat'; -import { ProfilingSetupStep } from '../lib/setup/types'; - -function checkSteps({ steps, logger }: { steps: ProfilingSetupStep[]; logger: Logger }) { - return Promise.all( - steps.map(async (step) => { - try { - return { name: step.name, completed: await step.hasCompleted() }; - } catch (error) { - logger.error(error); - return { name: step.name, completed: false, error: error.toString() }; - } - }) - ); -} +import { setSecurityRole, validateSecurityRole } from '../lib/setup/security_role'; +import { ProfilingSetupOptions } from '../lib/setup/types'; +import { handleRouteHandlerError } from '../utils/handle_route_error_handler'; +import { getRoutePaths } from '../../common'; +import { + areResourcesSetup, + createDefaultSetupState, + mergePartialSetupStates, +} from '../../common/setup'; export function registerSetupRoute({ router, @@ -35,7 +41,7 @@ export function registerSetupRoute({ dependencies, }: RouteRegisterParameters) { const paths = getRoutePaths(); - // Check if ES resources needed for Universal Profiling to work exist + // Check if Elasticsearch and Fleet are setup for Universal Profiling router.get( { path: paths.HasSetupESResources, @@ -44,53 +50,54 @@ export function registerSetupRoute({ async (context, request, response) => { try { const esClient = await getClient(context); - logger.debug('checking if profiling ES configurations are installed'); const core = await context.core; - - const steps = getProfilingSetupSteps({ - client: createProfilingEsClient({ - esClient, - request, - useDefaultAuth: true, - }), + const clientWithDefaultAuth = createProfilingEsClient({ + esClient, + request, + useDefaultAuth: true, + }); + const setupOptions: ProfilingSetupOptions = { + client: clientWithDefaultAuth, logger, packagePolicyClient: dependencies.start.fleet.packagePolicyService, soClient: core.savedObjects.client, spaceId: dependencies.setup.spaces.spacesService.getSpaceId(request), isCloudEnabled: dependencies.setup.cloud.isCloudEnabled, - }); - - const hasDataPromise = hasProfilingData({ - client: createProfilingEsClient({ - esClient, - request, - }), - }); + }; - const stepCompletionResultsPromises = checkSteps({ steps, logger }); + logger.info('Checking if Elasticsearch and Fleet are setup for Universal Profiling'); - const hasData = await hasDataPromise; + const state = createDefaultSetupState(); + state.cloud.available = dependencies.setup.cloud.isCloudEnabled; - if (hasData) { - return response.ok({ + if (!state.cloud.available) { + const msg = `Elastic Cloud is required to set up Elasticsearch and Fleet for Universal Profiling`; + logger.error(msg); + return response.custom({ + statusCode: 500, body: { - has_data: true, - has_setup: true, - steps: [], + message: msg, }, }); } - const stepCompletionResults = await stepCompletionResultsPromises; + const verifyFunctions = [ + hasProfilingData, + isApmPackageInstalled, + validateApmPolicy, + validateCollectorPackagePolicy, + validateMaximumBuckets, + validateResourceManagement, + validateSecurityRole, + validateSymbolizerPackagePolicy, + ]; + const partialStates = await Promise.all(verifyFunctions.map((fn) => fn(setupOptions))); + const mergedState = mergePartialSetupStates(state, partialStates); - // Reply to clients if we have already created all 12 events template indices. - // This is kind of simplistic but can be a good first step to ensure - // Profiling resources will be created. return response.ok({ body: { - has_setup: stepCompletionResults.every((step) => step.completed), - has_data: false, - steps: stepCompletionResults, + has_setup: areResourcesSetup(mergedState), + has_data: mergedState.data.available, }, }); } catch (error) { @@ -98,7 +105,7 @@ export function registerSetupRoute({ } } ); - // Configure ES resources needed by Universal Profiling using the mappings + // Set up Elasticsearch and Fleet for Universal Profiling router.post( { path: paths.HasSetupESResources, @@ -107,37 +114,68 @@ export function registerSetupRoute({ async (context, request, response) => { try { const esClient = await getClient(context); - logger.info('Applying initial setup of Elasticsearch resources'); - const steps = getProfilingSetupSteps({ - client: createProfilingEsClient({ esClient, request, useDefaultAuth: true }), + const core = await context.core; + const clientWithDefaultAuth = createProfilingEsClient({ + esClient, + request, + useDefaultAuth: true, + }); + const setupOptions: ProfilingSetupOptions = { + client: clientWithDefaultAuth, logger, packagePolicyClient: dependencies.start.fleet.packagePolicyService, - soClient: (await context.core).savedObjects.client, + soClient: core.savedObjects.client, spaceId: dependencies.setup.spaces.spacesService.getSpaceId(request), isCloudEnabled: dependencies.setup.cloud.isCloudEnabled, - }); + }; - await eachSeries(steps, (step, cb) => { - logger.debug(`Executing step ${step.name}`); - step - .init() - .then(() => cb()) - .catch(cb); - }); + logger.info('Setting up Elasticsearch and Fleet for Universal Profiling'); - const checkedSteps = await checkSteps({ steps, logger }); + const state = createDefaultSetupState(); + state.cloud.available = dependencies.setup.cloud.isCloudEnabled; - if (checkedSteps.every((step) => step.completed)) { + if (!state.cloud.available) { + const msg = `Elastic Cloud is required to set up Elasticsearch and Fleet for Universal Profiling`; + logger.error(msg); + return response.custom({ + statusCode: 500, + body: { + message: msg, + }, + }); + } + + const verifyFunctions = [ + isApmPackageInstalled, + validateApmPolicy, + validateCollectorPackagePolicy, + validateMaximumBuckets, + validateResourceManagement, + validateSecurityRole, + validateSymbolizerPackagePolicy, + ]; + const partialStates = await Promise.all(verifyFunctions.map((fn) => fn(setupOptions))); + const mergedState = mergePartialSetupStates(state, partialStates); + + if (areResourcesSetup(mergedState)) { return response.ok(); } - return response.custom({ - statusCode: 500, - body: { - message: `Failed to complete all steps`, - steps: checkedSteps, - }, - }); + const executeFunctions = [ + installLatestApmPackage, + updateApmPolicy, + createCollectorPackagePolicy, + createSymbolizerPackagePolicy, + enableResourceManagement, + setSecurityRole, + setMaximumBuckets, + ]; + await Promise.all(executeFunctions.map((fn) => fn(setupOptions))); + + // We return a status code of 202 instead of 200 because enabling + // resource management in Elasticsearch is an asynchronous action + // and is not guaranteed to complete before Kibana sends a response. + return response.accepted(); } catch (error) { return handleRouteHandlerError({ error, logger, response }); } diff --git a/x-pack/plugins/profiling/server/routes/topn.test.ts b/x-pack/plugins/profiling/server/routes/topn.test.ts index c6fb4c7aa4d78..1276c5dcc94dd 100644 --- a/x-pack/plugins/profiling/server/routes/topn.test.ts +++ b/x-pack/plugins/profiling/server/routes/topn.test.ts @@ -38,10 +38,6 @@ describe('TopN data from Elasticsearch', () => { (operationName, request) => context.elasticsearch.client.asCurrentUser.search(request) as Promise ), - mget: jest.fn( - (operationName, request) => - context.elasticsearch.client.asCurrentUser.search(request) as Promise - ), profilingStacktraces: jest.fn( (request) => context.elasticsearch.client.asCurrentUser.transport.request({ @@ -53,6 +49,14 @@ describe('TopN data from Elasticsearch', () => { }, }) as Promise ), + profilingStatus: jest.fn( + () => + context.elasticsearch.client.asCurrentUser.transport.request({ + method: 'GET', + path: encodeURI('_profiling/status'), + body: {}, + }) as Promise + ), getEsClient: jest.fn(() => context.elasticsearch.client.asCurrentUser), }; const logger = loggerMock.create(); @@ -62,7 +66,7 @@ describe('TopN data from Elasticsearch', () => { }); describe('when fetching Stack Traces', () => { - it('should search first then skip mget', async () => { + it('should call search twice', async () => { await topNElasticSearchQuery({ client, logger, @@ -73,9 +77,7 @@ describe('TopN data from Elasticsearch', () => { kuery: '', }); - // Calls to mget are skipped since data doesn't exist expect(client.search).toHaveBeenCalledTimes(2); - expect(client.mget).toHaveBeenCalledTimes(0); }); }); }); diff --git a/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts b/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts index 05df1e9d37fde..8fa67eb23ecfe 100644 --- a/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts +++ b/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts @@ -9,11 +9,9 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import type { KibanaRequest } from '@kbn/core/server'; import { unwrapEsResponse } from '@kbn/observability-plugin/server'; -import { MgetRequest, MgetResponse } from '@elastic/elasticsearch/lib/api/types'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { ProfilingESEvent } from '../../common/elasticsearch'; import { withProfilingSpan } from './with_profiling_span'; -import { StackTraceResponse } from '../../common/stack_traces'; +import { ProfilingStatusResponse, StackTraceResponse } from '../../common/stack_traces'; export function cancelEsRequestOnAbort>( promise: T, @@ -32,14 +30,11 @@ export interface ProfilingESClient { operationName: string, searchRequest: TSearchRequest ): Promise>; - mget( - operationName: string, - mgetRequest: MgetRequest - ): Promise>; profilingStacktraces({}: { query: QueryDslQueryContainer; sampleSize: number; }): Promise; + profilingStatus(): Promise; getEsClient(): ElasticsearchClient; } @@ -72,25 +67,6 @@ export function createProfilingEsClient({ return unwrapEsResponse(promise); }, - mget( - operationName: string, - mgetRequest: MgetRequest - ): Promise> { - const controller = new AbortController(); - - const promise = withProfilingSpan(operationName, () => { - return cancelEsRequestOnAbort( - esClient.mget(mgetRequest, { - signal: controller.signal, - meta: true, - }), - request, - controller - ); - }); - - return unwrapEsResponse(promise); - }, profilingStacktraces({ query, sampleSize }) { const controller = new AbortController(); @@ -117,6 +93,28 @@ export function createProfilingEsClient({ return unwrapEsResponse(promise) as Promise; }, + profilingStatus() { + const controller = new AbortController(); + + const promise = withProfilingSpan('_profiling/status', () => { + return cancelEsRequestOnAbort( + esClient.transport.request( + { + method: 'GET', + path: encodeURI('/_profiling/status'), + }, + { + signal: controller.signal, + meta: true, + } + ), + request, + controller + ); + }); + + return unwrapEsResponse(promise) as Promise; + }, getEsClient() { return esClient; }, diff --git a/x-pack/plugins/profiling/tsconfig.json b/x-pack/plugins/profiling/tsconfig.json index effbfc83a3716..2380450b136be 100644 --- a/x-pack/plugins/profiling/tsconfig.json +++ b/x-pack/plugins/profiling/tsconfig.json @@ -36,7 +36,6 @@ "@kbn/core-elasticsearch-server", "@kbn/fleet-plugin", "@kbn/shared-ux-page-no-data-types", - "@kbn/es-errors", "@kbn/core-http-request-handler-context-server", "@kbn/spaces-plugin", "@kbn/cloud-plugin", @@ -46,6 +45,7 @@ "@kbn/share-plugin", "@kbn/observability-shared-plugin", "@kbn/licensing-plugin", + "@kbn/apm-plugin", // add references to other TypeScript projects the plugin depends on // requiredPlugins from ./kibana.json