[Profiling] Manage indices via Elasticsearch (#157949)

## Summary

This PR sets up the profiling indices via Elasticsearch instead of using
the single-click installation in Kibana.

### Notable changes

* Set Elasticsearch flag (`xpack.profiling.templates.enabled`) to `true`
so that indices are bootstrapped in Elasticsearch
* Remove component templates and mappings (handled in Elasticsearch
plugin now)
* Remove six steps from Kibana's single-click installation (handled in
Elasticsearch plugin now)
* Use new status API in Elasticsearch to check if resources (templates,
indices, etc) are ready
* Invoke creation of collector and symbolizer package policies in
parallel to updating APM policy
* Refactor initialization and verification steps so that we can later
give the user more granular insight into the status of steps

x-pack/plugins/profiling/common/setup.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { RecursivePartial } from '@kbn/apm-plugin/typings/common';
+
+export interface SetupState {
+  cloud: {
+    available: boolean;
+    required: boolean;
+  };
+  data: {
+    available: boolean;
+  };
+  packages: {
+    installed: boolean;
+  };
+  permissions: {
+    configured: boolean;
+  };
+  policies: {
+    apm: {
+      installed: boolean;
+    };
+    collector: {
+      installed: boolean;
+    };
+    symbolizer: {
+      installed: boolean;
+    };
+  };
+  resource_management: {
+    enabled: boolean;
+  };
+  resources: {
+    created: boolean;
+  };
+  settings: {
+    configured: boolean;
+  };
+}
+
+export type PartialSetupState = RecursivePartial<SetupState>;
+
+export function createDefaultSetupState(): SetupState {
+  return {
+    cloud: {
+      available: false,
+      required: true,
+    },
+    data: {
+      available: false,
+    },
+    packages: {
+      installed: false,
+    },
+    permissions: {
+      configured: false,
+    },
+    policies: {
+      apm: {
+        installed: false,
+      },
+      collector: {
+        installed: false,
+      },
+      symbolizer: {
+        installed: false,
+      },
+    },
+    resource_management: {
+      enabled: false,
+    },
+    resources: {
+      created: false,
+    },
+    settings: {
+      configured: false,
+    },
+  };
+}
+
+export function areResourcesSetup(state: SetupState): boolean {
+  return (
+    state.resource_management.enabled &&
+    state.resources.created &&
+    state.packages.installed &&
+    state.permissions.configured &&
+    state.policies.apm.installed &&
+    state.policies.collector.installed &&
+    state.policies.symbolizer.installed &&
+    state.settings.configured
+  );
+}
+
+function mergeRecursivePartial<T>(base: T, partial: RecursivePartial<T>): T {
+  return { ...base, ...partial };
+}
+
+export function mergePartialSetupStates(
+  base: SetupState,
+  partials: PartialSetupState[]
+): SetupState {
+  return partials.reduce<SetupState>(mergeRecursivePartial, base);
+}

x-pack/plugins/profiling/common/stack_traces.ts

@@ -7,6 +7,18 @@
 
 import { ProfilingESField } from './elasticsearch';
 
+export interface ProfilingStatusResponse {
+  profiling: {
+    enabled: boolean;
+  };
+  resource_management: {
+    enabled: boolean;
+  };
+  resources: {
+    created: boolean;
+  };
+}
+
 interface ProfilingEvents {
   [key: string]: number;
 }

x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_data_false.json

@@ -1,46 +1,4 @@
 {
   "has_setup": true,
-  "has_data": false,
-  "steps": [
-    {
-      "name": "is_cloud",
-      "completed": false
-    },
-    {
-      "name": "apm_package",
-      "completed": false
-    },
-    {
-      "name": "cluster_settings",
-      "completed": false
-    },
-    {
-      "name": "ilm",
-      "completed": false
-    },
-    {
-      "name": "component_templates",
-      "completed": false
-    },
-    {
-      "name": "index_templates",
-      "completed": false
-    },
-    {
-      "name": "create_events_data_streams",
-      "completed": false
-    },
-    {
-      "name": "create_indices",
-      "completed": false
-    },
-    {
-      "name": "security",
-      "completed": false
-    },
-    {
-      "name": "fleet_policy",
-      "completed": false
-    }
-  ]
-}
+  "has_data": false
+}

x-pack/plugins/profiling/e2e/cypress/fixtures/es_resources_setup_false.json

@@ -1,46 +1,4 @@
 {
   "has_setup": false,
-  "has_data": false,
-  "steps": [
-    {
-      "name": "is_cloud",
-      "completed": false
-    },
-    {
-      "name": "apm_package",
-      "completed": false
-    },
-    {
-      "name": "cluster_settings",
-      "completed": false
-    },
-    {
-      "name": "ilm",
-      "completed": false
-    },
-    {
-      "name": "component_templates",
-      "completed": false
-    },
-    {
-      "name": "index_templates",
-      "completed": false
-    },
-    {
-      "name": "create_events_data_streams",
-      "completed": false
-    },
-    {
-      "name": "create_indices",
-      "completed": false
-    },
-    {
-      "name": "security",
-      "completed": false
-    },
-    {
-      "name": "fleet_policy",
-      "completed": false
-    }
-  ]
-}
+  "has_data": false
+}

x-pack/plugins/profiling/e2e/cypress_test_runner.ts

@@ -41,6 +41,9 @@ export async function cypressTestRunner({
     auth: `${username}:${password}`,
   });
 
+  // Ensure Fleet setup is complete
+  await axios.post(`${kibanaUrlWithAuth}/api/fleet/setup`, {}, { headers: { 'kbn-xsrf': true } });
+
   const profilingResources = await axios.get<{ has_setup: boolean; has_data: boolean }>(
     `${kibanaUrlWithAuth}/api/profiling/v1/setup/es_resources`,
     { headers: { 'kbn-xsrf': true } }

x-pack/plugins/profiling/server/lib/setup/apm_package.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { installPackage, getInstallation } from '@kbn/fleet-plugin/server/services/epm/packages';
+import {
+  fetchFindLatestPackageOrThrow,
+  pkgToPkgKey,
+} from '@kbn/fleet-plugin/server/services/epm/registry';
+import { ProfilingSetupOptions } from './types';
+import { PartialSetupState } from '../../../common/setup';
+
+export async function isApmPackageInstalled({
+  soClient,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const installation = await getInstallation({
+    pkgName: 'apm',
+    savedObjectsClient: soClient,
+  });
+  return {
+    packages: {
+      installed: !!installation,
+    },
+  };
+}
+
+export async function installLatestApmPackage({
+  client,
+  soClient,
+  spaceId,
+}: ProfilingSetupOptions) {
+  const esClient = client.getEsClient();
+  const { name, version } = await fetchFindLatestPackageOrThrow('apm');
+
+  await installPackage({
+    installSource: 'registry',
+    esClient,
+    savedObjectsClient: soClient,
+    pkgkey: pkgToPkgKey({ name, version }),
+    spaceId,
+    force: true,
+  });
+}

x-pack/plugins/profiling/server/lib/setup/cluster_settings.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ProfilingSetupOptions } from './types';
+import { PartialSetupState } from '../../../common/setup';
+
+const MAX_BUCKETS = 150000;
+
+export async function validateMaximumBuckets({
+  client,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const settings = await client.getEsClient().cluster.getSettings({});
+  const maxBuckets = settings.persistent.search?.max_buckets;
+  return {
+    settings: {
+      configured: maxBuckets === MAX_BUCKETS.toString(),
+    },
+  };
+}
+
+export async function setMaximumBuckets({ client }: ProfilingSetupOptions) {
+  await client.getEsClient().cluster.putSettings({
+    persistent: {
+      search: {
+        max_buckets: MAX_BUCKETS,
+      },
+    },
+  });
+}
+
+export async function validateResourceManagement({
+  client,
+}: ProfilingSetupOptions): Promise<PartialSetupState> {
+  const statusResponse = await client.profilingStatus();
+  return {
+    resource_management: {
+      enabled: statusResponse.resource_management.enabled,
+    },
+    resources: {
+      created: statusResponse.resources.created,
+    },
+  };
+}
+
+export async function enableResourceManagement({ client }: ProfilingSetupOptions) {
+  await client.getEsClient().cluster.putSettings({
+    persistent: {
+      xpack: {
+        profiling: {
+          templates: {
+            enabled: true,
+          },
+        },
+      },
+    },
+  });
+}