[APM] Add data access plugin (#162367)

Closes #161906
Related to:
elastic/observability-dev#2787
(_internal_)

This add a new plugin `apm_data_access` that contains the APM query
targets (indices to query for APM data).
This plugin can be consumed by apm and any other plugin, making it
possible for other plugins to know about the configured APM query
targets.



## Example:

APM query targets can be specified in kibana[.dev].yml using
`xpack.apm.indices.{dataset}: some-index-*` for instances:

```yml
xpack.apm.indices.transaction: apm-*
```

See all config options on:
https://www.elastic.co/guide/en/kibana/current/apm-settings-kb.html#general-apm-settings-kb

Query targets can also be specified via the UI (and persisted in a saved
object) via the settings page: `/app/apm/settings/apm-indices`

**Retrieving the query targets**
Query targets can be retrieved from other plugins via `getApmIndices`:
```ts
const apmIndices = await plugins.apmDataAccess.setup.getApmIndices(savedObjects.client); 
```

TODO:

- [x] Add SO client and fetch space aware index config (see
https://github.com/sqren/kibana/blob/4d7f626da020c156500f8bf52bf8861cde54b4f1/x-pack/plugins/apm/server/routes/settings/apm_indices/get_apm_indices.ts#L32-L44)
- [ ] Add simple APM client for querying apm data

---------

Co-authored-by: kibanamachine <[email protected]>

.github/CODEOWNERS

@@ -36,6 +36,7 @@ packages/analytics/shippers/elastic_v3/server @elastic/kibana-core
 packages/analytics/shippers/fullstory @elastic/kibana-core
 packages/analytics/shippers/gainsight @elastic/kibana-core
 packages/kbn-apm-config-loader @elastic/kibana-core @vigneshshanmugam
+x-pack/plugins/apm_data_access @elastic/apm-ui
 x-pack/plugins/apm @elastic/apm-ui
 packages/kbn-apm-synthtrace @elastic/apm-ui
 packages/kbn-apm-synthtrace-client @elastic/apm-ui

docs/developer/plugin-list.asciidoc

@@ -434,6 +434,10 @@ The plugin exposes the static DefaultEditorController class to consume.
 |This plugin provides access to App Monitoring features provided by Elastic. It allows you to monitor your software services and applications in real-time; visualize detailed performance information on your services, identify and analyze errors, and monitor host-level and APM agent-specific metrics like JVM and Go runtime metrics.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/apm_data_access[apmDataAccess]
+|WARNING: Missing README.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/asset_manager/README.md[assetManager]
 |This plugin provides access to the asset data stored in assets-* indices, primarily
 for inventory and topology purposes.

package.json

@@ -152,6 +152,7 @@
     "@kbn/analytics-shippers-fullstory": "link:packages/analytics/shippers/fullstory",
     "@kbn/analytics-shippers-gainsight": "link:packages/analytics/shippers/gainsight",
     "@kbn/apm-config-loader": "link:packages/kbn-apm-config-loader",
+    "@kbn/apm-data-access-plugin": "link:x-pack/plugins/apm_data_access",
     "@kbn/apm-plugin": "link:x-pack/plugins/apm",
     "@kbn/apm-utils": "link:packages/kbn-apm-utils",
     "@kbn/app-link-test-plugin": "link:test/plugin_functional/plugins/app_link_test",

tsconfig.base.json

@@ -66,6 +66,8 @@
       "@kbn/analytics-shippers-gainsight/*": ["packages/analytics/shippers/gainsight/*"],
       "@kbn/apm-config-loader": ["packages/kbn-apm-config-loader"],
       "@kbn/apm-config-loader/*": ["packages/kbn-apm-config-loader/*"],
+      "@kbn/apm-data-access-plugin": ["x-pack/plugins/apm_data_access"],
+      "@kbn/apm-data-access-plugin/*": ["x-pack/plugins/apm_data_access/*"],
       "@kbn/apm-plugin": ["x-pack/plugins/apm"],
       "@kbn/apm-plugin/*": ["x-pack/plugins/apm/*"],
       "@kbn/apm-synthtrace": ["packages/kbn-apm-synthtrace"],

x-pack/.i18nrc.json

@@ -8,6 +8,7 @@
     "xpack.stackAlerts": "plugins/stack_alerts",
     "xpack.stackConnectors": "plugins/stack_connectors",
     "xpack.apm": "plugins/apm",
+    "xpack.apmDataAccess": "plugins/apm_data_access",
     "xpack.banners": "plugins/banners",
     "xpack.canvas": "plugins/canvas",
     "xpack.cases": "plugins/cases",

x-pack/plugins/apm/common/apm_saved_object_constants.ts

@@ -5,13 +5,6 @@
  * 2.0.
  */
 
-// the types have to match the names of the saved object mappings
-// in /x-pack/plugins/apm/mappings.json
-
-// APM index settings
-export const APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE = 'apm-indices';
-export const APM_INDEX_SETTINGS_SAVED_OBJECT_ID = 'apm-indices';
-
 // APM telemetry
 export const APM_TELEMETRY_SAVED_OBJECT_TYPE = 'apm-telemetry';
 export const APM_TELEMETRY_SAVED_OBJECT_ID = 'apm-telemetry';

x-pack/plugins/apm/kibana.jsonc

@@ -9,6 +9,7 @@
     "browser": true,
     "configPath": ["xpack", "apm"],
     "requiredPlugins": [
+      "apmDataAccess",
       "data",
       "dashboard",
       "controls",

x-pack/plugins/apm/scripts/diagnostics_bundle/diagnostics_bundle.ts

@@ -10,9 +10,9 @@
 import { Client } from '@elastic/elasticsearch';
 import fs from 'fs/promises';
 import axios, { AxiosInstance } from 'axios';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { APIReturnType } from '../../public/services/rest/create_call_apm_api';
 import { getDiagnosticsBundle } from '../../server/routes/diagnostics/get_diagnostics_bundle';
-import { ApmIndicesConfig } from '../../server/routes/settings/apm_indices/get_apm_indices';
 
 type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
 
@@ -100,7 +100,7 @@ async function getApmIndices(kibanaClient: AxiosInstance) {
         savedValue ?? defaultValue,
       ]
     )
-  ) as ApmIndicesConfig;
+  ) as APMIndices;
 }
 
 async function getFleetPackageInfo(kibanaClient: AxiosInstance) {

x-pack/plugins/apm/server/deprecations/deprecations.test.ts

@@ -10,8 +10,8 @@ import { kibanaPackageJson } from '@kbn/repo-info';
 import { GetDeprecationsContext } from '@kbn/core/server';
 import { CloudSetup } from '@kbn/cloud-plugin/server';
 import { getDeprecations } from '.';
-import { APMRouteHandlerResources } from '..';
 import { AgentPolicy } from '@kbn/fleet-plugin/common';
+import { APMRouteHandlerResources } from '../routes/apm_routes/register_apm_server_routes';
 
 const deprecationContext = {
   esClient: {},

x-pack/plugins/apm/server/deprecations/index.ts

@@ -13,7 +13,7 @@ import {
   getCloudAgentPolicy,
   getApmPackagePolicy,
 } from '../routes/fleet/get_cloud_apm_package_policy';
-import { APMRouteHandlerResources } from '..';
+import { APMRouteHandlerResources } from '../routes/apm_routes/register_apm_server_routes';
 
 export function getDeprecations({
   cloudSetup,

x-pack/plugins/apm/server/feature.ts

@@ -12,11 +12,11 @@ import {
   LicensingPluginSetup,
   LicensingApiRequestHandlerContext,
 } from '@kbn/licensing-plugin/server';
+import { APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE } from '@kbn/apm-data-access-plugin/server/saved_objects/apm_indices';
 import {
   ApmRuleType,
   APM_SERVER_FEATURE_ID,
 } from '../common/rules/apm_rule_types';
-import { APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE } from '../common/apm_saved_object_constants';
 
 export const APM_FEATURE = {
   id: APM_SERVER_FEATURE_ID,

x-pack/plugins/apm/server/index.ts

@@ -56,13 +56,6 @@ const configSchema = schema.object({
       enabled: schema.boolean({ defaultValue: false }),
     }),
   }),
-  indices: schema.object({
-    transaction: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
-    span: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
-    error: schema.string({ defaultValue: 'logs-apm*,apm-*' }),
-    metric: schema.string({ defaultValue: 'metrics-apm*,apm-*' }),
-    onboarding: schema.string({ defaultValue: 'apm-*' }),
-  }),
   forceSyntheticSource: schema.boolean({ defaultValue: false }),
   latestAgentVersionsUrl: schema.string({
     defaultValue: 'https://apm-agent-versions.elastic.co/versions.json',
@@ -109,35 +102,12 @@ export const config: PluginConfigDescriptor<APMConfig> = {
     deprecateFromRoot,
     unusedFromRoot,
   }) => [
-    unused('indices.sourcemap', { level: 'warning' }),
     unused('ui.transactionGroupBucketSize', {
       level: 'warning',
     }),
     rename('autocreateApmIndexPattern', 'autoCreateApmDataView', {
       level: 'warning',
     }),
-    renameFromRoot(
-      'apm_oss.transactionIndices',
-      'xpack.apm.indices.transaction',
-      { level: 'warning' }
-    ),
-    renameFromRoot('apm_oss.spanIndices', 'xpack.apm.indices.span', {
-      level: 'warning',
-    }),
-    renameFromRoot('apm_oss.errorIndices', 'xpack.apm.indices.error', {
-      level: 'warning',
-    }),
-    renameFromRoot('apm_oss.metricsIndices', 'xpack.apm.indices.metric', {
-      level: 'warning',
-    }),
-    renameFromRoot('apm_oss.sourcemapIndices', 'xpack.apm.indices.sourcemap', {
-      level: 'warning',
-    }),
-    renameFromRoot(
-      'apm_oss.onboardingIndices',
-      'xpack.apm.indices.onboarding',
-      { level: 'warning' }
-    ),
     deprecateFromRoot('apm_oss.enabled', '8.0.0', { level: 'warning' }),
     unusedFromRoot('apm_oss.fleetMode', { level: 'warning' }),
     unusedFromRoot('apm_oss.indexPattern', { level: 'warning' }),
@@ -165,7 +135,6 @@ export const config: PluginConfigDescriptor<APMConfig> = {
 };
 
 export type APMConfig = TypeOf<typeof configSchema>;
-export type ApmIndicesConfigName = keyof APMConfig['indices'];
 
 export const plugin = (initContext: PluginInitializerContext) =>
   new APMPlugin(initContext);
@@ -177,4 +146,3 @@ export type {
   APMServerRouteRepository,
   APIEndpoint,
 } from './routes/apm_routes/get_global_apm_server_route_repository';
-export type { APMRouteHandlerResources } from './routes/typings';

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -12,6 +12,7 @@ import moment from 'moment';
 import { v4 as uuidv4 } from 'uuid';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { waitForIndexStatus } from '@kbn/core-saved-objects-migration-server-internal';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ML_ERRORS } from '../../../common/anomaly_detection';
 import { METRICSET_NAME, PROCESSOR_EVENT } from '../../../common/es_fields/apm';
 import { Environment } from '../../../common/environment_rt';
@@ -20,7 +21,6 @@ import { withApmSpan } from '../../utils/with_apm_span';
 import { MlClient } from '../helpers/get_ml_client';
 import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants';
 import { getAnomalyDetectionJobs } from './get_anomaly_detection_jobs';
-import { ApmIndicesConfig } from '../../routes/settings/apm_indices/get_apm_indices';
 
 const DEFAULT_TIMEOUT = '60s';
 
@@ -33,7 +33,7 @@ export async function createAnomalyDetectionJobs({
 }: {
   mlClient?: MlClient;
   esClient: ElasticsearchClient;
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
   environments: Environment[];
   logger: Logger;
 }) {

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
-import { ApmIndicesConfig } from '../../../routes/settings/apm_indices/get_apm_indices';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { tasks } from './tasks';
 import {
   SERVICE_NAME,
@@ -19,7 +19,7 @@ describe('data telemetry collection tasks', () => {
     metric: 'apm-8.0.0-metric',
     span: 'apm-8.0.0-span',
     transaction: 'apm-8.0.0-transaction',
-  } as ApmIndicesConfig;
+  } as APMIndices;
 
   describe('environments', () => {
     const task = tasks.find((t) => t.name === 'environments');

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -10,6 +10,7 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { createHash } from 'crypto';
 import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash';
 import { SavedObjectsClient } from '@kbn/core/server';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name';
 import {
   AGENT_ACTIVATION_METHOD,
@@ -56,10 +57,7 @@ import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
 import { Span } from '../../../../typings/es_schemas/ui/span';
 import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import { APMTelemetry, APMPerService, APMDataTelemetry } from '../types';
-import {
-  ApmIndicesConfig,
-  APM_AGENT_CONFIGURATION_INDEX,
-} from '../../../routes/settings/apm_indices/get_apm_indices';
+import { APM_AGENT_CONFIGURATION_INDEX } from '../../../routes/settings/apm_indices/apm_system_index_constants';
 import { TelemetryClient } from '../telemetry_client';
 
 type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
@@ -76,7 +74,7 @@ interface TelemetryTask {
 
 export interface TelemetryTaskExecutorParams {
   telemetryClient: TelemetryClient;
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
   savedObjectsClient: ISavedObjectsClient;
 }
 

x-pack/plugins/apm/server/lib/apm_telemetry/index.ts

@@ -6,12 +6,17 @@
  */
 
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
-import { CoreSetup, Logger, SavedObjectsErrorHelpers } from '@kbn/core/server';
+import {
+  CoreSetup,
+  Logger,
+  SavedObjectsClientContract,
+  SavedObjectsErrorHelpers,
+} from '@kbn/core/server';
 import {
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '@kbn/task-manager-plugin/server';
-import { APMConfig } from '../..';
+import { APMDataAccessConfig } from '@kbn/apm-data-access-plugin/server';
 import {
   APM_TELEMETRY_SAVED_OBJECT_ID,
   APM_TELEMETRY_SAVED_OBJECT_TYPE,
@@ -20,22 +25,23 @@ import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_obj
 import { collectDataTelemetry } from './collect_data_telemetry';
 import { APMUsage } from './types';
 import { apmSchema } from './schema';
-import { getApmIndices } from '../../routes/settings/apm_indices/get_apm_indices';
 import { getTelemetryClient } from './telemetry_client';
 
 export const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task';
 
 export async function createApmTelemetry({
   core,
-  config,
+  getApmIndices,
   usageCollector,
   taskManager,
   logger,
   kibanaVersion,
   isProd,
 }: {
   core: CoreSetup;
-  config: APMConfig;
+  getApmIndices: (
+    soClient: SavedObjectsClientContract
+  ) => Promise<APMDataAccessConfig['indices']>;
   usageCollector: UsageCollectionSetup;
   taskManager: TaskManagerSetupContract;
   logger: Logger;
@@ -56,14 +62,14 @@ export async function createApmTelemetry({
     },
   });
 
+  const telemetryClient = await getTelemetryClient({ core });
   const [coreStart] = await core.getStartServices();
   const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
-  const indices = await getApmIndices({ config, savedObjectsClient });
-  const telemetryClient = await getTelemetryClient({ core });
+  const apmIndices = await getApmIndices(savedObjectsClient);
 
   const collectAndStore = async () => {
     const dataTelemetry = await collectDataTelemetry({
-      indices,
+      indices: apmIndices,
       telemetryClient,
       logger,
       savedObjectsClient,

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/get_request_base.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { APMEventESSearchRequest } from '.';
-import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { getRequestBase } from './get_request_base';
 
 describe('getRequestBase', () => {
@@ -26,7 +26,7 @@ describe('getRequestBase', () => {
       error: 'my-apm-*-error-*',
       span: 'my-apm-*-span-*',
       onboarding: 'my-apm-*-onboarding-*',
-    } as ApmIndicesConfig;
+    } as APMIndices;
 
     res = getRequestBase({ ...request, indices });
   });

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/get_request_base.ts

@@ -6,12 +6,12 @@
  */
 
 import type { ESFilter } from '@kbn/es-types';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { uniq } from 'lodash';
 import { ApmDataSource } from '../../../../../common/data_source';
 import {} from '../../../../../common/document_type';
 import { PROCESSOR_EVENT } from '../../../../../common/es_fields/apm';
-import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
 import {
   getConfigForDocumentType,
   getProcessorEventForDocumentType,
@@ -26,7 +26,7 @@ const processorEventIndexMap = {
 
 export function processorEventsToIndex(
   events: ProcessorEvent[],
-  indices: ApmIndicesConfig
+  indices: APMIndices
 ) {
   return uniq(
     events.flatMap((event) =>
@@ -37,7 +37,7 @@ export function processorEventsToIndex(
 
 export function getRequestBase(options: {
   apm: { events: ProcessorEvent[] } | { sources: ApmDataSource[] };
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
 }) {
   const events =
     'events' in options.apm

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts

@@ -20,12 +20,12 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { unwrapEsResponse } from '@kbn/observability-plugin/server';
 import { compact, omit } from 'lodash';
 import { ValuesType } from 'utility-types';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ApmDataSource } from '../../../../../common/data_source';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
 import { Metric } from '../../../../../typings/es_schemas/ui/metric';
 import { Span } from '../../../../../typings/es_schemas/ui/span';
 import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
-import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
 import { withApmSpan } from '../../../../utils/with_apm_span';
 import {
   callAsyncWithDebug,
@@ -87,7 +87,7 @@ export interface APMEventClientConfig {
   esClient: ElasticsearchClient;
   debug: boolean;
   request: KibanaRequest;
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
   options: {
     includeFrozen: boolean;
     forceSyntheticSource: boolean;
@@ -98,7 +98,7 @@ export class APMEventClient {
   private readonly esClient: ElasticsearchClient;
   private readonly debug: boolean;
   private readonly request: KibanaRequest;
-  public readonly indices: ApmIndicesConfig;
+  public readonly indices: APMIndices;
   private readonly includeFrozen: boolean;
   private readonly forceSyntheticSource: boolean;