[APM] Add data access plugin

.github/CODEOWNERS

@@ -36,6 +36,7 @@ packages/analytics/shippers/elastic_v3/server @elastic/kibana-core
 packages/analytics/shippers/fullstory @elastic/kibana-core
 packages/analytics/shippers/gainsight @elastic/kibana-core
 packages/kbn-apm-config-loader @elastic/kibana-core @vigneshshanmugam
+x-pack/plugins/apm_data_access @elastic/apm-ui
 x-pack/plugins/apm @elastic/apm-ui
 packages/kbn-apm-synthtrace @elastic/apm-ui
 packages/kbn-apm-synthtrace-client @elastic/apm-ui

docs/developer/plugin-list.asciidoc

@@ -434,6 +434,10 @@ The plugin exposes the static DefaultEditorController class to consume.
 |This plugin provides access to App Monitoring features provided by Elastic. It allows you to monitor your software services and applications in real-time; visualize detailed performance information on your services, identify and analyze errors, and monitor host-level and APM agent-specific metrics like JVM and Go runtime metrics.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/apm_data_access[apmDataAccess]
+|WARNING: Missing README.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/asset_manager/README.md[assetManager]
 |This plugin provides access to the asset data stored in assets-* indices, primarily
 for inventory and topology purposes.

package.json

@@ -152,6 +152,7 @@
     "@kbn/analytics-shippers-fullstory": "link:packages/analytics/shippers/fullstory",
     "@kbn/analytics-shippers-gainsight": "link:packages/analytics/shippers/gainsight",
     "@kbn/apm-config-loader": "link:packages/kbn-apm-config-loader",
+    "@kbn/apm-data-access-plugin": "link:x-pack/plugins/apm_data_access",
     "@kbn/apm-plugin": "link:x-pack/plugins/apm",
     "@kbn/apm-utils": "link:packages/kbn-apm-utils",
     "@kbn/app-link-test-plugin": "link:test/plugin_functional/plugins/app_link_test",

tsconfig.base.json

@@ -66,6 +66,8 @@
       "@kbn/analytics-shippers-gainsight/*": ["packages/analytics/shippers/gainsight/*"],
       "@kbn/apm-config-loader": ["packages/kbn-apm-config-loader"],
       "@kbn/apm-config-loader/*": ["packages/kbn-apm-config-loader/*"],
+      "@kbn/apm-data-access-plugin": ["x-pack/plugins/apm_data_access"],
+      "@kbn/apm-data-access-plugin/*": ["x-pack/plugins/apm_data_access/*"],
       "@kbn/apm-plugin": ["x-pack/plugins/apm"],
       "@kbn/apm-plugin/*": ["x-pack/plugins/apm/*"],
       "@kbn/apm-synthtrace": ["packages/kbn-apm-synthtrace"],

x-pack/.i18nrc.json

@@ -8,6 +8,7 @@
     "xpack.stackAlerts": "plugins/stack_alerts",
     "xpack.stackConnectors": "plugins/stack_connectors",
     "xpack.apm": "plugins/apm",
+    "xpack.apmDataAccess": "plugins/apm_data_access",
     "xpack.banners": "plugins/banners",
     "xpack.canvas": "plugins/canvas",
     "xpack.cases": "plugins/cases",

x-pack/plugins/apm/common/apm_saved_object_constants.ts

@@ -5,13 +5,6 @@
  * 2.0.
  */
 
-// the types have to match the names of the saved object mappings
-// in /x-pack/plugins/apm/mappings.json
-
-// APM index settings
-export const APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE = 'apm-indices';
-export const APM_INDEX_SETTINGS_SAVED_OBJECT_ID = 'apm-indices';
-
 // APM telemetry
 export const APM_TELEMETRY_SAVED_OBJECT_TYPE = 'apm-telemetry';
 export const APM_TELEMETRY_SAVED_OBJECT_ID = 'apm-telemetry';

x-pack/plugins/apm/kibana.jsonc

@@ -9,6 +9,7 @@
     "browser": true,
     "configPath": ["xpack", "apm"],
     "requiredPlugins": [
+      "apmDataAccess",
       "data",
       "dashboard",
       "controls",

x-pack/plugins/apm/scripts/diagnostics_bundle/diagnostics_bundle.ts

@@ -10,9 +10,9 @@
 import { Client } from '@elastic/elasticsearch';
 import fs from 'fs/promises';
 import axios, { AxiosInstance } from 'axios';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { APIReturnType } from '../../public/services/rest/create_call_apm_api';
 import { getDiagnosticsBundle } from '../../server/routes/diagnostics/get_diagnostics_bundle';
-import { ApmIndicesConfig } from '../../server/routes/settings/apm_indices/get_apm_indices';
 
 type DiagnosticsBundle = APIReturnType<'GET /internal/apm/diagnostics'>;
 
@@ -100,7 +100,7 @@ async function getApmIndices(kibanaClient: AxiosInstance) {
         savedValue ?? defaultValue,
       ]
     )
-  ) as ApmIndicesConfig;
+  ) as APMIndices;
 }
 
 async function getFleetPackageInfo(kibanaClient: AxiosInstance) {

x-pack/plugins/apm/server/deprecations/deprecations.test.ts

@@ -10,8 +10,8 @@ import { kibanaPackageJson } from '@kbn/repo-info';
 import { GetDeprecationsContext } from '@kbn/core/server';
 import { CloudSetup } from '@kbn/cloud-plugin/server';
 import { getDeprecations } from '.';
-import { APMRouteHandlerResources } from '..';
 import { AgentPolicy } from '@kbn/fleet-plugin/common';
+import { APMRouteHandlerResources } from '../routes/apm_routes/register_apm_server_routes';
 
 const deprecationContext = {
   esClient: {},

x-pack/plugins/apm/server/deprecations/index.ts

@@ -13,7 +13,7 @@ import {
   getCloudAgentPolicy,
   getApmPackagePolicy,
 } from '../routes/fleet/get_cloud_apm_package_policy';
-import { APMRouteHandlerResources } from '..';
+import { APMRouteHandlerResources } from '../routes/apm_routes/register_apm_server_routes';
 
 export function getDeprecations({
   cloudSetup,

x-pack/plugins/apm/server/feature.ts

@@ -12,11 +12,11 @@ import {
   LicensingPluginSetup,
   LicensingApiRequestHandlerContext,
 } from '@kbn/licensing-plugin/server';
+import { APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE } from '@kbn/apm-data-access-plugin/server/saved_objects/apm_indices';
 import {
   ApmRuleType,
   APM_SERVER_FEATURE_ID,
 } from '../common/rules/apm_rule_types';
-import { APM_INDEX_SETTINGS_SAVED_OBJECT_TYPE } from '../common/apm_saved_object_constants';
 
 export const APM_FEATURE = {
   id: APM_SERVER_FEATURE_ID,

x-pack/plugins/apm/server/index.ts

@@ -56,13 +56,6 @@ const configSchema = schema.object({
       enabled: schema.boolean({ defaultValue: false }),
     }),
   }),
-  indices: schema.object({
-    transaction: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
-    span: schema.string({ defaultValue: 'traces-apm*,apm-*' }),
-    error: schema.string({ defaultValue: 'logs-apm*,apm-*' }),
-    metric: schema.string({ defaultValue: 'metrics-apm*,apm-*' }),
-    onboarding: schema.string({ defaultValue: 'apm-*' }),
-  }),
   forceSyntheticSource: schema.boolean({ defaultValue: false }),
   latestAgentVersionsUrl: schema.string({
     defaultValue: 'https://apm-agent-versions.elastic.co/versions.json',
@@ -109,35 +102,12 @@ export const config: PluginConfigDescriptor<APMConfig> = {
     deprecateFromRoot,
     unusedFromRoot,
   }) => [
-    unused('indices.sourcemap', { level: 'warning' }),
     unused('ui.transactionGroupBucketSize', {
       level: 'warning',
     }),
     rename('autocreateApmIndexPattern', 'autoCreateApmDataView', {
       level: 'warning',
     }),
-    renameFromRoot(
-      'apm_oss.transactionIndices',
-      'xpack.apm.indices.transaction',
-      { level: 'warning' }
-    ),
-    renameFromRoot('apm_oss.spanIndices', 'xpack.apm.indices.span', {
-      level: 'warning',
-    }),
-    renameFromRoot('apm_oss.errorIndices', 'xpack.apm.indices.error', {
-      level: 'warning',
-    }),
-    renameFromRoot('apm_oss.metricsIndices', 'xpack.apm.indices.metric', {
-      level: 'warning',
-    }),
-    renameFromRoot('apm_oss.sourcemapIndices', 'xpack.apm.indices.sourcemap', {
-      level: 'warning',
-    }),
-    renameFromRoot(
-      'apm_oss.onboardingIndices',
-      'xpack.apm.indices.onboarding',
-      { level: 'warning' }
-    ),
     deprecateFromRoot('apm_oss.enabled', '8.0.0', { level: 'warning' }),
     unusedFromRoot('apm_oss.fleetMode', { level: 'warning' }),
     unusedFromRoot('apm_oss.indexPattern', { level: 'warning' }),
@@ -165,7 +135,6 @@ export const config: PluginConfigDescriptor<APMConfig> = {
 };
 
 export type APMConfig = TypeOf<typeof configSchema>;
-export type ApmIndicesConfigName = keyof APMConfig['indices'];
 
 export const plugin = (initContext: PluginInitializerContext) =>
   new APMPlugin(initContext);
@@ -177,4 +146,3 @@ export type {
   APMServerRouteRepository,
   APIEndpoint,
 } from './routes/apm_routes/get_global_apm_server_route_repository';
-export type { APMRouteHandlerResources } from './routes/typings';

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -12,6 +12,7 @@ import moment from 'moment';
 import { v4 as uuidv4 } from 'uuid';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { waitForIndexStatus } from '@kbn/core-saved-objects-migration-server-internal';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ML_ERRORS } from '../../../common/anomaly_detection';
 import { METRICSET_NAME, PROCESSOR_EVENT } from '../../../common/es_fields/apm';
 import { Environment } from '../../../common/environment_rt';
@@ -20,7 +21,6 @@ import { withApmSpan } from '../../utils/with_apm_span';
 import { MlClient } from '../helpers/get_ml_client';
 import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants';
 import { getAnomalyDetectionJobs } from './get_anomaly_detection_jobs';
-import { ApmIndicesConfig } from '../../routes/settings/apm_indices/get_apm_indices';
 
 const DEFAULT_TIMEOUT = '60s';
 
@@ -33,7 +33,7 @@ export async function createAnomalyDetectionJobs({
 }: {
   mlClient?: MlClient;
   esClient: ElasticsearchClient;
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
   environments: Environment[];
   logger: Logger;
 }) {

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks';
-import { ApmIndicesConfig } from '../../../routes/settings/apm_indices/get_apm_indices';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { tasks } from './tasks';
 import {
   SERVICE_NAME,
@@ -19,7 +19,7 @@ describe('data telemetry collection tasks', () => {
     metric: 'apm-8.0.0-metric',
     span: 'apm-8.0.0-span',
     transaction: 'apm-8.0.0-transaction',
-  } as ApmIndicesConfig;
+  } as APMIndices;
 
   describe('environments', () => {
     const task = tasks.find((t) => t.name === 'environments');

x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts

@@ -10,6 +10,7 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { createHash } from 'crypto';
 import { flatten, merge, pickBy, sortBy, sum, uniq } from 'lodash';
 import { SavedObjectsClient } from '@kbn/core/server';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { AGENT_NAMES, RUM_AGENT_NAMES } from '../../../../common/agent_name';
 import {
   AGENT_ACTIVATION_METHOD,
@@ -56,10 +57,7 @@ import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
 import { Span } from '../../../../typings/es_schemas/ui/span';
 import { Transaction } from '../../../../typings/es_schemas/ui/transaction';
 import { APMTelemetry, APMPerService, APMDataTelemetry } from '../types';
-import {
-  ApmIndicesConfig,
-  APM_AGENT_CONFIGURATION_INDEX,
-} from '../../../routes/settings/apm_indices/get_apm_indices';
+import { APM_AGENT_CONFIGURATION_INDEX } from '../../../routes/settings/apm_indices/apm_system_index_constants';
 import { TelemetryClient } from '../telemetry_client';
 
 type ISavedObjectsClient = Pick<SavedObjectsClient, 'find'>;
@@ -76,7 +74,7 @@ interface TelemetryTask {
 
 export interface TelemetryTaskExecutorParams {
   telemetryClient: TelemetryClient;
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
   savedObjectsClient: ISavedObjectsClient;
 }
 

x-pack/plugins/apm/server/lib/apm_telemetry/index.ts

@@ -6,12 +6,17 @@
  */
 
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
-import { CoreSetup, Logger, SavedObjectsErrorHelpers } from '@kbn/core/server';
+import {
+  CoreSetup,
+  Logger,
+  SavedObjectsClientContract,
+  SavedObjectsErrorHelpers,
+} from '@kbn/core/server';
 import {
   TaskManagerSetupContract,
   TaskManagerStartContract,
 } from '@kbn/task-manager-plugin/server';
-import { APMConfig } from '../..';
+import { APMDataAccessConfig } from '@kbn/apm-data-access-plugin/server';
 import {
   APM_TELEMETRY_SAVED_OBJECT_ID,
   APM_TELEMETRY_SAVED_OBJECT_TYPE,
@@ -20,22 +25,23 @@ import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_obj
 import { collectDataTelemetry } from './collect_data_telemetry';
 import { APMUsage } from './types';
 import { apmSchema } from './schema';
-import { getApmIndices } from '../../routes/settings/apm_indices/get_apm_indices';
 import { getTelemetryClient } from './telemetry_client';
 
 export const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task';
 
 export async function createApmTelemetry({
   core,
-  config,
+  getApmIndices,
   usageCollector,
   taskManager,
   logger,
   kibanaVersion,
   isProd,
 }: {
   core: CoreSetup;
-  config: APMConfig;
+  getApmIndices: (
+    soClient: SavedObjectsClientContract
+  ) => Promise<APMDataAccessConfig['indices']>;
   usageCollector: UsageCollectionSetup;
   taskManager: TaskManagerSetupContract;
   logger: Logger;
@@ -56,14 +62,14 @@ export async function createApmTelemetry({
     },
   });
 
+  const telemetryClient = await getTelemetryClient({ core });
   const [coreStart] = await core.getStartServices();
   const savedObjectsClient = await getInternalSavedObjectsClient(coreStart);
-  const indices = await getApmIndices({ config, savedObjectsClient });
-  const telemetryClient = await getTelemetryClient({ core });
+  const apmIndices = await getApmIndices(savedObjectsClient);
 
   const collectAndStore = async () => {
     const dataTelemetry = await collectDataTelemetry({
-      indices,
+      indices: apmIndices,
       telemetryClient,
       logger,
       savedObjectsClient,

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/get_request_base.test.ts

@@ -6,7 +6,7 @@
  */
 
 import { APMEventESSearchRequest } from '.';
-import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { getRequestBase } from './get_request_base';
 
 describe('getRequestBase', () => {
@@ -26,7 +26,7 @@ describe('getRequestBase', () => {
       error: 'my-apm-*-error-*',
       span: 'my-apm-*-span-*',
       onboarding: 'my-apm-*-onboarding-*',
-    } as ApmIndicesConfig;
+    } as APMIndices;
 
     res = getRequestBase({ ...request, indices });
   });

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/get_request_base.ts

@@ -6,12 +6,12 @@
  */
 
 import type { ESFilter } from '@kbn/es-types';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { uniq } from 'lodash';
 import { ApmDataSource } from '../../../../../common/data_source';
 import {} from '../../../../../common/document_type';
 import { PROCESSOR_EVENT } from '../../../../../common/es_fields/apm';
-import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
 import {
   getConfigForDocumentType,
   getProcessorEventForDocumentType,
@@ -26,7 +26,7 @@ const processorEventIndexMap = {
 
 export function processorEventsToIndex(
   events: ProcessorEvent[],
-  indices: ApmIndicesConfig
+  indices: APMIndices
 ) {
   return uniq(
     events.flatMap((event) =>
@@ -37,7 +37,7 @@ export function processorEventsToIndex(
 
 export function getRequestBase(options: {
   apm: { events: ProcessorEvent[] } | { sources: ApmDataSource[] };
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
 }) {
   const events =
     'events' in options.apm

x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts

@@ -20,12 +20,12 @@ import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { unwrapEsResponse } from '@kbn/observability-plugin/server';
 import { compact, omit } from 'lodash';
 import { ValuesType } from 'utility-types';
+import type { APMIndices } from '@kbn/apm-data-access-plugin/server';
 import { ApmDataSource } from '../../../../../common/data_source';
 import { APMError } from '../../../../../typings/es_schemas/ui/apm_error';
 import { Metric } from '../../../../../typings/es_schemas/ui/metric';
 import { Span } from '../../../../../typings/es_schemas/ui/span';
 import { Transaction } from '../../../../../typings/es_schemas/ui/transaction';
-import { ApmIndicesConfig } from '../../../../routes/settings/apm_indices/get_apm_indices';
 import { withApmSpan } from '../../../../utils/with_apm_span';
 import {
   callAsyncWithDebug,
@@ -87,7 +87,7 @@ export interface APMEventClientConfig {
   esClient: ElasticsearchClient;
   debug: boolean;
   request: KibanaRequest;
-  indices: ApmIndicesConfig;
+  indices: APMIndices;
   options: {
     includeFrozen: boolean;
     forceSyntheticSource: boolean;
@@ -98,7 +98,7 @@ export class APMEventClient {
   private readonly esClient: ElasticsearchClient;
   private readonly debug: boolean;
   private readonly request: KibanaRequest;
-  public readonly indices: ApmIndicesConfig;
+  public readonly indices: APMIndices;
   private readonly includeFrozen: boolean;
   private readonly forceSyntheticSource: boolean;