From bb9fc4b06bdbc27d4fd06dce836379b86c33e644 Mon Sep 17 00:00:00 2001 From: mgiota Date: Thu, 21 Apr 2022 10:59:09 +0200 Subject: [PATCH 1/4] [Infra Monitoring]: add rac to feature privileges for logs and metrics (#130762) --- x-pack/plugins/infra/server/features.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/infra/server/features.ts b/x-pack/plugins/infra/server/features.ts index 73e247c495fc2..a1b1a7b729193 100644 --- a/x-pack/plugins/infra/server/features.ts +++ b/x-pack/plugins/infra/server/features.ts @@ -33,7 +33,7 @@ export const METRICS_FEATURE = { all: { app: ['infra', 'metrics', 'kibana'], catalogue: ['infraops', 'metrics'], - api: ['infra'], + api: ['infra', 'rac'], savedObject: { all: ['infrastructure-ui-source'], read: ['index-pattern'], @@ -54,7 +54,7 @@ export const METRICS_FEATURE = { read: { app: ['infra', 'metrics', 'kibana'], catalogue: ['infraops', 'metrics'], - api: ['infra'], + api: ['infra', 'rac'], savedObject: { all: [], read: ['infrastructure-ui-source', 'index-pattern'], @@ -92,7 +92,7 @@ export const LOGS_FEATURE = { all: { app: ['infra', 'logs', 'kibana'], catalogue: ['infralogging', 'logs'], - api: ['infra'], + api: ['infra', 'rac'], savedObject: { all: [infraSourceConfigurationSavedObjectName, logViewSavedObjectName], read: [], @@ -113,7 +113,7 @@ export const LOGS_FEATURE = { read: { app: ['infra', 'logs', 'kibana'], catalogue: ['infralogging', 'logs'], - api: ['infra'], + api: ['infra', 'rac'], alerting: { rule: { read: [LOG_DOCUMENT_COUNT_RULE_TYPE_ID], From 6e8d198c060d5dad17a0c0ab654b048bacd99779 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov <53621505+mibragimov@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:25:32 +0500 Subject: [PATCH 2/4] [Dev Tools] Fix cat APIs returning as escaped string (#130638) * Fix escaped strings * Add tests Co-authored-by: Muhammad Ibragimov --- .../send_request_to_es.test.ts | 32 +++++++++++++++++++ .../send_request_to_es.ts | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.test.ts b/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.test.ts index a308cf150f804..8578e271f37b3 100644 --- a/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.test.ts +++ b/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.test.ts @@ -98,4 +98,36 @@ describe('sendRequestToES', () => { expect(mockedSendRequestToES).toHaveBeenCalledTimes(1); } }); + describe('successful response value', () => { + describe('with text', () => { + it('should return value with lines separated', async () => { + mockedSendRequestToES.mockResolvedValue('\ntest_index-1 [] \ntest_index-2 []\n'); + const response = await sendRequestToES({ + http: mockContextValue.services.http, + requests: [{ method: 'GET', url: 'test-1', data: [] }], + }); + + expect(response).toMatchInlineSnapshot(` + " + test_index-1 [] + test_index-2 [] + " + `); + expect(mockedSendRequestToES).toHaveBeenCalledTimes(1); + }); + }); + + describe('with parsed json', () => { + it('should stringify value', async () => { + mockedSendRequestToES.mockResolvedValue(JSON.stringify({ test: 'some value' })); + const response = await sendRequestToES({ + http: mockContextValue.services.http, + requests: [{ method: 'GET', url: 'test-2', data: [] }], + }); + + expect(typeof response).toBe('string'); + expect(mockedSendRequestToES).toHaveBeenCalledTimes(1); + }); + }); + }); }); diff --git a/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.ts b/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.ts index 94e184748908c..451198aaf2d2b 100644 --- a/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.ts +++ b/src/plugins/console/public/application/hooks/use_send_current_request_to_es/send_request_to_es.ts @@ -93,7 +93,7 @@ export function sendRequestToES(args: EsRequestArgs): Promise if (body instanceof ArrayBuffer) { value = body; } else { - value = JSON.stringify(body, null, 2); + value = typeof body === 'string' ? body : JSON.stringify(body, null, 2); } const warnings = response.headers.get('warning'); From 7af6915581f0733d3dd82c283d1cc0f67a939e12 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 21 Apr 2022 12:11:46 +0200 Subject: [PATCH 3/4] [APM] Progressive fetching (experimental) (#127598) Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .eslintrc.js | 5 +- src/core/types/elasticsearch/search.ts | 5 + .../app/service_inventory/index.tsx | 20 ++- .../components/app/trace_overview/index.tsx | 5 +- .../public/hooks/use_progressive_fetcher.tsx | 155 ++++++++++++++++++ .../get_global_apm_server_route_repository.ts | 4 +- .../apm/server/routes/default_api_types.ts | 5 +- .../__snapshots__/queries.test.ts.snap | 114 +++++++------ .../get_service_transaction_stats.ts | 41 +++-- ...ervices_from_error_and_metric_documents.ts | 33 ++-- .../get_services/get_services_items.ts | 3 + .../routes/services/get_services/index.ts | 3 + ...service_transaction_detailed_statistics.ts | 53 +++--- .../get_services_detailed_statistics/index.ts | 3 + .../server/routes/services/queries.test.ts | 1 + .../apm/server/routes/services/route.ts | 34 +++- .../traces/get_top_traces_primary_stats.ts | 87 +++++----- .../plugins/apm/server/routes/traces/route.ts | 12 +- x-pack/plugins/observability/common/index.ts | 6 + .../common/progressive_loading.ts | 31 ++++ .../observability/common/ui_settings_keys.ts | 1 + .../observability/server/ui_settings.ts | 55 ++++++- .../tests/error_rate/service_apis.spec.ts | 1 + .../tests/error_rate/service_maps.spec.ts | 1 + .../tests/feature_controls.spec.ts | 4 +- .../tests/latency/service_apis.spec.ts | 1 + .../tests/latency/service_maps.spec.ts | 1 + .../observability_overview.spec.ts | 1 + .../services_detailed_statistics.spec.ts | 95 ++++++----- .../tests/services/top_services.spec.ts | 82 ++++++--- .../tests/throughput/service_apis.spec.ts | 1 + .../tests/throughput/service_maps.spec.ts | 1 + .../tests/traces/top_traces.spec.ts | 35 +++- 33 files changed, 662 insertions(+), 237 deletions(-) create mode 100644 x-pack/plugins/apm/public/hooks/use_progressive_fetcher.tsx create mode 100644 x-pack/plugins/observability/common/progressive_loading.ts diff --git a/.eslintrc.js b/.eslintrc.js index 3c1c455fc3295..dfbdd4de96f0a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -840,7 +840,10 @@ module.exports = { }, ], 'react-hooks/rules-of-hooks': 'error', // Checks rules of Hooks - 'react-hooks/exhaustive-deps': ['error', { additionalHooks: '^useFetcher$' }], + 'react-hooks/exhaustive-deps': [ + 'error', + { additionalHooks: '^(useFetcher|useProgressiveFetcher)$' }, + ], }, }, { diff --git a/src/core/types/elasticsearch/search.ts b/src/core/types/elasticsearch/search.ts index 6f2c0c28e670b..96d1bec3c5f1e 100644 --- a/src/core/types/elasticsearch/search.ts +++ b/src/core/types/elasticsearch/search.ts @@ -455,6 +455,11 @@ export type AggregateOf< reverse_nested: { doc_count: number; } & SubAggregateOf; + random_sampler: { + seed: number; + probability: number; + doc_count: number; + } & SubAggregateOf; sampler: { doc_count: number; } & SubAggregateOf; diff --git a/x-pack/plugins/apm/public/components/app/service_inventory/index.tsx b/x-pack/plugins/apm/public/components/app/service_inventory/index.tsx index 5d6dfef10d9dc..5baabca805b7c 100644 --- a/x-pack/plugins/apm/public/components/app/service_inventory/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_inventory/index.tsx @@ -19,6 +19,7 @@ import { useTimeRange } from '../../../hooks/use_time_range'; import { SearchBar } from '../../shared/search_bar'; import { ServiceList } from './service_list'; import { MLCallout, shouldDisplayMlCallout } from '../../shared/ml_callout'; +import { useProgressiveFetcher } from '../../../hooks/use_progressive_fetcher'; import { joinByKey } from '../../../../common/utils/join_by_key'; import { ServiceInventoryFieldName } from '../../../../common/service_inventory'; import { orderServiceItems } from './service_list/order_service_items'; @@ -62,7 +63,7 @@ function useServicesFetcher() { [start, end, environment, kuery, serviceGroup] ); - const mainStatisticsFetch = useFetcher( + const mainStatisticsFetch = useProgressiveFetcher( (callApmApi) => { if (start && end) { return callApmApi('GET /internal/apm/services', { @@ -88,9 +89,14 @@ function useServicesFetcher() { const { data: mainStatisticsData = initialData } = mainStatisticsFetch; - const comparisonFetch = useFetcher( + const comparisonFetch = useProgressiveFetcher( (callApmApi) => { - if (start && end && mainStatisticsData.items.length) { + if ( + start && + end && + mainStatisticsData.items.length && + mainStatisticsFetch.status === FETCH_STATUS.SUCCESS + ) { return callApmApi('GET /internal/apm/services/detailed_statistics', { params: { query: { @@ -141,14 +147,16 @@ export function ServiceInventory() { !userHasDismissedCallout && shouldDisplayMlCallout(anomalyDetectionSetupState); - const useOptimizedSorting = useKibana().services.uiSettings?.get( - apmServiceInventoryOptimizedSorting - ); + const useOptimizedSorting = + useKibana().services.uiSettings?.get( + apmServiceInventoryOptimizedSorting + ) || false; let isLoading: boolean; if (useOptimizedSorting) { isLoading = + // ensures table is usable when sorted and filtered services have loaded sortedAndFilteredServicesFetch.status === FETCH_STATUS.LOADING || (sortedAndFilteredServicesFetch.status === FETCH_STATUS.SUCCESS && sortedAndFilteredServicesFetch.data?.services.length === 0 && diff --git a/x-pack/plugins/apm/public/components/app/trace_overview/index.tsx b/x-pack/plugins/apm/public/components/app/trace_overview/index.tsx index 141aaee1b826e..21ae0f9820890 100644 --- a/x-pack/plugins/apm/public/components/app/trace_overview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/trace_overview/index.tsx @@ -8,13 +8,14 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React from 'react'; import { useApmParams } from '../../../hooks/use_apm_params'; -import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher'; +import { FETCH_STATUS } from '../../../hooks/use_fetcher'; import { APIReturnType } from '../../../services/rest/create_call_apm_api'; import { SearchBar } from '../../shared/search_bar'; import { TraceList } from './trace_list'; import { useFallbackToTransactionsFetcher } from '../../../hooks/use_fallback_to_transactions_fetcher'; import { AggregatedTransactionsBadge } from '../../shared/aggregated_transactions_badge'; import { useTimeRange } from '../../../hooks/use_time_range'; +import { useProgressiveFetcher } from '../../../hooks/use_progressive_fetcher'; type TracesAPIResponse = APIReturnType<'GET /internal/apm/traces'>; const DEFAULT_RESPONSE: TracesAPIResponse = { @@ -31,7 +32,7 @@ export function TraceOverview() { const { start, end } = useTimeRange({ rangeFrom, rangeTo }); - const { status, data = DEFAULT_RESPONSE } = useFetcher( + const { status, data = DEFAULT_RESPONSE } = useProgressiveFetcher( (callApmApi) => { if (start && end) { return callApmApi('GET /internal/apm/traces', { diff --git a/x-pack/plugins/apm/public/hooks/use_progressive_fetcher.tsx b/x-pack/plugins/apm/public/hooks/use_progressive_fetcher.tsx new file mode 100644 index 0000000000000..a3268fe8d2958 --- /dev/null +++ b/x-pack/plugins/apm/public/hooks/use_progressive_fetcher.tsx @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { OmitByValue, Assign } from 'utility-types'; +import type { + ClientRequestParamsOf, + EndpointOf, + ReturnOf, +} from '@kbn/server-route-repository'; +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { + apmProgressiveLoading, + getProbabilityFromProgressiveLoadingQuality, + ProgressiveLoadingQuality, +} from '@kbn/observability-plugin/common'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import type { APMServerRouteRepository } from '../../server'; + +import type { + APMClient, + APMClientOptions, +} from '../services/rest/create_call_apm_api'; +import { FetcherResult, FETCH_STATUS, useFetcher } from './use_fetcher'; + +type APMProgressivelyLoadingServerRouteRepository = OmitByValue< + { + [key in keyof APMServerRouteRepository]: ClientRequestParamsOf< + APMServerRouteRepository, + key + > extends { + params: { query: { probability: any } }; + } + ? APMServerRouteRepository[key] + : undefined; + }, + undefined +>; + +type WithoutProbabilityParameter> = { + params: { query: {} }; +} & Assign< + T, + { + params: Omit & { + query: Omit; + }; + } +>; + +type APMProgressiveAPIClient = < + TEndpoint extends EndpointOf +>( + endpoint: TEndpoint, + options: Omit & + WithoutProbabilityParameter< + ClientRequestParamsOf< + APMProgressivelyLoadingServerRouteRepository, + TEndpoint + > + > +) => Promise>; + +function clientWithProbability( + regularCallApmApi: APMClient, + probability: number +) { + return < + TEndpoint extends EndpointOf + >( + endpoint: TEndpoint, + options: Omit & + WithoutProbabilityParameter< + ClientRequestParamsOf< + APMProgressivelyLoadingServerRouteRepository, + TEndpoint + > + > + ) => { + return regularCallApmApi(endpoint, { + ...options, + params: { + ...options.params, + query: { + ...options.params.query, + probability, + }, + }, + } as any); + }; +} + +export function useProgressiveFetcher( + callback: ( + callApmApi: APMProgressiveAPIClient + ) => Promise | undefined, + dependencies: any[], + options?: Parameters[2] +): FetcherResult { + const { + services: { uiSettings }, + } = useKibana(); + + const progressiveLoadingQuality = + uiSettings?.get(apmProgressiveLoading) ?? + ProgressiveLoadingQuality.off; + + const sampledProbability = getProbabilityFromProgressiveLoadingQuality( + progressiveLoadingQuality + ); + + const sampledFetch = useFetcher( + (regularCallApmApi) => { + if (progressiveLoadingQuality === ProgressiveLoadingQuality.off) { + return; + } + return callback( + clientWithProbability(regularCallApmApi, sampledProbability) + ); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + dependencies, + options + ); + + const unsampledFetch = useFetcher( + (regularCallApmApi) => { + return callback(clientWithProbability(regularCallApmApi, 1)); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + dependencies + ); + + const fetches = [unsampledFetch, sampledFetch]; + + const isError = unsampledFetch.status === FETCH_STATUS.FAILURE; + + const usedFetch = + (!isError && + fetches.find((fetch) => fetch.status === FETCH_STATUS.SUCCESS)) || + unsampledFetch; + + const status = + unsampledFetch.status === FETCH_STATUS.LOADING && + usedFetch.status === FETCH_STATUS.SUCCESS + ? FETCH_STATUS.LOADING + : usedFetch.status; + + return { + ...usedFetch, + status, + }; +} diff --git a/x-pack/plugins/apm/server/routes/apm_routes/get_global_apm_server_route_repository.ts b/x-pack/plugins/apm/server/routes/apm_routes/get_global_apm_server_route_repository.ts index b0869b7abcbce..5e6ac627364d8 100644 --- a/x-pack/plugins/apm/server/routes/apm_routes/get_global_apm_server_route_repository.ts +++ b/x-pack/plugins/apm/server/routes/apm_routes/get_global_apm_server_route_repository.ts @@ -38,7 +38,7 @@ import { eventMetadataRouteRepository } from '../event_metadata/route'; import { suggestionsRouteRepository } from '../suggestions/route'; import { agentKeysRouteRepository } from '../agent_keys/route'; -const getTypedGlobalApmServerRouteRepository = () => { +function getTypedGlobalApmServerRouteRepository() { const repository = { ...dataViewRouteRepository, ...environmentsRouteRepository, @@ -70,7 +70,7 @@ const getTypedGlobalApmServerRouteRepository = () => { }; return repository; -}; +} const getGlobalApmServerRouteRepository = (): ServerRouteRepository => { return getTypedGlobalApmServerRouteRepository(); diff --git a/x-pack/plugins/apm/server/routes/default_api_types.ts b/x-pack/plugins/apm/server/routes/default_api_types.ts index e2b969f00edc5..78668e0bf2472 100644 --- a/x-pack/plugins/apm/server/routes/default_api_types.ts +++ b/x-pack/plugins/apm/server/routes/default_api_types.ts @@ -6,7 +6,7 @@ */ import * as t from 'io-ts'; -import { isoToEpochRt } from '@kbn/io-ts-utils'; +import { isoToEpochRt, toNumberRt } from '@kbn/io-ts-utils'; export { environmentRt } from '../../common/environment_rt'; @@ -15,4 +15,7 @@ export const rangeRt = t.type({ end: isoToEpochRt, }); +export const probabilityRt = t.type({ + probability: toNumberRt, +}); export const kueryRt = t.type({ kuery: t.string }); diff --git a/x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap index 4014f2a4a2acc..e09c50708c476 100644 --- a/x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/routes/services/__snapshots__/queries.test.ts.snap @@ -81,50 +81,57 @@ Array [ }, "body": Object { "aggs": Object { - "services": Object { + "sample": Object { "aggs": Object { - "transactionType": Object { + "services": Object { "aggs": Object { - "avg_duration": Object { - "avg": Object { - "field": "transaction.duration.us", - }, - }, - "environments": Object { - "terms": Object { - "field": "service.environment", - }, - }, - "outcomes": Object { - "terms": Object { - "field": "event.outcome", - "include": Array [ - "failure", - "success", - ], - }, - }, - "sample": Object { - "top_metrics": Object { - "metrics": Array [ - Object { - "field": "agent.name", + "transactionType": Object { + "aggs": Object { + "avg_duration": Object { + "avg": Object { + "field": "transaction.duration.us", + }, + }, + "environments": Object { + "terms": Object { + "field": "service.environment", + }, + }, + "outcomes": Object { + "terms": Object { + "field": "event.outcome", + "include": Array [ + "failure", + "success", + ], + }, + }, + "sample": Object { + "top_metrics": Object { + "metrics": Array [ + Object { + "field": "agent.name", + }, + ], + "sort": Object { + "@timestamp": "desc", + }, }, - ], - "sort": Object { - "@timestamp": "desc", }, }, + "terms": Object { + "field": "transaction.type", + }, }, }, "terms": Object { - "field": "transaction.type", + "field": "service.name", + "size": 50, }, }, }, - "terms": Object { - "field": "service.name", - "size": 50, + "random_sampler": Object { + "probability": 1, }, }, }, @@ -155,29 +162,36 @@ Array [ }, "body": Object { "aggs": Object { - "services": Object { + "sample": Object { "aggs": Object { - "environments": Object { - "terms": Object { - "field": "service.environment", - }, - }, - "latest": Object { - "top_metrics": Object { - "metrics": Array [ - Object { - "field": "agent.name", + "services": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", }, - ], - "sort": Object { - "@timestamp": "desc", }, + "latest": Object { + "top_metrics": Object { + "metrics": Array [ + Object { + "field": "agent.name", + }, + ], + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "terms": Object { + "field": "service.name", + "size": 50, }, }, }, - "terms": Object { - "field": "service.name", - "size": 50, + "random_sampler": Object { + "probability": 1, }, }, }, diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts index 92014c05726c4..75322ebcd0551 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts @@ -35,6 +35,7 @@ import { ServiceGroup } from '../../../../common/service_groups'; interface AggregationParams { environment: string; kuery: string; + probability: number; setup: ServicesItemsSetup; searchAggregatedTransactions: boolean; maxNumServices: number; @@ -46,6 +47,7 @@ interface AggregationParams { export async function getServiceTransactionStats({ environment, kuery, + probability, setup, searchAggregatedTransactions, maxNumServices, @@ -90,28 +92,35 @@ export async function getServiceTransactionStats({ }, }, aggs: { - services: { - terms: { - field: SERVICE_NAME, - size: maxNumServices, + sample: { + random_sampler: { + probability, }, aggs: { - transactionType: { + services: { terms: { - field: TRANSACTION_TYPE, + field: SERVICE_NAME, + size: maxNumServices, }, aggs: { - ...metrics, - environments: { + transactionType: { terms: { - field: SERVICE_ENVIRONMENT, + field: TRANSACTION_TYPE, }, - }, - sample: { - top_metrics: { - metrics: [{ field: AGENT_NAME } as const], - sort: { - '@timestamp': 'desc' as const, + aggs: { + ...metrics, + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + sample: { + top_metrics: { + metrics: [{ field: AGENT_NAME } as const], + sort: { + '@timestamp': 'desc' as const, + }, + }, }, }, }, @@ -125,7 +134,7 @@ export async function getServiceTransactionStats({ ); return ( - response.aggregations?.services.buckets.map((bucket) => { + response.aggregations?.sample.services.buckets.map((bucket) => { const topTransactionTypeBucket = bucket.transactionType.buckets.find( ({ key }) => diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_services_from_error_and_metric_documents.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_services_from_error_and_metric_documents.ts index 63378095d72e2..45049db6ef8f1 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_services_from_error_and_metric_documents.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_services_from_error_and_metric_documents.ts @@ -21,6 +21,7 @@ import { ServiceGroup } from '../../../../common/service_groups'; export async function getServicesFromErrorAndMetricDocuments({ environment, setup, + probability, maxNumServices, kuery, start, @@ -29,6 +30,7 @@ export async function getServicesFromErrorAndMetricDocuments({ }: { setup: Setup; environment: string; + probability: number; maxNumServices: number; kuery: string; start: number; @@ -56,21 +58,28 @@ export async function getServicesFromErrorAndMetricDocuments({ }, }, aggs: { - services: { - terms: { - field: SERVICE_NAME, - size: maxNumServices, + sample: { + random_sampler: { + probability, }, aggs: { - environments: { + services: { terms: { - field: SERVICE_ENVIRONMENT, + field: SERVICE_NAME, + size: maxNumServices, }, - }, - latest: { - top_metrics: { - metrics: [{ field: AGENT_NAME } as const], - sort: { '@timestamp': 'desc' }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + latest: { + top_metrics: { + metrics: [{ field: AGENT_NAME } as const], + sort: { '@timestamp': 'desc' }, + }, + }, }, }, }, @@ -81,7 +90,7 @@ export async function getServicesFromErrorAndMetricDocuments({ ); return ( - response.aggregations?.services.buckets.map((bucket) => { + response.aggregations?.sample.services.buckets.map((bucket) => { return { serviceName: bucket.key as string, environments: bucket.environments.buckets.map( diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts index 1235af756b76e..a5936dd68d026 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts @@ -21,6 +21,7 @@ const MAX_NUMBER_OF_SERVICES = 50; export async function getServicesItems({ environment, kuery, + probability, setup, searchAggregatedTransactions, logger, @@ -30,6 +31,7 @@ export async function getServicesItems({ }: { environment: string; kuery: string; + probability: number; setup: ServicesItemsSetup; searchAggregatedTransactions: boolean; logger: Logger; @@ -41,6 +43,7 @@ export async function getServicesItems({ const params = { environment, kuery, + probability, setup, searchAggregatedTransactions, maxNumServices: MAX_NUMBER_OF_SERVICES, diff --git a/x-pack/plugins/apm/server/routes/services/get_services/index.ts b/x-pack/plugins/apm/server/routes/services/get_services/index.ts index 223c0b3f613ef..fb133e4f5ad1a 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/index.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/index.ts @@ -14,6 +14,7 @@ import { ServiceGroup } from '../../../../common/service_groups'; export async function getServices({ environment, kuery, + probability, setup, searchAggregatedTransactions, logger, @@ -23,6 +24,7 @@ export async function getServices({ }: { environment: string; kuery: string; + probability: number; setup: Setup; searchAggregatedTransactions: boolean; logger: Logger; @@ -34,6 +36,7 @@ export async function getServices({ const items = await getServicesItems({ environment, kuery, + probability, setup, searchAggregatedTransactions, logger, diff --git a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts index 0e3310efd400c..5f1b4faad8dc3 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts @@ -39,6 +39,7 @@ export async function getServiceTransactionDetailedStatistics({ offset, start, end, + probability, }: { serviceNames: string[]; environment: string; @@ -48,6 +49,7 @@ export async function getServiceTransactionDetailedStatistics({ offset?: string; start: number; end: number; + probability: number; }) { const { apmEventClient } = setup; const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ @@ -91,33 +93,42 @@ export async function getServiceTransactionDetailedStatistics({ }, }, aggs: { - services: { - terms: { - field: SERVICE_NAME, + sample: { + random_sampler: { + probability, }, aggs: { - transactionType: { + services: { terms: { - field: TRANSACTION_TYPE, + field: SERVICE_NAME, + size: serviceNames.length, }, aggs: { - ...metrics, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSizeForAggregatedTransactions({ - start: startWithOffset, - end: endWithOffset, - numBuckets: 20, - searchAggregatedTransactions, - }).intervalString, - min_doc_count: 0, - extended_bounds: { - min: startWithOffset, - max: endWithOffset, + transactionType: { + terms: { + field: TRANSACTION_TYPE, + }, + aggs: { + ...metrics, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: + getBucketSizeForAggregatedTransactions({ + start: startWithOffset, + end: endWithOffset, + numBuckets: 20, + searchAggregatedTransactions, + }).intervalString, + min_doc_count: 0, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, + }, + }, + aggs: metrics, }, }, - aggs: metrics, }, }, }, @@ -129,7 +140,7 @@ export async function getServiceTransactionDetailedStatistics({ ); return keyBy( - response.aggregations?.services.buckets.map((bucket) => { + response.aggregations?.sample.services.buckets.map((bucket) => { const topTransactionTypeBucket = bucket.transactionType.buckets.find( ({ key }) => diff --git a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts index d0fa24913a214..3009f9214ac31 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts @@ -18,6 +18,7 @@ export async function getServicesDetailedStatistics({ offset, start, end, + probability, }: { serviceNames: string[]; environment: string; @@ -27,6 +28,7 @@ export async function getServicesDetailedStatistics({ offset?: string; start: number; end: number; + probability: number; }) { return withApmSpan('get_service_detailed_statistics', async () => { const commonProps = { @@ -37,6 +39,7 @@ export async function getServicesDetailedStatistics({ searchAggregatedTransactions, start, end, + probability, }; const [currentPeriod, previousPeriod] = await Promise.all([ diff --git a/x-pack/plugins/apm/server/routes/services/queries.test.ts b/x-pack/plugins/apm/server/routes/services/queries.test.ts index a8a65d0388297..8faa7b9857bf7 100644 --- a/x-pack/plugins/apm/server/routes/services/queries.test.ts +++ b/x-pack/plugins/apm/server/routes/services/queries.test.ts @@ -60,6 +60,7 @@ describe('services queries', () => { start: 0, end: 50000, serviceGroup: null, + probability: 1, }) ); diff --git a/x-pack/plugins/apm/server/routes/services/route.ts b/x-pack/plugins/apm/server/routes/services/route.ts index e8de9ba8948d9..0bcdca6e3cb89 100644 --- a/x-pack/plugins/apm/server/routes/services/route.ts +++ b/x-pack/plugins/apm/server/routes/services/route.ts @@ -36,7 +36,12 @@ import { getServiceProfilingTimeline } from './profiling/get_service_profiling_t import { getServiceInfrastructure } from './get_service_infrastructure'; import { withApmSpan } from '../../utils/with_apm_span'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; -import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; +import { + environmentRt, + kueryRt, + rangeRt, + probabilityRt, +} from '../default_api_types'; import { offsetPreviousPeriodCoordinates } from '../../../common/utils/offset_previous_period_coordinate'; import { getServicesDetailedStatistics } from './get_services_detailed_statistics'; import { getServiceDependenciesBreakdown } from './get_service_dependencies_breakdown'; @@ -57,6 +62,7 @@ const servicesRoute = createApmServerRoute({ kueryRt, rangeRt, t.partial({ serviceGroup: t.string }), + probabilityRt, ]), }), options: { tags: ['access:apm'] }, @@ -105,6 +111,7 @@ const servicesRoute = createApmServerRoute({ start, end, serviceGroup: serviceGroupId, + probability, } = params.query; const savedObjectsClient = context.core.savedObjects.client; @@ -123,6 +130,7 @@ const servicesRoute = createApmServerRoute({ return getServices({ environment, kuery, + probability, setup, searchAggregatedTransactions, logger, @@ -137,10 +145,14 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/services/detailed_statistics', params: t.type({ query: t.intersection([ - environmentRt, - kueryRt, - rangeRt, - offsetRt, + // t.intersection seemingly only supports 5 arguments so let's wrap them in another intersection + t.intersection([ + environmentRt, + kueryRt, + rangeRt, + offsetRt, + probabilityRt, + ]), t.type({ serviceNames: jsonRt.pipe(t.array(t.string)) }), ]), }), @@ -181,8 +193,15 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({ }> => { const setup = await setupRequest(resources); const { params } = resources; - const { environment, kuery, offset, serviceNames, start, end } = - params.query; + const { + environment, + kuery, + offset, + serviceNames, + start, + end, + probability, + } = params.query; const searchAggregatedTransactions = await getSearchAggregatedTransactions({ ...setup, start, @@ -203,6 +222,7 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({ serviceNames, start, end, + probability, }); }, }); diff --git a/x-pack/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts b/x-pack/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts index 7cc415a4673c8..f6903eba7476c 100644 --- a/x-pack/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts +++ b/x-pack/plugins/apm/server/routes/traces/get_top_traces_primary_stats.ts @@ -41,6 +41,7 @@ export type BucketKey = Record< interface TopTracesParams { environment: string; kuery: string; + probability: number; transactionName?: string; searchAggregatedTransactions: boolean; start: number; @@ -50,6 +51,7 @@ interface TopTracesParams { export function getTopTracesPrimaryStats({ environment, kuery, + probability, transactionName, searchAggregatedTransactions, start, @@ -101,47 +103,52 @@ export function getTopTracesPrimaryStats({ }, }, aggs: { - transaction_groups: { - composite: { - sources: asMutableArray([ - { [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } }, - { - [TRANSACTION_NAME]: { - terms: { field: TRANSACTION_NAME }, - }, - }, - ] as const), - // traces overview is hardcoded to 10000 - size: 10000, - }, + sample: { + random_sampler: { probability }, aggs: { - transaction_type: { - top_metrics: { - sort: { - '@timestamp': 'desc' as const, - }, - metrics: [ - { - field: TRANSACTION_TYPE, - } as const, + transaction_groups: { + composite: { + sources: asMutableArray([ + { [SERVICE_NAME]: { terms: { field: SERVICE_NAME } } }, { - field: AGENT_NAME, - } as const, - ], - }, - }, - avg: { - avg: { - field: getDurationFieldForTransactions( - searchAggregatedTransactions - ), + [TRANSACTION_NAME]: { + terms: { field: TRANSACTION_NAME }, + }, + }, + ] as const), + // traces overview is hardcoded to 10000 + size: 10000, }, - }, - sum: { - sum: { - field: getDurationFieldForTransactions( - searchAggregatedTransactions - ), + aggs: { + transaction_type: { + top_metrics: { + sort: { + '@timestamp': 'desc' as const, + }, + metrics: [ + { + field: TRANSACTION_TYPE, + } as const, + { + field: AGENT_NAME, + } as const, + ], + }, + }, + avg: { + avg: { + field: getDurationFieldForTransactions( + searchAggregatedTransactions + ), + }, + }, + sum: { + sum: { + field: getDurationFieldForTransactions( + searchAggregatedTransactions + ), + }, + }, }, }, }, @@ -152,12 +159,12 @@ export function getTopTracesPrimaryStats({ ); const calculateImpact = calculateImpactBuilder( - response.aggregations?.transaction_groups.buckets.map( + response.aggregations?.sample.transaction_groups.buckets.map( ({ sum }) => sum.value ) ); - const items = response.aggregations?.transaction_groups.buckets.map( + const items = response.aggregations?.sample.transaction_groups.buckets.map( (bucket) => { return { key: bucket.key as BucketKey, diff --git a/x-pack/plugins/apm/server/routes/traces/route.ts b/x-pack/plugins/apm/server/routes/traces/route.ts index 05aa27f27196d..c767a4e67aa63 100644 --- a/x-pack/plugins/apm/server/routes/traces/route.ts +++ b/x-pack/plugins/apm/server/routes/traces/route.ts @@ -10,7 +10,12 @@ import { setupRequest } from '../../lib/helpers/setup_request'; import { getTraceItems } from './get_trace_items'; import { getTopTracesPrimaryStats } from './get_top_traces_primary_stats'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; -import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; +import { + environmentRt, + kueryRt, + probabilityRt, + rangeRt, +} from '../default_api_types'; import { getSearchAggregatedTransactions } from '../../lib/helpers/transactions'; import { getRootTransactionByTraceId } from '../transactions/get_transaction_by_trace'; import { getTransaction } from '../transactions/get_transaction'; @@ -18,7 +23,7 @@ import { getTransaction } from '../transactions/get_transaction'; const tracesRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/traces', params: t.type({ - query: t.intersection([environmentRt, kueryRt, rangeRt]), + query: t.intersection([environmentRt, kueryRt, rangeRt, probabilityRt]), }), options: { tags: ['access:apm'] }, handler: async ( @@ -37,7 +42,7 @@ const tracesRoute = createApmServerRoute({ }> => { const setup = await setupRequest(resources); const { params } = resources; - const { environment, kuery, start, end } = params.query; + const { environment, kuery, start, end, probability } = params.query; const searchAggregatedTransactions = await getSearchAggregatedTransactions({ ...setup, kuery, @@ -48,6 +53,7 @@ const tracesRoute = createApmServerRoute({ return await getTopTracesPrimaryStats({ environment, kuery, + probability, setup, searchAggregatedTransactions, start, diff --git a/x-pack/plugins/observability/common/index.ts b/x-pack/plugins/observability/common/index.ts index 8a2ee7c0f1718..e01b9ba3f9922 100644 --- a/x-pack/plugins/observability/common/index.ts +++ b/x-pack/plugins/observability/common/index.ts @@ -16,8 +16,14 @@ export { enableInfrastructureView, defaultApmServiceEnvironment, apmServiceInventoryOptimizedSorting, + apmProgressiveLoading, } from './ui_settings_keys'; +export { + ProgressiveLoadingQuality, + getProbabilityFromProgressiveLoadingQuality, +} from './progressive_loading'; + export const casesFeatureId = 'observabilityCases'; // The ID of the observability app. Should more appropriately be called diff --git a/x-pack/plugins/observability/common/progressive_loading.ts b/x-pack/plugins/observability/common/progressive_loading.ts new file mode 100644 index 0000000000000..04063c8a9cf79 --- /dev/null +++ b/x-pack/plugins/observability/common/progressive_loading.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const enum ProgressiveLoadingQuality { + low = 'low', + medium = 'medium', + high = 'high', + off = 'off', +} + +export function getProbabilityFromProgressiveLoadingQuality( + quality: ProgressiveLoadingQuality +): number { + switch (quality) { + case ProgressiveLoadingQuality.high: + return 0.1; + + case ProgressiveLoadingQuality.medium: + return 0.01; + + case ProgressiveLoadingQuality.low: + return 0.001; + + case ProgressiveLoadingQuality.off: + return 1; + } +} diff --git a/x-pack/plugins/observability/common/ui_settings_keys.ts b/x-pack/plugins/observability/common/ui_settings_keys.ts index 54eaa9046d874..4c1b1dc729fea 100644 --- a/x-pack/plugins/observability/common/ui_settings_keys.ts +++ b/x-pack/plugins/observability/common/ui_settings_keys.ts @@ -10,6 +10,7 @@ export const maxSuggestions = 'observability:maxSuggestions'; export const enableComparisonByDefault = 'observability:enableComparisonByDefault'; export const enableInfrastructureView = 'observability:enableInfrastructureView'; export const defaultApmServiceEnvironment = 'observability:apmDefaultServiceEnvironment'; +export const apmProgressiveLoading = 'observability:apmProgressiveLoading'; export const enableServiceGroups = 'observability:enableServiceGroups'; export const apmServiceInventoryOptimizedSorting = 'observability:apmServiceInventoryOptimizedSorting'; diff --git a/x-pack/plugins/observability/server/ui_settings.ts b/x-pack/plugins/observability/server/ui_settings.ts index 1fec1883cd6f4..02c519b10d19c 100644 --- a/x-pack/plugins/observability/server/ui_settings.ts +++ b/x-pack/plugins/observability/server/ui_settings.ts @@ -8,13 +8,14 @@ import { schema } from '@kbn/config-schema'; import { i18n } from '@kbn/i18n'; import { UiSettingsParams } from '@kbn/core/types'; -import { observabilityFeatureId } from '../common'; +import { observabilityFeatureId, ProgressiveLoadingQuality } from '../common'; import { enableComparisonByDefault, enableInspectEsQueries, maxSuggestions, enableInfrastructureView, defaultApmServiceEnvironment, + apmProgressiveLoading, enableServiceGroups, apmServiceInventoryOptimizedSorting, } from '../common/ui_settings_keys'; @@ -86,6 +87,58 @@ export const uiSettings: Record[${technicalPreviewLabel}]` }, + }), + value: ProgressiveLoadingQuality.off, + schema: schema.oneOf([ + schema.literal(ProgressiveLoadingQuality.off), + schema.literal(ProgressiveLoadingQuality.low), + schema.literal(ProgressiveLoadingQuality.medium), + schema.literal(ProgressiveLoadingQuality.high), + ]), + requiresPageReload: false, + type: 'select', + options: [ + ProgressiveLoadingQuality.off, + ProgressiveLoadingQuality.low, + ProgressiveLoadingQuality.medium, + ProgressiveLoadingQuality.high, + ], + optionLabels: { + [ProgressiveLoadingQuality.off]: i18n.translate( + 'xpack.observability.apmProgressiveLoadingQualityOff', + { + defaultMessage: 'Off', + } + ), + [ProgressiveLoadingQuality.low]: i18n.translate( + 'xpack.observability.apmProgressiveLoadingQualityLow', + { + defaultMessage: 'Low sampling rate (fastest, least accurate)', + } + ), + [ProgressiveLoadingQuality.medium]: i18n.translate( + 'xpack.observability.apmProgressiveLoadingQualityMedium', + { + defaultMessage: 'Medium sampling rate', + } + ), + [ProgressiveLoadingQuality.high]: i18n.translate( + 'xpack.observability.apmProgressiveLoadingQualityHigh', + { + defaultMessage: 'High sampling rate (slower, most accurate)', + } + ), + }, + }, [enableServiceGroups]: { category: [observabilityFeatureId], name: i18n.translate('xpack.observability.enableServiceGroups', { diff --git a/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts b/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts index 205b5838e307d..77853b78685ec 100644 --- a/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts +++ b/x-pack/test/apm_api_integration/tests/error_rate/service_apis.spec.ts @@ -41,6 +41,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { ...commonQuery, + probability: 1, kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`, }, }, diff --git a/x-pack/test/apm_api_integration/tests/error_rate/service_maps.spec.ts b/x-pack/test/apm_api_integration/tests/error_rate/service_maps.spec.ts index 7f7aa7a85d407..011165c874421 100644 --- a/x-pack/test/apm_api_integration/tests/error_rate/service_maps.spec.ts +++ b/x-pack/test/apm_api_integration/tests/error_rate/service_maps.spec.ts @@ -31,6 +31,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { query: { ...commonQuery, kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`, + probability: 1, }, }, }), diff --git a/x-pack/test/apm_api_integration/tests/feature_controls.spec.ts b/x-pack/test/apm_api_integration/tests/feature_controls.spec.ts index 77b8faf781eb9..21d2ad617f112 100644 --- a/x-pack/test/apm_api_integration/tests/feature_controls.spec.ts +++ b/x-pack/test/apm_api_integration/tests/feature_controls.spec.ts @@ -79,7 +79,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext) }, { req: { - url: `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`, + url: `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=&probability=1`, }, expectForbidden: expect403, expectResponse: expect200, @@ -98,7 +98,7 @@ export default function featureControlsTests({ getService }: FtrProviderContext) }, { req: { - url: `/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=`, + url: `/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=&probability=1`, }, expectForbidden: expect403, expectResponse: expect200, diff --git a/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts b/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts index 3ba46802d6ee7..893a9f81d2526 100644 --- a/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts +++ b/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts @@ -44,6 +44,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { query: { ...commonQuery, kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`, + probability: 1, }, }, }), diff --git a/x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts b/x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts index dc150d598b7a7..2338e9ea128b2 100644 --- a/x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts +++ b/x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts @@ -31,6 +31,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { query: { ...commonQuery, kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`, + probability: 1, }, }, }), diff --git a/x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts b/x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts index 90c08433b6ab5..cb8fecb1cdf59 100644 --- a/x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts +++ b/x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts @@ -29,6 +29,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { ...commonQuery, + probability: 1, environment: 'ENVIRONMENT_ALL', kuery: '', }, diff --git a/x-pack/test/apm_api_integration/tests/services/services_detailed_statistics.spec.ts b/x-pack/test/apm_api_integration/tests/services/services_detailed_statistics.spec.ts index feb9aa8b8e5c4..25672acc76227 100644 --- a/x-pack/test/apm_api_integration/tests/services/services_detailed_statistics.spec.ts +++ b/x-pack/test/apm_api_integration/tests/services/services_detailed_statistics.spec.ts @@ -5,19 +5,20 @@ * 2.0. */ import expect from '@kbn/expect'; -import url from 'url'; import moment from 'moment'; import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number'; import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { ApmApiError } from '../../common/apm_api_supertest'; type ServicesDetailedStatisticsReturn = APIReturnType<'GET /internal/apm/services/detailed_statistics'>; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); - const supertest = getService('legacySupertestAsApmReadUser'); + + const apmApiClient = getService('apmApiClient'); const archiveName = 'apm_8.0.0'; const metadata = archives_metadata[archiveName]; @@ -29,9 +30,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { { config: 'basic', archives: [] }, () => { it('handles the empty state', async () => { - const response = await supertest.get( - url.format({ - pathname: `/internal/apm/services/detailed_statistics`, + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/detailed_statistics`, + params: { query: { start, end, @@ -39,9 +40,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { environment: 'ENVIRONMENT_ALL', kuery: '', offset: '1d', + probability: 1, }, - }) - ); + }, + }); + expect(response.status).to.be(200); expect(response.body.currentPeriod).to.be.empty(); expect(response.body.previousPeriod).to.be.empty(); @@ -55,18 +58,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { () => { let servicesDetailedStatistics: ServicesDetailedStatisticsReturn; before(async () => { - const response = await supertest.get( - url.format({ - pathname: `/internal/apm/services/detailed_statistics`, + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/detailed_statistics`, + params: { query: { start, end, serviceNames: JSON.stringify(serviceNames), environment: 'ENVIRONMENT_ALL', kuery: '', + probability: 1, }, - }) - ); + }, + }); expect(response.status).to.be(200); servicesDetailedStatistics = response.body; }); @@ -106,52 +110,61 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('returns empty when empty service names is passed', async () => { - const response = await supertest.get( - url.format({ - pathname: `/internal/apm/services/detailed_statistics`, - query: { - start, - end, - serviceNames: JSON.stringify([]), - environment: 'ENVIRONMENT_ALL', - kuery: '', + try { + await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/detailed_statistics`, + params: { + query: { + start, + end, + serviceNames: JSON.stringify([]), + environment: 'ENVIRONMENT_ALL', + kuery: '', + probability: 1, + }, }, - }) - ); - expect(response.status).to.be(400); - expect(response.body.message).to.equal('serviceNames cannot be empty'); + }); + expect().fail('Expected API call to throw an error'); + } catch (error: unknown) { + const apiError = error as ApmApiError; + expect(apiError.res.status).eql(400); + + expect(apiError.res.body.message).eql('serviceNames cannot be empty'); + } }); it('filters by environment', async () => { - const response = await supertest.get( - url.format({ - pathname: `/internal/apm/services/detailed_statistics`, + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/detailed_statistics`, + params: { query: { start, end, serviceNames: JSON.stringify(serviceNames), environment: 'production', kuery: '', + probability: 1, }, - }) - ); + }, + }); expect(response.status).to.be(200); expect(Object.keys(response.body.currentPeriod).length).to.be(1); expect(response.body.currentPeriod['opbeans-java']).not.to.be.empty(); }); it('filters by kuery', async () => { - const response = await supertest.get( - url.format({ - pathname: `/internal/apm/services/detailed_statistics`, + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/detailed_statistics`, + params: { query: { start, end, serviceNames: JSON.stringify(serviceNames), environment: 'ENVIRONMENT_ALL', kuery: 'transaction.type : "invalid_transaction_type"', + probability: 1, }, - }) - ); + }, + }); expect(response.status).to.be(200); expect(Object.keys(response.body.currentPeriod)).to.be.empty(); }); @@ -164,9 +177,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { () => { let servicesDetailedStatistics: ServicesDetailedStatisticsReturn; before(async () => { - const response = await supertest.get( - url.format({ - pathname: `/internal/apm/services/detailed_statistics`, + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/detailed_statistics`, + params: { query: { start: moment(end).subtract(15, 'minutes').toISOString(), end, @@ -174,9 +187,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { offset: '15m', environment: 'ENVIRONMENT_ALL', kuery: '', + probability: 1, }, - }) - ); + }, + }); + expect(response.status).to.be(200); servicesDetailedStatistics = response.body; }); diff --git a/x-pack/test/apm_api_integration/tests/services/top_services.spec.ts b/x-pack/test/apm_api_integration/tests/services/top_services.spec.ts index 2642553b2dfd8..3a27221304bc5 100644 --- a/x-pack/test/apm_api_integration/tests/services/top_services.spec.ts +++ b/x-pack/test/apm_api_integration/tests/services/top_services.spec.ts @@ -12,25 +12,21 @@ import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_ import { ENVIRONMENT_ALL } from '@kbn/apm-plugin/common/environment_filter_values'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import archives_metadata from '../../common/fixtures/es_archiver/archives_metadata'; +import { SupertestReturnType } from '../../common/apm_api_supertest'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); - const supertest = getService('legacySupertestAsApmReadUser'); const apmApiClient = getService('apmApiClient'); const synthtrace = getService('synthtraceEsClient'); - const supertestAsApmReadUserWithoutMlAccess = getService( - 'legacySupertestAsApmReadUserWithoutMlAccess' - ); - const archiveName = 'apm_8.0.0'; const archiveRange = archives_metadata[archiveName]; // url parameters - const archiveStart = encodeURIComponent(archiveRange.start); - const archiveEnd = encodeURIComponent(archiveRange.end); + const archiveStart = archiveRange.start; + const archiveEnd = archiveRange.end; const start = '2021-10-01T00:00:00.000Z'; const end = '2021-10-01T00:05:00.000Z'; @@ -40,9 +36,18 @@ export default function ApiTest({ getService }: FtrProviderContext) { { config: 'basic', archives: ['apm_mappings_only_8.0.0'] }, () => { it('handles the empty state', async () => { - const response = await supertest.get( - `/internal/apm/services?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=` - ); + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services`, + params: { + query: { + start, + end, + environment: ENVIRONMENT_ALL.value, + kuery: '', + probability: 1, + }, + }, + }); expect(response.status).to.be(200); expect(response.body.items.length).to.be(0); @@ -153,6 +158,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { end, environment: ENVIRONMENT_ALL.value, kuery: '', + probability: 1, }, }, }); @@ -204,6 +210,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { end, environment: 'production', kuery: '', + probability: 1, }, }, }); @@ -238,6 +245,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { end, environment: ENVIRONMENT_ALL.value, kuery: 'service.node.name:"multiple-env-service-development"', + probability: 1, }, }, }); @@ -272,6 +280,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { end, environment: ENVIRONMENT_ALL.value, kuery: 'not (transaction.type:request)', + probability: 1, }, }, }); @@ -300,9 +309,18 @@ export default function ApiTest({ getService }: FtrProviderContext) { }; before(async () => { - response = await supertest.get( - `/internal/apm/services?start=${archiveStart}&end=${archiveEnd}&environment=ENVIRONMENT_ALL&kuery=` - ); + response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services`, + params: { + query: { + start: archiveStart, + end: archiveEnd, + environment: ENVIRONMENT_ALL.value, + kuery: '', + probability: 1, + }, + }, + }); }); it('the response is successful', () => { @@ -344,11 +362,20 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('with a user that does not have access to ML', () => { - let response: Awaited>; + let response: SupertestReturnType<'GET /internal/apm/services'>; before(async () => { - response = await supertestAsApmReadUserWithoutMlAccess.get( - `/internal/apm/services?start=${archiveStart}&end=${archiveEnd}&environment=ENVIRONMENT_ALL&kuery=` - ); + response = await apmApiClient.noMlAccessUser({ + endpoint: 'GET /internal/apm/services', + params: { + query: { + start: archiveStart, + end: archiveEnd, + environment: ENVIRONMENT_ALL.value, + kuery: '', + probability: 1, + }, + }, + }); }); it('the response is successful', () => { @@ -361,7 +388,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('contains no health statuses', () => { const definedHealthStatuses = response.body.items - .map((item: any) => item.healthStatus) + .map((item) => item.healthStatus) .filter(Boolean); expect(definedHealthStatuses.length).to.be(0); @@ -369,13 +396,20 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('and fetching a list of services with a filter', () => { - let response: Awaited>; + let response: SupertestReturnType<'GET /internal/apm/services'>; before(async () => { - response = await supertest.get( - `/internal/apm/services?environment=ENVIRONMENT_ALL&start=${archiveStart}&end=${archiveEnd}&kuery=${encodeURIComponent( - 'service.name:opbeans-java' - )}` - ); + response = await apmApiClient.noMlAccessUser({ + endpoint: 'GET /internal/apm/services', + params: { + query: { + start: archiveStart, + end: archiveEnd, + environment: ENVIRONMENT_ALL.value, + kuery: 'service.name:opbeans-java', + probability: 1, + }, + }, + }); }); it('does not return health statuses for services that are not found in APM data', () => { diff --git a/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts b/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts index 566281b59a6af..e3b2c348a7f14 100644 --- a/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts +++ b/x-pack/test/apm_api_integration/tests/throughput/service_apis.spec.ts @@ -37,6 +37,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { ...commonQuery, + probability: 1, kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`, }, }, diff --git a/x-pack/test/apm_api_integration/tests/throughput/service_maps.spec.ts b/x-pack/test/apm_api_integration/tests/throughput/service_maps.spec.ts index 670881d783514..0530d166cc3af 100644 --- a/x-pack/test/apm_api_integration/tests/throughput/service_maps.spec.ts +++ b/x-pack/test/apm_api_integration/tests/throughput/service_maps.spec.ts @@ -42,6 +42,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { query: { ...commonQuery, kuery: `service.name : "${serviceName}" and processor.event : "${processorEvent}"`, + probability: 1, }, }, }), diff --git a/x-pack/test/apm_api_integration/tests/traces/top_traces.spec.ts b/x-pack/test/apm_api_integration/tests/traces/top_traces.spec.ts index 06a24cbd34a4b..b49133240c865 100644 --- a/x-pack/test/apm_api_integration/tests/traces/top_traces.spec.ts +++ b/x-pack/test/apm_api_integration/tests/traces/top_traces.spec.ts @@ -12,20 +12,28 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; export default function ApiTest({ getService }: FtrProviderContext) { const registry = getService('registry'); - const supertest = getService('legacySupertestAsApmReadUser'); + const apmApiClient = getService('apmApiClient'); const archiveName = 'apm_8.0.0'; const metadata = archives_metadata[archiveName]; // url parameters - const start = encodeURIComponent(metadata.start); - const end = encodeURIComponent(metadata.end); + const { start, end } = metadata; registry.when('Top traces when data is not loaded', { config: 'basic', archives: [] }, () => { it('handles empty state', async () => { - const response = await supertest.get( - `/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=` - ); + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/traces`, + params: { + query: { + start, + end, + kuery: '', + environment: 'ENVIRONMENT_ALL', + probability: 1, + }, + }, + }); expect(response.status).to.be(200); expect(response.body.items.length).to.be(0); @@ -38,9 +46,18 @@ export default function ApiTest({ getService }: FtrProviderContext) { () => { let response: any; before(async () => { - response = await supertest.get( - `/internal/apm/traces?start=${start}&end=${end}&environment=ENVIRONMENT_ALL&kuery=` - ); + response = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/traces', + params: { + query: { + start, + end, + kuery: '', + environment: 'ENVIRONMENT_ALL', + probability: 1, + }, + }, + }); }); it('returns the correct status code', async () => { From 64479235912eecd8364a7dfd4231931839c61cf9 Mon Sep 17 00:00:00 2001 From: Muhammad Ibragimov <53621505+mibragimov@users.noreply.github.com> Date: Thu, 21 Apr 2022 15:45:18 +0500 Subject: [PATCH 4/4] [Console] Fix condition auto-completion for templates (#126881) * Fix condition autocompletion for templates * Added block level matching logic * Fix lint * Fixed types * Resolved comments * Added a custom type guard * Minor refactor * Add type to type imports * Add functional tests and comments Co-authored-by: Muhammad Ibragimov Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/lib/autocomplete/autocomplete.ts | 78 ++++++++++++++++++- .../console/public/lib/autocomplete/types.ts | 9 +++ test/functional/apps/console/_autocomplete.ts | 44 +++++++++++ test/functional/page_objects/console_page.ts | 5 +- 4 files changed, 130 insertions(+), 6 deletions(-) diff --git a/src/plugins/console/public/lib/autocomplete/autocomplete.ts b/src/plugins/console/public/lib/autocomplete/autocomplete.ts index 1dc41430e3855..3e59c4e6bb023 100644 --- a/src/plugins/console/public/lib/autocomplete/autocomplete.ts +++ b/src/plugins/console/public/lib/autocomplete/autocomplete.ts @@ -11,22 +11,22 @@ import { i18n } from '@kbn/i18n'; // TODO: All of these imports need to be moved to the core editor so that it can inject components from there. import { - getTopLevelUrlCompleteComponents, getEndpointBodyCompleteComponents, getGlobalAutocompleteComponents, + getTopLevelUrlCompleteComponents, getUnmatchedEndpointComponents, // @ts-ignore } from '../kb/kb'; import { createTokenIterator } from '../../application/factories'; -import { Position, Token, Range, CoreEditor } from '../../types'; +import type { CoreEditor, Position, Range, Token } from '../../types'; import type RowParser from '../row_parser'; import * as utils from '../utils'; // @ts-ignore import { populateContext } from './engine'; -import { AutoCompleteContext, ResultTerm } from './types'; +import type { AutoCompleteContext, DataAutoCompleteRulesOneOf, ResultTerm } from './types'; // @ts-ignore import { URL_PATH_END_MARKER } from './components'; @@ -349,14 +349,84 @@ export default function ({ }); } + /** + * Get a different set of templates based on the value configured in the request. + * For example, when creating a snapshot repository of different types (`fs`, `url` etc), + * different properties are inserted in the textarea based on the type. + * E.g. https://github.com/elastic/kibana/blob/main/src/plugins/console/server/lib/spec_definitions/json/overrides/snapshot.create_repository.json + */ + function getConditionalTemplate( + name: string, + autocompleteRules: Record | null | undefined + ) { + const obj = autocompleteRules && autocompleteRules[name]; + + if (obj) { + const currentLineNumber = editor.getCurrentPosition().lineNumber; + + if (hasOneOfIn(obj)) { + // Get the line number of value that should provide different templates based on that + const startLine = getStartLineNumber(currentLineNumber, obj.__one_of); + // Join line values from start to current line + const lines = editor.getLines(startLine, currentLineNumber).join('\n'); + // Get the correct template by comparing the autocomplete rules against the lines + const prop = getProperty(lines, obj.__one_of); + if (prop && prop.__template) { + return prop.__template; + } + } + } + } + + /** + * Check if object has a property of '__one_of' + */ + function hasOneOfIn(value: unknown): value is { __one_of: DataAutoCompleteRulesOneOf[] } { + return typeof value === 'object' && value !== null && '__one_of' in value; + } + + /** + * Get the start line of value that matches the autocomplete rules condition + */ + function getStartLineNumber(currentLine: number, rules: DataAutoCompleteRulesOneOf[]): number { + if (currentLine === 1) { + return currentLine; + } + const value = editor.getLineValue(currentLine); + const prop = getProperty(value, rules); + if (prop) { + return currentLine; + } + return getStartLineNumber(currentLine - 1, rules); + } + + /** + * Get the matching property based on the given condition + */ + function getProperty(condition: string, rules: DataAutoCompleteRulesOneOf[]) { + return rules.find((rule) => { + if (rule.__condition && rule.__condition.lines_regex) { + return new RegExp(rule.__condition.lines_regex, 'm').test(condition); + } + return false; + }); + } + function applyTerm(term: { value?: string; context?: AutoCompleteContext; - template?: { __raw: boolean; value: string }; + template?: { __raw?: boolean; value?: string; [key: string]: unknown }; insertValue?: string; }) { const context = term.context!; + if (context?.endpoint && term.value) { + const { data_autocomplete_rules: autocompleteRules } = context.endpoint; + const template = getConditionalTemplate(term.value, autocompleteRules); + if (template) { + term.template = template; + } + } // make sure we get up to date replacement info. addReplacementInfoToContext(context, editor.getCurrentPosition(), term.insertValue); diff --git a/src/plugins/console/public/lib/autocomplete/types.ts b/src/plugins/console/public/lib/autocomplete/types.ts index 33c543f43be9e..15d32e6426a6c 100644 --- a/src/plugins/console/public/lib/autocomplete/types.ts +++ b/src/plugins/console/public/lib/autocomplete/types.ts @@ -15,6 +15,14 @@ export interface ResultTerm { value?: string; } +export interface DataAutoCompleteRulesOneOf { + __condition?: { + lines_regex: string; + }; + __template: Record; + [key: string]: unknown; +} + export interface AutoCompleteContext { autoCompleteSet?: null | ResultTerm[]; endpoint?: null | { @@ -24,6 +32,7 @@ export interface AutoCompleteContext { bodyAutocompleteRootComponents: unknown; id?: string; documentation?: string; + data_autocomplete_rules?: Record | null; }; urlPath?: null | unknown; urlParamsTokenPath?: Array> | null; diff --git a/test/functional/apps/console/_autocomplete.ts b/test/functional/apps/console/_autocomplete.ts index 4b424b2a79c66..57c59793f69f6 100644 --- a/test/functional/apps/console/_autocomplete.ts +++ b/test/functional/apps/console/_autocomplete.ts @@ -7,10 +7,12 @@ */ import expect from '@kbn/expect'; +import { asyncForEach } from '@kbn/std'; import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const log = getService('log'); + const retry = getService('retry'); const PageObjects = getPageObjects(['common', 'console']); describe('console autocomplete feature', function describeIndexTests() { @@ -62,5 +64,47 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(lastChar).to.be.eql(','); }); }); + + describe('with conditional templates', async () => { + const CONDITIONAL_TEMPLATES = [ + { + type: 'fs', + template: `"location": "path"`, + }, + { + type: 'url', + template: `"url": ""`, + }, + { type: 's3', template: `"bucket": ""` }, + { + type: 'azure', + template: `"path": ""`, + }, + ]; + + beforeEach(async () => { + await PageObjects.console.clearTextArea(); + await PageObjects.console.enterRequest('\n POST _snapshot/test_repo'); + }); + + await asyncForEach(CONDITIONAL_TEMPLATES, async ({ type, template }) => { + it('should insert different templates depending on the value of type', async () => { + await PageObjects.console.enterText(`{\n\t"type": "${type}"`); + await PageObjects.console.pressEnter(); + // Prompt autocomplete for 'settings' + await PageObjects.console.promptAutocomplete('s'); + + await retry.waitFor('autocomplete to be visible', () => + PageObjects.console.isAutocompleteVisible() + ); + await PageObjects.console.pressEnter(); + await retry.try(async () => { + const request = await PageObjects.console.getRequest(); + log.debug(request); + expect(request).to.contain(`${template}`); + }); + }); + }); + }); }); } diff --git a/test/functional/page_objects/console_page.ts b/test/functional/page_objects/console_page.ts index 32c859cc1aed9..281c49a789acf 100644 --- a/test/functional/page_objects/console_page.ts +++ b/test/functional/page_objects/console_page.ts @@ -83,10 +83,11 @@ export class ConsolePageObject extends FtrService { } } - public async promptAutocomplete() { + // Prompt autocomplete window and provide a initial letter of properties to narrow down the results. E.g. 'b' = 'bool' + public async promptAutocomplete(letter = 'b') { const textArea = await this.testSubjects.find('console-textarea'); await textArea.clickMouseButton(); - await textArea.type('b'); + await textArea.type(letter); await this.retry.waitFor('autocomplete to be visible', () => this.isAutocompleteVisible()); }