diff --git a/.eslintrc.js b/.eslintrc.js index da5dfb4eccb5..e907d104a3c1 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -897,7 +897,14 @@ module.exports = { ], }, }, - + // Profiling + { + files: ['x-pack/plugins/profiling/**/*.{js,mjs,ts,tsx}'], + rules: { + 'react-hooks/rules-of-hooks': 'error', // Checks rules of Hooks + 'react-hooks/exhaustive-deps': ['error', { additionalHooks: '^(useAsync)$' }], + }, + }, { // disable imports from legacy uptime plugin files: ['x-pack/plugins/synthetics/public/apps/synthetics/**/*.{js,mjs,ts,tsx}'], diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0a6577ce46c6..189dc56985cb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -158,6 +158,9 @@ x-pack/examples/files_example @elastic/kibana-app-services /x-pack/plugins/apm/public/components/app/rum_dashboard @elastic/uptime /x-pack/test/apm_api_integration/tests/csm/ @elastic/uptime +# Profiling +/x-pack/plugins/profiling @elastic/profiling-ui + # Observability onboarding tour /x-pack/plugins/observability/public/components/shared/tour @elastic/platform-onboarding /x-pack/test/functional/apps/infra/tour.ts @elastic/platform-onboarding @@ -362,6 +365,10 @@ x-pack/examples/files_example @elastic/kibana-app-services /x-pack/plugins/ingest_pipelines/ @elastic/platform-deployment-management #CC# /x-pack/plugins/cross_cluster_replication/ @elastic/platform-deployment-management +# Management Experience - Platform Onboarding +/src/plugins/guided_onboarding/ @elastic/platform-onboarding +/examples/guided_onboarding_example/ @elastic/platform-onboarding + # Security Solution /x-pack/test/endpoint_api_integration_no_ingest/ @elastic/security-solution /x-pack/test/security_solution_endpoint/ @elastic/security-solution diff --git a/STYLEGUIDE.mdx b/STYLEGUIDE.mdx index 8e043cba9224..10815b1f0002 100644 --- a/STYLEGUIDE.mdx +++ b/STYLEGUIDE.mdx @@ -630,7 +630,7 @@ Doing so ensures the styles are never separated or lost on import and allows for better modularization (smaller individual plugin asset footprint). All SASS (.scss) files will automatically build with the [EUI](https://elastic.github.io/eui/#/guidelines/sass) -& Kibana invisibles (SASS variables, mixins, functions) from the [`globals_[theme].scss` file](src/core/public/core_app/styles/_globals_v8light.scss). +& Kibana invisibles (SASS variables, mixins, functions) from the [`globals_[theme].scss` file](src/core/public/styles/core_app/_globals_v8light.scss). While the styles for this component will only be loaded if the component exists on the page, the styles **will** be global and so it is recommended to use a three letter prefix on your diff --git a/api_docs/security_solution.devdocs.json b/api_docs/security_solution.devdocs.json index f6e7dcb00168..de0462d8c99d 100644 --- a/api_docs/security_solution.devdocs.json +++ b/api_docs/security_solution.devdocs.json @@ -64,7 +64,7 @@ "label": "experimentalFeatures", "description": [], "signature": [ - "{ readonly tGridEnabled: boolean; readonly tGridEventRenderedViewEnabled: boolean; readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly disableIsolationUIPendingStatuses: boolean; readonly riskyHostsEnabled: boolean; readonly riskyUsersEnabled: boolean; readonly pendingActionResponsesWithAck: boolean; readonly policyListEnabled: boolean; readonly policyResponseInFleetEnabled: boolean; readonly threatIntelligenceEnabled: boolean; readonly entityAnalyticsDashboardEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly responseActionsConsoleEnabled: boolean; readonly insightsRelatedAlertsByProcessAncestry: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; }" + "{ readonly tGridEnabled: boolean; readonly tGridEventRenderedViewEnabled: boolean; readonly excludePoliciesInFilterEnabled: boolean; readonly kubernetesEnabled: boolean; readonly disableIsolationUIPendingStatuses: boolean; readonly pendingActionResponsesWithAck: boolean; readonly policyListEnabled: boolean; readonly policyResponseInFleetEnabled: boolean; readonly threatIntelligenceEnabled: boolean; readonly previewTelemetryUrlEnabled: boolean; readonly responseActionsConsoleEnabled: boolean; readonly insightsRelatedAlertsByProcessAncestry: boolean; readonly extendedRuleExecutionLoggingEnabled: boolean; readonly socTrendsEnabled: boolean; }" ], "path": "x-pack/plugins/security_solution/public/plugin.tsx", "deprecated": false, @@ -1095,7 +1095,7 @@ "label": "ConfigType", "description": [], "signature": [ - "Readonly<{} & { signalsIndex: string; maxRuleImportExportSize: number; maxRuleImportPayloadBytes: number; maxTimelineImportExportSize: number; maxTimelineImportPayloadBytes: number; alertMergeStrategy: \"allFields\" | \"missingFields\" | \"noFields\"; alertIgnoreFields: string[]; enableExperimental: string[]; packagerTaskInterval: string; prebuiltRulesFromFileSystem: boolean; prebuiltRulesFromSavedObjects: boolean; }> & { experimentalFeatures: Readonly<{ tGridEnabled: boolean; tGridEventRenderedViewEnabled: boolean; excludePoliciesInFilterEnabled: boolean; kubernetesEnabled: boolean; disableIsolationUIPendingStatuses: boolean; riskyHostsEnabled: boolean; riskyUsersEnabled: boolean; pendingActionResponsesWithAck: boolean; policyListEnabled: boolean; policyResponseInFleetEnabled: boolean; threatIntelligenceEnabled: boolean; entityAnalyticsDashboardEnabled: boolean; previewTelemetryUrlEnabled: boolean; responseActionsConsoleEnabled: boolean; insightsRelatedAlertsByProcessAncestry: boolean; extendedRuleExecutionLoggingEnabled: boolean; socTrendsEnabled: boolean; }>; }" + "Readonly<{} & { signalsIndex: string; maxRuleImportExportSize: number; maxRuleImportPayloadBytes: number; maxTimelineImportExportSize: number; maxTimelineImportPayloadBytes: number; alertMergeStrategy: \"allFields\" | \"missingFields\" | \"noFields\"; alertIgnoreFields: string[]; enableExperimental: string[]; packagerTaskInterval: string; prebuiltRulesFromFileSystem: boolean; prebuiltRulesFromSavedObjects: boolean; }> & { experimentalFeatures: Readonly<{ tGridEnabled: boolean; tGridEventRenderedViewEnabled: boolean; excludePoliciesInFilterEnabled: boolean; kubernetesEnabled: boolean; disableIsolationUIPendingStatuses: boolean; pendingActionResponsesWithAck: boolean; policyListEnabled: boolean; policyResponseInFleetEnabled: boolean; threatIntelligenceEnabled: boolean; previewTelemetryUrlEnabled: boolean; responseActionsConsoleEnabled: boolean; insightsRelatedAlertsByProcessAncestry: boolean; extendedRuleExecutionLoggingEnabled: boolean; socTrendsEnabled: boolean; }>; }" ], "path": "x-pack/plugins/security_solution/server/config.ts", "deprecated": false, @@ -1141,4 +1141,4 @@ "misc": [], "objects": [] } -} \ No newline at end of file +} diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 08b409609882..3716d64909f7 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -577,6 +577,10 @@ Elastic. |This plugin helps users learn how to use the Painless scripting language. +|{kib-repo}blob/{branch}/x-pack/plugins/profiling/README.md[profiling] +|undefined + + |{kib-repo}blob/{branch}/x-pack/plugins/remote_clusters/README.md[remoteClusters] |This plugin helps users manage their remote clusters, which enable cross-cluster search and cross-cluster replication. diff --git a/package.json b/package.json index e7e9092fa0f1..a12c4494d7eb 100644 --- a/package.json +++ b/package.json @@ -461,6 +461,7 @@ "fast-deep-equal": "^3.1.1", "fflate": "^0.6.9", "file-saver": "^1.3.8", + "fnv-plus": "^1.3.1", "font-awesome": "4.7.0", "formik": "^2.2.9", "fp-ts": "^2.3.1", @@ -534,6 +535,7 @@ "pbf": "3.2.1", "pdfjs-dist": "^2.13.216", "pdfmake": "^0.2.5", + "peggy": "^1.2.0", "pluralize": "3.1.0", "polished": "^3.7.2", "pretty-ms": "6.0.0", @@ -774,6 +776,7 @@ "@types/d3-time": "^1.0.10", "@types/d3-time-format": "^2.1.1", "@types/d3-transition": "^3.0.1", + "@types/dagre": "^0.7.47", "@types/dedent": "^0.7.0", "@types/deep-freeze-strict": "^1.1.0", "@types/delete-empty": "^2.0.0", @@ -786,6 +789,7 @@ "@types/fetch-mock": "^7.3.1", "@types/file-saver": "^2.0.0", "@types/flot": "^0.0.31", + "@types/fnv-plus": "^1.3.0", "@types/geojson": "7946.0.7", "@types/getos": "^3.0.0", "@types/gulp": "^4.0.6", diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts b/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts index 690e6c3563f2..7736aa8e8879 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/apm_fields.ts @@ -18,6 +18,10 @@ export type ApmApplicationMetricFields = Partial<{ 'jvm.memory.heap.used': number; 'jvm.memory.non_heap.used': number; 'jvm.thread.count': number; + 'faas.billed_duration': number; + 'faas.timeout': number; + 'faas.coldstart_duration': number; + 'faas.duration': number; }>; export type ApmUserAgentFields = Partial<{ @@ -104,6 +108,7 @@ export type ApmFields = Fields & 'cloud.region': string; 'host.os.platform': string; 'faas.id': string; + 'faas.name': string; 'faas.coldstart': boolean; 'faas.execution': string; 'faas.trigger.type': string; diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/index.ts b/packages/kbn-apm-synthtrace/src/lib/apm/index.ts index a136daabee8f..84e6bfc9e812 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/index.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/index.ts @@ -7,6 +7,7 @@ */ import { service } from './service'; import { browser } from './browser'; +import { serverlessFunction } from './serverless_function'; import { getTransactionMetrics } from './processors/get_transaction_metrics'; import { getSpanDestinationMetrics } from './processors/get_span_destination_metrics'; import { getChromeUserAgentDefaults } from './defaults/get_chrome_user_agent_defaults'; @@ -27,6 +28,7 @@ export const apm = { getApmWriteTargets, ApmSynthtraceEsClient, ApmSynthtraceKibanaClient, + serverlessFunction, }; export type { ApmSynthtraceEsClient, ApmException }; diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/serverless.ts b/packages/kbn-apm-synthtrace/src/lib/apm/serverless.ts new file mode 100644 index 000000000000..b67586c18a07 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/apm/serverless.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { generateLongId, generateShortId } from '../utils/generate_id'; +import { ApmFields } from './apm_fields'; +import { BaseSpan } from './base_span'; +import { Metricset } from './metricset'; + +export type FaasTriggerType = 'http' | 'pubsub' | 'datasource' | 'timer' | 'other'; + +export class Serverless extends BaseSpan { + private readonly metric: Metricset; + + constructor(fields: ApmFields) { + const faasExection = generateLongId(); + const triggerType = 'other'; + super({ + ...fields, + 'processor.event': 'transaction', + 'transaction.id': generateShortId(), + 'transaction.sampled': true, + 'faas.execution': faasExection, + 'faas.trigger.type': triggerType, + 'transaction.name': fields['transaction.name'] || fields['faas.name'], + 'transaction.type': 'request', + }); + this.metric = new Metricset({ + ...fields, + 'metricset.name': 'app', + 'faas.execution': faasExection, + 'faas.id': fields['service.name'], + }); + } + + duration(duration: number) { + this.fields['transaction.duration.us'] = duration * 1000; + return this; + } + + coldStart(coldstart: boolean) { + this.fields['faas.coldstart'] = coldstart; + this.metric.fields['faas.coldstart'] = coldstart; + return this; + } + + billedDuration(billedDuration: number) { + this.metric.fields['faas.billed_duration'] = billedDuration; + return this; + } + + faasTimeout(faasTimeout: number) { + this.metric.fields['faas.timeout'] = faasTimeout; + return this; + } + + memory({ total, free }: { total: number; free: number }) { + this.metric.fields['system.memory.total'] = total; + this.metric.fields['system.memory.actual.free'] = free; + return this; + } + + coldStartDuration(coldStartDuration: number) { + this.metric.fields['faas.coldstart_duration'] = coldStartDuration; + return this; + } + + faasDuration(faasDuration: number) { + this.metric.fields['faas.duration'] = faasDuration; + return this; + } + + timestamp(time: number): this { + super.timestamp(time); + this.metric.fields['@timestamp'] = time; + return this; + } + + serialize(): ApmFields[] { + const transaction = super.serialize(); + const metric = this.metric.serialize(); + return [...transaction, ...metric]; + } +} diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/serverless_function.ts b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_function.ts new file mode 100644 index 000000000000..e10bb23b1f93 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_function.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Entity } from '../entity'; +import { generateShortId } from '../utils/generate_id'; +import { ApmFields } from './apm_fields'; +import { ServerlessInstance } from './serverless_instance'; + +export class ServerlessFunction extends Entity { + instance({ instanceName, ...apmFields }: { instanceName: string } & ApmFields) { + return new ServerlessInstance({ + ...this.fields, + ['service.node.name']: instanceName, + 'host.name': instanceName, + ...apmFields, + }); + } +} + +export function serverlessFunction({ + functionName, + serviceName, + environment, + agentName, +}: { + functionName: string; + environment: string; + agentName: string; + serviceName?: string; +}) { + const faasId = `arn:aws:lambda:us-west-2:${generateShortId()}:function:${functionName}`; + return new ServerlessFunction({ + 'service.name': serviceName || faasId, + 'faas.id': faasId, + 'faas.name': functionName, + 'service.environment': environment, + 'agent.name': agentName, + 'service.runtime.name': 'AWS_lambda', + }); +} diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/serverless_instance.ts b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_instance.ts new file mode 100644 index 000000000000..2d1626add771 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/apm/serverless_instance.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Entity } from '../entity'; +import { ApmFields } from './apm_fields'; +import { FaasTriggerType, Serverless } from './serverless'; + +export class ServerlessInstance extends Entity { + invocation(params: { transactionName?: string; faasTriggerType?: FaasTriggerType } = {}) { + const { transactionName, faasTriggerType = 'other' } = params; + return new Serverless({ + ...this.fields, + 'transaction.name': transactionName, + 'faas.trigger.type': faasTriggerType, + }); + } +} diff --git a/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts b/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts index 2dcce23ab2a2..b89b0b5ea5f1 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/aws_lambda.ts @@ -7,55 +7,90 @@ */ import { apm, timerange } from '../..'; -import { ApmFields } from '../lib/apm/apm_fields'; import { Scenario } from '../cli/scenario'; -import { getLogger } from '../cli/utils/get_common_services'; import { RunOptions } from '../cli/utils/parse_run_cli_flags'; +import { ApmFields } from '../lib/apm/apm_fields'; import { getSynthtraceEnvironment } from '../lib/utils/get_synthtrace_environment'; const ENVIRONMENT = getSynthtraceEnvironment(__filename); const scenario: Scenario = async (runOptions: RunOptions) => { - const logger = getLogger(runOptions); - return { generate: ({ from, to }) => { const range = timerange(from, to); const timestamps = range.ratePerMinute(180); - const instance = apm - .service({ name: 'lambda-python', environment: ENVIRONMENT, agentName: 'python' }) - .instance('instance'); - - const traceEventsSetups = [ - { functionName: 'lambda-python-1', coldStart: true }, - { functionName: 'lambda-python-2', coldStart: false }, - ]; - - const traceEvents = ({ functionName, coldStart }: typeof traceEventsSetups[0]) => { - return timestamps.generator((timestamp) => - instance - .transaction({ transactionName: 'GET /order/{id}' }) - .defaults({ - 'service.runtime.name': 'AWS_Lambda_python3.8', - 'cloud.provider': 'aws', - 'cloud.service.name': 'lambda', - 'cloud.region': 'us-east-1', - 'faas.id': `arn:aws:lambda:us-west-2:123456789012:function:${functionName}`, - 'faas.coldstart': coldStart, - 'faas.trigger.type': 'other', - }) + const cloudFields: ApmFields = { + 'cloud.provider': 'aws', + 'cloud.service.name': 'lambda', + 'cloud.region': 'us-west-2', + }; + + const instanceALambdaPython = apm + .serverlessFunction({ + serviceName: 'aws-lambdas', + environment: ENVIRONMENT, + agentName: 'python', + functionName: 'fn-python-1', + }) + .instance({ instanceName: 'instance_A', ...cloudFields }); + + const instanceALambdaNode = apm + .serverlessFunction({ + serviceName: 'aws-lambdas', + environment: ENVIRONMENT, + agentName: 'nodejs', + functionName: 'fn-node-1', + }) + .instance({ instanceName: 'instance_A', ...cloudFields }); + + const instanceALambdaNode2 = apm + .serverlessFunction({ + environment: ENVIRONMENT, + agentName: 'nodejs', + functionName: 'fn-node-2', + }) + .instance({ instanceName: 'instance_A', ...cloudFields }); + + const memory = { + total: 536870912, // 0.5gb + free: 94371840, // ~0.08 gb + }; + + const awsLambdaEvents = timestamps.generator((timestamp) => { + return [ + instanceALambdaPython + .invocation() + .duration(1000) .timestamp(timestamp) + .coldStart(true) + .billedDuration(4000) + .faasTimeout(10000) + .memory(memory) + .coldStartDuration(4000) + .faasDuration(4000), + instanceALambdaNode + .invocation() .duration(1000) - .success() - ); - }; + .timestamp(timestamp) + .coldStart(false) + .billedDuration(4000) + .faasTimeout(10000) + .memory(memory) + .faasDuration(4000), + instanceALambdaNode2 + .invocation() + .duration(1000) + .timestamp(timestamp) + .coldStart(false) + .billedDuration(4000) + .faasTimeout(10000) + .memory(memory) + .faasDuration(4000), + ]; + }); - return traceEventsSetups - .map((traceEventsSetup) => - logger.perf('generating_apm_events', () => traceEvents(traceEventsSetup)) - ) - .reduce((p, c) => p.merge(c)); + return awsLambdaEvents; }, }; }; diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 4412e271e93f..74114c97c7b2 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -86,6 +86,7 @@ pageLoadAssetSize: osquery: 107090 painlessLab: 179748 presentationUtil: 58834 + profiling: 18628 remoteClusters: 51327 reporting: 57003 rollup: 97204 diff --git a/packages/kbn-typed-react-router-config/src/create_router.test.tsx b/packages/kbn-typed-react-router-config/src/create_router.test.tsx index e0582de320a9..fe4ada218b0b 100644 --- a/packages/kbn-typed-react-router-config/src/create_router.test.tsx +++ b/packages/kbn-typed-react-router-config/src/create_router.test.tsx @@ -10,6 +10,7 @@ import * as t from 'io-ts'; import { toNumberRt } from '@kbn/io-ts-utils'; import { createRouter } from './create_router'; import { createMemoryHistory } from 'history'; +import { last } from 'lodash'; describe('createRouter', () => { const routes = { @@ -382,4 +383,12 @@ describe('createRouter', () => { ); }); }); + + describe('getRoutePath', () => { + it('returns the correct route path', () => { + expect( + router.getRoutePath(last(router.getRoutesToMatch('/services/opbeans-java/errors'))!) + ).toBe('/services/{serviceName}/errors'); + }); + }); }); diff --git a/packages/kbn-typed-react-router-config/src/create_router.ts b/packages/kbn-typed-react-router-config/src/create_router.ts index f545fa8ed63e..4430c852b19e 100644 --- a/packages/kbn-typed-react-router-config/src/create_router.ts +++ b/packages/kbn-typed-react-router-config/src/create_router.ts @@ -208,7 +208,7 @@ export function createRouter(routes: TRoutes): Router< return matchRoutes(...args) as any; }, getRoutePath: (route) => { - return reactRouterConfigsByRoute.get(route)!.path as string; + return route.path; }, getRoutesToMatch: (path: string) => { return getRoutesToMatch(path) as unknown as FlattenRoutesOf; diff --git a/packages/kbn-typed-react-router-config/src/types/index.ts b/packages/kbn-typed-react-router-config/src/types/index.ts index a5121ba0d72c..f4655c7022db 100644 --- a/packages/kbn-typed-react-router-config/src/types/index.ts +++ b/packages/kbn-typed-react-router-config/src/types/index.ts @@ -36,7 +36,7 @@ export interface RouteWithPath extends Route { } export interface RouteMatch { - route: TRoute; + route: TRoute & { path: string }; match: { isExact: boolean; path: string; @@ -146,7 +146,7 @@ export interface Router { path: TPath, ...args: TypeAsArgs> ): string; - getRoutePath(route: Route): string; + getRoutePath(route: RouteWithPath): string; getRoutesToMatch(path: string): FlattenRoutesOf; } diff --git a/renovate.json b/renovate.json index 8ec367e91ee7..508cdcc684fc 100644 --- a/renovate.json +++ b/renovate.json @@ -207,6 +207,15 @@ "matchBaseBranches": ["main"], "labels": ["release_note:skip", "backport:skip", "ci:all-cypress-suites"], "enabled": true + }, + { + "groupName": "Profiling", + "matchPackageNames": ["fnv-plus", "peggy", "@types/dagre", "@types/fnv-plus"], + "reviewers": ["team:profiling-ui"], + "matchBaseBranches": ["main"], + "labels": ["release_note:skip", "backport:skip"], + "enabled": true, + "prCreation": "immediate" } ] } diff --git a/src/core/types/elasticsearch/search.ts b/src/core/types/elasticsearch/search.ts index 036f85177d30..1543add4c215 100644 --- a/src/core/types/elasticsearch/search.ts +++ b/src/core/types/elasticsearch/search.ts @@ -607,6 +607,8 @@ export type InferSearchResponseOf< > = Omit, 'aggregations' | 'hits'> & (TSearchRequest['body'] extends TopLevelAggregationRequest ? WrapAggregationResponse> + : TSearchRequest extends TopLevelAggregationRequest + ? WrapAggregationResponse> : { aggregations?: InvalidAggregationRequest }) & { hits: Omit['hits'], 'total' | 'hits'> & (TOptions['restTotalHitsAsInt'] extends true @@ -618,5 +620,12 @@ export type InferSearchResponseOf< value: number; relation: 'eq' | 'gte'; }; - }) & { hits: HitsOf }; + }) & { + hits: HitsOf< + TSearchRequest['body'] extends estypes.SearchRequest['body'] + ? TSearchRequest['body'] + : TSearchRequest, + TDocument + >; + }; }; diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 418a8f448b2c..f79c53e11514 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -62,6 +62,8 @@ export const IGNORE_FILE_GLOBS = [ 'x-pack/plugins/maps/server/fonts/**/*', + 'x-pack/plugins/profiling/Makefile', + // Bazel default files '**/WORKSPACE.bazel', '**/BUILD.bazel', diff --git a/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx b/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx index 7476d43f773e..917c9afdb304 100644 --- a/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx +++ b/src/plugins/chart_expressions/expression_xy/public/helpers/data_layers.tsx @@ -481,7 +481,7 @@ export const getSeriesProps: GetSeriesPropsFn = ({ return { splitSeriesAccessors: splitColumnIds.length ? splitColumnIds : [], - stackAccessors: isStacked && xColumnId ? [xColumnId] : [], + stackAccessors: isStacked ? [xColumnId || 'unifiedX'] : [], id: generateSeriesId( layer, splitColumnIds.length ? splitColumnIds : [EMPTY_ACCESSOR], diff --git a/src/plugins/charts/public/static/components/warnings.tsx b/src/plugins/charts/public/static/components/warnings.tsx index 9681cef1d6ac..f2ca61e3e1ed 100644 --- a/src/plugins/charts/public/static/components/warnings.tsx +++ b/src/plugins/charts/public/static/components/warnings.tsx @@ -21,7 +21,13 @@ export function Warnings({ warnings }: { warnings: React.ReactNode[] }) { panelPaddingSize="none" closePopover={() => setOpen(false)} button={ - setOpen(!open)} size="xs"> + setOpen(!open)} + size="xs" + data-test-subj="chart-inline-warning-button" + > {i18n.translate('charts.warning.warningLabel', { defaultMessage: '{numberWarnings, number} {numberWarnings, plural, one {warning} other {warnings}}', @@ -39,6 +45,7 @@ export function Warnings({ warnings }: { warnings: React.ReactNode[] }) { css={{ padding: euiThemeVars.euiSizeS, }} + data-test-subj="chart-inline-warning" > {w} diff --git a/src/plugins/discover/public/application/main/components/sidebar/lib/visualize_trigger_utils.ts b/src/plugins/discover/public/application/main/components/sidebar/lib/visualize_trigger_utils.ts index 81f097aacf70..005accf3021f 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/lib/visualize_trigger_utils.ts +++ b/src/plugins/discover/public/application/main/components/sidebar/lib/visualize_trigger_utils.ts @@ -15,6 +15,7 @@ import { import type { DataViewField, DataView } from '@kbn/data-views-plugin/public'; import { KBN_FIELD_TYPES } from '@kbn/data-plugin/public'; import { getUiActions } from '../../../../../kibana_services'; +import { PLUGIN_ID } from '../../../../../../common'; export function getTriggerConstant(type: string) { return type === KBN_FIELD_TYPES.GEO_POINT || type === KBN_FIELD_TYPES.GEO_SHAPE @@ -53,6 +54,7 @@ export function triggerVisualizeActions( dataViewSpec: dataView.toSpec(false), fieldName: field.name, contextualFields, + originatingApp: PLUGIN_ID, }; getUiActions().getTrigger(trigger).exec(triggerOptions); } diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts index 6a5b99973dad..4b393a7a1c24 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts @@ -152,6 +152,7 @@ export const applicationUsageSchema = { monitoring: commonSchema, 'observability-overview': commonSchema, osquery: commonSchema, + profiling: commonSchema, security_account: commonSchema, reportingRedirect: commonSchema, security_access_agreement: commonSchema, diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 10dbdf69fb29..c5d90ba52c1c 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -4527,6 +4527,137 @@ } } }, + "profiling": { + "properties": { + "appId": { + "type": "keyword", + "_meta": { + "description": "The application being tracked" + } + }, + "viewId": { + "type": "keyword", + "_meta": { + "description": "Always `main`" + } + }, + "clicks_total": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application since we started counting them" + } + }, + "clicks_7_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 7 days" + } + }, + "clicks_30_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 30 days" + } + }, + "clicks_90_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application over the last 90 days" + } + }, + "minutes_on_screen_total": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen since we started counting them." + } + }, + "minutes_on_screen_7_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 7 days" + } + }, + "minutes_on_screen_30_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 30 days" + } + }, + "minutes_on_screen_90_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen over the last 90 days" + } + }, + "views": { + "type": "array", + "items": { + "properties": { + "appId": { + "type": "keyword", + "_meta": { + "description": "The application being tracked" + } + }, + "viewId": { + "type": "keyword", + "_meta": { + "description": "The application view being tracked" + } + }, + "clicks_total": { + "type": "long", + "_meta": { + "description": "General number of clicks in the application sub view since we started counting them" + } + }, + "clicks_7_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 7 days" + } + }, + "clicks_30_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 30 days" + } + }, + "clicks_90_days": { + "type": "long", + "_meta": { + "description": "General number of clicks in the active application sub view over the last 90 days" + } + }, + "minutes_on_screen_total": { + "type": "float", + "_meta": { + "description": "Minutes the application sub view is active and on-screen since we started counting them." + } + }, + "minutes_on_screen_7_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 7 days" + } + }, + "minutes_on_screen_30_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 30 days" + } + }, + "minutes_on_screen_90_days": { + "type": "float", + "_meta": { + "description": "Minutes the application is active and on-screen active application sub view over the last 90 days" + } + } + } + } + } + } + }, "security_account": { "properties": { "appId": { diff --git a/tsconfig.base.json b/tsconfig.base.json index 431d898f3773..29a4a52c7351 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -389,6 +389,8 @@ "@kbn/osquery-plugin/*": ["x-pack/plugins/osquery/*"], "@kbn/painless-lab-plugin": ["x-pack/plugins/painless_lab"], "@kbn/painless-lab-plugin/*": ["x-pack/plugins/painless_lab/*"], + "@kbn/profiling-plugin": ["x-pack/plugins/profiling"], + "@kbn/profiling-plugin/*": ["x-pack/plugins/profiling/*"], "@kbn/remote-clusters-plugin": ["x-pack/plugins/remote_clusters"], "@kbn/remote-clusters-plugin/*": ["x-pack/plugins/remote_clusters/*"], "@kbn/reporting-plugin": ["x-pack/plugins/reporting"], diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index c527e0a3d955..569c5f6fddbe 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -45,6 +45,7 @@ "xpack.monitoring": ["plugins/monitoring"], "xpack.osquery": ["plugins/osquery"], "xpack.painlessLab": "plugins/painless_lab", + "xpack.profiling": [ "plugins/profiling" ], "xpack.remoteClusters": "plugins/remote_clusters", "xpack.reporting": ["plugins/reporting"], "xpack.rollupJobs": ["plugins/rollup"], diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index 83bcac0bfa70..f0fadf9476e7 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -7,6 +7,8 @@ import { chunk } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + import { i18n } from '@kbn/i18n'; import { asyncForEach } from '@kbn/std'; import type { IRouter } from '@kbn/core/server'; @@ -212,6 +214,7 @@ export const defineExplainLogRateSpikesRoute = ( const { fields, df } = await fetchFrequentItems( client, request.body.index, + JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer, changePoints, request.body.timeFieldName, request.body.deviationMin, diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts index fc834e2951db..02d20ba18795 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts +++ b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts @@ -25,6 +25,7 @@ function dropDuplicates(cp: ChangePoint[], uniqueFields: string[]) { export async function fetchFrequentItems( client: ElasticsearchClient, index: string, + searchQuery: estypes.QueryDslQueryContainer, changePoints: ChangePoint[], timeFieldName: string, deviationMin: number, @@ -45,7 +46,9 @@ export async function fetchFrequentItems( // TODO add query params const query = { bool: { + minimum_should_match: 2, filter: [ + searchQuery, { range: { [timeFieldName]: { @@ -83,6 +86,7 @@ export async function fetchFrequentItems( fi: { // @ts-expect-error `frequent_items` is not yet part of `AggregationsAggregationContainer` frequent_items: { + minimum_set_size: 2, size: 200, minimum_support: 0.1, fields: aggFields, diff --git a/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts b/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts index 9e0b82b341d1..706d2b6aa5c7 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_query_with_params.ts @@ -7,7 +7,6 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { Query } from '@kbn/es-query'; import type { FieldValuePair } from '@kbn/ml-agg-utils'; import type { AiopsExplainLogRateSpikesSchema } from '../../../common/api/explain_log_rate_spikes'; @@ -23,7 +22,7 @@ interface QueryParams { termFilters?: FieldValuePair[]; } export const getQueryWithParams = ({ params, termFilters }: QueryParams) => { - const searchQuery = JSON.parse(params.searchQuery) as Query['query']; + const searchQuery = JSON.parse(params.searchQuery) as estypes.QueryDslQueryContainer; return { bool: { filter: [ diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 0065245e507e..fddee59d9c6c 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -61,8 +61,14 @@ exports[`Error ERROR_PAGE_URL 1`] = `undefined`; exports[`Error EVENT_OUTCOME 1`] = `undefined`; +exports[`Error FAAS_BILLED_DURATION 1`] = `undefined`; + exports[`Error FAAS_COLDSTART 1`] = `undefined`; +exports[`Error FAAS_COLDSTART_DURATION 1`] = `undefined`; + +exports[`Error FAAS_DURATION 1`] = `undefined`; + exports[`Error FAAS_ID 1`] = `undefined`; exports[`Error FAAS_TRIGGER_TYPE 1`] = `undefined`; @@ -284,8 +290,14 @@ exports[`Span ERROR_PAGE_URL 1`] = `undefined`; exports[`Span EVENT_OUTCOME 1`] = `"unknown"`; +exports[`Span FAAS_BILLED_DURATION 1`] = `undefined`; + exports[`Span FAAS_COLDSTART 1`] = `undefined`; +exports[`Span FAAS_COLDSTART_DURATION 1`] = `undefined`; + +exports[`Span FAAS_DURATION 1`] = `undefined`; + exports[`Span FAAS_ID 1`] = `undefined`; exports[`Span FAAS_TRIGGER_TYPE 1`] = `undefined`; @@ -499,8 +511,14 @@ exports[`Transaction ERROR_PAGE_URL 1`] = `undefined`; exports[`Transaction EVENT_OUTCOME 1`] = `"unknown"`; +exports[`Transaction FAAS_BILLED_DURATION 1`] = `undefined`; + exports[`Transaction FAAS_COLDSTART 1`] = `undefined`; +exports[`Transaction FAAS_COLDSTART_DURATION 1`] = `undefined`; + +exports[`Transaction FAAS_DURATION 1`] = `undefined`; + exports[`Transaction FAAS_ID 1`] = `undefined`; exports[`Transaction FAAS_TRIGGER_TYPE 1`] = `undefined`; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index d8460bcad9b7..1e227713f0db 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -134,6 +134,9 @@ export const USER_AGENT_OS = 'user_agent.os.name'; export const FAAS_ID = 'faas.id'; export const FAAS_COLDSTART = 'faas.coldstart'; export const FAAS_TRIGGER_TYPE = 'faas.trigger.type'; +export const FAAS_DURATION = 'faas.duration'; +export const FAAS_COLDSTART_DURATION = 'faas.coldstart_duration'; +export const FAAS_BILLED_DURATION = 'faas.billed_duration'; // Metadata export const TIER = '_tier'; diff --git a/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx b/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx index cf51fe6cf75e..da4c603ff283 100644 --- a/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx +++ b/x-pack/plugins/apm/public/components/app/dependency_operation_detail_view/dependency_operation_detail_trace_list.tsx @@ -127,6 +127,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Trace' } ), field: 'traceId', + truncateText: true, render: ( _, { @@ -158,6 +159,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Originating service' } ), field: 'serviceName', + truncateText: true, render: (_, { serviceName, agentName }) => { const serviceLinkQuery = { comparisonEnabled, @@ -187,6 +189,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Transaction name' } ), field: 'transactionName', + truncateText: true, render: ( _, { @@ -227,6 +230,7 @@ export function DependencyOperationDetailTraceList() { { defaultMessage: 'Timestamp' } ), field: '@timestamp', + truncateText: true, render: (_, { '@timestamp': timestamp }) => { return ; }, @@ -286,7 +290,6 @@ export function DependencyOperationDetailTraceList() { status === FETCH_STATUS.LOADING || status === FETCH_STATUS.NOT_INITIATED } - tableLayout="auto" /> diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/active_instances.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/active_instances.ts new file mode 100644 index 000000000000..426132b39cc0 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/active_instances.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { euiLightVars as theme } from '@kbn/ui-theme'; +import { + SERVICE_NAME, + SERVICE_NODE_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; +import { environmentQuery } from '../../../../../common/utils/environment_query'; +import { getVizColorForIndex } from '../../../../../common/viz_colors'; +import { getMetricsDateHistogramParams } from '../../../../lib/helpers/metrics'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { + getDocumentTypeFilterForTransactions, + getProcessorEventForTransactions, +} from '../../../../lib/helpers/transactions'; +import { GenericMetricsChart } from '../../fetch_and_transform_metrics'; + +export async function getActiveInstances({ + environment, + kuery, + setup, + serviceName, + start, + end, + searchAggregatedTransactions, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; + searchAggregatedTransactions: boolean; +}): Promise { + const { apmEventClient, config } = setup; + + const aggs = { + activeInstances: { + cardinality: { + field: SERVICE_NODE_NAME, + }, + }, + }; + + const params = { + apm: { + events: [getProcessorEventForTransactions(searchAggregatedTransactions)], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...getDocumentTypeFilterForTransactions( + searchAggregatedTransactions + ), + ], + }, + }, + aggs: { + ...aggs, + timeseriesData: { + date_histogram: getMetricsDateHistogramParams({ + start, + end, + metricsInterval: config.metricsInterval, + }), + aggs, + }, + }, + }, + }; + + const { aggregations } = await apmEventClient.search( + 'get_active_instances', + params + ); + + return { + title: i18n.translate('xpack.apm.agentMetrics.serverless.activeInstances', { + defaultMessage: 'Active instances', + }), + key: 'active_instances', + yUnit: 'number', + series: [ + { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.series.activeInstances', + { defaultMessage: 'Active instances' } + ), + key: 'active_instances', + type: 'linemark', + color: getVizColorForIndex(0, theme), + overallValue: aggregations?.activeInstances.value ?? 0, + data: + aggregations?.timeseriesData.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: timeseriesBucket.activeInstances.value, + })) || [], + }, + ], + }; +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_count.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_count.ts new file mode 100644 index 000000000000..fc94a59da9a2 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_count.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { termQuery } from '@kbn/observability-plugin/server'; +import { + FAAS_COLDSTART, + METRICSET_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { fetchAndTransformMetrics } from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.coldStart', { + defaultMessage: 'Cold start', + }), + key: 'cold_start_count', + type: 'linemark', + yUnit: 'number', + series: { + coldStart: { + title: i18n.translate('xpack.apm.agentMetrics.serverless.coldStart', { + defaultMessage: 'Cold start', + }), + }, + }, +}; + +export function getColdStartCount({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}) { + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + start, + end, + chartBase, + aggs: { coldStart: { sum: { field: FAAS_COLDSTART } } }, + additionalFilters: [ + ...termQuery(FAAS_COLDSTART, true), + ...termQuery(METRICSET_NAME, 'transaction'), + ], + operationName: 'get_cold_start_count', + }); +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_duration.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_duration.ts new file mode 100644 index 000000000000..84fac7f99303 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/cold_start_duration.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { FAAS_COLDSTART_DURATION } from '../../../../../common/elasticsearch_fieldnames'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { fetchAndTransformMetrics } from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.coldStartDuration', { + defaultMessage: 'Cold start duration', + }), + key: 'cold_start_duration', + type: 'linemark', + yUnit: 'time', + series: { + coldStart: { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.coldStartDuration', + { defaultMessage: 'Cold start duration' } + ), + }, + }, +}; + +export function getColdStartDuration({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}) { + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + start, + end, + chartBase, + aggs: { coldStart: { avg: { field: FAAS_COLDSTART_DURATION } } }, + additionalFilters: [{ exists: { field: FAAS_COLDSTART_DURATION } }], + operationName: 'get_cold_start_duration', + }); +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/compute_usage.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/compute_usage.ts new file mode 100644 index 000000000000..da57498c8af0 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/compute_usage.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { + kqlQuery, + rangeQuery, + termQuery, +} from '@kbn/observability-plugin/server'; +import { euiLightVars as theme } from '@kbn/ui-theme'; +import { + FAAS_BILLED_DURATION, + METRICSET_NAME, + METRIC_SYSTEM_TOTAL_MEMORY, + SERVICE_NAME, +} from '../../../../../common/elasticsearch_fieldnames'; +import { environmentQuery } from '../../../../../common/utils/environment_query'; +import { isFiniteNumber } from '../../../../../common/utils/is_finite_number'; +import { getVizColorForIndex } from '../../../../../common/viz_colors'; +import { getMetricsDateHistogramParams } from '../../../../lib/helpers/metrics'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { GenericMetricsChart } from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.computeUsage', { + defaultMessage: 'Compute usage', + }), + key: 'compute_usage', + type: 'linemark', + yUnit: 'number', + series: { + computeUsage: { + title: i18n.translate('xpack.apm.agentMetrics.serverless.computeUsage', { + defaultMessage: 'Compute usage', + }), + }, + }, +}; + +/** + * To calculate the compute usage we need to multiply the "system.memory.total" by "faas.billed_duration". + * But the result of this calculation is in Bytes-milliseconds, as the "system.memory.total" is stored in bytes and the "faas.billed_duration" is stored in milliseconds. + * But to calculate the overall cost AWS uses GB-second, so we need to convert the result to this unit. + */ +const GB = 1024 ** 3; +function calculateComputeUsageGBSeconds({ + faasBilledDuration, + totalMemory, +}: { + faasBilledDuration?: number | null; + totalMemory?: number | null; +}) { + if (!isFiniteNumber(faasBilledDuration) || !isFiniteNumber(totalMemory)) { + return 0; + } + + const totalMemoryGB = totalMemory / GB; + const faasBilledDurationSec = faasBilledDuration / 1000; + return totalMemoryGB * faasBilledDurationSec; +} + +export async function getComputeUsage({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}): Promise { + const { apmEventClient, config } = setup; + + const aggs = { + avgFaasBilledDuration: { avg: { field: FAAS_BILLED_DURATION } }, + avgTotalMemory: { avg: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + }; + + const params = { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + { exists: { field: FAAS_BILLED_DURATION } }, + ...termQuery(METRICSET_NAME, 'app'), + ], + }, + }, + aggs: { + timeseriesData: { + date_histogram: getMetricsDateHistogramParams({ + start, + end, + metricsInterval: config.metricsInterval, + }), + aggs, + }, + ...aggs, + }, + }, + }; + + const { aggregations } = await apmEventClient.search( + 'get_compute_usage', + params + ); + const timeseriesData = aggregations?.timeseriesData; + + return { + title: chartBase.title, + key: chartBase.key, + yUnit: chartBase.yUnit, + series: + !timeseriesData || timeseriesData.buckets.length === 0 + ? [] + : [ + { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.computeUsage', + { defaultMessage: 'Compute usage' } + ), + key: 'compute_usage', + type: 'linemark', + overallValue: calculateComputeUsageGBSeconds({ + faasBilledDuration: aggregations?.avgFaasBilledDuration.value, + totalMemory: aggregations?.avgTotalMemory.value, + }), + color: getVizColorForIndex(0, theme), + data: timeseriesData.buckets.map((bucket) => { + const computeUsage = calculateComputeUsageGBSeconds({ + faasBilledDuration: bucket.avgFaasBilledDuration.value, + totalMemory: bucket.avgTotalMemory.value, + }); + return { + x: bucket.key, + y: computeUsage, + }; + }), + }, + ], + }; +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/index.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/index.ts new file mode 100644 index 000000000000..a34165dc95f4 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/index.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { withApmSpan } from '../../../../utils/with_apm_span'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { getServerlessFunctionLatency } from './serverless_function_latency'; +import { getColdStartDuration } from './cold_start_duration'; +import { getMemoryChartData } from '../shared/memory'; +import { getComputeUsage } from './compute_usage'; +import { getActiveInstances } from './active_instances'; +import { getColdStartCount } from './cold_start_count'; +import { getSearchAggregatedTransactions } from '../../../../lib/helpers/transactions'; + +export function getServerlessAgentMetricCharts({ + environment, + kuery, + setup, + serviceName, + start, + end, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; +}) { + return withApmSpan('get_serverless_agent_metric_charts', async () => { + const searchAggregatedTransactions = await getSearchAggregatedTransactions({ + ...setup, + kuery, + start, + end, + }); + + const options = { + environment, + kuery, + setup, + serviceName, + start, + end, + }; + return await Promise.all([ + getServerlessFunctionLatency({ + ...options, + searchAggregatedTransactions, + }), + getColdStartDuration(options), + getColdStartCount(options), + getMemoryChartData(options), + getComputeUsage(options), + getActiveInstances({ ...options, searchAggregatedTransactions }), + ]); + }); +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/serverless_function_latency.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/serverless_function_latency.ts new file mode 100644 index 000000000000..99a09c86e954 --- /dev/null +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/serverless/serverless_function_latency.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { euiLightVars as theme } from '@kbn/ui-theme'; +import { FAAS_BILLED_DURATION } from '../../../../../common/elasticsearch_fieldnames'; +import { LatencyAggregationType } from '../../../../../common/latency_aggregation_types'; +import { getVizColorForIndex } from '../../../../../common/viz_colors'; +import { Setup } from '../../../../lib/helpers/setup_request'; +import { getLatencyTimeseries } from '../../../transactions/get_latency_charts'; +import { + fetchAndTransformMetrics, + GenericMetricsChart, +} from '../../fetch_and_transform_metrics'; +import { ChartBase } from '../../types'; + +const billedDurationAvg = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.billedDurationAvg', { + defaultMessage: 'Billed Duration', + }), +}; + +const chartBase: ChartBase = { + title: i18n.translate('xpack.apm.agentMetrics.serverless.avgDuration', { + defaultMessage: 'Avg. Duration', + }), + key: 'avg_duration', + type: 'linemark', + yUnit: 'time', + series: {}, +}; + +async function getServerlessLantecySeries({ + environment, + kuery, + setup, + serviceName, + start, + end, + searchAggregatedTransactions, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; + searchAggregatedTransactions: boolean; +}): Promise { + const transactionLatency = await getLatencyTimeseries({ + environment, + kuery, + serviceName, + setup, + searchAggregatedTransactions, + latencyAggregationType: LatencyAggregationType.avg, + start, + end, + }); + + return [ + { + title: i18n.translate( + 'xpack.apm.agentMetrics.serverless.transactionDuration', + { defaultMessage: 'Transaction Duration' } + ), + key: 'transaction_duration', + type: 'linemark', + color: getVizColorForIndex(1, theme), + overallValue: transactionLatency.overallAvgDuration ?? 0, + data: transactionLatency.latencyTimeseries, + }, + ]; +} + +export async function getServerlessFunctionLatency({ + environment, + kuery, + setup, + serviceName, + start, + end, + searchAggregatedTransactions, +}: { + environment: string; + kuery: string; + setup: Setup; + serviceName: string; + start: number; + end: number; + searchAggregatedTransactions: boolean; +}): Promise { + const options = { + environment, + kuery, + setup, + serviceName, + start, + end, + }; + + const [billedDurationMetrics, serverlessDurationSeries] = await Promise.all([ + fetchAndTransformMetrics({ + ...options, + chartBase: { ...chartBase, series: { billedDurationAvg } }, + aggs: { + billedDurationAvg: { avg: { field: FAAS_BILLED_DURATION } }, + }, + additionalFilters: [{ exists: { field: FAAS_BILLED_DURATION } }], + operationName: 'get_billed_duration', + }), + getServerlessLantecySeries({ ...options, searchAggregatedTransactions }), + ]); + + return { + ...billedDurationMetrics, + series: [...billedDurationMetrics.series, ...serverlessDurationSeries], + }; +} diff --git a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts index 714ac5cdf38d..a7e41ea71b72 100644 --- a/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/routes/metrics/by_agent/shared/memory/index.ts @@ -6,8 +6,10 @@ */ import { i18n } from '@kbn/i18n'; +import { termQuery } from '@kbn/observability-plugin/server'; import { withApmSpan } from '../../../../../utils/with_apm_span'; import { + FAAS_ID, METRIC_CGROUP_MEMORY_LIMIT_BYTES, METRIC_CGROUP_MEMORY_USAGE_BYTES, METRIC_SYSTEM_FREE_MEMORY, @@ -84,6 +86,7 @@ export async function getMemoryChartData({ setup, serviceName, serviceNodeName, + faasId, start, end, }: { @@ -92,6 +95,7 @@ export async function getMemoryChartData({ setup: Setup; serviceName: string; serviceNodeName?: string; + faasId?: string; start: number; end: number; }) { @@ -111,6 +115,7 @@ export async function getMemoryChartData({ }, additionalFilters: [ { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, + ...termQuery(FAAS_ID, faasId), ], operationName: 'get_cgroup_memory_metrics_charts', }); @@ -132,6 +137,7 @@ export async function getMemoryChartData({ additionalFilters: [ { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ...termQuery(FAAS_ID, faasId), ], operationName: 'get_system_memory_metrics_charts', }); diff --git a/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts b/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts index c6927417687d..2aa5312d66b9 100644 --- a/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts +++ b/x-pack/plugins/apm/server/routes/metrics/get_metrics_chart_data_by_agent.ts @@ -8,8 +8,9 @@ import { Setup } from '../../lib/helpers/setup_request'; import { getJavaMetricsCharts } from './by_agent/java'; import { getDefaultMetricsCharts } from './by_agent/default'; -import { isJavaAgentName } from '../../../common/agent_name'; +import { isJavaAgentName, isServerlessAgent } from '../../../common/agent_name'; import { GenericMetricsChart } from './fetch_and_transform_metrics'; +import { getServerlessAgentMetricCharts } from './by_agent/serverless'; export async function getMetricsChartDataByAgent({ environment, @@ -20,6 +21,7 @@ export async function getMetricsChartDataByAgent({ agentName, start, end, + serviceRuntimeName, }: { environment: string; kuery: string; @@ -29,25 +31,26 @@ export async function getMetricsChartDataByAgent({ agentName: string; start: number; end: number; + serviceRuntimeName?: string; }): Promise { - if (isJavaAgentName(agentName)) { - return getJavaMetricsCharts({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - start, - end, - }); - } - - return getDefaultMetricsCharts({ + const options = { environment, kuery, setup, serviceName, start, end, - }); + }; + if (isJavaAgentName(agentName)) { + return getJavaMetricsCharts({ + ...options, + serviceNodeName, + }); + } + + if (isServerlessAgent(serviceRuntimeName)) { + return getServerlessAgentMetricCharts(options); + } + + return getDefaultMetricsCharts(options); } diff --git a/x-pack/plugins/apm/server/routes/metrics/route.ts b/x-pack/plugins/apm/server/routes/metrics/route.ts index 3ee0c3b0fac8..f8b20ae4e029 100644 --- a/x-pack/plugins/apm/server/routes/metrics/route.ts +++ b/x-pack/plugins/apm/server/routes/metrics/route.ts @@ -7,9 +7,9 @@ import * as t from 'io-ts'; import { setupRequest } from '../../lib/helpers/setup_request'; -import { getMetricsChartDataByAgent } from './get_metrics_chart_data_by_agent'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; +import { getMetricsChartDataByAgent } from './get_metrics_chart_data_by_agent'; const metricsChartsRoute = createApmServerRoute({ endpoint: 'GET /internal/apm/services/{serviceName}/metrics/charts', @@ -23,6 +23,7 @@ const metricsChartsRoute = createApmServerRoute({ }), t.partial({ serviceNodeName: t.string, + serviceRuntimeName: t.string, }), environmentRt, kueryRt, @@ -50,8 +51,15 @@ const metricsChartsRoute = createApmServerRoute({ const { params } = resources; const setup = await setupRequest(resources); const { serviceName } = params.path; - const { agentName, environment, kuery, serviceNodeName, start, end } = - params.query; + const { + agentName, + environment, + kuery, + serviceNodeName, + start, + end, + serviceRuntimeName, + } = params.query; const charts = await getMetricsChartDataByAgent({ environment, @@ -62,6 +70,7 @@ const metricsChartsRoute = createApmServerRoute({ serviceNodeName, start, end, + serviceRuntimeName, }); return { charts }; diff --git a/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts b/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts index 82da75e56043..325642db16f9 100644 --- a/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts +++ b/x-pack/plugins/apm/server/routes/transactions/get_latency_charts/index.ts @@ -136,8 +136,8 @@ export async function getLatencyTimeseries({ environment: string; kuery: string; serviceName: string; - transactionType: string | undefined; - transactionName: string | undefined; + transactionType?: string; + transactionName?: string; setup: Setup; searchAggregatedTransactions: boolean; latencyAggregationType: LatencyAggregationType; diff --git a/x-pack/plugins/enterprise_search/common/constants.ts b/x-pack/plugins/enterprise_search/common/constants.ts index 3310ed5d69e4..6c254b8ea6ff 100644 --- a/x-pack/plugins/enterprise_search/common/constants.ts +++ b/x-pack/plugins/enterprise_search/common/constants.ts @@ -109,6 +109,7 @@ export const ENTERPRISE_SEARCH_KIBANA_COOKIE = '_enterprise_search'; export const ENTERPRISE_SEARCH_RELEVANCE_LOGS_SOURCE_ID = 'ent-search-logs'; export const ENTERPRISE_SEARCH_AUDIT_LOGS_SOURCE_ID = 'ent-search-audit-logs'; +export const ENTERPRISE_SEARCH_ANALYTICS_LOGS_SOURCE_ID = 'ent-search-analytics-logs'; export const APP_SEARCH_URL = '/app/enterprise_search/app_search'; export const ENTERPRISE_SEARCH_ELASTICSEARCH_URL = '/app/enterprise_search/elasticsearch'; diff --git a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_events.test.tsx b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_events.test.tsx new file mode 100644 index 000000000000..9a2366384cfc --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_events.test.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import '../../../__mocks__/shallow_useeffect.mock'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { AnalyticsCollection } from '../../../../../common/types/analytics'; +import { EntSearchLogStream } from '../../../shared/log_stream'; + +import { AnalyticsCollectionEvents } from './analytics_collection_events'; + +describe('AnalyticsCollectionEvents', () => { + const analyticsCollections: AnalyticsCollection = { + event_retention_day_length: 180, + id: '1', + name: 'example', + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders', () => { + const expectedQuery = '_index: logs-elastic_analytics.events-example*'; + + const wrapper = shallow(); + expect(wrapper.find(EntSearchLogStream).prop('query')).toEqual(expectedQuery); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_events.tsx b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_events.tsx new file mode 100644 index 000000000000..25feb0ffcd98 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_events.tsx @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { i18n } from '@kbn/i18n'; + +import { ENTERPRISE_SEARCH_ANALYTICS_LOGS_SOURCE_ID } from '../../../../../common/constants'; +import { AnalyticsCollection } from '../../../../../common/types/analytics'; + +import { EntSearchLogStream } from '../../../shared/log_stream'; + +interface AnalyticsCollectionEventsProps { + collection: AnalyticsCollection; +} + +export const AnalyticsCollectionEvents: React.FC = ({ + collection, +}) => { + return ( + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_view.tsx b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_view.tsx index 4c8e5a36e4e0..231dd04893bd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_view.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/analytics/components/analytics_collection_view/analytics_collection_view.tsx @@ -27,6 +27,7 @@ import { COLLECTION_CREATION_PATH, COLLECTION_VIEW_PATH } from '../../routes'; import { EnterpriseSearchAnalyticsPageTemplate } from '../layout/page_template'; +import { AnalyticsCollectionEvents } from './analytics_collection_events'; import { AnalyticsCollectionIntegrate } from './analytics_collection_integrate'; import { AnalyticsCollectionSettings } from './analytics_collection_settings'; @@ -147,6 +148,7 @@ export const AnalyticsCollectionView: React.FC = () => { {section === 'integrate' && ( )} + {section === 'events' && } ) : ( ; diff --git a/x-pack/plugins/fleet/server/services/package_policies/__snapshots__/simplified_package_policy_helper.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap similarity index 100% rename from x-pack/plugins/fleet/server/services/package_policies/__snapshots__/simplified_package_policy_helper.test.ts.snap rename to x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap diff --git a/x-pack/plugins/fleet/server/services/package_policies/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts similarity index 94% rename from x-pack/plugins/fleet/server/services/package_policies/simplified_package_policy_helper.test.ts rename to x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 15fd7c902b02..eec59a647c39 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -5,13 +5,14 @@ * 2.0. */ -import type { NewPackagePolicy, PackageInfo } from '../../types'; +import type { NewPackagePolicy, PackageInfo } from '../../server/types'; + +import nginxPackageInfo from '../../server/services/package_policies/fixtures/package_info/nginx_1.5.0.json'; import { simplifiedPackagePolicytoNewPackagePolicy, generateInputId, } from './simplified_package_policy_helper'; -import nginxPackageInfo from './fixtures/package_info/nginx_1.5.0.json'; function getEnabledInputsAndStreams(newPackagePolicy: NewPackagePolicy) { return newPackagePolicy.inputs diff --git a/x-pack/plugins/fleet/server/services/package_policies/simplified_package_policy_helper.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts similarity index 88% rename from x-pack/plugins/fleet/server/services/package_policies/simplified_package_policy_helper.ts rename to x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts index 12db049aef71..c0d05d1eeab2 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/simplified_package_policy_helper.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts @@ -5,16 +5,26 @@ * 2.0. */ -import { packageToPackagePolicy } from '../../../common/services'; import type { NewPackagePolicyInput, NewPackagePolicyInputStream, PackagePolicyConfigRecord, -} from '../../../common/types'; -import { PackagePolicyValidationError } from '../../errors'; -import type { NewPackagePolicy, PackageInfo } from '../../types'; + NewPackagePolicy, + PackageInfo, +} from '../types'; +import { PackagePolicyValidationError } from '../errors'; -type SimplifiedVars = Record; +import { packageToPackagePolicy } from '.'; + +export type SimplifiedVars = Record; + +export type SimplifiedPackagePolicyStreams = Record< + string, + { + enabled?: undefined | boolean; + vars?: SimplifiedVars; + } +>; export interface SimplifiedPackagePolicy { id?: string; @@ -28,13 +38,7 @@ export interface SimplifiedPackagePolicy { { enabled?: boolean | undefined; vars?: SimplifiedVars; - streams?: Record< - string, - { - enabled?: undefined | boolean; - vars?: SimplifiedVars; - } - >; + streams?: SimplifiedPackagePolicyStreams; } >; } diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx index 51a8826456fc..cc49d03f1f1c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx @@ -43,12 +43,21 @@ import { useGetPackageInfoByKey, sendCreateAgentPolicy, } from '../../../../hooks'; -import { Loading, Error, ExtensionWrapper } from '../../../../components'; +import { + Loading, + Error, + ExtensionWrapper, + DevtoolsRequestFlyoutButton, +} from '../../../../components'; import { agentPolicyFormValidation, ConfirmDeployAgentPolicyModal } from '../../components'; import { useUIExtension } from '../../../../hooks'; import type { PackagePolicyEditExtensionComponentProps } from '../../../../types'; -import { pkgKeyFromPackageInfo, isVerificationError } from '../../../../services'; +import { + pkgKeyFromPackageInfo, + isVerificationError, + ExperimentalFeaturesService, +} from '../../../../services'; import type { PackagePolicyFormState, @@ -60,13 +69,13 @@ import { IntegrationBreadcrumb } from '../components'; import type { PackagePolicyValidationResults } from '../services'; import { validatePackagePolicy, validationHasErrors } from '../services'; - import { StepConfigurePackagePolicy, StepDefinePackagePolicy, SelectedPolicyTab, StepSelectHosts, } from '../components'; +import { generateCreatePackagePolicyDevToolsRequest } from '../../services'; import { CreatePackagePolicySinglePageLayout, PostInstallAddAgentModal } from './components'; @@ -548,6 +557,15 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({ }, ]; + const { showDevtoolsRequest } = ExperimentalFeaturesService.get(); + const devtoolRequest = useMemo( + () => + generateCreatePackagePolicyDevToolsRequest({ + ...packagePolicy, + }), + [packagePolicy] + ); + // Display package error if there is one if (packageInfoError) { return ( @@ -562,6 +580,7 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({ /> ); } + return ( @@ -617,6 +636,22 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({ /> + {showDevtoolsRequest ? ( + + + + ) : null} onSubmit()} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/settings/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/settings/index.tsx index 87614fc6413c..c23f29a81c39 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/settings/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/settings/index.tsx @@ -5,9 +5,10 @@ * 2.0. */ -import React, { memo, useState } from 'react'; +import React, { memo, useMemo, useState } from 'react'; import { useHistory } from 'react-router-dom'; import styled from 'styled-components'; +import { pick } from 'lodash'; import { EuiBottomBar, EuiFlexGroup, @@ -35,6 +36,9 @@ import { agentPolicyFormValidation, ConfirmDeployAgentPolicyModal, } from '../../../components'; +import { DevtoolsRequestFlyoutButton } from '../../../../../components'; +import { ExperimentalFeaturesService } from '../../../../../services'; +import { generateUpdateAgentPolicyDevToolsRequest } from '../../../services'; const FormWrapper = styled.div` max-width: 800px; @@ -126,6 +130,26 @@ export const SettingsView = memo<{ agentPolicy: AgentPolicy }>( setIsLoading(false); }; + const { showDevtoolsRequest } = ExperimentalFeaturesService.get(); + const devtoolRequest = useMemo( + () => + generateUpdateAgentPolicyDevToolsRequest( + agentPolicy.id, + pick( + agentPolicy, + 'name', + 'description', + 'namespace', + 'monitoring_enabled', + 'unenroll_timeout', + 'data_output_id', + 'monitoring_output_id', + 'download_source_id' + ) + ), + [agentPolicy] + ); + const onSubmit = async () => { // Retrieve agent count if fleet is enabled if (isFleetEnabled) { @@ -197,6 +221,23 @@ export const SettingsView = memo<{ agentPolicy: AgentPolicy }>( /> + {showDevtoolsRequest ? ( + + 0} + btnProps={{ + color: 'ghost', + }} + description={i18n.translate( + 'xpack.fleet.editAgentPolicy.devtoolsRequestDescription', + { + defaultMessage: 'This Kibana request updates an agent policy.', + } + )} + request={devtoolRequest} + /> + + ) : null} + generateUpdatePackagePolicyDevToolsRequest( + packagePolicyId, + omit(packagePolicy, 'elasticsearch') + ), + [packagePolicyId, packagePolicy] + ); + return ( @@ -638,6 +656,23 @@ export const EditPackagePolicyForm = memo<{ /> + {showDevtoolsRequest ? ( + + + + ) : null} props.theme.eui.euiZLevel5}; @@ -98,6 +101,11 @@ export const CreateAgentPolicyFlyout: React.FunctionComponent = ({ /> ); + const { showDevtoolsRequest } = ExperimentalFeaturesService.get(); + const agentPolicyContent = useMemo( + () => generateCreateAgentPolicyDevToolsRequest(agentPolicy, withSysMonitoring), + [agentPolicy, withSysMonitoring] + ); const footer = ( @@ -111,48 +119,68 @@ export const CreateAgentPolicyFlyout: React.FunctionComponent = ({ - 0} - onClick={async () => { - setIsLoading(true); - try { - const { data, error } = await createAgentPolicy(); - setIsLoading(false); - if (data) { - notifications.toasts.addSuccess( - i18n.translate('xpack.fleet.createAgentPolicy.successNotificationTitle', { - defaultMessage: "Agent policy '{name}' created", - values: { name: agentPolicy.name }, - }) - ); - onClose(data.item); - } else { - notifications.toasts.addDanger( - error - ? error.message - : i18n.translate('xpack.fleet.createAgentPolicy.errorNotificationTitle', { - defaultMessage: 'Unable to create agent policy', - }) - ); + + {showDevtoolsRequest ? ( + + 0} + description={i18n.translate( + 'xpack.fleet.createAgentPolicy.devtoolsRequestDescription', + { + defaultMessage: 'This Kibana request creates a new agent policy.', + } + )} + request={agentPolicyContent} + /> + + ) : null} + + 0 } - } catch (e) { - setIsLoading(false); - notifications.toasts.addDanger( - i18n.translate('xpack.fleet.createAgentPolicy.errorNotificationTitle', { - defaultMessage: 'Unable to create agent policy', - }) - ); - } - }} - data-test-subj="createAgentPolicyFlyoutBtn" - > - - + onClick={async () => { + setIsLoading(true); + try { + const { data, error } = await createAgentPolicy(); + setIsLoading(false); + if (data) { + notifications.toasts.addSuccess( + i18n.translate('xpack.fleet.createAgentPolicy.successNotificationTitle', { + defaultMessage: "Agent policy '{name}' created", + values: { name: agentPolicy.name }, + }) + ); + onClose(data.item); + } else { + notifications.toasts.addDanger( + error + ? error.message + : i18n.translate('xpack.fleet.createAgentPolicy.errorNotificationTitle', { + defaultMessage: 'Unable to create agent policy', + }) + ); + } + } catch (e) { + setIsLoading(false); + notifications.toasts.addDanger( + i18n.translate('xpack.fleet.createAgentPolicy.errorNotificationTitle', { + defaultMessage: 'Unable to create agent policy', + }) + ); + } + }} + data-test-subj="createAgentPolicyFlyoutBtn" + > + + + + diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx new file mode 100644 index 000000000000..875d43d1f0ea --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { omit } from 'lodash'; + +import { agentPolicyRouteService, packagePolicyRouteService } from '../../../services'; +import { generateInputId } from '../../../../../../common/services/simplified_package_policy_helper'; +import type { + SimplifiedPackagePolicy, + SimplifiedVars, + SimplifiedPackagePolicyStreams, +} from '../../../../../../common/services/simplified_package_policy_helper'; +import type { + NewAgentPolicy, + NewPackagePolicy, + UpdatePackagePolicy, + UpdateAgentPolicyRequest, +} from '../../../types'; + +function generateKibanaDevToolsRequest(method: string, path: string, body: any) { + return `${method} kbn:${path}\n${JSON.stringify(body, null, 2)}\n`; +} + +/** + * Generate a request to create an agent policy that can be used in Kibana Dev tools + * @param agentPolicy + * @param withSysMonitoring + * @returns + */ +export function generateCreateAgentPolicyDevToolsRequest( + agentPolicy: NewAgentPolicy, + withSysMonitoring?: boolean +) { + return generateKibanaDevToolsRequest( + 'POST', + `${agentPolicyRouteService.getCreatePath()}${withSysMonitoring ? '?sys_monitoring=true' : ''}`, + agentPolicy + ); +} + +/** + * Generate a request to create a package policy that can be used in Kibana Dev tools + * @param packagePolicy + * @param withSysMonitoring + * @returns + */ +export function generateCreatePackagePolicyDevToolsRequest( + packagePolicy: NewPackagePolicy & { force?: boolean } +) { + return generateKibanaDevToolsRequest('POST', packagePolicyRouteService.getCreatePath(), { + policy_id: packagePolicy.policy_id ? packagePolicy.policy_id : '', + package: formatPackage(packagePolicy.package), + ...omit(packagePolicy, 'policy_id', 'package', 'enabled'), + inputs: formatInputs(packagePolicy.inputs), + vars: formatVars(packagePolicy.vars), + }); +} + +/** + * Generate a request to update a package policy that can be used in Kibana Dev tools + * @param packagePolicyId + * @param packagePolicy + * @returns + */ +export function generateUpdatePackagePolicyDevToolsRequest( + packagePolicyId: string, + packagePolicy: UpdatePackagePolicy +) { + return generateKibanaDevToolsRequest( + 'PUT', + packagePolicyRouteService.getUpdatePath(packagePolicyId), + { + package: formatPackage(packagePolicy.package), + ...omit(packagePolicy, 'version', 'package', 'enabled'), + inputs: formatInputs(packagePolicy.inputs), + vars: formatVars(packagePolicy.vars), + } + ); +} + +/** + * Generate a request to update an agent policy that can be used in Kibana Dev tools + * @param agentPolicyId + * @param agentPolicy + * @returns + */ +export function generateUpdateAgentPolicyDevToolsRequest( + agentPolicyId: string, + agentPolicy: UpdateAgentPolicyRequest['body'] +) { + return generateKibanaDevToolsRequest( + 'PUT', + agentPolicyRouteService.getUpdatePath(agentPolicyId), + omit(agentPolicy, 'version') + ); +} + +function formatVars(vars: NewPackagePolicy['inputs'][number]['vars']) { + if (!vars) { + return; + } + + return Object.entries(vars).reduce((acc, [varKey, varRecord]) => { + acc[varKey] = varRecord.value; + + return acc; + }, {} as SimplifiedVars); +} + +function formatInputs(inputs: NewPackagePolicy['inputs']) { + return inputs.reduce((acc, input) => { + const inputId = generateInputId(input); + if (!acc) { + acc = {}; + } + acc[inputId] = { + enabled: input.enabled, + vars: formatVars(input.vars), + streams: formatStreams(input.streams), + }; + + return acc; + }, {} as SimplifiedPackagePolicy['inputs']); +} + +function formatStreams(streams: NewPackagePolicy['inputs'][number]['streams']) { + return streams.reduce((acc, stream) => { + if (!acc) { + acc = {}; + } + acc[stream.data_stream.dataset] = { + enabled: stream.enabled, + vars: formatVars(stream.vars), + }; + + return acc; + }, {} as SimplifiedPackagePolicyStreams); +} + +function formatPackage(pkg: NewPackagePolicy['package']) { + return omit(pkg, 'title'); +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/index.tsx new file mode 100644 index 000000000000..25266fb5326c --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/index.tsx @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { + generateCreateAgentPolicyDevToolsRequest, + generateCreatePackagePolicyDevToolsRequest, + generateUpdatePackagePolicyDevToolsRequest, + generateUpdateAgentPolicyDevToolsRequest, +} from './devtools_request'; diff --git a/x-pack/plugins/fleet/public/components/devtools_request_flyout/devtools_request_flyout.tsx b/x-pack/plugins/fleet/public/components/devtools_request_flyout/devtools_request_flyout.tsx new file mode 100644 index 000000000000..5de1a42fc26e --- /dev/null +++ b/x-pack/plugins/fleet/public/components/devtools_request_flyout/devtools_request_flyout.tsx @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useRef } from 'react'; + +import { EuiBetaBadge, EuiButtonEmpty, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import type { EuiButtonEmptyProps } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { ViewApiRequestFlyout } from '@kbn/es-ui-shared-plugin/public'; +import { KibanaContextProvider, toMountPoint } from '@kbn/kibana-react-plugin/public'; + +import { useStartServices } from '../../hooks'; + +interface DevtoolsRequestFlyoutButtonProps { + title?: string; + description?: string; + isDisabled?: boolean; + request: string; + btnProps?: EuiButtonEmptyProps; +} + +export const DevtoolsRequestFlyoutButton: React.FunctionComponent< + DevtoolsRequestFlyoutButtonProps +> = ({ isDisabled, request, title, description, btnProps = {} }) => { + const flyoutRef = useRef>(); + + const services = useStartServices(); + const onClick = useCallback(() => { + const flyout = services.overlays.openFlyout( + toMountPoint( + + flyout.close()} + request={request} + title={title} + description={description} + /> + , + { theme$: services.theme.theme$ } + ) + ); + + flyoutRef.current = flyout; + }, [services, request, title, description]); + + React.useEffect(() => { + return () => { + flyoutRef.current?.close(); + }; + }, []); + + return ( + + + + ); +}; + +export interface ApiRequestFlyoutProps { + title?: string; + description?: string; + isDisabled?: string; + request: string; + closeFlyout: () => void; +} + +export const ApiRequestFlyout: React.FunctionComponent = ({ + closeFlyout, + title = i18n.translate('xpack.fleet.apiRequestFlyout.title', { + defaultMessage: 'Kibana API Request', + }), + request, + description = i18n.translate('xpack.fleet.apiRequestFlyout.description', { + defaultMessage: 'Perform these request against Kibana', + }), +}) => { + const { application, share } = useStartServices(); + + return ( + + {title} + + + + + } + description={description} + request={request} + closeFlyout={closeFlyout} + application={application} + urlService={share.url} + /> + ); +}; diff --git a/x-pack/plugins/fleet/public/components/devtools_request_flyout/index.tsx b/x-pack/plugins/fleet/public/components/devtools_request_flyout/index.tsx new file mode 100644 index 000000000000..33d7aa37b60a --- /dev/null +++ b/x-pack/plugins/fleet/public/components/devtools_request_flyout/index.tsx @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ApiRequestFlyout, DevtoolsRequestFlyoutButton } from './devtools_request_flyout'; diff --git a/x-pack/plugins/fleet/public/components/index.ts b/x-pack/plugins/fleet/public/components/index.ts index 4e9d9f0f021b..7eb22e4c459b 100644 --- a/x-pack/plugins/fleet/public/components/index.ts +++ b/x-pack/plugins/fleet/public/components/index.ts @@ -25,4 +25,5 @@ export * from './link_and_revision'; export * from './agent_enrollment_flyout'; export * from './platform_selector'; export { ConfirmForceInstallModal } from './confirm_force_install_modal'; +export { DevtoolsRequestFlyoutButton } from './devtools_request_flyout'; export { HeaderReleaseBadge, InlineReleaseBadge } from './release_badge'; diff --git a/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx b/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx index 4c8d7dba7176..51ff2cb0f824 100644 --- a/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx +++ b/x-pack/plugins/fleet/public/mock/create_test_renderer.tsx @@ -24,6 +24,7 @@ import { FleetAppContext } from '../applications/fleet/app'; import { IntegrationsAppContext } from '../applications/integrations/app'; import type { FleetConfigType } from '../plugin'; import type { UIExtensionsStorage } from '../types'; +import { ExperimentalFeaturesService } from '../services'; import { createConfigurationMock } from './plugin_configuration'; import { createStartMock } from './plugin_interfaces'; @@ -63,6 +64,12 @@ export const createFleetTestRendererMock = (): TestRenderer => { const history = createMemoryHistory({ initialEntries: [basePath] }); const mountHistory = new CoreScopedHistory(history, basePath); + ExperimentalFeaturesService.init({ + createPackagePolicyMultiPageLayout: true, + packageVerification: true, + showDevtoolsRequest: false, + }); + const HookWrapper = memo(({ children }) => { return ( diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index 713804d8a36f..7ed8e2daa04a 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -8,20 +8,14 @@ /* eslint-disable max-classes-per-file */ import type { ElasticsearchErrorDetails } from '@kbn/es-errors'; -import type { FleetErrorType } from '../../common/types'; +import { IngestManagerError } from '../../common/errors'; import { isESClientError } from './utils'; export { defaultIngestErrorHandler, ingestErrorToResponseOptions } from './handlers'; export { isESClientError } from './utils'; -export class IngestManagerError extends Error { - attributes?: { type: FleetErrorType }; - constructor(message?: string, public readonly meta?: unknown) { - super(message); - this.name = this.constructor.name; // for stack traces - } -} +export { IngestManagerError } from '../../common/errors'; export class RegistryError extends IngestManagerError {} export class RegistryConnectionError extends RegistryError {} diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 2dd49d505a27..94721a48df3f 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -39,9 +39,9 @@ import { installationStatuses } from '../../../common/constants'; import { defaultIngestErrorHandler, PackagePolicyNotFoundError } from '../../errors'; import { getInstallations, getPackageInfo } from '../../services/epm/packages'; import { PACKAGES_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../constants'; -import { simplifiedPackagePolicytoNewPackagePolicy } from '../../services/package_policies/simplified_package_policy_helper'; +import { simplifiedPackagePolicytoNewPackagePolicy } from '../../../common/services/simplified_package_policy_helper'; -import type { SimplifiedPackagePolicy } from '../../services/package_policies/simplified_package_policy_helper'; +import type { SimplifiedPackagePolicy } from '../../../common/services/simplified_package_policy_helper'; export const getPackagePoliciesHandler: RequestHandler< undefined, diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 560fd8a3f79d..62568a3cbca5 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -211,6 +211,13 @@ export async function mountApp( ? historyLocationState.payload : undefined; + if (historyLocationState && historyLocationState.type === ACTION_VISUALIZE_LENS_FIELD) { + // remove originatingApp from context when visualizing a field in Lens + // so Lens does not try to return to the original app on Save + // see https://github.com/elastic/kibana/issues/128695 + delete initialContext?.originatingApp; + } + if (embeddableEditorIncomingState?.searchSessionId) { data.search.session.continue(embeddableEditorIncomingState.searchSessionId); } diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx index 5c1b5c4d9d8a..65c714c1731b 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.test.tsx @@ -166,7 +166,7 @@ describe('ConfigPanel', () => { .first() .instance(); act(() => { - instance.find('[data-test-subj="lnsLayerRemove"]').first().simulate('click'); + instance.find('[data-test-subj="lnsLayerRemove--0"]').first().simulate('click'); }); instance.update(); act(() => { @@ -193,7 +193,7 @@ describe('ConfigPanel', () => { .first() .instance(); act(() => { - instance.find('[data-test-subj="lnsLayerRemove"]').at(0).simulate('click'); + instance.find('[data-test-subj="lnsLayerRemove--0"]').first().simulate('click'); }); instance.update(); act(() => { @@ -219,7 +219,7 @@ describe('ConfigPanel', () => { .first() .instance(); act(() => { - instance.find('[data-test-subj="lnsLayerRemove"]').at(2).simulate('click'); + instance.find('[data-test-subj="lnsLayerRemove--1"]').first().simulate('click'); }); instance.update(); act(() => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx index cb45519250a8..3b0f46e5e615 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.test.tsx @@ -137,7 +137,7 @@ describe('LayerPanel', () => { it('should show the reset button when single layer', async () => { const { instance } = await mountWithProvider(); expect( - instance.find('[data-test-subj="lnsLayerRemove"]').first().props()['aria-label'] + instance.find('[data-test-subj="lnsLayerRemove--0"]').first().props()['aria-label'] ).toContain('Reset layer'); }); @@ -146,7 +146,7 @@ describe('LayerPanel', () => { ); expect( - instance.find('[data-test-subj="lnsLayerRemove"]').first().props()['aria-label'] + instance.find('[data-test-subj="lnsLayerRemove--0"]').first().props()['aria-label'] ).toContain('Delete layer'); }); @@ -155,7 +155,7 @@ describe('LayerPanel', () => { delete layerPanelAttributes.activeVisualization.removeLayer; const { instance } = await mountWithProvider(); expect( - instance.find('[data-test-subj="lnsLayerRemove"]').first().props()['aria-label'] + instance.find('[data-test-subj="lnsLayerRemove--0"]').first().props()['aria-label'] ).toContain('Reset visualization'); }); @@ -165,7 +165,7 @@ describe('LayerPanel', () => { ); act(() => { - instance.find('[data-test-subj="lnsLayerRemove"]').first().simulate('click'); + instance.find('[data-test-subj="lnsLayerRemove--0"]').first().simulate('click'); }); instance.update(); act(() => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/remove_layer_button.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/remove_layer_button.tsx index 64c4d808f255..651ad3f2526a 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/remove_layer_button.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/remove_layer_button.tsx @@ -158,7 +158,7 @@ export function RemoveLayerButton({ size="xs" iconType={isOnlyLayer ? 'eraser' : 'trash'} color="danger" - data-test-subj="lnsLayerRemove" + data-test-subj={`lnsLayerRemove--${layerIndex}`} aria-label={ariaLabel} title={ariaLabel} onClick={() => { diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx index 6f90bd340c1e..75d22b2feea8 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/warnings_popover.tsx @@ -35,6 +35,7 @@ export const WarningsPopover = ({ onClick={onButtonClick} iconType="alert" className="lnsWorkspaceWarning__button" + data-test-subj="lens-editor-warning-button" > {warningsCount} @@ -53,7 +54,11 @@ export const WarningsPopover = ({ >
    {React.Children.map(children, (child, index) => ( -
  • +
  • {child}
    • ))} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 2017ef9f6328..fbd9a5650013 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -341,7 +341,7 @@ export function DimensionEditor(props: DimensionEditorProps) { const partialIcon = compatibleWithCurrentField && referencedField?.partiallyApplicableFunctions?.[operationType] && ( - <> + {' '} - + ); let label: EuiListGroupItemProps['label'] = ( <> diff --git a/x-pack/plugins/lens/public/shared_components/filter_query_input.tsx b/x-pack/plugins/lens/public/shared_components/filter_query_input.tsx index db585f5f2820..9a2ec76d4c79 100644 --- a/x-pack/plugins/lens/public/shared_components/filter_query_input.tsx +++ b/x-pack/plugins/lens/public/shared_components/filter_query_input.tsx @@ -37,6 +37,7 @@ export function FilterQueryInput({ helpMessage, label = filterByLabel, initiallyOpen, + ['data-test-subj']: dataTestSubj, }: { inputFilter: Query | undefined; onChange: (query: Query) => void; @@ -44,6 +45,7 @@ export function FilterQueryInput({ helpMessage?: string | null; label?: string; initiallyOpen?: boolean; + ['data-test-subj']?: string; }) { const [filterPopoverOpen, setFilterPopoverOpen] = useState(Boolean(initiallyOpen)); const { inputValue: queryInput, handleInputChange: setQueryInput } = useDebouncedValue({ @@ -133,6 +135,7 @@ export function FilterQueryInput({ onChange={setQueryInput} isInvalid={!isQueryInputValid} onSubmit={() => {}} + data-test-subj={dataTestSubj} /> diff --git a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/annotations_panel.tsx b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/annotations_panel.tsx index 5d68e29a88d0..8b36ee762a23 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/annotations_panel.tsx +++ b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/annotations_panel.tsx @@ -264,6 +264,7 @@ export const AnnotationsPanel = ( } }} fieldIsInvalid={!fieldIsValid} + data-test-subj="lnsXY-annotation-query-based-text-decoration-field-picker" autoFocus={!selectedField} /> diff --git a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/index.test.tsx b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/index.test.tsx index 5514e3062213..f405985748fd 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/index.test.tsx +++ b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/index.test.tsx @@ -284,16 +284,16 @@ describe('AnnotationsPanel', () => { ); expect( - component.find('[data-test-subj="annotation-query-based-field-picker"]').exists() + component.find('[data-test-subj="lnsXY-annotation-query-based-field-picker"]').exists() ).toBeTruthy(); expect( - component.find('[data-test-subj="annotation-query-based-query-input"]').exists() + component.find('[data-test-subj="lnsXY-annotation-query-based-query-input"]').exists() ).toBeTruthy(); // The provided indexPattern has 2 date fields expect( component - .find('[data-test-subj="annotation-query-based-field-picker"]') + .find('[data-test-subj="lnsXY-annotation-query-based-field-picker"]') .at(0) .prop('options') ).toHaveLength(2); diff --git a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/query_annotation_panel.tsx b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/query_annotation_panel.tsx index 69cd398c562b..8bf926a70e40 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/query_annotation_panel.tsx +++ b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/query_annotation_panel.tsx @@ -84,6 +84,7 @@ export const ConfigPanelQueryAnnotation = ({ onChange={(query: Query) => { onChange({ filter: { type: 'kibana_query', ...query } }); }} + data-test-subj="lnsXY-annotation-query-based-query-input" indexPattern={currentIndexPattern} /> @@ -114,7 +115,7 @@ export const ConfigPanelQueryAnnotation = ({ } }} fieldIsInvalid={!fieldIsValid} - data-test-subj="annotation-query-based-field-picker" + data-test-subj="lnsXY-annotation-query-based-field-picker" /> diff --git a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/tooltip_annotation_panel.tsx b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/tooltip_annotation_panel.tsx index c8ea7a0ed2ec..586d83b993b9 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/tooltip_annotation_panel.tsx +++ b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/tooltip_annotation_panel.tsx @@ -198,6 +198,7 @@ export function TooltipSection({ onFieldSelectChange(choice, index); }} fieldIsInvalid={!fieldIsValid} + data-test-subj={`lnsXY-annotation-tooltip-field-picker--${index}`} autoFocus={isNew && value == null} /> diff --git a/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx b/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx index 9278f08bd4d2..37a2f75df154 100644 --- a/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx +++ b/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx @@ -7,6 +7,7 @@ import React, { MouseEvent } from 'react'; import { SavedObjectReference } from '@kbn/core/types'; +import { METRIC_TYPE } from '@kbn/analytics'; import { i18n } from '@kbn/i18n'; import { EuiLink } from '@elastic/eui'; import { EuiBasicTableColumn } from '@elastic/eui/src/components/basic_table/basic_table'; @@ -24,6 +25,7 @@ import { getUiSettings, getTheme, getApplication, + getUsageCollection, } from '../../kibana_services'; import { getAppTitle } from '../../../common/i18n_getters'; import { MapSavedObjectAttributes } from '../../../common/map_saved_object_type'; @@ -77,6 +79,7 @@ if (savedObjectsTagging) { function navigateToNewMap() { const navigateToApp = getNavigateToApp(); + getUsageCollection()?.reportUiCounter(APP_ID, METRIC_TYPE.CLICK, 'create_maps_vis_editor'); navigateToApp(APP_ID, { path: MAP_PATH, }); diff --git a/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts b/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts index ae104c08034f..f073c7335eb0 100644 --- a/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts +++ b/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts @@ -9,6 +9,7 @@ import uuid from 'uuid/v4'; import { i18n } from '@kbn/i18n'; import type { Query } from '@kbn/es-query'; import type { SerializableRecord } from '@kbn/utility-types'; +import { METRIC_TYPE } from '@kbn/analytics'; import { createAction, ACTION_VISUALIZE_GEO_FIELD, @@ -50,8 +51,8 @@ export const visualizeGeoFieldAction = createAction({ const usageCollection = getUsageCollection(); usageCollection?.reportUiCounter( APP_ID, - 'visualize_geo_field', - context.originatingApp ? context.originatingApp : 'unknownOriginatingApp' + METRIC_TYPE.CLICK, + `create_maps_vis_${context.originatingApp ? context.originatingApp : 'unknownOriginatingApp'}` ); getCore().application.navigateToApp(app, { diff --git a/x-pack/plugins/observability/public/index.ts b/x-pack/plugins/observability/public/index.ts index b4e23ce0e83a..3b5d34e87df7 100644 --- a/x-pack/plugins/observability/public/index.ts +++ b/x-pack/plugins/observability/public/index.ts @@ -114,3 +114,5 @@ export { } from './components/shared/exploratory_view/configurations/constants'; export { ExploratoryViewContextProvider } from './components/shared/exploratory_view/contexts/exploratory_view_config'; export { fromQuery, toQuery } from './utils/url'; + +export type { NavigationSection } from './services/navigation_registry'; diff --git a/x-pack/plugins/profiling/.gitignore b/x-pack/plugins/profiling/.gitignore new file mode 100644 index 000000000000..eb5a316cbd19 --- /dev/null +++ b/x-pack/plugins/profiling/.gitignore @@ -0,0 +1 @@ +target diff --git a/x-pack/plugins/profiling/.i18nrc.json b/x-pack/plugins/profiling/.i18nrc.json new file mode 100644 index 000000000000..de8ac3249413 --- /dev/null +++ b/x-pack/plugins/profiling/.i18nrc.json @@ -0,0 +1,7 @@ +{ + "prefix": "profiling", + "paths": { + "profiling": "." + }, + "translations": [] +} diff --git a/x-pack/plugins/profiling/DOCKER.md b/x-pack/plugins/profiling/DOCKER.md new file mode 100644 index 000000000000..632c6b7a38c0 --- /dev/null +++ b/x-pack/plugins/profiling/DOCKER.md @@ -0,0 +1,68 @@ +# Containerized development environment for Kibana + +Kibana development moves quickly and has specific NodeJS requirements, which can +change depending on the current branch or commit. This makes developing a plugin +challenging for local development. + +We created a containerized environment to encapsulate the necessary dependencies +and reduce issues, especially when switching between Kibana versions. + +## Assumptions + +By default, this setup assumes that you are running Elasticsearch via the +`apm-integration-testing` repo. However, you can override this as described +below. + +## Usage + +You will first need to build the Docker image for the Kibana environment: + +``` +make build +``` + +This will create a Docker image with the tag `kibana-dev:latest`. + +If you wish to change the image version, you can run this instead: + +``` +make build KIBANA_VERSION=8.4 +``` + +If you need to do a full refresh of the Docker image, you can rebuild from +scratch using this: + +``` +make build-nocache +``` + +Next, you can start the container using this (assumes Docker image is +`kibana-dev:latest` and Elasticsearch is running on the `apm-integration-testing` +Docker network): + +``` +make run +``` + +If your Elasticsearch instance is not running via Docker, you can run this: + +``` +make run-networkless +``` + +## Configuration + +* `KIBANA_VERSION` + +This is the version of the Docker image. This can be used to build separate +images for different Kibana versions. + +* `NETWORK` + +This is the Docker network to use so that Kibana can connect to a local running +instance of Elasticsearch. + +* `PORT` + +This is the exposed port for the Kibana instance. This is useful if you have +conflicts with another running instance. diff --git a/x-pack/plugins/profiling/Dockerfile b/x-pack/plugins/profiling/Dockerfile new file mode 100644 index 000000000000..777a7ba9cce2 --- /dev/null +++ b/x-pack/plugins/profiling/Dockerfile @@ -0,0 +1,48 @@ +FROM ubuntu:22.04 + +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# Install base dependencies +RUN apt-get update + +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata + +RUN apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + git \ + libssl-dev \ + python3 \ + wget \ + && rm -rf /var/lib/apt/lists/* + +RUN useradd -ms /bin/bash -d /kibana kibana + +# Install and setup nvm and node +USER kibana + +ARG NODE_VERSION + +ENV NVM_VERSION v0.39.1 +ENV NVM_DIR /kibana/nvm + +RUN mkdir $NVM_DIR \ + && cd $NVM_DIR \ + && curl https://raw.githubusercontent.com/nvm-sh/nvm/$NVM_VERSION/install.sh | bash \ + && . ./nvm.sh \ + && nvm install $NODE_VERSION \ + && nvm alias default $NODE_VERSION \ + && nvm use default + +ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules +ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH + +# Install yarn +USER kibana +WORKDIR /kibana +RUN npm install -g yarn + +WORKDIR /kibana/src +CMD ["bash"] diff --git a/x-pack/plugins/profiling/INTERNALS.md b/x-pack/plugins/profiling/INTERNALS.md new file mode 100644 index 000000000000..9612a64380ef --- /dev/null +++ b/x-pack/plugins/profiling/INTERNALS.md @@ -0,0 +1,97 @@ +This plugin is divided into two components: the UI and the server. + +The UI is responsible for rendering the charts and flamegraphs. It will make +API calls to the server component depending on the user interactions with the +UI. You will find most of the source code in the `public` directory. + +The server is responsible for retrieving data from the necessary sources and +sending it in the expected format to the UI in response to the UI's API calls. +You will find most of the source code in the `server` directory. + +## Server API + +Depending on how the plugin is configured, there are two different sets of API +calls. + +If the server uses local fixtures as a data source, then the following API is +served by the server and called by the UI: + +* /api/prodfiler/v1/topn/containers +* /api/prodfiler/v1/topn/deployments +* /api/prodfiler/v1/topn/hosts +* /api/prodfiler/v1/topn/threads +* /api/prodfiler/v1/topn/traces +* /api/prodfiler/v1/flamechart/elastic + +If the server uses an Elasticsearch cluster as a data source, then the following +API is served by the server and called by the UI: + +* /api/prodfiler/v2/topn/containers +* /api/prodfiler/v2/topn/deployments +* /api/prodfiler/v2/topn/hosts +* /api/prodfiler/v2/topn/threads +* /api/prodfiler/v2/topn/traces +* /api/prodfiler/v2/flamechart/elastic + +By default, the plugin is configured to use the second API set. See README.md to +configure the plugin to use the first API set (aka local fixtures as a data +source). + +Both API sets are expected to return the same response format. + +The design to have separate API sets for local vs Elasticsearch was partly +because the UI and server components were originally developed separately and +later merged. However, it also allows the server methods to have a single +responsibility, making it easier to test and verify that the server returns +the expected responses for the given data sources. + +## Server API Responses + +### /api/prodfiler/*/flamechart/elastic + +The response returned from this API is used by the Elastic flamegraph. + +The following example is the expected response: + +```json +{ + leaves: [ + { + id: 'pf-collection-agent: runtime.releaseSudog() in runtime2.go#282', + value: 1, + depth: 19, + pathFromRoot: { + '0': 'root', + '1': 'pf-collection-agent: runtime.goexit() in asm_amd64.s#1581', + '2': 'pf-collection-agent: github.com/optimyze/prodfiler/pf-storage-backend/storagebackend/storagebackendv1.(*ScyllaExecutor).Start.func1 in scyllaexecutor.go#102', + '3': 'pf-collection-agent: github.com/optimyze/prodfiler/pf-storage-backend/storagebackend/storagebackendv1.(*ScyllaExecutor).executeQueryAndReadResults in scyllaexecutor.go#158', + '4': 'pf-collection-agent: github.com/gocql/gocql.(*Query).Iter in session.go#1246', + '5': 'pf-collection-agent: github.com/gocql/gocql.(*Session).executeQuery in session.go#463', + '6': 'pf-collection-agent: github.com/gocql/gocql.(*queryExecutor).executeQuery in query_executor.go#66', + '7': 'pf-collection-agent: github.com/gocql/gocql.(*queryExecutor).do in query_executor.go#127', + '8': 'pf-collection-agent: github.com/gocql/gocql.(*queryExecutor).attemptQuery in query_executor.go#32', + '9': 'pf-collection-agent: github.com/gocql/gocql.(*Query).execute in session.go#1044', + '10': 'pf-collection-agent: github.com/gocql/gocql.(*Conn).executeQuery in conn.go#1129', + '11': 'pf-collection-agent: github.com/gocql/gocql.(*Conn).exec in conn.go#916', + '12': 'pf-collection-agent: github.com/gocql/gocql.(*writeExecuteFrame).writeFrame in frame.go#1618', + '13': 'pf-collection-agent: github.com/gocql/gocql.(*framer).writeExecuteFrame in frame.go#1643', + '14': 'pf-collection-agent: github.com/gocql/gocql.(*framer).finishWrite in frame.go#788', + '15': 'pf-collection-agent: github.com/gocql/gocql.(*Conn).Write in conn.go#319', + '16': 'pf-collection-agent: github.com/gocql/gocql.(*writeCoalescer).Write in conn.go#829', + '17': 'pf-collection-agent: sync.(*Cond).Wait in cond.go#83', + '18': 'pf-collection-agent: sync.runtime_notifyListWait() in sema.go#498', + '19': 'pf-collection-agent: runtime.releaseSudog() in runtime2.go#282', + }, + }, + ... + ] +} +``` + +Here is a basic description of the response format: + +* Each object in the `leaves` list represents a leaf node in the flamegraph +* `id` represents the name of the flamegraph node +* `value` represents the number of samples for that node +* `depth` represents the depth of the node in the flamegraph, starting from zero +* `pathFromRoot` represents the full path from the flamegraph root to the given node diff --git a/x-pack/plugins/profiling/Makefile b/x-pack/plugins/profiling/Makefile new file mode 100644 index 000000000000..3f624ceea340 --- /dev/null +++ b/x-pack/plugins/profiling/Makefile @@ -0,0 +1,24 @@ +KIBANA_REPO = $(shell git rev-parse --show-toplevel) +NODE_VERSION = $(shell cat ${KIBANA_REPO}/.node-version) + +KIBANA_VERSION ?= latest +NETWORK ?= apm-integration-testing +PORT ?= 5601 + +DOCKER_IMAGE = kibana-dev:${KIBANA_VERSION} +DOCKER_BUILD_ARGS = --build-arg NODE_VERSION="${NODE_VERSION}" -t ${DOCKER_IMAGE} . +DOCKER_RUN_ARGS = --rm -p ${PORT}:5601 -p 9229-9231:9229-9231/tcp -v ${KIBANA_REPO}:/kibana/src -it ${DOCKER_IMAGE} + +.PHONY: build build-nocache run run-networkless + +build: + docker build ${DOCKER_BUILD_ARGS} + +build-nocache: + docker build --no-cache ${DOCKER_BUILD_ARGS} + +run: + docker run --network ${NETWORK} ${DOCKER_RUN_ARGS} + +run-networkless: + docker run ${DOCKER_RUN_ARGS} diff --git a/x-pack/plugins/profiling/README.md b/x-pack/plugins/profiling/README.md new file mode 100644 index 000000000000..4b2108cc6779 --- /dev/null +++ b/x-pack/plugins/profiling/README.md @@ -0,0 +1 @@ +### TODO diff --git a/x-pack/plugins/profiling/common/__fixtures__/stacktraces.ts b/x-pack/plugins/profiling/common/__fixtures__/stacktraces.ts new file mode 100644 index 000000000000..73b383059583 --- /dev/null +++ b/x-pack/plugins/profiling/common/__fixtures__/stacktraces.ts @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createStackFrameID } from '../profiling'; + +enum stackTraceID { + A = 'yU2Oct2ct0HkxJ7-pRcPkg==', + B = 'Xt8aKN70PDXpMDLCOmojzQ==', + C = '8OauxYq2WK4_tBqM4xkIwA==', + D = 'nQWGdRxvqVjwlLmQWH1Phw==', + E = '2KciEEWALlol3b6x95PHcw==', + F = 'BxRgiXa4h9Id6BjdPPHK8Q==', +} + +enum fileID { + A = 'Ncujji3wC1nL73TTEyFBhA==', + B = 'T2vdys5d7j85az1aP86zCg==', + C = 'jMaTVVjYv7cecd0C4HguGw==', + D = 'RLkjnlfcvSJN2Wph9WUuOQ==', + E = 'gnEsgxvvEODj6iFYMQWYlA==', + F = 'Gf4xoLc8QuAHU49Ch_CFOA==', + G = 'ZCOCZlls7r2cbG1HchkbVg==', + H = 'Og7kGWGe9qiCunkaXDffHQ==', + I = 'WAE6T1TeDsjDMOuwX4Ynxg==', + J = 'ZNiZco1zgh0nJI6hPllMaQ==', + K = 'abl5r8Vvvb2Y7NaDZW1QLQ==', +} + +enum addressOrLine { + A = 26278522, + B = 6712518, + C = 105806025, + D = 105806854, + E = 107025202, + F = 107044169, + G = 18353156, + H = 3027, + I = 5201, + J = 67384048, + K = 8888, +} + +const frameID: Record = { + A: createStackFrameID(fileID.A, addressOrLine.A), + B: createStackFrameID(fileID.B, addressOrLine.B), + C: createStackFrameID(fileID.C, addressOrLine.C), + D: createStackFrameID(fileID.D, addressOrLine.D), + E: createStackFrameID(fileID.E, addressOrLine.C), + F: createStackFrameID(fileID.E, addressOrLine.D), + G: createStackFrameID(fileID.E, addressOrLine.E), + H: createStackFrameID(fileID.E, addressOrLine.F), + I: createStackFrameID(fileID.E, addressOrLine.G), + J: createStackFrameID(fileID.F, addressOrLine.H), + K: createStackFrameID(fileID.F, addressOrLine.I), + L: createStackFrameID(fileID.F, addressOrLine.J), + M: createStackFrameID(fileID.F, addressOrLine.K), + N: createStackFrameID(fileID.G, addressOrLine.G), + O: createStackFrameID(fileID.H, addressOrLine.H), + P: createStackFrameID(fileID.I, addressOrLine.I), + Q: createStackFrameID(fileID.F, addressOrLine.A), + R: createStackFrameID(fileID.E, addressOrLine.B), + S: createStackFrameID(fileID.E, addressOrLine.C), +}; + +export const events = new Map([ + [stackTraceID.A, 16], + [stackTraceID.B, 9], + [stackTraceID.C, 7], + [stackTraceID.D, 5], + [stackTraceID.E, 2], + [stackTraceID.F, 1], +]); + +export const stackTraces = new Map([ + [ + stackTraceID.A, + { + FileIDs: [fileID.D, fileID.C, fileID.B, fileID.A], + AddressOrLines: [addressOrLine.D, addressOrLine.C, addressOrLine.B, addressOrLine.A], + FrameIDs: [frameID.D, frameID.C, frameID.B, frameID.A], + Types: [3, 3, 3, 3], + }, + ], + [ + stackTraceID.B, + { + FileIDs: [fileID.E, fileID.E, fileID.E, fileID.E, fileID.E], + AddressOrLines: [ + addressOrLine.G, + addressOrLine.F, + addressOrLine.E, + addressOrLine.D, + addressOrLine.C, + ], + FrameIDs: [frameID.I, frameID.H, frameID.G, frameID.F, frameID.E], + Types: [3, 3, 3, 3, 3], + }, + ], + [ + stackTraceID.C, + { + FileIDs: [fileID.F, fileID.F, fileID.F, fileID.F], + AddressOrLines: [addressOrLine.K, addressOrLine.J, addressOrLine.I, addressOrLine.H], + FrameIDs: [frameID.M, frameID.L, frameID.K, frameID.J], + Types: [3, 3, 3, 3], + }, + ], + [ + stackTraceID.D, + { + FileIDs: [fileID.I, fileID.H, fileID.G], + AddressOrLines: [addressOrLine.I, addressOrLine.H, addressOrLine.G], + FrameIDs: [frameID.P, frameID.O, frameID.N], + Types: [3, 8, 8], + }, + ], + [ + stackTraceID.E, + { + FileIDs: [fileID.F, fileID.F, fileID.F], + AddressOrLines: [addressOrLine.K, addressOrLine.J, addressOrLine.I], + FrameIDs: [frameID.M, frameID.L, frameID.K], + Types: [3, 3, 3], + }, + ], + [ + stackTraceID.F, + { + FileIDs: [fileID.E, fileID.E], + AddressOrLines: [addressOrLine.F, addressOrLine.E], + FrameIDs: [frameID.H, frameID.G], + Types: [3, 3], + }, + ], +]); + +const defaultStackFrame = { + FileName: '', + FunctionName: '', + FunctionOffset: 0, + LineNumber: 0, + SourceType: 0, +}; + +export const stackFrames = new Map([ + [ + frameID.A, + { + FileName: 'ThreadPoolExecutor.java', + FunctionName: 'java.lang.Runnable java.util.concurrent.ThreadPoolExecutor.getTask()', + FunctionOffset: 26, + LineNumber: 1061, + SourceType: 5, + }, + ], + [ + frameID.B, + { FileName: '', FunctionName: 'sock_sendmsg', FunctionOffset: 0, LineNumber: 0, SourceType: 0 }, + ], + [frameID.C, defaultStackFrame], + [frameID.D, defaultStackFrame], + [frameID.E, defaultStackFrame], + [frameID.F, defaultStackFrame], + [frameID.G, defaultStackFrame], + [frameID.H, defaultStackFrame], + [frameID.I, defaultStackFrame], + [frameID.J, defaultStackFrame], + [frameID.K, defaultStackFrame], + [ + frameID.L, + { FileName: '', FunctionName: 'udp_sendmsg', FunctionOffset: 0, LineNumber: 0, SourceType: 0 }, + ], + [frameID.M, defaultStackFrame], + [frameID.N, defaultStackFrame], + [frameID.O, defaultStackFrame], + [frameID.P, defaultStackFrame], + [frameID.Q, defaultStackFrame], + [frameID.R, defaultStackFrame], + [frameID.S, defaultStackFrame], +]); + +export const executables = new Map([ + [fileID.A, { FileName: '' }], + [fileID.B, { FileName: '' }], + [fileID.C, { FileName: '' }], + [fileID.D, { FileName: 'libglapi.so.0.0.0' }], + [fileID.E, { FileName: '' }], + [fileID.F, { FileName: '' }], + [fileID.G, { FileName: '' }], + [fileID.H, { FileName: '' }], + [fileID.I, { FileName: '' }], +]); diff --git a/x-pack/plugins/profiling/common/callercallee.test.ts b/x-pack/plugins/profiling/common/callercallee.test.ts new file mode 100644 index 000000000000..af07da439a3e --- /dev/null +++ b/x-pack/plugins/profiling/common/callercallee.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { sum } from 'lodash'; +import { + createCallerCalleeDiagram, + createCallerCalleeIntermediateNode, + fromCallerCalleeIntermediateNode, +} from './callercallee'; +import { createFrameGroupID } from './frame_group'; +import { createStackFrameMetadata } from './profiling'; + +import { events, stackTraces, stackFrames, executables } from './__fixtures__/stacktraces'; + +describe('Caller-callee operations', () => { + test('1', () => { + const parentFrame = createStackFrameMetadata({ + FileID: '6bc50d345244d5956f93a1b88f41874d', + FrameType: 3, + AddressOrLine: 971740, + FunctionName: 'epoll_wait', + SourceID: 'd670b496cafcaea431a23710fb5e4f58', + SourceLine: 30, + ExeFileName: 'libc-2.26.so', + }); + const parent = createCallerCalleeIntermediateNode(parentFrame, 10, 'parent'); + + const childFrame = createStackFrameMetadata({ + FileID: '8d8696a4fd51fa88da70d3fde138247d', + FrameType: 3, + AddressOrLine: 67000, + FunctionName: 'epoll_poll', + SourceID: 'f0a7901dcefed6cc8992a324b9df733c', + SourceLine: 150, + ExeFileName: 'auditd', + }); + const child = createCallerCalleeIntermediateNode(childFrame, 10, 'child'); + + const root = createCallerCalleeIntermediateNode(createStackFrameMetadata(), 10, 'root'); + root.callees.set(createFrameGroupID(child.frameGroup), child); + root.callees.set(createFrameGroupID(parent.frameGroup), parent); + + const graph = fromCallerCalleeIntermediateNode(root); + + // Modify original frames to verify graph does not contain references + parent.samples = 30; + child.samples = 20; + + expect(graph.Callees[0].Samples).toEqual(10); + expect(graph.Callees[1].Samples).toEqual(10); + }); + + test('2', () => { + const totalSamples = sum([...events.values()]); + + const root = createCallerCalleeDiagram(events, stackTraces, stackFrames, executables); + expect(root.Samples).toEqual(totalSamples); + expect(root.CountInclusive).toEqual(totalSamples); + expect(root.CountExclusive).toEqual(0); + }); +}); diff --git a/x-pack/plugins/profiling/common/callercallee.ts b/x-pack/plugins/profiling/common/callercallee.ts new file mode 100644 index 000000000000..e4472968c4d3 --- /dev/null +++ b/x-pack/plugins/profiling/common/callercallee.ts @@ -0,0 +1,322 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { clone } from 'lodash'; +import { + compareFrameGroup, + createFrameGroup, + createFrameGroupID, + FrameGroup, + FrameGroupID, +} from './frame_group'; +import { + createStackFrameMetadata, + Executable, + FileID, + groupStackFrameMetadataByStackTrace, + StackFrame, + StackFrameID, + StackFrameMetadata, + StackTrace, + StackTraceID, +} from './profiling'; + +export interface CallerCalleeIntermediateNode { + frameGroup: FrameGroup; + frameGroupID: string; + callers: Map; + callees: Map; + frameMetadata: Set; + samples: number; + countInclusive: number; + countExclusive: number; +} + +export function createCallerCalleeIntermediateNode( + frameMetadata: StackFrameMetadata, + samples: number, + frameGroupID: string +): CallerCalleeIntermediateNode { + return { + frameGroup: createFrameGroup(frameMetadata), + callers: new Map(), + callees: new Map(), + frameMetadata: new Set([frameMetadata]), + samples, + countInclusive: 0, + countExclusive: 0, + frameGroupID, + }; +} + +interface RelevantTrace { + frames: StackFrameMetadata[]; + index: number; +} + +// selectRelevantTraces searches through a map that maps trace hashes to their +// frames and only returns those traces that have a frame that are equivalent +// to the rootFrame provided. It also sets the "index" in the sequence of +// traces at which the rootFrame is found. +// +// If the rootFrame is "empty" (e.g. fileID is empty and line number is 0), all +// traces in the given time frame are deemed relevant, and the "index" is set +// to the length of the trace -- since there is no root frame, the frame should +// be considered "calls-to" only going. +function selectRelevantTraces( + rootFrame: StackFrameMetadata, + frames: Map +): Map { + const result = new Map(); + const rootString = createFrameGroupID(createFrameGroup(rootFrame)); + for (const [stackTraceID, frameMetadata] of frames) { + if (rootFrame.FileID === '' && rootFrame.AddressOrLine === 0) { + // If the root frame is empty, every trace is relevant, and all elements + // of the trace are relevant. This means that the index is set to the + // length of the frameMetadata, implying that in the absence of a root + // frame the "topmost" frame is the root frame. + result.set(stackTraceID, { + frames: frameMetadata, + index: frameMetadata.length, + } as RelevantTrace); + } else { + // Search for the right index of the root frame in the frameMetadata, and + // set it in the result. + for (let i = 0; i < frameMetadata.length; i++) { + if (rootString === createFrameGroupID(createFrameGroup(frameMetadata[i]))) { + result.set(stackTraceID, { + frames: frameMetadata, + index: i, + } as RelevantTrace); + } + } + } + } + return result; +} + +function sortRelevantTraces(relevantTraces: Map): StackTraceID[] { + const sortedRelevantTraces = new Array(); + for (const trace of relevantTraces.keys()) { + sortedRelevantTraces.push(trace); + } + return sortedRelevantTraces.sort((t1, t2) => { + if (t1 < t2) return -1; + if (t1 > t2) return 1; + return 0; + }); +} + +// createCallerCalleeIntermediateRoot creates a graph in the internal +// representation from a StackFrameMetadata that identifies the "centered" +// function and the trace results that provide traces and the number of times +// that the trace has been seen. +// +// The resulting data structure contains all of the data, but is not yet in the +// form most easily digestible by others. +export function createCallerCalleeIntermediateRoot( + rootFrame: StackFrameMetadata, + traces: Map, + frames: Map +): CallerCalleeIntermediateNode { + // Create a node for the centered frame + const root = createCallerCalleeIntermediateNode(rootFrame, 0, 'root'); + + // Obtain only the relevant frames (e.g. frames that contain the root frame + // somewhere). If the root frame is "empty" (e.g. fileID is zero and line + // number is zero), all frames are deemed relevant. + const relevantTraces = selectRelevantTraces(rootFrame, frames); + + // For a deterministic result we have to walk the traces in a deterministic + // order. A deterministic result allows for deterministic UI views, something + // that users expect. + const relevantTracesSorted = sortRelevantTraces(relevantTraces); + + // Walk through all traces that contain the root. Increment the count of the + // root by the count of that trace. Walk "up" the trace (through the callers) + // and add the count of the trace to each caller. Then walk "down" the trace + // (through the callees) and add the count of the trace to each callee. + + for (const traceHash of relevantTracesSorted) { + const trace = relevantTraces.get(traceHash)!; + + // The slice of frames is ordered so that the leaf function is at index 0. + // This means that the "second part" of the slice are the callers, and the + // "first part" are the callees. + // + // We currently assume there are no callers. + const callees = trace.frames; + const samples = traces.get(traceHash)!; + + // Go through the callees, reverse iteration + let currentNode = clone(root); + root.samples += samples; + + for (let i = 0; i < callees.length; i++) { + const callee = callees[i]; + const calleeName = createFrameGroupID(createFrameGroup(callee)); + let node = currentNode.callees.get(calleeName); + if (node === undefined) { + node = createCallerCalleeIntermediateNode(callee, samples, calleeName); + currentNode.callees.set(calleeName, node); + } else { + node.samples += samples; + } + + node.countInclusive += samples; + + if (i === callees.length - 1) { + // Leaf frame: sum up counts for exclusive CPU. + node.countExclusive += samples; + } + currentNode = node; + } + } + + root.countExclusive = 0; + root.countInclusive = root.samples; + + return root; +} + +export interface CallerCalleeNode { + FrameID: string; + FrameGroupID: string; + Callers: CallerCalleeNode[]; + Callees: CallerCalleeNode[]; + FileID: string; + FrameType: number; + ExeFileName: string; + FunctionID: string; + FunctionName: string; + AddressOrLine: number; + FunctionSourceLine: number; + FunctionSourceID: string; + FunctionSourceURL: string; + SourceFilename: string; + SourceLine: number; + Samples: number; + CountInclusive: number; + CountExclusive: number; +} + +export function createCallerCalleeNode(options: Partial = {}): CallerCalleeNode { + const node = {} as CallerCalleeNode; + + node.FrameID = options.FrameID ?? ''; + node.FrameGroupID = options.FrameGroupID ?? ''; + node.Callers = clone(options.Callers ?? []); + node.Callees = clone(options.Callees ?? []); + node.FileID = options.FileID ?? ''; + node.FrameType = options.FrameType ?? 0; + node.ExeFileName = options.ExeFileName ?? ''; + node.FunctionID = options.FunctionID ?? ''; + node.FunctionName = options.FunctionName ?? ''; + node.AddressOrLine = options.AddressOrLine ?? 0; + node.FunctionSourceLine = options.FunctionSourceLine ?? 0; + node.FunctionSourceID = options.FunctionSourceID ?? ''; + node.FunctionSourceURL = options.FunctionSourceURL ?? ''; + node.SourceFilename = options.SourceFilename ?? ''; + node.SourceLine = options.SourceLine ?? 0; + node.Samples = options.Samples ?? 0; + node.CountInclusive = options.CountInclusive ?? 0; + node.CountExclusive = options.CountExclusive ?? 0; + + return node; +} + +// selectCallerCalleeData is the "standard" way of merging multiple frames into +// one node. It simply takes the data from the first frame. +function selectCallerCalleeData(frameMetadata: Set, node: CallerCalleeNode) { + for (const metadata of frameMetadata) { + node.FileID = metadata.FileID; + node.FrameType = metadata.FrameType; + node.ExeFileName = metadata.ExeFileName; + node.FunctionID = metadata.FunctionName; + node.FunctionName = metadata.FunctionName; + node.AddressOrLine = metadata.AddressOrLine; + node.FrameID = metadata.FrameID; + + // Unknown/invalid offsets are currently set to 0. + // + // In this case we leave FunctionSourceLine=0 as a flag for the UI that the + // FunctionSourceLine should not be displayed. + // + // As FunctionOffset=0 could also be a legit value, this work-around needs + // a real fix. The idea for after GA is to change FunctionOffset=-1 to + // indicate unknown/invalid. + if (metadata.FunctionOffset > 0) { + node.FunctionSourceLine = metadata.SourceLine - metadata.FunctionOffset; + } else { + node.FunctionSourceLine = 0; + } + + node.FunctionSourceID = metadata.SourceID; + node.FunctionSourceURL = metadata.SourceCodeURL; + node.SourceFilename = metadata.SourceFilename; + node.SourceLine = metadata.SourceLine; + break; + } +} + +function sortNodes( + nodes: Map +): CallerCalleeIntermediateNode[] { + const sortedNodes = new Array(); + for (const node of nodes.values()) { + sortedNodes.push(node); + } + return sortedNodes.sort((n1, n2) => { + return compareFrameGroup(n1.frameGroup, n2.frameGroup); + }); +} + +// fromCallerCalleeIntermediateNode is used to convert the intermediate representation +// of the diagram into the format that is easily JSONified and more easily consumed by +// others. +export function fromCallerCalleeIntermediateNode( + root: CallerCalleeIntermediateNode +): CallerCalleeNode { + const node = createCallerCalleeNode({ + FrameGroupID: root.frameGroupID, + Samples: root.samples, + CountInclusive: root.countInclusive, + CountExclusive: root.countExclusive, + }); + + // Populate the other fields with data from the root node. Selectors are not supposed + // to be able to fail. + selectCallerCalleeData(root.frameMetadata, node); + + // Now fill the caller and callee arrays. + // For a deterministic result we have to walk the callers / callees in a deterministic + // order. A deterministic result allows deterministic UI views, something that users expect. + for (const caller of sortNodes(root.callers)) { + node.Callers.push(fromCallerCalleeIntermediateNode(caller)); + } + for (const callee of sortNodes(root.callees)) { + node.Callees.push(fromCallerCalleeIntermediateNode(callee)); + } + + return node; +} + +export function createCallerCalleeDiagram( + events: Map, + stackTraces: Map, + stackFrames: Map, + executables: Map +): CallerCalleeNode { + const rootFrame = createStackFrameMetadata(); + const frameMetadataForTraces = groupStackFrameMetadataByStackTrace( + stackTraces, + stackFrames, + executables + ); + const root = createCallerCalleeIntermediateRoot(rootFrame, events, frameMetadataForTraces); + return fromCallerCalleeIntermediateNode(root); +} diff --git a/x-pack/plugins/profiling/common/commonly_used_ranges.ts b/x-pack/plugins/profiling/common/commonly_used_ranges.ts new file mode 100644 index 000000000000..2323915d17b4 --- /dev/null +++ b/x-pack/plugins/profiling/common/commonly_used_ranges.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface CommonlyUsedRange { + start: string; + end: string; + label: string; +} + +export const commonlyUsedRanges: CommonlyUsedRange[] = [ + { + start: 'now-30m', + end: 'now', + label: 'Last 30 minutes', + }, + { + start: 'now-1h', + end: 'now', + label: 'Last hour', + }, + { + start: 'now-24h', + end: 'now', + label: 'Last 24 hours', + }, + { + start: 'now-1w', + end: 'now', + label: 'Last 7 days', + }, + { + start: 'now-30d', + end: 'now', + label: 'Last 30 days', + }, +]; diff --git a/x-pack/plugins/profiling/common/elasticsearch.ts b/x-pack/plugins/profiling/common/elasticsearch.ts new file mode 100644 index 000000000000..637f722fa7ef --- /dev/null +++ b/x-pack/plugins/profiling/common/elasticsearch.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UnionToIntersection, ValuesType } from 'utility-types'; + +export enum ProfilingESField { + Timestamp = '@timestamp', + ContainerName = 'container.name', + ProcessThreadName = 'process.thread.name', + StacktraceCount = 'Stacktrace.count', + HostID = 'host.id', + OrchestratorResourceName = 'orchestrator.resource.name', + ServiceName = 'service.name', + StacktraceID = 'Stacktrace.id', + StacktraceFrameIDs = 'Stacktrace.frame.ids', + StacktraceFrameTypes = 'Stacktrace.frame.types', + StackframeFileName = 'Stackframe.file.name', + StackframeFunctionName = 'Stackframe.function.name', + StackframeLineNumber = 'Stackframe.line.number', + StackframeFunctionOffset = 'Stackframe.function.offset', + StackframeSourceType = 'Stackframe.source.type', + ExecutableBuildID = 'Executable.build.id', + ExecutableFileName = 'Executable.file.name', +} + +type DedotKey< + TKey extends string | number | symbol, + TValue +> = TKey extends `${infer THead}.${infer TTail}` + ? { + [key in THead]: DedotKey; + } + : { [key in TKey]: TValue }; + +export type DedotObject> = UnionToIntersection< + ValuesType<{ + [TKey in keyof TObject]: DedotKey; + }> +>; + +export type FlattenObject< + TObject extends Record, + TPrefix extends string = '' +> = UnionToIntersection< + ValuesType<{ + [TKey in keyof TObject & string]: TObject[TKey] extends Record + ? FlattenObject + : { [key in `${TPrefix}${TKey}`]: TObject[TKey] }; + }> +>; + +type FlattenedKeysOf> = keyof FlattenObject; + +export type PickFlattened< + TObject extends Record, + TPickKey extends FlattenedKeysOf +> = DedotObject, TPickKey>>; + +export type ProfilingESEvent = DedotObject<{ + [ProfilingESField.Timestamp]: string; + [ProfilingESField.ContainerName]: string; + [ProfilingESField.ProcessThreadName]: string; + [ProfilingESField.StacktraceCount]: number; + [ProfilingESField.HostID]: string; + [ProfilingESField.OrchestratorResourceName]: string; + [ProfilingESField.ServiceName]: string; + [ProfilingESField.StacktraceID]: string; +}>; + +export type ProfilingStackTrace = DedotObject<{ + [ProfilingESField.Timestamp]: number; + [ProfilingESField.StacktraceFrameIDs]: string; + [ProfilingESField.StacktraceFrameTypes]: string; +}>; + +export type ProfilingStackFrame = DedotObject<{ + [ProfilingESField.StackframeFileName]: string; + [ProfilingESField.StackframeFunctionName]: string; + [ProfilingESField.StackframeLineNumber]: number; + [ProfilingESField.StackframeFunctionOffset]: number; + [ProfilingESField.StackframeSourceType]: number; +}>; + +export type ProfilingExecutable = DedotObject<{ + [ProfilingESField.ExecutableBuildID]: string; + [ProfilingESField.ExecutableFileName]: string; + [ProfilingESField.Timestamp]: string; +}>; diff --git a/x-pack/plugins/profiling/common/flamegraph.ts b/x-pack/plugins/profiling/common/flamegraph.ts new file mode 100644 index 000000000000..b06154dde9c5 --- /dev/null +++ b/x-pack/plugins/profiling/common/flamegraph.ts @@ -0,0 +1,303 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import fnv from 'fnv-plus'; +import { CallerCalleeNode, createCallerCalleeDiagram } from './callercallee'; +import { + describeFrameType, + Executable, + FileID, + StackFrame, + StackFrameID, + StackTrace, + StackTraceID, +} from './profiling'; + +interface ColumnarCallerCallee { + Label: string[]; + Value: number[]; + X: number[]; + Y: number[]; + Color: number[]; + CountInclusive: number[]; + CountExclusive: number[]; + ID: string[]; + FrameID: string[]; + ExecutableID: string[]; +} + +export interface ElasticFlameGraph { + Label: string[]; + Value: number[]; + Position: number[]; + Size: number[]; + Color: number[]; + CountInclusive: number[]; + CountExclusive: number[]; + ID: string[]; + FrameID: string[]; + ExecutableID: string[]; + TotalSeconds: number; + TotalTraces: number; + SampledTraces: number; +} + +export enum FlameGraphComparisonMode { + Absolute = 'absolute', + Relative = 'relative', +} + +/* + * Helper to calculate the color of a given block to be drawn. The desirable outcomes of this are: + * Each of the following frame types should get a different set of color hues: + * + * 0 = Unsymbolized frame + * 1 = Python + * 2 = PHP + * 3 = Native + * 4 = Kernel + * 5 = JVM/Hotspot + * 6 = Ruby + * 7 = Perl + * 8 = JavaScript + * + * This is most easily achieved by mapping frame types to different color variations, using + * the x-position we can use different colors for adjacent blocks while keeping a similar hue + * + * Taken originally from prodfiler_ui/src/helpers/Pixi/frameTypeToColors.tsx + */ +const frameTypeToColors = [ + [0xfd8484, 0xfd9d9d, 0xfeb5b5, 0xfecece], + [0xfcae6b, 0xfdbe89, 0xfdcea6, 0xfedfc4], + [0xfcdb82, 0xfde29b, 0xfde9b4, 0xfef1cd], + [0x6dd0dc, 0x8ad9e3, 0xa7e3ea, 0xc5ecf1], + [0x7c9eff, 0x96b1ff, 0xb0c5ff, 0xcbd8ff], + [0x65d3ac, 0x84dcbd, 0xa3e5cd, 0xc1edde], + [0xd79ffc, 0xdfb2fd, 0xe7c5fd, 0xefd9fe], + [0xf98bb9, 0xfaa2c7, 0xfbb9d5, 0xfdd1e3], + [0xcbc3e3, 0xd5cfe8, 0xdfdbee, 0xeae7f3], +]; + +function frameTypeToRGB(frameType: number, x: number): number { + return frameTypeToColors[frameType][x % 4]; +} + +export function rgbToRGBA(rgb: number): number[] { + return [ + Math.floor(rgb / 65536) / 255, + (Math.floor(rgb / 256) % 256) / 255, + (rgb % 256) / 255, + 1.0, + ]; +} + +function normalize(n: number, lower: number, upper: number): number { + return (n - lower) / (upper - lower); +} + +function checkIfStringHasParentheses(s: string) { + return /\(|\)/.test(s); +} + +function getFunctionName(node: CallerCalleeNode) { + return node.FunctionName !== '' && !checkIfStringHasParentheses(node.FunctionName) + ? `${node.FunctionName}()` + : node.FunctionName; +} + +function getExeFileName(node: CallerCalleeNode) { + if (node?.ExeFileName === undefined) { + return ''; + } + if (node.ExeFileName !== '') { + return node.ExeFileName; + } + return describeFrameType(node.FrameType); +} + +function getLabel(node: CallerCalleeNode) { + if (node.FunctionName !== '') { + const sourceFilename = node.SourceFilename; + const sourceURL = sourceFilename ? sourceFilename.split('/').pop() : ''; + return `${getExeFileName(node)}: ${getFunctionName(node)} in ${sourceURL} #${node.SourceLine}`; + } + return getExeFileName(node); +} + +export class FlameGraph { + // sampleRate is 1/5^N, with N being the downsampled index the events were fetched from. + // N=0: full events table (sampleRate is 1) + // N=1: downsampled by 5 (sampleRate is 0.2) + // ... + sampleRate: number; + + // totalCount is the sum(Count) of all events in the filter range in the + // downsampled index we were looking at. + // To estimate how many events we have in the full events index: totalCount / sampleRate. + // Do the same for single entries in the events array. + totalCount: number; + + totalSeconds: number; + + events: Map; + stacktraces: Map; + stackframes: Map; + executables: Map; + + constructor({ + sampleRate, + totalCount, + events, + stackTraces, + stackFrames, + executables, + totalSeconds, + }: { + sampleRate: number; + totalCount: number; + events: Map; + stackTraces: Map; + stackFrames: Map; + executables: Map; + totalSeconds: number; + }) { + this.sampleRate = sampleRate; + this.totalCount = totalCount; + this.events = events; + this.stacktraces = stackTraces; + this.stackframes = stackFrames; + this.executables = executables; + this.totalSeconds = totalSeconds; + } + + private countCallees(root: CallerCalleeNode): number { + let numCallees = 1; + for (const callee of root.Callees) { + numCallees += this.countCallees(callee); + } + return numCallees; + } + + // createColumnarCallerCallee flattens the intermediate representation of the diagram + // into a columnar format that is more compact than JSON. This representation will later + // need to be normalized into the response ultimately consumed by the flamegraph. + private createColumnarCallerCallee(root: CallerCalleeNode): ColumnarCallerCallee { + const numCallees = this.countCallees(root); + const columnar: ColumnarCallerCallee = { + Label: new Array(numCallees), + Value: new Array(numCallees), + X: new Array(numCallees), + Y: new Array(numCallees), + Color: new Array(numCallees * 4), + CountInclusive: new Array(numCallees), + CountExclusive: new Array(numCallees), + ID: new Array(numCallees), + FrameID: new Array(numCallees), + ExecutableID: new Array(numCallees), + }; + + const queue = [{ x: 0, depth: 1, node: root, parentID: 'root' }]; + + let idx = 0; + while (queue.length > 0) { + const { x, depth, node, parentID } = queue.pop()!; + + if (x === 0 && depth === 1) { + columnar.Label[idx] = 'root: Represents 100% of CPU time.'; + } else { + columnar.Label[idx] = getLabel(node); + } + columnar.Value[idx] = node.Samples; + columnar.X[idx] = x; + columnar.Y[idx] = depth; + + const [red, green, blue, alpha] = rgbToRGBA(frameTypeToRGB(node.FrameType, x)); + const j = 4 * idx; + columnar.Color[j] = red; + columnar.Color[j + 1] = green; + columnar.Color[j + 2] = blue; + columnar.Color[j + 3] = alpha; + + columnar.CountInclusive[idx] = node.CountInclusive; + columnar.CountExclusive[idx] = node.CountExclusive; + + const id = fnv.fast1a64utf(`${parentID}${node.FrameGroupID}`).toString(); + + columnar.ID[idx] = id; + columnar.FrameID[idx] = node.FrameID; + columnar.ExecutableID[idx] = node.FileID; + + node.Callees.sort((a: CallerCalleeNode, b: CallerCalleeNode) => b.Samples - a.Samples); + + let delta = 0; + for (const callee of node.Callees) { + delta += callee.Samples; + } + + for (let i = node.Callees.length - 1; i >= 0; i--) { + delta -= node.Callees[i].Samples; + queue.push({ x: x + delta, depth: depth + 1, node: node.Callees[i], parentID: id }); + } + + idx++; + } + + return columnar; + } + + // createElasticFlameGraph normalizes the intermediate columnar representation into the + // response ultimately consumed by the flamegraph. + private createElasticFlameGraph(columnar: ColumnarCallerCallee): ElasticFlameGraph { + const graph: ElasticFlameGraph = { + Label: [], + Value: [], + Position: [], + Size: [], + Color: [], + CountInclusive: [], + CountExclusive: [], + ID: [], + FrameID: [], + ExecutableID: [], + TotalSeconds: this.totalSeconds, + TotalTraces: Math.floor(this.totalCount / this.sampleRate), + SampledTraces: this.totalCount, + }; + + graph.Label = columnar.Label; + graph.Value = columnar.Value; + graph.Color = columnar.Color; + graph.CountInclusive = columnar.CountInclusive; + graph.CountExclusive = columnar.CountExclusive; + graph.ID = columnar.ID; + graph.FrameID = columnar.FrameID; + graph.ExecutableID = columnar.ExecutableID; + + const maxX = columnar.Value[0]; + const maxY = columnar.Y.reduce((max, n) => (n > max ? n : max), 0); + + for (let i = 0; i < columnar.X.length; i++) { + const x = normalize(columnar.X[i], 0, maxX); + const y = normalize(maxY - columnar.Y[i], 0, maxY); + graph.Position.push(x, y); + } + + graph.Size = graph.Value.map((n) => normalize(n, 0, maxX)); + + return graph; + } + + toElastic(): ElasticFlameGraph { + const root = createCallerCalleeDiagram( + this.events, + this.stacktraces, + this.stackframes, + this.executables + ); + return this.createElasticFlameGraph(this.createColumnarCallerCallee(root)); + } +} diff --git a/x-pack/plugins/profiling/common/frame_group.test.ts b/x-pack/plugins/profiling/common/frame_group.test.ts new file mode 100644 index 000000000000..57ed29001ccd --- /dev/null +++ b/x-pack/plugins/profiling/common/frame_group.test.ts @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { compareFrameGroup, createFrameGroup, createFrameGroupID } from './frame_group'; +import { createStackFrameMetadata } from './profiling'; + +const nonSymbolizedFrameGroups = [ + createFrameGroup( + createStackFrameMetadata({ + FileID: '0x0123456789ABCDEF', + AddressOrLine: 102938, + }) + ), + createFrameGroup( + createStackFrameMetadata({ + FileID: '0x0123456789ABCDEF', + AddressOrLine: 1234, + }) + ), + createFrameGroup( + createStackFrameMetadata({ + FileID: '0x0102030405060708', + AddressOrLine: 1234, + }) + ), +]; + +const elfSymbolizedFrameGroups = [ + createFrameGroup( + createStackFrameMetadata({ + FileID: '0x0123456789ABCDEF', + FunctionName: 'strlen()', + }) + ), + createFrameGroup( + createStackFrameMetadata({ + FileID: '0xFEDCBA9876543210', + FunctionName: 'strtok()', + }) + ), + createFrameGroup( + createStackFrameMetadata({ + FileID: '0xFEDCBA9876543210', + FunctionName: 'main()', + }) + ), +]; + +const symbolizedFrameGroups = [ + createFrameGroup( + createStackFrameMetadata({ + ExeFileName: 'chrome', + SourceFilename: 'strlen()', + FunctionName: 'strlen()', + }) + ), + createFrameGroup( + createStackFrameMetadata({ + ExeFileName: 'dockerd', + SourceFilename: 'main()', + FunctionName: 'createTask()', + }) + ), + createFrameGroup( + createStackFrameMetadata({ + ExeFileName: 'oom_reaper', + SourceFilename: 'main()', + FunctionName: 'crash()', + }) + ), +]; + +describe('Frame group operations', () => { + describe('check if a non-symbolized frame group is', () => { + test('less than another non-symbolized frame group', () => { + expect(compareFrameGroup(nonSymbolizedFrameGroups[1], nonSymbolizedFrameGroups[0])).toEqual( + -1 + ); + }); + + test('equal to another non-symbolized frame group', () => { + expect(compareFrameGroup(nonSymbolizedFrameGroups[0], nonSymbolizedFrameGroups[0])).toEqual( + 0 + ); + }); + + test('greater than another non-symbolized frame group', () => { + expect(compareFrameGroup(nonSymbolizedFrameGroups[1], nonSymbolizedFrameGroups[2])).toEqual( + 1 + ); + }); + + test('less than an ELF-symbolized frame group', () => { + expect(compareFrameGroup(nonSymbolizedFrameGroups[1], elfSymbolizedFrameGroups[0])).toEqual( + -1 + ); + }); + + test('less than a symbolized frame group', () => { + expect(compareFrameGroup(nonSymbolizedFrameGroups[1], symbolizedFrameGroups[0])).toEqual(-1); + }); + }); + + describe('check if an ELF-symbolized frame group is', () => { + test('less than another ELF-symbolized frame group', () => { + expect(compareFrameGroup(elfSymbolizedFrameGroups[0], elfSymbolizedFrameGroups[1])).toEqual( + -1 + ); + }); + + test('equal to another ELF-symbolized frame group', () => { + expect(compareFrameGroup(elfSymbolizedFrameGroups[0], elfSymbolizedFrameGroups[0])).toEqual( + 0 + ); + }); + + test('greater than another ELF-symbolized frame group', () => { + expect(compareFrameGroup(elfSymbolizedFrameGroups[1], elfSymbolizedFrameGroups[0])).toEqual( + 1 + ); + }); + + test('greater than a non-symbolized frame group', () => { + expect(compareFrameGroup(elfSymbolizedFrameGroups[0], nonSymbolizedFrameGroups[0])).toEqual( + 1 + ); + }); + + test('less than a symbolized frame group', () => { + expect(compareFrameGroup(elfSymbolizedFrameGroups[2], symbolizedFrameGroups[0])).toEqual(-1); + }); + }); + + describe('check if a symbolized frame group is', () => { + test('less than another symbolized frame group', () => { + expect(compareFrameGroup(symbolizedFrameGroups[0], symbolizedFrameGroups[1])).toEqual(-1); + }); + + test('equal to another symbolized frame group', () => { + expect(compareFrameGroup(symbolizedFrameGroups[0], symbolizedFrameGroups[0])).toEqual(0); + }); + + test('greater than another symbolized frame group', () => { + expect(compareFrameGroup(symbolizedFrameGroups[1], symbolizedFrameGroups[0])).toEqual(1); + }); + + test('greater than a non-symbolized frame group', () => { + expect(compareFrameGroup(symbolizedFrameGroups[0], nonSymbolizedFrameGroups[0])).toEqual(1); + }); + + test('greater than an ELF-symbolized frame group', () => { + expect(compareFrameGroup(symbolizedFrameGroups[0], elfSymbolizedFrameGroups[2])).toEqual(1); + }); + }); + + describe('check serialization for', () => { + test('non-symbolized frame', () => { + expect(createFrameGroupID(nonSymbolizedFrameGroups[0])).toEqual( + 'empty;0x0123456789ABCDEF;102938' + ); + }); + + test('non-symbolized ELF frame', () => { + expect(createFrameGroupID(elfSymbolizedFrameGroups[0])).toEqual( + 'elf;0x0123456789ABCDEF;strlen()' + ); + }); + + test('symbolized frame', () => { + expect(createFrameGroupID(symbolizedFrameGroups[0])).toEqual('full;chrome;strlen();strlen()'); + }); + }); +}); diff --git a/x-pack/plugins/profiling/common/frame_group.ts b/x-pack/plugins/profiling/common/frame_group.ts new file mode 100644 index 000000000000..776eccaa2e79 --- /dev/null +++ b/x-pack/plugins/profiling/common/frame_group.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StackFrameMetadata } from './profiling'; + +export type FrameGroupID = string; + +enum FrameGroupName { + EMPTY = 'empty', + ELF = 'elf', + FULL = 'full', +} + +interface BaseFrameGroup { + readonly name: FrameGroupName; +} + +interface EmptyFrameGroup extends BaseFrameGroup { + readonly name: FrameGroupName.EMPTY; + readonly fileID: StackFrameMetadata['FileID']; + readonly addressOrLine: StackFrameMetadata['AddressOrLine']; +} + +interface ElfFrameGroup extends BaseFrameGroup { + readonly name: FrameGroupName.ELF; + readonly fileID: StackFrameMetadata['FileID']; + readonly functionName: StackFrameMetadata['FunctionName']; +} + +interface FullFrameGroup extends BaseFrameGroup { + readonly name: FrameGroupName.FULL; + readonly exeFilename: StackFrameMetadata['ExeFileName']; + readonly functionName: StackFrameMetadata['FunctionName']; + readonly sourceFilename: StackFrameMetadata['SourceFilename']; +} + +export type FrameGroup = EmptyFrameGroup | ElfFrameGroup | FullFrameGroup; + +// createFrameGroup is the "standard" way of grouping frames, by commonly +// shared group identifiers. +// +// For ELF-symbolized frames, group by FunctionName and FileID. +// For non-symbolized frames, group by FileID and AddressOrLine. +// otherwise group by ExeFileName, SourceFilename and FunctionName. +export function createFrameGroup(frame: StackFrameMetadata): FrameGroup { + if (frame.FunctionName === '') { + return { + name: FrameGroupName.EMPTY, + fileID: frame.FileID, + addressOrLine: frame.AddressOrLine, + } as EmptyFrameGroup; + } + + if (frame.SourceFilename === '') { + return { + name: FrameGroupName.ELF, + fileID: frame.FileID, + functionName: frame.FunctionName, + } as ElfFrameGroup; + } + + return { + name: FrameGroupName.FULL, + exeFilename: frame.ExeFileName, + functionName: frame.FunctionName, + sourceFilename: frame.SourceFilename, + } as FullFrameGroup; +} + +// compareFrameGroup compares any two frame groups +// +// In general, frame groups are ordered using the following steps: +// +// * If frame groups are the same type, then we compare using their same +// properties +// * If frame groups have different types, then we compare using overlapping +// properties +// * If frame groups do not share properties, then we compare using the frame +// group type +// +// The union of the properties across all frame group types are ordered below +// from highest to lowest. For instance, given any two frame groups, shared +// properties are compared in the given order: +// +// * exeFilename +// * sourceFilename +// * functionName +// * fileID +// * addressOrLine +// +// Frame group types are ordered according to how much symbolization metadata +// is available, starting from most to least: +// +// * Symbolized frame group +// * ELF-symbolized frame group +// * Unsymbolized frame group +export function compareFrameGroup(a: FrameGroup, b: FrameGroup): number { + if (a.name === FrameGroupName.EMPTY) { + if (b.name === FrameGroupName.EMPTY) { + if (a.fileID < b.fileID) return -1; + if (a.fileID > b.fileID) return 1; + if (a.addressOrLine < b.addressOrLine) return -1; + if (a.addressOrLine > b.addressOrLine) return 1; + return 0; + } + if (b.name === FrameGroupName.ELF) { + if (a.fileID < b.fileID) return -1; + if (a.fileID > b.fileID) return 1; + } + return -1; + } + + if (a.name === FrameGroupName.ELF) { + if (b.name === FrameGroupName.EMPTY) { + if (a.fileID < b.fileID) return -1; + if (a.fileID > b.fileID) return 1; + return 1; + } + if (b.name === FrameGroupName.ELF) { + if (a.functionName < b.functionName) return -1; + if (a.functionName > b.functionName) return 1; + if (a.fileID < b.fileID) return -1; + if (a.fileID > b.fileID) return 1; + return 0; + } + if (a.functionName < b.functionName) return -1; + if (a.functionName > b.functionName) return 1; + return -1; + } + + if (b.name === FrameGroupName.FULL) { + if (a.exeFilename < b.exeFilename) return -1; + if (a.exeFilename > b.exeFilename) return 1; + if (a.sourceFilename < b.sourceFilename) return -1; + if (a.sourceFilename > b.sourceFilename) return 1; + if (a.functionName < b.functionName) return -1; + if (a.functionName > b.functionName) return 1; + return 0; + } + if (b.name === FrameGroupName.ELF) { + if (a.functionName < b.functionName) return -1; + if (a.functionName > b.functionName) return 1; + } + return 1; +} + +export function createFrameGroupID(frameGroup: FrameGroup): FrameGroupID { + switch (frameGroup.name) { + case FrameGroupName.EMPTY: + return `${frameGroup.name};${frameGroup.fileID};${frameGroup.addressOrLine}`; + break; + case FrameGroupName.ELF: + return `${frameGroup.name};${frameGroup.fileID};${frameGroup.functionName}`; + break; + case FrameGroupName.FULL: + return `${frameGroup.name};${frameGroup.exeFilename};${frameGroup.functionName};${frameGroup.sourceFilename}`; + break; + } +} diff --git a/x-pack/plugins/profiling/common/functions.test.ts b/x-pack/plugins/profiling/common/functions.test.ts new file mode 100644 index 000000000000..c17687453c2f --- /dev/null +++ b/x-pack/plugins/profiling/common/functions.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createTopNFunctions } from './functions'; + +import { events, stackTraces, stackFrames, executables } from './__fixtures__/stacktraces'; +import { sum } from 'lodash'; + +describe('TopN function operations', () => { + test('1', () => { + const maxTopN = 5; + const totalSamples = sum([...events.values()]); + const topNFunctions = createTopNFunctions( + events, + stackTraces, + stackFrames, + executables, + 0, + maxTopN + ); + + expect(topNFunctions.TotalCount).toEqual(totalSamples); + expect(topNFunctions.TopN.length).toEqual(maxTopN); + + const exclusiveCounts = topNFunctions.TopN.map((value) => value.CountExclusive); + expect(exclusiveCounts).toEqual([16, 9, 7, 5, 2]); + }); +}); diff --git a/x-pack/plugins/profiling/common/functions.ts b/x-pack/plugins/profiling/common/functions.ts new file mode 100644 index 000000000000..5b45e3fcfaa7 --- /dev/null +++ b/x-pack/plugins/profiling/common/functions.ts @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import * as t from 'io-ts'; +import { + compareFrameGroup, + createFrameGroup, + createFrameGroupID, + FrameGroup, + FrameGroupID, +} from './frame_group'; +import { + Executable, + FileID, + groupStackFrameMetadataByStackTrace, + StackFrame, + StackFrameID, + StackFrameMetadata, + StackTrace, + StackTraceID, +} from './profiling'; + +interface TopNFunctionAndFrameGroup { + Frame: StackFrameMetadata; + FrameGroup: FrameGroup; + CountExclusive: number; + CountInclusive: number; +} + +type TopNFunction = Pick< + TopNFunctionAndFrameGroup, + 'Frame' | 'CountExclusive' | 'CountInclusive' +> & { Id: string; Rank: number }; + +export interface TopNFunctions { + TotalCount: number; + TopN: TopNFunction[]; +} + +export function createTopNFunctions( + events: Map, + stackTraces: Map, + stackFrames: Map, + executables: Map, + startIndex: number, + endIndex: number +): TopNFunctions { + const metadata = groupStackFrameMetadataByStackTrace(stackTraces, stackFrames, executables); + + // The `count` associated with a frame provides the total number of + // traces in which that node has appeared at least once. However, a + // frame may appear multiple times in a trace, and thus to avoid + // counting it multiple times we need to record the frames seen so + // far in each trace. + let totalCount = 0; + const topNFunctions = new Map(); + + // Collect metadata and inclusive + exclusive counts for each distinct frame. + for (const [traceHash, count] of events) { + const uniqueFrameGroupsPerEvent = new Set(); + + totalCount += count; + + // It is possible that we do not have a stacktrace for an event, + // e.g. when stopping the host agent or on network errors. + const frames = metadata.get(traceHash) ?? []; + for (let i = 0; i < frames.length; i++) { + const frameGroup = createFrameGroup(frames[i]); + const frameGroupID = createFrameGroupID(frameGroup); + + if (!topNFunctions.has(frameGroupID)) { + topNFunctions.set(frameGroupID, { + Frame: frames[i], + FrameGroup: frameGroup, + CountExclusive: 0, + CountInclusive: 0, + }); + } + + const topNFunction = topNFunctions.get(frameGroupID)!; + + if (!uniqueFrameGroupsPerEvent.has(frameGroupID)) { + uniqueFrameGroupsPerEvent.add(frameGroupID); + topNFunction.CountInclusive += count; + } + + if (i === frames.length - 1) { + // Leaf frame: sum up counts for exclusive CPU. + topNFunction.CountExclusive += count; + } + } + } + + // Sort in descending order by exclusive CPU. Same values should appear in a + // stable order, so compare the FrameGroup in this case. + const topN = [...topNFunctions.values()]; + topN + .sort((a: TopNFunctionAndFrameGroup, b: TopNFunctionAndFrameGroup) => { + if (a.CountExclusive > b.CountExclusive) { + return 1; + } + if (a.CountExclusive < b.CountExclusive) { + return -1; + } + return compareFrameGroup(a.FrameGroup, b.FrameGroup); + }) + .reverse(); + + if (startIndex > topN.length) { + startIndex = topN.length; + } + if (endIndex > topN.length) { + endIndex = topN.length; + } + + const framesAndCountsAndIds = topN.slice(startIndex, endIndex).map((frameAndCount, i) => ({ + Rank: i + 1, + Frame: frameAndCount.Frame, + CountExclusive: frameAndCount.CountExclusive, + CountInclusive: frameAndCount.CountInclusive, + Id: createFrameGroupID(frameAndCount.FrameGroup), + })); + + return { + TotalCount: totalCount, + TopN: framesAndCountsAndIds, + }; +} + +export enum TopNFunctionSortField { + Rank = 'rank', + Frame = 'frame', + Samples = 'samples', + ExclusiveCPU = 'exclusiveCPU', + InclusiveCPU = 'inclusiveCPU', + Diff = 'diff', +} + +export const topNFunctionSortFieldRt = t.union([ + t.literal(TopNFunctionSortField.Rank), + t.literal(TopNFunctionSortField.Frame), + t.literal(TopNFunctionSortField.Samples), + t.literal(TopNFunctionSortField.ExclusiveCPU), + t.literal(TopNFunctionSortField.InclusiveCPU), + t.literal(TopNFunctionSortField.Diff), +]); diff --git a/x-pack/plugins/profiling/common/histogram.ts b/x-pack/plugins/profiling/common/histogram.ts new file mode 100644 index 000000000000..1d18e0342891 --- /dev/null +++ b/x-pack/plugins/profiling/common/histogram.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { range } from 'lodash'; + +export function computeBucketWidthFromTimeRangeAndBucketCount( + timeFrom: number, + timeTo: number, + numBuckets: number +): number { + return Math.max(Math.floor((timeTo - timeFrom) / numBuckets), 1); +} + +// Given a possibly empty set of timestamps, a time range, and a bucket width, +// we create an increasing list of timestamps that are uniformally spaced and +// cover the given time range. +// +// The smallest timestamp, t0, should match this invariant: +// timeFrom - bucketWidth < t0 <= timeFrom +// +// The largest timestamp, t1, should match this invariant: +// timeTo - bucketWidth < t1 <= timeTo +export function createUniformBucketsForTimeRange( + timestamps: number[], + timeFrom: number, + timeTo: number, + bucketWidth: number +): number[] { + if (timestamps.length > 0) { + // We only need one arbitrary timestamp to generate the buckets covering + // the given time range + const t = timestamps[0]; + const left = t - bucketWidth * Math.ceil((t - timeFrom) / bucketWidth); + const right = t + bucketWidth * Math.floor((timeTo - t) / bucketWidth); + return range(left, right + 1, bucketWidth); + } + return range(timeFrom, timeTo + 1, bucketWidth); +} diff --git a/x-pack/plugins/profiling/common/index.test.ts b/x-pack/plugins/profiling/common/index.test.ts new file mode 100644 index 000000000000..8fd482ce9759 --- /dev/null +++ b/x-pack/plugins/profiling/common/index.test.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timeRangeFromRequest } from '.'; + +describe('Common profiling helpers', () => { + test('convert query parameters time ranges into tuple', () => { + const request = { + query: { + timeFrom: 123, + timeTo: 456, + }, + }; + expect(timeRangeFromRequest(request)).toEqual([123, 456]); + }); +}); diff --git a/x-pack/plugins/profiling/common/index.ts b/x-pack/plugins/profiling/common/index.ts new file mode 100644 index 000000000000..871bc9ee1cd9 --- /dev/null +++ b/x-pack/plugins/profiling/common/index.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const PLUGIN_ID = 'profiling'; +export const PLUGIN_NAME = 'profiling'; + +export const INDEX_EVENTS = 'profiling-events-all'; +export const INDEX_TRACES = 'profiling-stacktraces'; +export const INDEX_FRAMES = 'profiling-stackframes'; +export const INDEX_EXECUTABLES = 'profiling-executables'; + +const BASE_ROUTE_PATH = '/api/profiling/v1'; + +export function getRoutePaths() { + return { + TopN: `${BASE_ROUTE_PATH}/topn`, + TopNContainers: `${BASE_ROUTE_PATH}/topn/containers`, + TopNDeployments: `${BASE_ROUTE_PATH}/topn/deployments`, + TopNFunctions: `${BASE_ROUTE_PATH}/topn/functions`, + TopNHosts: `${BASE_ROUTE_PATH}/topn/hosts`, + TopNThreads: `${BASE_ROUTE_PATH}/topn/threads`, + TopNTraces: `${BASE_ROUTE_PATH}/topn/traces`, + Flamechart: `${BASE_ROUTE_PATH}/flamechart`, + FrameInformation: `${BASE_ROUTE_PATH}/frame_information`, + }; +} + +export function timeRangeFromRequest(request: any): [number, number] { + const timeFrom = parseInt(request.query.timeFrom!, 10); + const timeTo = parseInt(request.query.timeTo!, 10); + return [timeFrom, timeTo]; +} + +// Converts from a Map object to a Record object since Map objects are not +// serializable to JSON by default +export function fromMapToRecord(m: Map): Record { + const output: Record = {}; + + for (const [key, value] of m) { + output[key] = value; + } + + return output; +} + +export const NOT_AVAILABLE_LABEL = i18n.translate('xpack.profiling.notAvailableLabel', { + defaultMessage: 'N/A', +}); diff --git a/x-pack/plugins/profiling/common/profiling.test.ts b/x-pack/plugins/profiling/common/profiling.test.ts new file mode 100644 index 000000000000..4f7b1fb3b8fb --- /dev/null +++ b/x-pack/plugins/profiling/common/profiling.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createStackFrameMetadata, + FrameType, + getCalleeFunction, + getCalleeSource, +} from './profiling'; + +describe('Stack frame metadata operations', () => { + test('metadata has executable and function names', () => { + const metadata = createStackFrameMetadata({ + ExeFileName: 'chrome', + FrameType: FrameType.Native, + FunctionName: 'strlen()', + }); + expect(getCalleeFunction(metadata)).toEqual('chrome: strlen()'); + }); + + test('metadata only has executable name', () => { + const metadata = createStackFrameMetadata({ + ExeFileName: 'promtail', + FrameType: FrameType.Native, + }); + expect(getCalleeFunction(metadata)).toEqual('promtail'); + }); + + test('metadata has executable name but no function name or source line', () => { + const metadata = createStackFrameMetadata({ + ExeFileName: 'promtail', + FrameType: FrameType.Native, + }); + expect(getCalleeSource(metadata)).toEqual('promtail+0x0'); + }); + + test('metadata has no executable name, function name, or source line', () => { + const metadata = createStackFrameMetadata({}); + expect(getCalleeSource(metadata)).toEqual(''); + }); + + test('metadata has source name but no source line', () => { + const metadata = createStackFrameMetadata({ + ExeFileName: 'dockerd', + FrameType: FrameType.Native, + SourceFilename: 'dockerd', + FunctionOffset: 0x183a5b0, + }); + expect(getCalleeSource(metadata)).toEqual('dockerd+0x0'); + }); + + test('metadata has source name and function offset', () => { + const metadata = createStackFrameMetadata({ + ExeFileName: 'python3.9', + FrameType: FrameType.Python, + FunctionName: 'PyDict_GetItemWithError', + FunctionOffset: 2567, + SourceFilename: '/build/python3.9-RNBry6/python3.9-3.9.2/Objects/dictobject.c', + SourceLine: 1456, + }); + expect(getCalleeSource(metadata)).toEqual( + '/build/python3.9-RNBry6/python3.9-3.9.2/Objects/dictobject.c#1456' + ); + }); + + test('metadata has source name but no function offset', () => { + const metadata = createStackFrameMetadata({ + ExeFileName: 'agent', + FrameType: FrameType.Native, + FunctionName: 'runtime.mallocgc', + SourceFilename: 'runtime/malloc.go', + }); + expect(getCalleeSource(metadata)).toEqual('runtime/malloc.go'); + }); +}); diff --git a/x-pack/plugins/profiling/common/profiling.ts b/x-pack/plugins/profiling/common/profiling.ts new file mode 100644 index 000000000000..003f6565677c --- /dev/null +++ b/x-pack/plugins/profiling/common/profiling.ts @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export type StackTraceID = string; +export type StackFrameID = string; +export type FileID = string; + +export function createStackFrameID(fileID: FileID, addressOrLine: number): StackFrameID { + const buf = Buffer.alloc(24); + Buffer.from(fileID, 'base64url').copy(buf); + buf.writeBigUInt64BE(BigInt(addressOrLine), 16); + return buf.toString('base64url'); +} + +export enum FrameType { + Unsymbolized = 0, + Python, + PHP, + Native, + Kernel, + JVM, + Ruby, + Perl, + JavaScript, +} + +export function describeFrameType(ft: FrameType): string { + return { + [FrameType.Unsymbolized]: '', + [FrameType.Python]: 'Python', + [FrameType.PHP]: 'PHP', + [FrameType.Native]: 'Native', + [FrameType.Kernel]: 'Kernel', + [FrameType.JVM]: 'JVM/Hotspot', + [FrameType.Ruby]: 'Ruby', + [FrameType.Perl]: 'Perl', + [FrameType.JavaScript]: 'JavaScript', + }[ft]; +} + +export interface StackTraceEvent { + StackTraceID: StackTraceID; + Count: number; +} + +export interface StackTrace { + FrameIDs: string[]; + FileIDs: string[]; + AddressOrLines: number[]; + Types: number[]; +} + +export interface StackFrame { + FileName: string; + FunctionName: string; + FunctionOffset: number; + LineNumber: number; + SourceType: number; +} + +export interface Executable { + FileName: string; +} + +export interface StackFrameMetadata { + // StackTrace.FrameID + FrameID: string; + // StackTrace.FileID + FileID: FileID; + // StackTrace.Type + FrameType: FrameType; + + // StackFrame.LineNumber? + AddressOrLine: number; + // StackFrame.FunctionName + FunctionName: string; + // StackFrame.FunctionOffset + FunctionOffset: number; + // should this be StackFrame.SourceID? + SourceID: FileID; + // StackFrame.Filename + SourceFilename: string; + // StackFrame.LineNumber + SourceLine: number; + + // Executable.FileName + ExeFileName: string; + + // unused atm due to lack of symbolization metadata + CommitHash: string; + // unused atm due to lack of symbolization metadata + SourceCodeURL: string; + // unused atm due to lack of symbolization metadata + SourcePackageHash: string; + // unused atm due to lack of symbolization metadata + SourcePackageURL: string; + // unused atm due to lack of symbolization metadata + SourceType: number; +} + +export function createStackFrameMetadata( + options: Partial = {} +): StackFrameMetadata { + const metadata = {} as StackFrameMetadata; + + metadata.FrameID = options.FrameID ?? ''; + metadata.FileID = options.FileID ?? ''; + metadata.FrameType = options.FrameType ?? 0; + metadata.AddressOrLine = options.AddressOrLine ?? 0; + metadata.FunctionName = options.FunctionName ?? ''; + metadata.FunctionOffset = options.FunctionOffset ?? 0; + metadata.SourceID = options.SourceID ?? ''; + metadata.SourceLine = options.SourceLine ?? 0; + metadata.ExeFileName = options.ExeFileName ?? ''; + metadata.CommitHash = options.CommitHash ?? ''; + metadata.SourceCodeURL = options.SourceCodeURL ?? ''; + metadata.SourceFilename = options.SourceFilename ?? ''; + metadata.SourcePackageHash = options.SourcePackageHash ?? ''; + metadata.SourcePackageURL = options.SourcePackageURL ?? ''; + metadata.SourceType = options.SourceType ?? 0; + + return metadata; +} + +export function getCalleeFunction(frame: StackFrameMetadata): string { + // In the best case scenario, we have the file names, source lines, + // and function names. However we need to deal with missing function or + // executable info. + const exeDisplayName = frame.ExeFileName ? frame.ExeFileName : describeFrameType(frame.FrameType); + + // When there is no function name, only use the executable name + return frame.FunctionName ? exeDisplayName + ': ' + frame.FunctionName : exeDisplayName; +} + +export function getCalleeSource(frame: StackFrameMetadata): string { + if (frame.FunctionName === '' && frame.SourceLine === 0) { + if (frame.ExeFileName) { + // If no source line or filename available, display the executable offset + return frame.ExeFileName + '+0x' + frame.AddressOrLine.toString(16); + } + + // If we don't have the executable filename, display + return ''; + } + + if (frame.SourceFilename !== '' && frame.SourceLine === 0) { + return frame.SourceFilename; + } + + return frame.SourceFilename + (frame.SourceLine !== 0 ? `#${frame.SourceLine}` : ''); +} + +// groupStackFrameMetadataByStackTrace collects all of the per-stack-frame +// metadata for a given set of trace IDs and their respective stack frames. +// +// This is similar to GetTraceMetaData in pf-storage-backend/storagebackend/storagebackendv1/reads_webservice.go +export function groupStackFrameMetadataByStackTrace( + stackTraces: Map, + stackFrames: Map, + executables: Map +): Map { + const frameMetadataForTraces = new Map(); + for (const [stackTraceID, trace] of stackTraces) { + const frameMetadata = new Array(); + for (let i = 0; i < trace.FrameIDs.length; i++) { + const frameID = trace.FrameIDs[i]; + const fileID = trace.FileIDs[i]; + const addressOrLine = trace.AddressOrLines[i]; + const frame = stackFrames.get(frameID)!; + const executable = executables.get(fileID)!; + + const metadata = createStackFrameMetadata({ + FrameID: frameID, + FileID: fileID, + AddressOrLine: addressOrLine, + FrameType: trace.Types[i], + FunctionName: frame.FunctionName, + FunctionOffset: frame.FunctionOffset, + SourceLine: frame.LineNumber, + SourceFilename: frame.FileName, + ExeFileName: executable.FileName, + }); + + frameMetadata.push(metadata); + } + frameMetadataForTraces.set(stackTraceID, frameMetadata); + } + return frameMetadataForTraces; +} diff --git a/x-pack/plugins/profiling/common/runtime_types/range_rt.ts b/x-pack/plugins/profiling/common/runtime_types/range_rt.ts new file mode 100644 index 000000000000..9231f1c49219 --- /dev/null +++ b/x-pack/plugins/profiling/common/runtime_types/range_rt.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import * as t from 'io-ts'; + +export const rangeRt = t.type({ + rangeFrom: t.string, + rangeTo: t.string, +}); diff --git a/x-pack/plugins/profiling/common/stack_traces.ts b/x-pack/plugins/profiling/common/stack_traces.ts new file mode 100644 index 000000000000..8879728c2153 --- /dev/null +++ b/x-pack/plugins/profiling/common/stack_traces.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ProfilingESField } from './elasticsearch'; + +export enum StackTracesDisplayOption { + StackTraces = 'stackTraces', + Percentage = 'percentage', +} + +export enum TopNType { + Containers = 'containers', + Deployments = 'deployments', + Threads = 'threads', + Hosts = 'hosts', + Traces = 'traces', +} + +export function getFieldNameForTopNType(type: TopNType): string { + return { + [TopNType.Containers]: ProfilingESField.ContainerName, + [TopNType.Deployments]: ProfilingESField.OrchestratorResourceName, + [TopNType.Threads]: ProfilingESField.ProcessThreadName, + [TopNType.Hosts]: ProfilingESField.HostID, + [TopNType.Traces]: ProfilingESField.StacktraceID, + }[type]; +} diff --git a/x-pack/plugins/profiling/common/topn.ts b/x-pack/plugins/profiling/common/topn.ts new file mode 100644 index 000000000000..b012bb3c1412 --- /dev/null +++ b/x-pack/plugins/profiling/common/topn.ts @@ -0,0 +1,248 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { euiPaletteColorBlind } from '@elastic/eui'; +import { InferSearchResponseOf } from '@kbn/core/types/elasticsearch'; +import { i18n } from '@kbn/i18n'; +import { orderBy } from 'lodash'; +import { ProfilingESField } from './elasticsearch'; +import { createUniformBucketsForTimeRange } from './histogram'; +import { StackFrameMetadata } from './profiling'; + +export const OTHER_BUCKET_LABEL = i18n.translate('xpack.profiling.topn.otherBucketLabel', { + defaultMessage: 'Other', +}); + +export interface CountPerTime { + Timestamp: number; + Count: number | null; +} + +export interface TopNSample extends CountPerTime { + Category: string; +} + +export interface TopNSamples { + TopN: TopNSample[]; +} + +export interface TopNResponse extends TopNSamples { + TotalCount: number; + Metadata: Record; +} + +export interface TopNSamplesHistogramResponse { + sum_other_doc_count: number; + buckets: Array<{ + key: string | number; + doc_count: number; + count: { value: number | null }; + over_time: { + buckets: Array<{ doc_count: number; key: string | number; count: { value: number | null } }>; + }; + }>; +} + +export function getTopNAggregationRequest({ + searchField, + highCardinality, + fixedInterval, +}: { + searchField: string; + highCardinality: boolean; + fixedInterval: string; +}) { + return { + group_by: { + terms: { + field: searchField, + order: { count: 'desc' as const }, + size: 99, + execution_hint: highCardinality ? ('map' as const) : ('global_ordinals' as const), + }, + aggs: { + over_time: { + date_histogram: { + field: ProfilingESField.Timestamp, + fixed_interval: fixedInterval, + }, + aggs: { + count: { + sum: { + field: ProfilingESField.StacktraceCount, + }, + }, + }, + }, + count: { + sum: { + field: ProfilingESField.StacktraceCount, + }, + }, + }, + }, + over_time: { + date_histogram: { + field: ProfilingESField.Timestamp, + fixed_interval: fixedInterval, + }, + aggs: { + count: { + sum: { + field: ProfilingESField.StacktraceCount, + }, + }, + }, + }, + total_count: { + sum_bucket: { + buckets_path: 'over_time>count', + }, + }, + }; +} + +export function createTopNSamples( + response: Required< + InferSearchResponseOf } }> + >['aggregations'], + startMilliseconds: number, + endMilliseconds: number, + bucketWidth: number +): TopNSample[] { + const bucketsByCategories = new Map(); + const uniqueTimestamps = new Set(); + const groupByBuckets = response.group_by.buckets ?? []; + + // Keep track of the sum per timestamp to subtract it from the 'other' bucket + const sumsOfKnownFieldsByTimestamp = new Map(); + + // Convert the buckets into nested maps and record the unique timestamps + for (let i = 0; i < groupByBuckets.length; i++) { + const frameCountsByTimestamp = new Map(); + const items = groupByBuckets[i].over_time.buckets; + + for (let j = 0; j < items.length; j++) { + const timestamp = Number(items[j].key); + const count = items[j].count.value ?? 0; + uniqueTimestamps.add(timestamp); + const sumAtTimestamp = (sumsOfKnownFieldsByTimestamp.get(timestamp) ?? 0) + count; + sumsOfKnownFieldsByTimestamp.set(timestamp, sumAtTimestamp); + frameCountsByTimestamp.set(timestamp, count); + } + bucketsByCategories.set(groupByBuckets[i].key, frameCountsByTimestamp); + } + + // Create the 'other' bucket by subtracting the sum of all known buckets + // from the total + const otherFrameCountsByTimestamp = new Map(); + + let addOtherBucket = false; + + for (let i = 0; i < response.over_time.buckets.length; i++) { + const bucket = response.over_time.buckets[i]; + const timestamp = Number(bucket.key); + const valueForOtherBucket = + (bucket.count.value ?? 0) - (sumsOfKnownFieldsByTimestamp.get(timestamp) ?? 0); + + if (valueForOtherBucket > 0) { + addOtherBucket = true; + } + + otherFrameCountsByTimestamp.set(timestamp, valueForOtherBucket); + } + + // Only add the 'other' bucket if at least one value per timestamp is > 0 + if (addOtherBucket) { + bucketsByCategories.set(OTHER_BUCKET_LABEL, otherFrameCountsByTimestamp); + } + + // Fill in missing timestamps so that the entire time range is covered + const timestamps = createUniformBucketsForTimeRange( + [...uniqueTimestamps], + startMilliseconds, + endMilliseconds, + bucketWidth + ); + + // Normalize samples so there are an equal number of data points per timestamp + const samples: TopNSample[] = []; + for (const category of bucketsByCategories.keys()) { + const frameCountsByTimestamp = bucketsByCategories.get(category); + for (const timestamp of timestamps) { + const sample: TopNSample = { + Timestamp: timestamp, + Count: frameCountsByTimestamp.get(timestamp) ?? 0, + Category: category, + }; + samples.push(sample); + } + } + + return orderBy(samples, ['Timestamp', 'Count', 'Category'], ['asc', 'desc', 'asc']); +} + +export interface TopNSubchart { + Category: string; + Percentage: number; + Series: CountPerTime[]; + Color: string; + Index: number; + Metadata: StackFrameMetadata[]; +} + +export function groupSamplesByCategory({ + samples, + totalCount, + metadata, +}: { + samples: TopNSample[]; + totalCount: number; + metadata: Record; +}): TopNSubchart[] { + const seriesByCategory = new Map(); + + for (let i = 0; i < samples.length; i++) { + const sample = samples[i]; + + if (!seriesByCategory.has(sample.Category)) { + seriesByCategory.set(sample.Category, []); + } + const series = seriesByCategory.get(sample.Category)!; + series.push({ Timestamp: sample.Timestamp, Count: sample.Count }); + } + + const subcharts: Array> = []; + + for (const [category, series] of seriesByCategory) { + const totalPerCategory = series.reduce((sumOf, { Count }) => sumOf + (Count ?? 0), 0); + subcharts.push({ + Category: category, + Percentage: (totalPerCategory / totalCount) * 100, + Series: series, + Metadata: metadata[category] ?? [], + }); + } + + const colors = euiPaletteColorBlind({ + rotations: Math.ceil(subcharts.length / 10), + }); + + return orderBy(subcharts, ['Percentage', 'Category'], ['desc', 'asc']).map((chart, index) => { + return { + ...chart, + Color: colors[index], + Index: index + 1, + Series: chart.Series.map((value) => { + return { + ...value, + Category: chart.Category, + }; + }), + }; + }); +} diff --git a/x-pack/plugins/profiling/common/types.ts b/x-pack/plugins/profiling/common/types.ts new file mode 100644 index 000000000000..83efba8cb813 --- /dev/null +++ b/x-pack/plugins/profiling/common/types.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export interface TimeRange { + start: string; + end: string; +} diff --git a/x-pack/plugins/profiling/jest.config.js b/x-pack/plugins/profiling/jest.config.js new file mode 100644 index 000000000000..783ace59b9b3 --- /dev/null +++ b/x-pack/plugins/profiling/jest.config.js @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/profiling'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/profiling', + coverageReporters: ['text', 'html'], + collectCoverageFrom: ['/x-pack/plugins/profiling/{common,public,server}/**/*.{ts,tsx}'], +}; diff --git a/x-pack/plugins/profiling/kibana.json b/x-pack/plugins/profiling/kibana.json new file mode 100644 index 000000000000..0ead3e39f83f --- /dev/null +++ b/x-pack/plugins/profiling/kibana.json @@ -0,0 +1,29 @@ +{ + "id": "profiling", + "version": "1.0.0", + "kibanaVersion": "kibana", + "owner": { + "name": "profiling", + "githubTeam": "profiling-ui" + }, + "description": "", + "server": true, + "ui": true, + "requiredPlugins": [ + "navigation", + "data", + "kibanaUtils", + "share", + "observability", + "features", + "kibanaReact", + "unifiedSearch", + "dataViews", + "charts" + ], + "optionalPlugins": [], + "configPath": [ + "xpack", + "profiling" + ] +} diff --git a/x-pack/plugins/profiling/public/app.tsx b/x-pack/plugins/profiling/public/app.tsx new file mode 100644 index 000000000000..da92e839d05d --- /dev/null +++ b/x-pack/plugins/profiling/public/app.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AppMountParameters, CoreSetup, CoreStart } from '@kbn/core/public'; +import React, { useMemo } from 'react'; +import ReactDOM from 'react-dom'; + +import { KibanaContextProvider, KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; +import { Storage } from '@kbn/kibana-utils-plugin/public'; +import { RouteRenderer, RouterProvider } from '@kbn/typed-react-router-config'; + +import { RedirectAppLinks } from '@kbn/shared-ux-link-redirect-app'; +import { ProfilingDependenciesContextProvider } from './components/contexts/profiling_dependencies/profiling_dependencies_context'; +import { RedirectWithDefaultDateRange } from './components/redirect_with_default_date_range'; +import { profilingRouter } from './routing'; +import { Services } from './services'; +import { ProfilingPluginPublicSetupDeps, ProfilingPluginPublicStartDeps } from './types'; +import { RouteBreadcrumbsContextProvider } from './components/contexts/route_breadcrumbs_context'; +import { TimeRangeContextProvider } from './components/contexts/time_range_context'; + +interface Props { + profilingFetchServices: Services; + coreStart: CoreStart; + coreSetup: CoreSetup; + pluginsStart: ProfilingPluginPublicStartDeps; + pluginsSetup: ProfilingPluginPublicSetupDeps; + theme$: AppMountParameters['theme$']; + history: AppMountParameters['history']; +} + +const storage = new Storage(localStorage); + +function App({ + coreStart, + coreSetup, + pluginsStart, + pluginsSetup, + profilingFetchServices, + theme$, + history, +}: Props) { + const i18nCore = coreStart.i18n; + + const profilingDependencies = useMemo(() => { + return { + start: { + core: coreStart, + ...pluginsStart, + }, + setup: { + core: coreSetup, + ...pluginsSetup, + }, + services: profilingFetchServices, + }; + }, [coreStart, coreSetup, pluginsStart, pluginsSetup, profilingFetchServices]); + + return ( + + + + + + + + + + + + + + + + + + + + ); +} + +export const renderApp = (props: Props, element: AppMountParameters['element']) => { + ReactDOM.render(, element); + + return () => ReactDOM.unmountComponentAtNode(element); +}; diff --git a/x-pack/plugins/profiling/public/components/async_component.tsx b/x-pack/plugins/profiling/public/components/async_component.tsx new file mode 100644 index 000000000000..2dba7ed2ab3c --- /dev/null +++ b/x-pack/plugins/profiling/public/components/async_component.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiFlexItem, EuiLoadingChart, EuiText, EuiIcon } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { AsyncState, AsyncStatus } from '../hooks/use_async'; + +export function AsyncComponent({ + children, + status, + error, + mono, + size, + style, + alignTop, +}: AsyncState & { + style?: React.ComponentProps['style']; + children: React.ReactElement; + mono?: boolean; + size: 'm' | 'l' | 'xl'; + alignTop?: boolean; +}) { + if (status === AsyncStatus.Settled && !error) { + return children; + } + + return ( + + + {error && status === AsyncStatus.Settled ? ( + + + + + + + {i18n.translate('xpack.profiling.asyncComponent.errorLoadingData', { + defaultMessage: 'Could not load data', + })} + + + + ) : ( + + )} + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/chart_grid.tsx b/x-pack/plugins/profiling/public/components/chart_grid.tsx new file mode 100644 index 000000000000..643928ac51c4 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/chart_grid.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGrid, EuiFlexItem, EuiFlyout, EuiPanel, EuiSpacer, EuiTitle } from '@elastic/eui'; +import { take } from 'lodash'; +import React, { useState } from 'react'; +import { TopNSubchart } from '../../common/topn'; +import { SubChart } from './subchart'; + +export interface ChartGridProps { + limit: number; + charts: TopNSubchart[]; + showFrames: boolean; +} + +export const ChartGrid: React.FC = ({ limit, charts, showFrames }) => { + const maximum = Math.min(limit, charts.length ?? 0); + + const ncharts = Math.min(maximum, charts.length); + + const [selectedSubchart, setSelectedSubchart] = useState(undefined); + + return ( + <> + + +

    Top {charts.length}

    +     + + + {take(charts, ncharts).map((subchart, i) => ( + + + { + setSelectedSubchart(subchart); + }} + showFrames={showFrames} + /> + + + ))} + + {selectedSubchart && ( + { + setSelectedSubchart(undefined); + }} + > + + + )} + + ); +}; diff --git a/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/profiling_dependencies_context.tsx b/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/profiling_dependencies_context.tsx new file mode 100644 index 000000000000..240d34b8e18c --- /dev/null +++ b/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/profiling_dependencies_context.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { CoreStart, CoreSetup } from '@kbn/core/public'; +import { createContext } from 'react'; +import { Services } from '../../../services'; +import { ProfilingPluginPublicSetupDeps, ProfilingPluginPublicStartDeps } from '../../../types'; + +export interface ProfilingDependencies { + start: { + core: CoreStart; + } & ProfilingPluginPublicStartDeps; + setup: { + core: CoreSetup; + } & ProfilingPluginPublicSetupDeps; + services: Services; +} + +export const ProfilingDependenciesContext = createContext( + undefined +); + +export const ProfilingDependenciesContextProvider = ProfilingDependenciesContext.Provider; diff --git a/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/use_profiling_dependencies.tsx b/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/use_profiling_dependencies.tsx new file mode 100644 index 000000000000..7e6ca9324c9f --- /dev/null +++ b/x-pack/plugins/profiling/public/components/contexts/profiling_dependencies/use_profiling_dependencies.tsx @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useContext } from 'react'; +import { ProfilingDependenciesContext } from './profiling_dependencies_context'; + +export function useProfilingDependencies() { + const context = useContext(ProfilingDependenciesContext); + if (!context) { + throw new Error('ProfilingDependenciesContext not found'); + } + return context; +} diff --git a/x-pack/plugins/profiling/public/components/contexts/route_breadcrumbs_context/index.tsx b/x-pack/plugins/profiling/public/components/contexts/route_breadcrumbs_context/index.tsx new file mode 100644 index 000000000000..fff05f05a371 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/contexts/route_breadcrumbs_context/index.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Route, RouteMatch, useMatchRoutes } from '@kbn/typed-react-router-config'; +import { ChromeBreadcrumb } from '@kbn/core/public'; +import { compact, isEqual } from 'lodash'; +import React, { createContext, useMemo, useState } from 'react'; +import { useBreadcrumbs } from '@kbn/observability-plugin/public'; + +export interface Breadcrumb { + title: string; + href: string; +} + +interface BreadcrumbApi { + set(route: Route, breadcrumb: Breadcrumb[]): void; + unset(route: Route): void; + getBreadcrumbs(matches: RouteMatch[]): Breadcrumb[]; +} + +export const RouteBreadcrumbsContext = createContext(undefined); + +export function RouteBreadcrumbsContextProvider({ children }: { children: React.ReactElement }) { + const [, forceUpdate] = useState({}); + + const breadcrumbs = useMemo(() => { + return new Map(); + }, []); + + const matches: RouteMatch[] = useMatchRoutes(); + + const api = useMemo( + () => ({ + set(route, breadcrumb) { + if (!isEqual(breadcrumbs.get(route), breadcrumb)) { + breadcrumbs.set(route, breadcrumb); + forceUpdate({}); + } + }, + unset(route) { + if (breadcrumbs.has(route)) { + breadcrumbs.delete(route); + forceUpdate({}); + } + }, + getBreadcrumbs(currentMatches: RouteMatch[]) { + return compact( + currentMatches.flatMap((match) => { + const breadcrumb = breadcrumbs.get(match.route); + + return breadcrumb; + }) + ); + }, + }), + [breadcrumbs] + ); + + const formattedBreadcrumbs: ChromeBreadcrumb[] = api + .getBreadcrumbs(matches) + .map((breadcrumb, index, array) => { + return { + text: breadcrumb.title, + ...(index === array.length - 1 + ? {} + : { + href: breadcrumb.href, + }), + }; + }); + + useBreadcrumbs(formattedBreadcrumbs); + + return ( + {children} + ); +} diff --git a/x-pack/plugins/profiling/public/components/contexts/route_breadcrumbs_context/use_route_breadcrumb.ts b/x-pack/plugins/profiling/public/components/contexts/route_breadcrumbs_context/use_route_breadcrumb.ts new file mode 100644 index 000000000000..79a34e3ff79a --- /dev/null +++ b/x-pack/plugins/profiling/public/components/contexts/route_breadcrumbs_context/use_route_breadcrumb.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCurrentRoute } from '@kbn/typed-react-router-config'; +import { useContext, useEffect, useRef } from 'react'; +import { castArray } from 'lodash'; +import { RouteBreadcrumbsContext, Breadcrumb } from '.'; + +export function useRouteBreadcrumb(breadcrumb: Breadcrumb | Breadcrumb[]) { + const api = useContext(RouteBreadcrumbsContext); + + if (!api) { + throw new Error('Missing Breadcrumb API in context'); + } + + const { match } = useCurrentRoute(); + + const matchedRoute = useRef(match?.route); + + useEffect(() => { + if (matchedRoute.current && matchedRoute.current !== match?.route) { + api.unset(matchedRoute.current); + } + + matchedRoute.current = match?.route; + + if (matchedRoute.current) { + api.set(matchedRoute.current, castArray(breadcrumb)); + } + + return () => { + if (matchedRoute.current) { + api.unset(matchedRoute.current); + } + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [matchedRoute.current, match?.route]); +} diff --git a/x-pack/plugins/profiling/public/components/contexts/time_range_context/index.tsx b/x-pack/plugins/profiling/public/components/contexts/time_range_context/index.tsx new file mode 100644 index 000000000000..4bd38ea31a95 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/contexts/time_range_context/index.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { uniqueId } from 'lodash'; +import React, { useMemo, useState } from 'react'; + +export const TimeRangeContext = React.createContext< + { timeRangeId: string; refresh: () => void } | undefined +>(undefined); + +export function TimeRangeContextProvider({ children }: { children: React.ReactElement }) { + const [timeRangeId, setTimeRangeId] = useState(uniqueId()); + + const timeRangeContextValue = useMemo(() => { + return { + timeRangeId, + refresh: () => { + setTimeRangeId(uniqueId()); + }, + }; + }, [timeRangeId]); + + return ( + {children} + ); +} diff --git a/x-pack/plugins/profiling/public/components/flame_graphs_view/flamegraph_information_window.tsx b/x-pack/plugins/profiling/public/components/flame_graphs_view/flamegraph_information_window.tsx new file mode 100644 index 000000000000..824e6d1476a1 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/flame_graphs_view/flamegraph_information_window.tsx @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiLoadingSpinner, + EuiPanel, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { NOT_AVAILABLE_LABEL } from '../../../common'; +import { AsyncStatus } from '../../hooks/use_async'; +import { getImpactRows } from './get_impact_rows'; + +interface Props { + frame?: { + exeFileName: string; + functionName: string; + sourceFileName: string; + samples: number; + childSamples: number; + }; + sampledTraces: number; + totalTraces: number; + totalSeconds: number; + onClose: () => void; + status: AsyncStatus; +} + +function KeyValueList({ rows }: { rows: Array<{ label: string; value: React.ReactNode }> }) { + return ( + + {rows.map((row, index) => ( + <> + + + {row.label}: + + {row.value} + + + + {index < rows.length - 1 ? ( + + + + ) : undefined} + + ))} + + ); +} + +function FlamegraphFrameInformationPanel({ + children, + onClose, + status, +}: { + children: React.ReactNode; + onClose: () => void; + status: AsyncStatus; +}) { + return ( + + + + + + + + +

    + {i18n.translate('xpack.profiling.flameGraphInformationWindowTitle', { + defaultMessage: 'Frame information', + })} +

    +     +     + {status === AsyncStatus.Loading ? ( + + + + ) : undefined} +     +     + + onClose()} /> + +     +     + {children} +     +     + ); +} + +export function FlamegraphInformationWindow({ + onClose, + frame, + sampledTraces, + totalTraces, + totalSeconds, + status, +}: Props) { + if (!frame) { + return ( + + + {i18n.translate('xpack.profiling.flamegraphInformationWindow.selectFrame', { + defaultMessage: 'Click on a frame to display more information', + })} + + + ); + } + + const { childSamples, exeFileName, samples, functionName, sourceFileName } = frame; + + const impactRows = getImpactRows({ + samples, + childSamples, + sampledTraces, + totalSeconds, + totalTraces, + }); + + return ( + + + + + + + + + +

    + {i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.impactEstimatesTitle', + { defaultMessage: 'Impact estimates' } + )} +

    +     +     + + + +     +     +     +     + ); +} diff --git a/x-pack/plugins/profiling/public/components/flame_graphs_view/get_impact_rows.ts b/x-pack/plugins/profiling/public/components/flame_graphs_view/get_impact_rows.ts new file mode 100644 index 000000000000..8ca1347e4497 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/flame_graphs_view/get_impact_rows.ts @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { asCost } from '../../utils/formatters/as_cost'; +import { asDuration } from '../../utils/formatters/as_duration'; +import { asPercentage } from '../../utils/formatters/as_percentage'; +import { asWeight } from '../../utils/formatters/as_weight'; + +const ANNUAL_SECONDS = 60 * 60 * 24 * 365; + +// The assumed amortized per-core average power consumption. +const PER_CORE_WATT = 40; + +// The assumed CO2 emissions per KWH (sourced from www.eia.gov) +const CO2_PER_KWH = 0.92; + +// The cost of a CPU core per hour, in dollars +const CORE_COST_PER_HOUR = 0.0425; + +export function getImpactRows({ + samples, + childSamples, + sampledTraces, + totalTraces, + totalSeconds, +}: { + samples: number; + childSamples: number; + sampledTraces: number; + totalTraces: number; + totalSeconds: number; +}) { + const percentage = samples / sampledTraces; + const percentageNoChildren = (samples - childSamples) / sampledTraces; + const totalCoreSeconds = totalTraces / 20; + const coreSeconds = totalCoreSeconds * percentage; + const coreSecondsNoChildren = totalCoreSeconds * percentageNoChildren; + const coreHours = coreSeconds / (60 * 60); + const coreHoursNoChildren = coreSecondsNoChildren / (60 * 60); + const annualizedScaleUp = ANNUAL_SECONDS / totalSeconds; + const co2 = ((PER_CORE_WATT * coreHours) / 1000.0) * CO2_PER_KWH; + const co2NoChildren = ((PER_CORE_WATT * coreHoursNoChildren) / 1000.0) * CO2_PER_KWH; + const annualizedCo2 = co2 * annualizedScaleUp; + const annualizedCo2NoChildren = co2NoChildren * annualizedScaleUp; + const dollarCost = coreHours * CORE_COST_PER_HOUR; + const dollarCostNoChildren = coreHoursNoChildren * CORE_COST_PER_HOUR; + + const impactRows = [ + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.percentageCpuTimeInclusiveLabel', + { + defaultMessage: '% of CPU time', + } + ), + value: asPercentage(percentage), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.percentageCpuTimeExclusiveLabel', + { + defaultMessage: '% of CPU time (excl. children)', + } + ), + value: asPercentage(percentageNoChildren), + }, + { + label: i18n.translate('xpack.profiling.flameGraphInformationWindow.samplesLabel', { + defaultMessage: 'Samples', + }), + value: samples, + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.coreSecondsInclusiveLabel', + { + defaultMessage: 'Core-seconds', + } + ), + value: asDuration(coreSeconds), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.coreSecondsExclusiveLabel', + { + defaultMessage: 'Core-seconds (excl. children)', + } + ), + value: asDuration(coreSecondsNoChildren), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.annualizedCoreSecondsInclusiveLabel', + { + defaultMessage: 'Annualized core-seconds', + } + ), + value: asDuration(coreSeconds * annualizedScaleUp), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.annualizedCoreSecondsExclusiveLabel', + { + defaultMessage: 'Annualized core-seconds (excl. children)', + } + ), + value: asDuration(coreSecondsNoChildren * annualizedScaleUp), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.co2EmissionInclusiveLabel', + { + defaultMessage: 'CO2 emission', + } + ), + value: asWeight(co2), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.co2EmissionExclusiveLabel', + { + defaultMessage: 'CO2 emission (excl. children)', + } + ), + value: asWeight(co2NoChildren), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.annualizedCo2InclusiveLabel', + { + defaultMessage: 'Annualized CO2', + } + ), + value: asWeight(annualizedCo2), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.annualizedCo2ExclusiveLabel', + { + defaultMessage: 'Annualized CO2 (excl. children)', + } + ), + value: asWeight(annualizedCo2NoChildren), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.dollarCostInclusiveLabel', + { + defaultMessage: 'Dollar cost', + } + ), + value: asCost(dollarCost), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.dollarCostExclusiveLabel', + { + defaultMessage: 'Dollar cost (excl. children)', + } + ), + value: asCost(dollarCostNoChildren), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.annualizedDollarCostInclusiveLabel', + { + defaultMessage: 'Annualized dollar cost', + } + ), + value: asCost(dollarCost * annualizedScaleUp), + }, + { + label: i18n.translate( + 'xpack.profiling.flameGraphInformationWindow.annualizedDollarCostExclusiveLabel', + { + defaultMessage: 'Annualized dollar cost (excl. children)', + } + ), + value: asCost(dollarCostNoChildren * annualizedScaleUp), + }, + ]; + + return impactRows; +} diff --git a/x-pack/plugins/profiling/public/components/flame_graphs_view/index.tsx b/x-pack/plugins/profiling/public/components/flame_graphs_view/index.tsx new file mode 100644 index 000000000000..a5b0f6486726 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/flame_graphs_view/index.tsx @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiButtonGroup, EuiFlexGroup, EuiFlexItem, EuiPageHeaderContentProps } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { FlameGraphComparisonMode } from '../../../common/flamegraph'; +import { useProfilingParams } from '../../hooks/use_profiling_params'; +import { useProfilingRouter } from '../../hooks/use_profiling_router'; +import { useProfilingRoutePath } from '../../hooks/use_profiling_route_path'; +import { useTimeRange } from '../../hooks/use_time_range'; +import { useTimeRangeAsync } from '../../hooks/use_time_range_async'; +import { AsyncComponent } from '../async_component'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; +import { FlameGraph } from '../flamegraph'; +import { PrimaryAndComparisonSearchBar } from '../primary_and_comparison_search_bar'; +import { ProfilingAppPageTemplate } from '../profiling_app_page_template'; +import { RedirectTo } from '../redirect_to'; + +export function FlameGraphsView({ children }: { children: React.ReactElement }) { + const { + path, + query, + query: { rangeFrom, rangeTo, kuery }, + } = useProfilingParams('/flamegraphs/*'); + + const timeRange = useTimeRange({ rangeFrom, rangeTo }); + + const comparisonTimeRange = useTimeRange( + 'comparisonRangeFrom' in query + ? { rangeFrom: query.comparisonRangeFrom, rangeTo: query.comparisonRangeTo, optional: true } + : { rangeFrom: undefined, rangeTo: undefined, optional: true } + ); + + const comparisonKuery = 'comparisonKuery' in query ? query.comparisonKuery : ''; + const comparisonMode = + 'comparisonMode' in query ? query.comparisonMode : FlameGraphComparisonMode.Absolute; + + const { + services: { fetchElasticFlamechart }, + } = useProfilingDependencies(); + + const state = useTimeRangeAsync(() => { + return Promise.all([ + fetchElasticFlamechart({ + timeFrom: new Date(timeRange.start).getTime() / 1000, + timeTo: new Date(timeRange.end).getTime() / 1000, + kuery, + }), + comparisonTimeRange.start && comparisonTimeRange.end + ? fetchElasticFlamechart({ + timeFrom: new Date(comparisonTimeRange.start).getTime() / 1000, + timeTo: new Date(comparisonTimeRange.end).getTime() / 1000, + kuery: comparisonKuery, + }) + : Promise.resolve(undefined), + ]).then(([primaryFlamegraph, comparisonFlamegraph]) => { + return { + primaryFlamegraph, + comparisonFlamegraph, + }; + }); + }, [ + timeRange.start, + timeRange.end, + kuery, + comparisonTimeRange.start, + comparisonTimeRange.end, + comparisonKuery, + fetchElasticFlamechart, + ]); + + const { data } = state; + + const routePath = useProfilingRoutePath(); + + const profilingRouter = useProfilingRouter(); + + const isDifferentialView = routePath === '/flamegraphs/differential'; + + const tabs: Required['tabs'] = [ + { + label: i18n.translate('xpack.profiling.flameGraphsView.flameGraphTabLabel', { + defaultMessage: 'Flamegraph', + }), + isSelected: !isDifferentialView, + href: profilingRouter.link('/flamegraphs/flamegraph', { query }), + }, + { + label: i18n.translate('xpack.profiling.flameGraphsView.differentialFlameGraphTabLabel', { + defaultMessage: 'Differential flamegraph', + }), + isSelected: isDifferentialView, + href: profilingRouter.link('/flamegraphs/differential', { + query: { + ...query, + comparisonRangeFrom: query.rangeFrom, + comparisonRangeTo: query.rangeTo, + comparisonKuery: query.kuery, + comparisonMode, + }, + }), + }, + ]; + + if (routePath === '/flamegraphs') { + return ; + } + + return ( + + + {isDifferentialView ? ( + + + + + + + { + if (!('comparisonRangeFrom' in query)) { + return; + } + + profilingRouter.push(routePath, { + path, + query: { + ...query, + comparisonMode: nextComparisonMode as FlameGraphComparisonMode, + }, + }); + }} + options={[ + { + id: FlameGraphComparisonMode.Absolute, + label: i18n.translate( + 'xpack.profiling.flameGraphsView.differentialFlameGraphComparisonModeAbsoluteButtonLabel', + { + defaultMessage: 'Abs', + } + ), + }, + { + id: FlameGraphComparisonMode.Relative, + label: i18n.translate( + 'xpack.profiling.flameGraphsView.differentialFlameGraphComparisonModeRelativeButtonLabel', + { + defaultMessage: 'Rel', + } + ), + }, + ]} + /> + + + + ) : null} + + + + + {children} + + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/flamegraph.tsx b/x-pack/plugins/profiling/public/components/flamegraph.tsx new file mode 100644 index 000000000000..7077b98364d1 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/flamegraph.tsx @@ -0,0 +1,330 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Chart, Datum, Flame, FlameLayerValue, PartialTheme, Settings } from '@elastic/charts'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSwitch, useEuiTheme } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { Maybe } from '@kbn/observability-plugin/common/typings'; +import { isNumber } from 'lodash'; +import React, { useEffect, useMemo, useState } from 'react'; +import { ElasticFlameGraph, FlameGraphComparisonMode } from '../../common/flamegraph'; +import { useAsync } from '../hooks/use_async'; +import { asPercentage } from '../utils/formatters/as_percentage'; +import { getFlamegraphModel } from '../utils/get_flamegraph_model'; +import { useProfilingDependencies } from './contexts/profiling_dependencies/use_profiling_dependencies'; +import { FlamegraphInformationWindow } from './flame_graphs_view/flamegraph_information_window'; + +function TooltipRow({ + value, + label, + comparison, + formatAsPercentage, + showChange, +}: { + value: number; + label: string; + comparison?: number; + formatAsPercentage: boolean; + showChange: boolean; +}) { + const valueLabel = formatAsPercentage ? asPercentage(value, 2) : value.toString(); + const comparisonLabel = + formatAsPercentage && isNumber(comparison) + ? asPercentage(comparison, 2) + : comparison?.toString(); + + const diff = showChange && isNumber(comparison) ? comparison - value : undefined; + + let diffLabel: string | undefined = diff?.toString(); + + if (diff === 0) { + diffLabel = i18n.translate('xpack.profiling.flameGraphToolTip.diffNoChange', { + defaultMessage: 'no change', + }); + } else if (formatAsPercentage && diff !== undefined) { + diffLabel = asPercentage(diff, 2); + } + + return ( + + + + {label} + + + {comparison + ? i18n.translate('xpack.profiling.flameGraphTooltip.valueLabel', { + defaultMessage: `{value} vs {comparison}`, + values: { + value: valueLabel, + comparison: comparisonLabel, + }, + }) + : valueLabel} + {diffLabel ? ` (${diffLabel})` : ''} + + + + ); +} + +function FlameGraphTooltip({ + label, + countInclusive, + countExclusive, + samples, + totalSamples, + comparisonCountInclusive, + comparisonCountExclusive, + comparisonSamples, + comparisonTotalSamples, +}: { + samples: number; + label: string; + countInclusive: number; + countExclusive: number; + totalSamples: number; + comparisonCountInclusive?: number; + comparisonCountExclusive?: number; + comparisonSamples?: number; + comparisonTotalSamples?: number; +}) { + return ( + + + {label} + + + + + + + + + + ); +} + +export interface FlameGraphProps { + id: string; + height: number; + comparisonMode: FlameGraphComparisonMode; + primaryFlamegraph?: ElasticFlameGraph; + comparisonFlamegraph?: ElasticFlameGraph; +} + +export const FlameGraph: React.FC = ({ + id, + height, + comparisonMode, + primaryFlamegraph, + comparisonFlamegraph, +}) => { + const theme = useEuiTheme(); + + const { + services: { fetchFrameInformation }, + } = useProfilingDependencies(); + + const columnarData = useMemo(() => { + return getFlamegraphModel({ + primaryFlamegraph, + comparisonFlamegraph, + colorSuccess: theme.euiTheme.colors.success, + colorDanger: theme.euiTheme.colors.danger, + colorNeutral: theme.euiTheme.colors.lightShade, + comparisonMode, + }); + }, [ + primaryFlamegraph, + comparisonFlamegraph, + theme.euiTheme.colors.success, + theme.euiTheme.colors.danger, + theme.euiTheme.colors.lightShade, + comparisonMode, + ]); + + const chartTheme: PartialTheme = { + chartMargins: { top: 0, left: 0, bottom: 0, right: 0 }, + chartPaddings: { left: 0, right: 0, top: 0, bottom: 0 }, + }; + + const totalSamples = columnarData.viewModel.value[0]; + + const [highlightedVmIndex, setHighlightedVmIndex] = useState(undefined); + + const highlightedFrameQueryParams = useMemo(() => { + if (!primaryFlamegraph || highlightedVmIndex === undefined || highlightedVmIndex === 0) { + return undefined; + } + + const frameID = primaryFlamegraph.FrameID[highlightedVmIndex]; + const executableID = primaryFlamegraph.ExecutableID[highlightedVmIndex]; + + return { + frameID, + executableID, + }; + }, [primaryFlamegraph, highlightedVmIndex]); + + const { data: highlightedFrame, status: highlightedFrameStatus } = useAsync(() => { + if (!highlightedFrameQueryParams) { + return Promise.resolve(undefined); + } + + return fetchFrameInformation({ + frameID: highlightedFrameQueryParams.frameID, + executableID: highlightedFrameQueryParams.executableID, + }); + }, [highlightedFrameQueryParams, fetchFrameInformation]); + + const selected: undefined | React.ComponentProps['frame'] = + primaryFlamegraph && highlightedFrame && highlightedVmIndex !== undefined + ? { + exeFileName: highlightedFrame.ExeFileName, + sourceFileName: highlightedFrame.SourceFilename, + functionName: highlightedFrame.FunctionName, + samples: primaryFlamegraph.Value[highlightedVmIndex], + childSamples: + primaryFlamegraph.Value[highlightedVmIndex] - + primaryFlamegraph.CountExclusive[highlightedVmIndex], + } + : undefined; + + useEffect(() => { + setHighlightedVmIndex(undefined); + }, [columnarData.key]); + + const [showInformationWindow, setShowInformationWindow] = useState(false); + + return ( + + + { + setShowInformationWindow((prev) => !prev); + }} + label={i18n.translate('xpack.profiling.flameGraph.showInformationWindow', { + defaultMessage: 'Show information window', + })} + /> + + + + {columnarData.viewModel.label.length > 0 && ( + + + { + const selectedElement = elements[0] as Maybe; + if (Number.isNaN(selectedElement?.vmIndex)) { + setHighlightedVmIndex(undefined); + } else { + setHighlightedVmIndex(selectedElement!.vmIndex); + } + }} + tooltip={{ + customTooltip: (props) => { + if (!primaryFlamegraph) { + return <>; + } + + const valueIndex = props.values[0].valueAccessor as number; + const label = primaryFlamegraph.Label[valueIndex]; + const samples = primaryFlamegraph.Value[valueIndex]; + const countInclusive = primaryFlamegraph.CountInclusive[valueIndex]; + const countExclusive = primaryFlamegraph.CountExclusive[valueIndex]; + const nodeID = primaryFlamegraph.ID[valueIndex]; + + const comparisonNode = columnarData.comparisonNodesById[nodeID]; + + return ( + + ); + }, + }} + /> + d.value as number} + valueFormatter={(value) => `${value}`} + animation={{ duration: 100 }} + controlProviderCallback={{}} + /> + + + )} + {showInformationWindow ? ( + + { + setShowInformationWindow(false); + }} + /> + + ) : undefined} + + + + ); +}; diff --git a/x-pack/plugins/profiling/public/components/functions_view/index.tsx b/x-pack/plugins/profiling/public/components/functions_view/index.tsx new file mode 100644 index 000000000000..94410f961c46 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/functions_view/index.tsx @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiFlexItem, EuiPageHeaderContentProps } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { TypeOf } from '@kbn/typed-react-router-config'; +import React from 'react'; +import { useProfilingParams } from '../../hooks/use_profiling_params'; +import { useProfilingRouter } from '../../hooks/use_profiling_router'; +import { useProfilingRoutePath } from '../../hooks/use_profiling_route_path'; +import { useTimeRange } from '../../hooks/use_time_range'; +import { useTimeRangeAsync } from '../../hooks/use_time_range_async'; +import { ProfilingRoutes } from '../../routing'; +import { AsyncComponent } from '../async_component'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; +import { PrimaryAndComparisonSearchBar } from '../primary_and_comparison_search_bar'; +import { ProfilingAppPageTemplate } from '../profiling_app_page_template'; +import { RedirectTo } from '../redirect_to'; +import { TopNFunctionsTable } from '../topn_functions'; + +export function FunctionsView({ children }: { children: React.ReactElement }) { + const { + path, + query, + query: { rangeFrom, rangeTo, kuery, sortDirection, sortField }, + } = useProfilingParams('/functions/*'); + + const timeRange = useTimeRange({ rangeFrom, rangeTo }); + + const comparisonTimeRange = useTimeRange( + 'comparisonRangeFrom' in query + ? { rangeFrom: query.comparisonRangeFrom, rangeTo: query.comparisonRangeTo, optional: true } + : { rangeFrom: undefined, rangeTo: undefined, optional: true } + ); + + const comparisonKuery = 'comparisonKuery' in query ? query.comparisonKuery : ''; + + const { + services: { fetchTopNFunctions }, + } = useProfilingDependencies(); + + const state = useTimeRangeAsync(() => { + return fetchTopNFunctions({ + timeFrom: new Date(timeRange.start).getTime() / 1000, + timeTo: new Date(timeRange.end).getTime() / 1000, + startIndex: 0, + endIndex: 1000, + kuery, + }); + }, [timeRange.start, timeRange.end, kuery, fetchTopNFunctions]); + + const comparisonState = useTimeRangeAsync(() => { + if (!comparisonTimeRange.start || !comparisonTimeRange.end) { + return undefined; + } + return fetchTopNFunctions({ + timeFrom: new Date(comparisonTimeRange.start).getTime() / 1000, + timeTo: new Date(comparisonTimeRange.end).getTime() / 1000, + startIndex: 0, + endIndex: 1000, + kuery: comparisonKuery, + }); + }, [comparisonTimeRange.start, comparisonTimeRange.end, comparisonKuery, fetchTopNFunctions]); + + const routePath = useProfilingRoutePath() as + | '/functions' + | '/functions/topn' + | '/functions/differential'; + + const profilingRouter = useProfilingRouter(); + + const isDifferentialView = routePath === '/functions/differential'; + + const tabs: Required['tabs'] = [ + { + label: i18n.translate('xpack.profiling.functionsView.functionsTabLabel', { + defaultMessage: 'TopN functions', + }), + isSelected: !isDifferentialView, + href: profilingRouter.link('/functions/topn', { query }), + }, + { + label: i18n.translate('xpack.profiling.functionsView.differentialFunctionsTabLabel', { + defaultMessage: 'Differential TopN functions', + }), + isSelected: isDifferentialView, + href: profilingRouter.link('/functions/differential', { + query: { + ...query, + comparisonRangeFrom: query.rangeFrom, + comparisonRangeTo: query.rangeTo, + comparisonKuery: query.kuery, + }, + }), + }, + ]; + + if (routePath === '/functions') { + return ; + } + + return ( + + <> + + {isDifferentialView ? ( + + + + ) : null} + + + + + { + profilingRouter.push(routePath, { + path, + query: { + ...query, + sortField: nextSort.sortField, + sortDirection: nextSort.sortDirection, + }, + }); + }} + /> + + + {isDifferentialView && comparisonTimeRange.start && comparisonTimeRange.end ? ( + + + { + profilingRouter.push(routePath, { + path, + query: { + ...(query as TypeOf< + ProfilingRoutes, + '/functions/differential' + >['query']), + sortField: nextSort.sortField, + sortDirection: nextSort.sortDirection, + }, + }); + }} + topNFunctions={comparisonState.data} + comparisonTopNFunctions={state.data} + /> + + + ) : null} + + + + {children} + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/primary_and_comparison_search_bar.tsx b/x-pack/plugins/profiling/public/components/primary_and_comparison_search_bar.tsx new file mode 100644 index 000000000000..7a8a09e84558 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/primary_and_comparison_search_bar.tsx @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { TypeOf } from '@kbn/typed-react-router-config'; +import React from 'react'; +import { useAnyOfProfilingParams } from '../hooks/use_profiling_params'; +import { useProfilingRouter } from '../hooks/use_profiling_router'; +import { useProfilingRoutePath } from '../hooks/use_profiling_route_path'; +import { useTimeRangeContext } from '../hooks/use_time_range_context'; +import { ProfilingRoutes } from '../routing'; +import { PrimaryProfilingSearchBar } from './profiling_app_page_template/primary_profiling_search_bar'; +import { ProfilingSearchBar } from './profiling_app_page_template/profiling_search_bar'; + +export function PrimaryAndComparisonSearchBar() { + const { + path, + query, + query: { comparisonKuery, comparisonRangeFrom, comparisonRangeTo }, + } = useAnyOfProfilingParams('/flamegraphs/differential', '/functions/differential'); + + const { refresh } = useTimeRangeContext(); + + const profilingRouter = useProfilingRouter(); + const routePath = useProfilingRoutePath() as + | '/flamegraphs/differential' + | '/functions/differential'; + + function navigate(nextOptions: { rangeFrom: string; rangeTo: string; kuery?: string }) { + if (routePath === '/flamegraphs/differential') { + profilingRouter.push(routePath, { + path, + query: { + ...(query as TypeOf['query']), + comparisonRangeFrom: nextOptions.rangeFrom, + comparisonRangeTo: nextOptions.rangeTo, + comparisonKuery: nextOptions.kuery ?? query.comparisonKuery, + }, + }); + } else { + profilingRouter.push(routePath, { + path, + query: { + ...(query as TypeOf['query']), + comparisonRangeFrom: nextOptions.rangeFrom, + comparisonRangeTo: nextOptions.rangeTo, + comparisonKuery: nextOptions.kuery ?? query.comparisonKuery, + }, + }); + } + } + + return ( + + + + + + { + navigate({ + kuery: String(next.query?.query || ''), + rangeFrom: next.dateRange.from, + rangeTo: next.dateRange.to, + }); + }} + onRefresh={(nextDateRange) => { + navigate({ + rangeFrom: nextDateRange.dateRange.from, + rangeTo: nextDateRange.dateRange.to, + }); + }} + onRefreshClick={() => { + refresh(); + }} + /> + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx b/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx new file mode 100644 index 000000000000..d2eef22ba922 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/profiling_app_page_template/index.tsx @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiFlexGroup, EuiFlexItem, EuiPageHeaderContentProps } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { useEffect } from 'react'; +import { useHistory } from 'react-router-dom'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; +import { PrimaryProfilingSearchBar } from './primary_profiling_search_bar'; + +export function ProfilingAppPageTemplate({ + children, + tabs, + hideSearchBar = false, +}: { + children: React.ReactElement; + tabs: EuiPageHeaderContentProps['tabs']; + hideSearchBar?: boolean; +}) { + const { + start: { observability }, + } = useProfilingDependencies(); + + const { PageTemplate: ObservabilityPageTemplate } = observability.navigation; + + const history = useHistory(); + + useEffect(() => { + window.scrollTo(0, 0); + }, [history.location.pathname]); + + return ( + + + {!hideSearchBar && ( + + + + )} + {children} + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/profiling_app_page_template/primary_profiling_search_bar.tsx b/x-pack/plugins/profiling/public/components/profiling_app_page_template/primary_profiling_search_bar.tsx new file mode 100644 index 000000000000..fb4ee5d780f6 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/profiling_app_page_template/primary_profiling_search_bar.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect } from 'react'; +import { useProfilingParams } from '../../hooks/use_profiling_params'; +import { useProfilingRouter } from '../../hooks/use_profiling_router'; +import { useProfilingRoutePath } from '../../hooks/use_profiling_route_path'; +import { useTimeRangeContext } from '../../hooks/use_time_range_context'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; +import { ProfilingSearchBar } from './profiling_search_bar'; + +export function PrimaryProfilingSearchBar({ showSubmitButton }: { showSubmitButton?: boolean }) { + const { + start: { data }, + } = useProfilingDependencies(); + + const profilingRouter = useProfilingRouter(); + const routePath = useProfilingRoutePath(); + + const { + path, + query, + query: { rangeFrom, rangeTo, kuery }, + } = useProfilingParams('/*'); + + const { refresh } = useTimeRangeContext(); + + useEffect(() => { + // set time if both to and from are given in the url + if (rangeFrom && rangeTo) { + data.query.timefilter.timefilter.setTime({ + from: rangeFrom, + to: rangeTo, + }); + return; + } + }, [rangeFrom, rangeTo, data]); + + return ( + { + profilingRouter.push(routePath, { + path, + query: { + ...query, + kuery: String(next.query?.query || ''), + rangeFrom: next.dateRange.from, + rangeTo: next.dateRange.to, + }, + }); + }} + onRefresh={(nextDateRange) => { + profilingRouter.push(routePath, { + path, + query: { + ...query, + rangeFrom: nextDateRange.dateRange.from, + rangeTo: nextDateRange.dateRange.to, + }, + }); + }} + onRefreshClick={() => { + refresh(); + }} + showSubmitButton={showSubmitButton} + /> + ); +} diff --git a/x-pack/plugins/profiling/public/components/profiling_app_page_template/profiling_search_bar.tsx b/x-pack/plugins/profiling/public/components/profiling_app_page_template/profiling_search_bar.tsx new file mode 100644 index 000000000000..5d199c8a26cb --- /dev/null +++ b/x-pack/plugins/profiling/public/components/profiling_app_page_template/profiling_search_bar.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useEffect, useState } from 'react'; +import { SearchBar } from '@kbn/unified-search-plugin/public'; +import { DataView } from '@kbn/data-views-plugin/common'; +import { compact } from 'lodash'; +import { Query, TimeRange } from '@kbn/es-query'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; +import { INDEX_EVENTS } from '../../../common'; + +export function ProfilingSearchBar({ + kuery, + rangeFrom, + rangeTo, + onQuerySubmit, + onRefresh, + onRefreshClick, + showSubmitButton = true, +}: { + kuery: string; + rangeFrom: string; + rangeTo: string; + onQuerySubmit: ( + payload: { + dateRange: TimeRange; + query?: Query; + }, + isUpdate?: boolean + ) => void; + onRefresh: Required>['onRefresh']; + onRefreshClick: () => void; + showSubmitButton?: boolean; +}) { + const { + start: { dataViews }, + } = useProfilingDependencies(); + + const [dataView, setDataView] = useState(); + + useEffect(() => { + dataViews + .create({ + title: INDEX_EVENTS, + }) + .then((nextDataView) => setDataView(nextDataView)); + }, [dataViews]); + + const searchBarQuery: Required>['query'] = { + language: 'kuery', + query: kuery, + }; + + return ( + + onQuerySubmit={({ dateRange, query }) => { + if (dateRange.from === rangeFrom && dateRange.to === rangeTo && query?.query === kuery) { + onRefreshClick(); + return; + } + + onQuerySubmit({ dateRange, query }); + }} + showQueryBar + showQueryInput + showDatePicker + showFilterBar={false} + showSaveQuery={false} + showSubmitButton={showSubmitButton} + query={searchBarQuery} + dateRangeFrom={rangeFrom} + dateRangeTo={rangeTo} + indexPatterns={compact([dataView])} + onRefresh={onRefresh} + /> + ); +} diff --git a/x-pack/plugins/profiling/public/components/redirect_to.tsx b/x-pack/plugins/profiling/public/components/redirect_to.tsx new file mode 100644 index 000000000000..fc5020909289 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/redirect_to.tsx @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { useHistory, Redirect } from 'react-router-dom'; + +export function RedirectTo({ pathname }: { pathname: string }) { + const { location } = useHistory(); + + return ; +} diff --git a/x-pack/plugins/profiling/public/components/redirect_with_default_date_range.tsx b/x-pack/plugins/profiling/public/components/redirect_with_default_date_range.tsx new file mode 100644 index 000000000000..6a39aad78c6e --- /dev/null +++ b/x-pack/plugins/profiling/public/components/redirect_with_default_date_range.tsx @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { useDateRangeRedirect } from '../hooks/use_default_date_range_redirect'; + +export function RedirectWithDefaultDateRange({ children }: { children: React.ReactElement }) { + const { redirect, isDateRangeSet } = useDateRangeRedirect(); + + if (isDateRangeSet) { + return children; + } + + redirect(); + + return null; +} diff --git a/x-pack/plugins/profiling/public/components/route_breadcrumb.tsx b/x-pack/plugins/profiling/public/components/route_breadcrumb.tsx new file mode 100644 index 000000000000..78d358d8a7b6 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/route_breadcrumb.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { useProfilingDependencies } from './contexts/profiling_dependencies/use_profiling_dependencies'; +import { useRouteBreadcrumb } from './contexts/route_breadcrumbs_context/use_route_breadcrumb'; + +export const RouteBreadcrumb = ({ + title, + href, + children, +}: { + title: string; + href: string; + children: React.ReactElement; +}) => { + const { + start: { core }, + } = useProfilingDependencies(); + useRouteBreadcrumb({ title, href: core.http.basePath.prepend('/app/profiling/' + href) }); + + return children; +}; diff --git a/x-pack/plugins/profiling/public/components/stack_frame_summary.tsx b/x-pack/plugins/profiling/public/components/stack_frame_summary.tsx new file mode 100644 index 000000000000..9f25b4d9d890 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/stack_frame_summary.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; +import React from 'react'; +import { getCalleeFunction, getCalleeSource, StackFrameMetadata } from '../../common/profiling'; + +export function StackFrameSummary({ frame }: { frame: StackFrameMetadata }) { + return ( + + +
    + + {getCalleeFunction(frame)} + +
    +     + + {getCalleeSource(frame) || '‎'} + +     + ); +} diff --git a/x-pack/plugins/profiling/public/components/stack_traces_view/get_stack_traces_tabs.ts b/x-pack/plugins/profiling/public/components/stack_traces_view/get_stack_traces_tabs.ts new file mode 100644 index 000000000000..435333ce6898 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/stack_traces_view/get_stack_traces_tabs.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiPageHeaderContentProps } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { TypeOf } from '@kbn/typed-react-router-config'; +import { TopNType } from '../../../common/stack_traces'; +import { StatefulProfilingRouter } from '../../hooks/use_profiling_router'; +import { ProfilingRoutes } from '../../routing'; + +export function getStackTracesTabs({ + path, + query, + profilingRouter, +}: TypeOf & { + profilingRouter: StatefulProfilingRouter; +}): Required['tabs'] { + return [ + { + label: i18n.translate('xpack.profiling.stackTracesView.containersTabLabel', { + defaultMessage: 'Containers', + }), + topNType: TopNType.Containers, + }, + { + label: i18n.translate('xpack.profiling.stackTracesView.deploymentsTabLabel', { + defaultMessage: 'Deployments', + }), + topNType: TopNType.Deployments, + }, + { + label: i18n.translate('xpack.profiling.stackTracesView.threadsTabLabel', { + defaultMessage: 'Threads', + }), + topNType: TopNType.Threads, + }, + { + label: i18n.translate('xpack.profiling.stackTracesView.hostsTabLabel', { + defaultMessage: 'Hosts', + }), + topNType: TopNType.Hosts, + }, + { + label: i18n.translate('xpack.profiling.stackTracesView.tracesTabLabel', { + defaultMessage: 'Traces', + }), + topNType: TopNType.Traces, + }, + ].map((tab) => ({ + label: tab.label, + isSelected: tab.topNType === path.topNType, + href: profilingRouter.link(`/stacktraces/{topNType}`, { + path: { topNType: tab.topNType }, + query, + }), + })); +} diff --git a/x-pack/plugins/profiling/public/components/stack_traces_view/index.tsx b/x-pack/plugins/profiling/public/components/stack_traces_view/index.tsx new file mode 100644 index 000000000000..f263f235a75c --- /dev/null +++ b/x-pack/plugins/profiling/public/components/stack_traces_view/index.tsx @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiButton, EuiButtonGroup, EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { useState } from 'react'; +import { StackTracesDisplayOption, TopNType } from '../../../common/stack_traces'; +import { groupSamplesByCategory, TopNResponse, TopNSubchart } from '../../../common/topn'; +import { useProfilingParams } from '../../hooks/use_profiling_params'; +import { useProfilingRouter } from '../../hooks/use_profiling_router'; +import { useProfilingRoutePath } from '../../hooks/use_profiling_route_path'; +import { useTimeRange } from '../../hooks/use_time_range'; +import { useTimeRangeAsync } from '../../hooks/use_time_range_async'; +import { AsyncComponent } from '../async_component'; +import { ChartGrid } from '../chart_grid'; +import { useProfilingDependencies } from '../contexts/profiling_dependencies/use_profiling_dependencies'; +import { ProfilingAppPageTemplate } from '../profiling_app_page_template'; +import { StackedBarChart } from '../stacked_bar_chart'; +import { getStackTracesTabs } from './get_stack_traces_tabs'; + +export function StackTracesView() { + const routePath = useProfilingRoutePath(); + + const profilingRouter = useProfilingRouter(); + + const { + path, + query, + path: { topNType }, + query: { rangeFrom, rangeTo, kuery, displayAs, limit: limitFromQueryParams }, + } = useProfilingParams('/stacktraces/{topNType}'); + + const limit = limitFromQueryParams || 10; + + const tabs = getStackTracesTabs({ + path, + query, + profilingRouter, + }); + + const { + services: { fetchTopN }, + } = useProfilingDependencies(); + + const timeRange = useTimeRange({ + rangeFrom, + rangeTo, + }); + + const state = useTimeRangeAsync(() => { + if (!topNType) { + return Promise.resolve({ charts: [], metadata: {} }); + } + return fetchTopN({ + type: topNType, + timeFrom: new Date(timeRange.start).getTime() / 1000, + timeTo: new Date(timeRange.end).getTime() / 1000, + kuery, + }).then((response: TopNResponse) => { + const totalCount = response.TotalCount; + const samples = response.TopN; + const charts = groupSamplesByCategory({ samples, totalCount, metadata: response.Metadata }); + return { + charts, + }; + }); + }, [topNType, timeRange.start, timeRange.end, fetchTopN, kuery]); + + const [highlightedSubchart, setHighlightedSubchart] = useState( + undefined + ); + + const { data } = state; + + return ( + + + + + + + { + profilingRouter.push(routePath, { + path, + query: { + ...query, + displayAs: nextValue, + }, + }); + }} + options={[ + { + id: StackTracesDisplayOption.StackTraces, + iconType: 'visLine', + label: i18n.translate( + 'xpack.profiling.stackTracesView.stackTracesCountButton', + { + defaultMessage: 'Stack traces', + } + ), + }, + { + id: StackTracesDisplayOption.Percentage, + iconType: 'percent', + label: i18n.translate('xpack.profiling.stackTracesView.percentagesButton', { + defaultMessage: 'Percentages', + }), + }, + ]} + legend={i18n.translate('xpack.profiling.stackTracesView.displayOptionLegend', { + defaultMessage: 'Display option', + })} + /> + + + + { + profilingRouter.push(routePath, { + path, + query: { + ...query, + rangeFrom: nextRange.rangeFrom, + rangeTo: nextRange.rangeTo, + }, + }); + }} + onSampleClick={(sample) => { + setHighlightedSubchart( + data?.charts.find((subchart) => subchart.Category === sample.Category) + ); + }} + onSampleOut={() => { + setHighlightedSubchart(undefined); + }} + highlightedSubchart={highlightedSubchart} + showFrames={topNType === TopNType.Traces} + /> + + + + + + + + + + + {(data?.charts.length ?? 0) > limit ? ( + + { + profilingRouter.push(routePath, { + path, + query: { + ...query, + limit: limit + 10, + }, + }); + }} + > + {i18n.translate('xpack.profiling.stackTracesView.showMoreButton', { + defaultMessage: 'Show more', + })} + + + ) : null} + + + ); +} diff --git a/x-pack/plugins/profiling/public/components/stacked_bar_chart.tsx b/x-pack/plugins/profiling/public/components/stacked_bar_chart.tsx new file mode 100644 index 000000000000..d11fb799ff35 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/stacked_bar_chart.tsx @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + Axis, + BrushAxis, + Chart, + HistogramBarSeries, + ScaleType, + Settings, + StackMode, + timeFormatter, + Tooltip, + TooltipInfo, + XYChartElementEvent, +} from '@elastic/charts'; +import { EuiPanel } from '@elastic/eui'; +import React from 'react'; +import { TopNSample, TopNSubchart } from '../../common/topn'; +import { useKibanaTimeZoneSetting } from '../hooks/use_kibana_timezone_setting'; +import { useProfilingChartsTheme } from '../hooks/use_profiling_charts_theme'; +import { asPercentage } from '../utils/formatters/as_percentage'; +import { SubChart } from './subchart'; + +function SubchartTooltip({ + highlightedSubchart, + showFrames, +}: TooltipInfo & { highlightedSubchart: TopNSubchart; showFrames: boolean }) { + // max tooltip width - 2 * padding (16px) + const width = 224; + return ( + + + + ); +} + +export interface StackedBarChartProps { + height: number; + asPercentages: boolean; + onBrushEnd: (range: { rangeFrom: string; rangeTo: string }) => void; + onSampleClick: (sample: TopNSample) => void; + onSampleOut: () => void; + highlightedSubchart?: TopNSubchart; + charts: TopNSubchart[]; + showFrames: boolean; +} + +export const StackedBarChart: React.FC = ({ + height, + asPercentages, + onBrushEnd, + onSampleClick, + onSampleOut, + highlightedSubchart, + charts, + showFrames, +}) => { + const timeZone = useKibanaTimeZoneSetting(); + + const { chartsBaseTheme, chartsTheme } = useProfilingChartsTheme(); + + return ( + + { + const rangeFrom = new Date(brushEvent.x![0]).toISOString(); + const rangeTo = new Date(brushEvent.x![1]).toISOString(); + + onBrushEnd({ rangeFrom, rangeTo }); + }} + baseTheme={chartsBaseTheme} + theme={chartsTheme} + onElementClick={(events) => { + const [value] = events[0] as XYChartElementEvent; + onSampleClick(value.datum as TopNSample); + }} + onElementOver={() => { + onSampleOut(); + }} + onElementOut={() => { + onSampleOut(); + }} + /> + ( + + ) + : () => <> + } + /> + {charts.map((chart) => ( + + ))} + + (asPercentages ? asPercentage(d) : d.toFixed(0))} + /> + + ); +}; diff --git a/x-pack/plugins/profiling/public/components/subchart.tsx b/x-pack/plugins/profiling/public/components/subchart.tsx new file mode 100644 index 000000000000..1d20213f2498 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/subchart.tsx @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + AreaSeries, + Axis, + Chart, + CurveType, + ScaleType, + Settings, + timeFormatter, +} from '@elastic/charts'; +import { + EuiBadge, + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiLink, + EuiSpacer, + EuiText, + useEuiTheme, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { StackFrameMetadata } from '../../common/profiling'; +import { getFieldNameForTopNType, TopNType } from '../../common/stack_traces'; +import { CountPerTime, OTHER_BUCKET_LABEL } from '../../common/topn'; +import { useKibanaTimeZoneSetting } from '../hooks/use_kibana_timezone_setting'; +import { useProfilingChartsTheme } from '../hooks/use_profiling_charts_theme'; +import { useProfilingParams } from '../hooks/use_profiling_params'; +import { useProfilingRouter } from '../hooks/use_profiling_router'; +import { asPercentage } from '../utils/formatters/as_percentage'; +import { StackFrameSummary } from './stack_frame_summary'; + +export interface SubChartProps { + index: number; + color: string; + height: number; + width?: number; + category: string; + percentage: number; + data: CountPerTime[]; + showAxes: boolean; + metadata: StackFrameMetadata[]; + onShowMoreClick: (() => void) | null; + style?: React.ComponentProps['style']; + showFrames: boolean; +} + +const NUM_DISPLAYED_FRAMES = 5; + +export const SubChart: React.FC = ({ + index, + color, + category, + percentage, + height, + data, + width, + showAxes, + metadata, + onShowMoreClick, + style, + showFrames, +}) => { + const theme = useEuiTheme(); + + const profilingRouter = useProfilingRouter(); + + const { path, query } = useProfilingParams('/stacktraces/{topNType}'); + + const href = profilingRouter.link('/stacktraces/{topNType}', { + path: { + topNType: TopNType.Traces, + }, + query: { + ...query, + kuery: `${getFieldNameForTopNType(path.topNType)}:"${category}"`, + }, + }); + + const timeZone = useKibanaTimeZoneSetting(); + + const { chartsTheme, chartsBaseTheme } = useProfilingChartsTheme(); + + const compact = !!onShowMoreClick; + + const displayedFrames = compact ? metadata.slice(0, NUM_DISPLAYED_FRAMES) : metadata; + + const hasMoreFrames = displayedFrames.length < metadata.length; + + let bottomElement: React.ReactElement; + + if (metadata.length > 0) { + bottomElement = ( + <> + + + {displayedFrames.map((frame, frameIndex) => ( + <> + + + {frameIndex + 1} + + + + + + {frameIndex < displayedFrames.length - 1 || hasMoreFrames ? ( + + + + ) : null} + + ))} + + + {hasMoreFrames && !!onShowMoreClick && ( + + {i18n.translate('xpack.profiling.stackTracesView.showMoreTracesButton', { + defaultMessage: 'Show more', + })} + + )} + + + ); + } else if (category === OTHER_BUCKET_LABEL && showFrames) { + bottomElement = ( + + + + {i18n.translate('xpack.profiling.stackTracesView.otherTraces', { + defaultMessage: '[This summarizes all traces that are too small to display]', + })} + + + + ); + } else { + bottomElement = ; + } + + return ( + + + + + + + {index} + + + + + + {category} + + + + {asPercentage(percentage / 100, 2)} + + + + + + + + {showAxes ? ( + + ) : null} + (showAxes ? Number(d).toFixed(0) : '')} + style={ + showAxes + ? {} + : { + tickLine: { visible: false }, + tickLabel: { visible: false }, + axisTitle: { visible: false }, + } + } + /> + + {!showAxes ? ( +
    + {i18n.translate('xpack.profiling.maxValue', { + defaultMessage: 'Max: {max}', + values: { max: Math.max(...data.map((value) => value.Count ?? 0)) }, + })} +
    + ) : null} +     + {bottomElement} +     + ); +}; diff --git a/x-pack/plugins/profiling/public/components/topn_functions.tsx b/x-pack/plugins/profiling/public/components/topn_functions.tsx new file mode 100644 index 000000000000..3ad540983d90 --- /dev/null +++ b/x-pack/plugins/profiling/public/components/topn_functions.tsx @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiBadge, + EuiBasicTable, + EuiBasicTableColumn, + EuiFlexGroup, + EuiFlexItem, + EuiHorizontalRule, + EuiSpacer, + EuiText, + useEuiTheme, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { keyBy, orderBy } from 'lodash'; +import React, { useMemo } from 'react'; +import { TopNFunctions, TopNFunctionSortField } from '../../common/functions'; +import { getCalleeFunction, StackFrameMetadata } from '../../common/profiling'; +import { StackFrameSummary } from './stack_frame_summary'; + +interface Row { + rank: number; + frame: StackFrameMetadata; + samples: number; + exclusiveCPU: number; + inclusiveCPU: number; + diff?: { + rank: number; + exclusiveCPU: number; + inclusiveCPU: number; + }; +} + +function CPUStat({ cpu, diffCPU }: { cpu: number; diffCPU: number | undefined }) { + const cpuLabel = `${cpu.toFixed(2)}%`; + + if (diffCPU === undefined || diffCPU === 0) { + return <>{cpuLabel}; + } + const color = diffCPU < 0 ? 'success' : 'danger'; + const label = Math.abs(diffCPU) <= 0.01 ? '<0.01' : Math.abs(diffCPU).toFixed(2); + + return ( + + {cpuLabel} + + + ({label}) + + + + ); +} + +export const TopNFunctionsTable = ({ + sortDirection, + sortField, + onSortChange, + topNFunctions, + comparisonTopNFunctions, +}: { + sortDirection: 'asc' | 'desc'; + sortField: TopNFunctionSortField; + onSortChange: (options: { + sortDirection: 'asc' | 'desc'; + sortField: TopNFunctionSortField; + }) => void; + topNFunctions?: TopNFunctions; + comparisonTopNFunctions?: TopNFunctions; +}) => { + const totalCount: number = useMemo(() => { + if (!topNFunctions || !topNFunctions.TotalCount || topNFunctions.TotalCount === 0) { + return 0; + } + + return topNFunctions.TotalCount; + }, [topNFunctions]); + + const rows: Row[] = useMemo(() => { + if (!topNFunctions || !topNFunctions.TotalCount || topNFunctions.TotalCount === 0) { + return []; + } + + const comparisonDataById = comparisonTopNFunctions + ? keyBy(comparisonTopNFunctions.TopN, 'Id') + : {}; + + return topNFunctions.TopN.filter((topN) => topN.CountExclusive > 0).map((topN, i) => { + const comparisonRow = comparisonDataById?.[topN.Id]; + + const inclusiveCPU = (topN.CountInclusive / topNFunctions.TotalCount) * 100; + const exclusiveCPU = (topN.CountExclusive / topNFunctions.TotalCount) * 100; + + const diff = + comparisonTopNFunctions && comparisonRow + ? { + rank: topN.Rank - comparisonRow.Rank, + exclusiveCPU: + exclusiveCPU - + (comparisonRow.CountExclusive / comparisonTopNFunctions.TotalCount) * 100, + inclusiveCPU: + inclusiveCPU - + (comparisonRow.CountInclusive / comparisonTopNFunctions.TotalCount) * 100, + } + : undefined; + + return { + rank: topN.Rank, + frame: topN.Frame, + samples: topN.CountExclusive, + exclusiveCPU, + inclusiveCPU, + diff, + }; + }); + }, [topNFunctions, comparisonTopNFunctions]); + + const theme = useEuiTheme(); + + const columns: Array> = [ + { + field: TopNFunctionSortField.Rank, + name: i18n.translate('xpack.profiling.functionsView.rankColumnLabel', { + defaultMessage: 'Rank', + }), + align: 'right', + }, + { + field: TopNFunctionSortField.Frame, + name: i18n.translate('xpack.profiling.functionsView.functionColumnLabel', { + defaultMessage: 'Function', + }), + width: '100%', + render: (_, { frame }) => , + }, + { + field: TopNFunctionSortField.Samples, + name: i18n.translate('xpack.profiling.functionsView.samplesColumnLabel', { + defaultMessage: 'Samples', + }), + align: 'right', + }, + { + field: TopNFunctionSortField.ExclusiveCPU, + name: i18n.translate('xpack.profiling.functionsView.exclusiveCpuColumnLabel', { + defaultMessage: 'Exclusive CPU', + }), + render: (_, { exclusiveCPU, diff }) => { + return ; + }, + align: 'right', + }, + { + field: TopNFunctionSortField.InclusiveCPU, + name: i18n.translate('xpack.profiling.functionsView.inclusiveCpuColumnLabel', { + defaultMessage: 'Inclusive CPU', + }), + render: (_, { inclusiveCPU, diff }) => { + return ; + }, + align: 'right', + }, + ]; + + if (comparisonTopNFunctions) { + columns.push({ + field: TopNFunctionSortField.Diff, + name: i18n.translate('xpack.profiling.functionsView.diffColumnLabel', { + defaultMessage: 'Diff', + }), + align: 'right', + render: (_, { diff }) => { + if (!diff) { + return ( + + {i18n.translate('xpack.profiling.functionsView.newLabel', { defaultMessage: 'New' })} + + ); + } + + if (diff.rank === 0) { + return null; + } + + const color = diff.rank > 0 ? 'success' : 'danger'; + const icon = diff.rank > 0 ? 'sortDown' : 'sortUp'; + + return ( + + {diff.rank} + + ); + }, + }); + } + + const totalSampleCountLabel = i18n.translate( + 'xpack.profiling.functionsView.totalSampleCountLabel', + { + defaultMessage: 'Total sample count', + } + ); + + const sortedRows = orderBy( + rows, + (row) => { + return sortField === TopNFunctionSortField.Frame + ? getCalleeFunction(row.frame).toLowerCase() + : row[sortField]; + }, + [sortDirection] + ).slice(0, 100); + + return ( + <> + + {totalSampleCountLabel}: {totalCount} + + + + { + onSortChange({ + sortDirection: criteria.sort!.direction, + sortField: criteria.sort!.field as TopNFunctionSortField, + }); + }} + sorting={{ + enableAllColumns: true, + sort: { + direction: sortDirection, + field: sortField, + }, + }} + /> + + ); +}; diff --git a/x-pack/plugins/profiling/public/hooks/use_async.ts b/x-pack/plugins/profiling/public/hooks/use_async.ts new file mode 100644 index 000000000000..ea6da578c387 --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_async.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { HttpStart } from '@kbn/core-http-browser'; +import { useEffect, useState } from 'react'; +import { useProfilingDependencies } from '../components/contexts/profiling_dependencies/use_profiling_dependencies'; + +export enum AsyncStatus { + Loading = 'loading', + Init = 'init', + Settled = 'settled', +} + +export interface AsyncState { + data?: T; + error?: Error; + status: AsyncStatus; +} + +export type UseAsync = ( + fn: ({ http }: { http: HttpStart }) => Promise | undefined, + dependencies: any[] +) => AsyncState; + +export const useAsync: UseAsync = (fn, dependencies) => { + const { + start: { + core: { http }, + }, + } = useProfilingDependencies(); + const [asyncState, setAsyncState] = useState>({ + status: AsyncStatus.Init, + }); + + const { data, error } = asyncState; + + useEffect(() => { + const returnValue = fn({ http }); + + if (returnValue === undefined) { + setAsyncState({ + status: AsyncStatus.Init, + data: undefined, + error: undefined, + }); + return; + } + + setAsyncState({ + status: AsyncStatus.Loading, + data, + error, + }); + + returnValue.then((nextData) => { + setAsyncState({ + status: AsyncStatus.Settled, + data: nextData, + }); + }); + + returnValue.catch((nextError) => { + setAsyncState({ + status: AsyncStatus.Settled, + error: nextError, + }); + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [http, ...dependencies]); + + return asyncState; +}; diff --git a/x-pack/plugins/profiling/public/hooks/use_default_date_range_redirect.ts b/x-pack/plugins/profiling/public/hooks/use_default_date_range_redirect.ts new file mode 100644 index 000000000000..bb59b7d6b49f --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_default_date_range_redirect.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import qs from 'query-string'; +import { useHistory, useLocation } from 'react-router-dom'; +import { UI_SETTINGS } from '@kbn/data-plugin/public'; +import { useProfilingDependencies } from '../components/contexts/profiling_dependencies/use_profiling_dependencies'; + +export function useDateRangeRedirect() { + const history = useHistory(); + const location = useLocation(); + const query = qs.parse(location.search); + + const { + start: { core, data }, + } = useProfilingDependencies(); + + const timePickerTimeDefaults = core.uiSettings.get<{ from: string; to: string }>( + UI_SETTINGS.TIMEPICKER_TIME_DEFAULTS + ); + + const timePickerSharedState = data.query.timefilter.timefilter.getTime(); + + const isDateRangeSet = 'rangeFrom' in query && 'rangeTo' in query; + + const redirect = () => { + const nextQuery = { + rangeFrom: timePickerSharedState.from ?? timePickerTimeDefaults.from, + rangeTo: timePickerSharedState.to ?? timePickerTimeDefaults.to, + ...query, + }; + + history.replace({ + ...location, + search: qs.stringify(nextQuery), + }); + }; + + return { + isDateRangeSet, + redirect, + }; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_kibana_timezone_setting.ts b/x-pack/plugins/profiling/public/hooks/use_kibana_timezone_setting.ts new file mode 100644 index 000000000000..411b6013154c --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_kibana_timezone_setting.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useUiSetting$ } from '@kbn/kibana-react-plugin/public'; +import { UI_SETTINGS } from '@kbn/data-plugin/public'; + +export function useKibanaTimeZoneSetting() { + const [kibanaTimeZone] = useUiSetting$(UI_SETTINGS.DATEFORMAT_TZ); + + if (!kibanaTimeZone || kibanaTimeZone === 'Browser') { + return 'local'; + } + + return kibanaTimeZone; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_profiling_charts_theme.ts b/x-pack/plugins/profiling/public/hooks/use_profiling_charts_theme.ts new file mode 100644 index 000000000000..cd72d1ae3b4b --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_profiling_charts_theme.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { RecursivePartial, Theme } from '@elastic/charts'; +import { merge } from 'lodash'; +import { useProfilingDependencies } from '../components/contexts/profiling_dependencies/use_profiling_dependencies'; + +const profilingTheme: RecursivePartial = { + barSeriesStyle: { + rectBorder: { + strokeOpacity: 1, + strokeWidth: 1, + visible: true, + }, + rect: { + opacity: 0.6, + }, + }, + scales: { + barsPadding: 0, + histogramPadding: 0, + }, +}; + +export function useProfilingChartsTheme() { + const { + start: { charts }, + } = useProfilingDependencies(); + + const chartsBaseTheme = charts.theme.useChartsBaseTheme(); + const chartsTheme = charts.theme.useChartsTheme(); + + return { + chartsBaseTheme, + chartsTheme: merge({}, chartsTheme, profilingTheme), + }; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_profiling_params.ts b/x-pack/plugins/profiling/public/hooks/use_profiling_params.ts new file mode 100644 index 000000000000..7c4c1f8beaec --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_profiling_params.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { PathsOf, TypeOf, useParams } from '@kbn/typed-react-router-config'; +import { ValuesType } from 'utility-types'; +import { ProfilingRoutes } from '../routing'; + +export function useProfilingParams>( + path: T, + ...args: any[] +): TypeOf { + return useParams(path, ...args) as TypeOf; +} + +export function useAnyOfProfilingParams>>( + ...paths: TPaths +): TypeOf> { + return useParams(...paths)! as TypeOf>; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_profiling_route_path.ts b/x-pack/plugins/profiling/public/hooks/use_profiling_route_path.ts new file mode 100644 index 000000000000..91859984be45 --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_profiling_route_path.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PathsOf, useRoutePath } from '@kbn/typed-react-router-config'; +import { ProfilingRoutes } from '../routing'; + +export function useProfilingRoutePath(): PathsOf { + return useRoutePath() as PathsOf; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_profiling_router.ts b/x-pack/plugins/profiling/public/hooks/use_profiling_router.ts new file mode 100644 index 000000000000..0aa31af63dfa --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_profiling_router.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PathsOf, TypeOf, TypeAsArgs } from '@kbn/typed-react-router-config'; +import { useHistory } from 'react-router-dom'; +import { useProfilingDependencies } from '../components/contexts/profiling_dependencies/use_profiling_dependencies'; +import { ProfilingRouter, profilingRouter, ProfilingRoutes } from '../routing'; + +export interface StatefulProfilingRouter extends ProfilingRouter { + push>( + path: T, + ...params: TypeAsArgs> + ): void; + replace>( + path: T, + ...params: TypeAsArgs> + ): void; +} + +export function useProfilingRouter(): StatefulProfilingRouter { + const history = useHistory(); + + const { + start: { core }, + } = useProfilingDependencies(); + + const link = (...args: any[]) => { + // @ts-expect-error + return profilingRouter.link(...args); + }; + + return { + ...profilingRouter, + push: (...args) => { + const next = link(...args); + + history.push(next); + }, + replace: (path, ...args) => { + const next = link(path, ...args); + history.replace(next); + }, + link: (path, ...args) => { + return core.http.basePath.prepend('/app/profiling' + link(path, ...args)); + }, + }; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_time_range.ts b/x-pack/plugins/profiling/public/hooks/use_time_range.ts new file mode 100644 index 000000000000..32e1b2c2cfb7 --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_time_range.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import { TimeRange } from '../../common/types'; +import { getNextTimeRange } from '../utils/get_next_time_range'; +import { useTimeRangeContext } from './use_time_range_context'; + +interface TimeRangeAPI { + timeRangeId: string; +} + +type PartialTimeRange = Pick, 'start' | 'end'>; + +export function useTimeRange(range: { + rangeFrom?: string; + rangeTo?: string; + optional: true; +}): TimeRangeAPI & PartialTimeRange; + +export function useTimeRange(range: { + rangeFrom: string; + rangeTo: string; +}): TimeRangeAPI & TimeRange; + +export function useTimeRange({ + rangeFrom, + rangeTo, + optional, +}: { + rangeFrom?: string; + rangeTo?: string; + optional?: boolean; +}): TimeRangeAPI & (TimeRange | PartialTimeRange) { + const timeRangeApi = useTimeRangeContext(); + + const { start, end } = useMemo(() => { + return getNextTimeRange({ + state: {}, + rangeFrom, + rangeTo, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [rangeFrom, rangeTo, timeRangeApi.timeRangeId]); + + if ((!start || !end) && !optional) { + throw new Error('start and/or end were unexpectedly not set'); + } + + return { + start, + end, + timeRangeId: timeRangeApi.timeRangeId, + }; +} diff --git a/x-pack/plugins/profiling/public/hooks/use_time_range_async.ts b/x-pack/plugins/profiling/public/hooks/use_time_range_async.ts new file mode 100644 index 000000000000..2a4bffc639e5 --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_time_range_async.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { UseAsync, useAsync } from './use_async'; +import { useTimeRangeContext } from './use_time_range_context'; + +export const useTimeRangeAsync: UseAsync = (fn, dependencies) => { + const { timeRangeId } = useTimeRangeContext(); + // eslint-disable-next-line react-hooks/exhaustive-deps + return useAsync(fn, dependencies.concat(timeRangeId)); +}; diff --git a/x-pack/plugins/profiling/public/hooks/use_time_range_context.ts b/x-pack/plugins/profiling/public/hooks/use_time_range_context.ts new file mode 100644 index 000000000000..982f547eef21 --- /dev/null +++ b/x-pack/plugins/profiling/public/hooks/use_time_range_context.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useContext } from 'react'; +import { TimeRangeContext } from '../components/contexts/time_range_context'; + +export function useTimeRangeContext() { + const context = useContext(TimeRangeContext); + + if (!context) { + throw new Error('TimeRangeContext was not provided'); + } + + return context; +} diff --git a/x-pack/plugins/profiling/public/index.tsx b/x-pack/plugins/profiling/public/index.tsx new file mode 100644 index 000000000000..974fcc27397b --- /dev/null +++ b/x-pack/plugins/profiling/public/index.tsx @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ProfilingPlugin } from './plugin'; + +export function plugin() { + return new ProfilingPlugin(); +} diff --git a/x-pack/plugins/profiling/public/plugin.tsx b/x-pack/plugins/profiling/public/plugin.tsx new file mode 100644 index 000000000000..054a32da8efa --- /dev/null +++ b/x-pack/plugins/profiling/public/plugin.tsx @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + AppMountParameters, + CoreSetup, + CoreStart, + DEFAULT_APP_CATEGORIES, + Plugin, +} from '@kbn/core/public'; +import { i18n } from '@kbn/i18n'; +import type { NavigationSection } from '@kbn/observability-plugin/public'; +import { Location } from 'history'; +import { BehaviorSubject, combineLatest, from, map } from 'rxjs'; +import { getServices } from './services'; +import type { ProfilingPluginPublicSetupDeps, ProfilingPluginPublicStartDeps } from './types'; + +export class ProfilingPlugin implements Plugin { + public setup(coreSetup: CoreSetup, pluginsSetup: ProfilingPluginPublicSetupDeps) { + // Register an application into the side navigation menu + + const links = [ + { + id: 'stacktraces', + title: i18n.translate('xpack.profiling.navigation.stacktracesLinkLabel', { + defaultMessage: 'Stacktraces', + }), + path: '/stacktraces', + }, + { + id: 'flamegraphs', + title: i18n.translate('xpack.profiling.navigation.flameGraphsLinkLabel', { + defaultMessage: 'Flamegraphs', + }), + path: '/flamegraphs', + }, + { + id: 'functions', + title: i18n.translate('xpack.profiling.navigation.functionsLinkLabel', { + defaultMessage: 'Functions', + }), + path: '/functions', + }, + ]; + + const kuerySubject = new BehaviorSubject(''); + + const section$ = combineLatest([from(coreSetup.getStartServices()), kuerySubject]).pipe( + map(([_, kuery]) => { + const sections: NavigationSection[] = [ + { + // TODO: add beta badge to section label, needs support in Observability plugin + label: i18n.translate('xpack.profiling.navigation.sectionLabel', { + defaultMessage: 'Profiling', + }), + entries: links.map((link) => { + return { + app: 'profiling', + label: link.title, + path: `${link.path}?kuery=${kuery ?? ''}`, + matchPath: (path) => { + return path.startsWith(link.path); + }, + }; + }), + sortKey: 700, + }, + ]; + return sections; + }) + ); + + pluginsSetup.observability.navigation.registerSections(section$); + + coreSetup.application.register({ + id: 'profiling', + title: 'Profiling', + euiIconType: 'logoObservability', + appRoute: '/app/profiling', + category: DEFAULT_APP_CATEGORIES.observability, + deepLinks: links, + async mount({ element, history, theme$ }: AppMountParameters) { + const [coreStart, pluginsStart] = (await coreSetup.getStartServices()) as [ + CoreStart, + ProfilingPluginPublicStartDeps, + unknown + ]; + + const profilingFetchServices = getServices(coreStart); + const { renderApp } = await import('./app'); + + function pushKueryToSubject(location: Location) { + const query = new URLSearchParams(location.search); + kuerySubject.next(query.get('kuery') ?? ''); + } + + pushKueryToSubject(history.location); + + history.listen(pushKueryToSubject); + + const unmount = renderApp( + { + profilingFetchServices, + coreStart, + coreSetup, + pluginsStart, + pluginsSetup, + history, + theme$, + }, + element + ); + + return () => { + unmount(); + kuerySubject.next(''); + }; + }, + }); + } + + public start(core: CoreStart) { + return {}; + } + + public stop() {} +} diff --git a/x-pack/plugins/profiling/public/routing/index.tsx b/x-pack/plugins/profiling/public/routing/index.tsx new file mode 100644 index 000000000000..221ff10a5dad --- /dev/null +++ b/x-pack/plugins/profiling/public/routing/index.tsx @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { i18n } from '@kbn/i18n'; +import { toNumberRt } from '@kbn/io-ts-utils'; +import { createRouter, Outlet } from '@kbn/typed-react-router-config'; +import * as t from 'io-ts'; +import React from 'react'; +import { TopNFunctionSortField, topNFunctionSortFieldRt } from '../../common/functions'; +import { StackTracesDisplayOption, TopNType } from '../../common/stack_traces'; +import { FlameGraphComparisonMode } from '../../common/flamegraph'; +import { FlameGraphsView } from '../components/flame_graphs_view'; +import { FunctionsView } from '../components/functions_view'; +import { RedirectTo } from '../components/redirect_to'; +import { RouteBreadcrumb } from '../components/route_breadcrumb'; +import { StackTracesView } from '../components/stack_traces_view'; + +const routes = { + '/': { + element: ( + + + + ), + children: { + '/': { + children: { + '/stacktraces/{topNType}': { + element: , + params: t.type({ + path: t.type({ + topNType: t.union([ + t.literal(TopNType.Containers), + t.literal(TopNType.Deployments), + t.literal(TopNType.Hosts), + t.literal(TopNType.Threads), + t.literal(TopNType.Traces), + ]), + }), + query: t.type({ + displayAs: t.union([ + t.literal(StackTracesDisplayOption.StackTraces), + t.literal(StackTracesDisplayOption.Percentage), + ]), + limit: toNumberRt, + }), + }), + defaults: { + query: { + displayAs: StackTracesDisplayOption.StackTraces, + limit: '10', + }, + }, + }, + '/stacktraces': { + element: , + }, + '/flamegraphs': { + element: ( + + + + + + ), + children: { + '/flamegraphs/flamegraph': { + element: ( + + + + ), + }, + '/flamegraphs/differential': { + element: ( + + + + ), + params: t.type({ + query: t.type({ + comparisonRangeFrom: t.string, + comparisonRangeTo: t.string, + comparisonKuery: t.string, + comparisonMode: t.union([ + t.literal(FlameGraphComparisonMode.Absolute), + t.literal(FlameGraphComparisonMode.Relative), + ]), + }), + }), + defaults: { + query: { + comparisonMode: FlameGraphComparisonMode.Absolute, + }, + }, + }, + }, + }, + '/functions': { + element: ( + + + + + + ), + params: t.type({ + query: t.type({ + sortField: topNFunctionSortFieldRt, + sortDirection: t.union([t.literal('asc'), t.literal('desc')]), + }), + }), + defaults: { + query: { + sortField: TopNFunctionSortField.Rank, + sortDirection: 'asc', + }, + }, + children: { + '/functions/topn': { + element: ( + + + + ), + }, + '/functions/differential': { + element: ( + + + + ), + params: t.type({ + query: t.type({ + comparisonRangeFrom: t.string, + comparisonRangeTo: t.string, + comparisonKuery: t.string, + }), + }), + }, + }, + }, + '/': { + element: , + }, + }, + element: , + params: t.type({ + query: t.type({ + rangeFrom: t.string, + rangeTo: t.string, + kuery: t.string, + }), + }), + defaults: { + query: { + kuery: '', + }, + }, + }, + }, + }, +}; + +export const profilingRouter = createRouter(routes); +export type ProfilingRoutes = typeof routes; +export type ProfilingRouter = typeof profilingRouter; diff --git a/x-pack/plugins/profiling/public/services.ts b/x-pack/plugins/profiling/public/services.ts new file mode 100644 index 000000000000..07234ca124b3 --- /dev/null +++ b/x-pack/plugins/profiling/public/services.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart, HttpFetchQuery } from '@kbn/core/public'; +import { getRoutePaths } from '../common'; +import { ElasticFlameGraph } from '../common/flamegraph'; +import { TopNFunctions } from '../common/functions'; +import { StackFrameMetadata } from '../common/profiling'; +import { TopNResponse } from '../common/topn'; + +export interface Services { + fetchTopN: (params: { + type: string; + timeFrom: number; + timeTo: number; + kuery: string; + }) => Promise; + fetchTopNFunctions: (params: { + timeFrom: number; + timeTo: number; + startIndex: number; + endIndex: number; + kuery: string; + }) => Promise; + fetchElasticFlamechart: (params: { + timeFrom: number; + timeTo: number; + kuery: string; + }) => Promise; + fetchFrameInformation: (params: { + frameID: string; + executableID: string; + }) => Promise; +} + +export function getServices(core: CoreStart): Services { + const paths = getRoutePaths(); + + return { + fetchTopN: async ({ type, timeFrom, timeTo, kuery }) => { + try { + const query: HttpFetchQuery = { + timeFrom, + timeTo, + kuery, + }; + return await core.http.get(`${paths.TopN}/${type}`, { query }); + } catch (e) { + return e; + } + }, + + fetchTopNFunctions: async ({ + timeFrom, + timeTo, + startIndex, + endIndex, + kuery, + }: { + timeFrom: number; + timeTo: number; + startIndex: number; + endIndex: number; + kuery: string; + }) => { + try { + const query: HttpFetchQuery = { + timeFrom, + timeTo, + startIndex, + endIndex, + kuery, + }; + return await core.http.get(paths.TopNFunctions, { query }); + } catch (e) { + return e; + } + }, + + fetchElasticFlamechart: async ({ + timeFrom, + timeTo, + kuery, + }: { + timeFrom: number; + timeTo: number; + kuery: string; + }) => { + try { + const query: HttpFetchQuery = { + timeFrom, + timeTo, + kuery, + }; + return await core.http.get(paths.Flamechart, { query }); + } catch (e) { + return e; + } + }, + fetchFrameInformation: async ({ + frameID, + executableID, + }: { + frameID: string; + executableID: string; + }) => { + try { + const query: HttpFetchQuery = { + frameID, + executableID, + }; + return await core.http.get(paths.FrameInformation, { query }); + } catch (e) { + return e; + } + }, + }; +} diff --git a/x-pack/plugins/profiling/public/types.ts b/x-pack/plugins/profiling/public/types.ts new file mode 100644 index 000000000000..6c831e0b1350 --- /dev/null +++ b/x-pack/plugins/profiling/public/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; +import { + DataViewsPublicPluginSetup, + DataViewsPublicPluginStart, +} from '@kbn/data-views-plugin/public'; +import type { + ObservabilityPublicSetup, + ObservabilityPublicStart, +} from '@kbn/observability-plugin/public'; +import { ChartsPluginSetup, ChartsPluginStart } from '@kbn/charts-plugin/public'; + +export interface ProfilingPluginPublicSetupDeps { + observability: ObservabilityPublicSetup; + dataViews: DataViewsPublicPluginSetup; + data: DataPublicPluginSetup; + charts: ChartsPluginSetup; +} + +export interface ProfilingPluginPublicStartDeps { + observability: ObservabilityPublicStart; + dataViews: DataViewsPublicPluginStart; + data: DataPublicPluginStart; + charts: ChartsPluginStart; +} diff --git a/x-pack/plugins/profiling/public/utils/formatters/as_cost.ts b/x-pack/plugins/profiling/public/utils/formatters/as_cost.ts new file mode 100644 index 000000000000..148eba478526 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/formatters/as_cost.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function asCost(value: number, precision: number = 2, unit: string = '$') { + return `${value.toPrecision(precision)}${unit}`; +} diff --git a/x-pack/plugins/profiling/public/utils/formatters/as_duration.ts b/x-pack/plugins/profiling/public/utils/formatters/as_duration.ts new file mode 100644 index 000000000000..ba0839f06e77 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/formatters/as_duration.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +export function asDuration(valueInSeconds: number) { + return moment.duration(valueInSeconds * 1000).humanize(); +} diff --git a/x-pack/plugins/profiling/public/utils/formatters/as_percentage.ts b/x-pack/plugins/profiling/public/utils/formatters/as_percentage.ts new file mode 100644 index 000000000000..f4c3a84b6275 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/formatters/as_percentage.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function asPercentage(value: number, precision: number = 0) { + return `${Number(value * 100).toFixed(precision)}%`; +} diff --git a/x-pack/plugins/profiling/public/utils/formatters/as_weight.ts b/x-pack/plugins/profiling/public/utils/formatters/as_weight.ts new file mode 100644 index 000000000000..82a6cbd4f64b --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/formatters/as_weight.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +const ONE_POUND_TO_A_KILO = 0.45359237; + +export function asWeight(valueInPounds: number, precision: number = 2) { + const lbs = valueInPounds.toPrecision(precision); + const kgs = Number(valueInPounds * ONE_POUND_TO_A_KILO).toPrecision(precision); + + return i18n.translate('xpack.profiling.formatters.weight', { + defaultMessage: `{lbs} lbs / {kgs} kg`, + values: { + lbs, + kgs, + }, + }); +} diff --git a/x-pack/plugins/profiling/public/utils/get_flamegraph_model/get_interpolation_value.test.ts b/x-pack/plugins/profiling/public/utils/get_flamegraph_model/get_interpolation_value.test.ts new file mode 100644 index 000000000000..07b3e9ded923 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/get_flamegraph_model/get_interpolation_value.test.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getInterpolationValue } from './get_interpolation_value'; + +describe('getInterpolationValue', () => { + it('returns 0 for no change', () => { + expect(getInterpolationValue(100, 100)).toBe(0); + }); + + it('returns -1 when the background is undefined', () => { + expect(getInterpolationValue(100, undefined)).toBe(-1); + }); + + it('returns -1 when the background is 0', () => { + expect(getInterpolationValue(100, 0)).toBe(-1); + }); + + it('returns 0 when both values are 0', () => { + expect(getInterpolationValue(0, 0)).toBe(0); + }); + + it('returns the correct value on positive changes', () => { + expect(getInterpolationValue(100, 120)).toBeCloseTo(0.1); + expect(getInterpolationValue(80, 100)).toBeCloseTo(0.125); + + expect(getInterpolationValue(90, 270)).toBeCloseTo(1); + }); + + it('returns the correct value on negative changes', () => { + expect(getInterpolationValue(160, 120)).toBeCloseTo(-0.5); + expect(getInterpolationValue(150, 100)).toBeCloseTo(-2 / 3); + }); + + it('clamps the value', () => { + expect(getInterpolationValue(90, 360)).toBeCloseTo(1); + expect(getInterpolationValue(360, 90)).toBeCloseTo(-1); + }); +}); diff --git a/x-pack/plugins/profiling/public/utils/get_flamegraph_model/get_interpolation_value.ts b/x-pack/plugins/profiling/public/utils/get_flamegraph_model/get_interpolation_value.ts new file mode 100644 index 000000000000..66c325869cfd --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/get_flamegraph_model/get_interpolation_value.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { clamp } from 'lodash'; + +const MAX_POSITIVE_CHANGE = 2; +const MAX_NEGATIVE_CHANGE = 0.5; + +export function getInterpolationValue(foreground: number, background: number | null | undefined) { + if (background === null || background === undefined) { + return -1; + } + + const change = clamp(background / foreground - 1, -MAX_NEGATIVE_CHANGE, MAX_POSITIVE_CHANGE) || 0; + + return change >= 0 ? change / MAX_POSITIVE_CHANGE : change / MAX_NEGATIVE_CHANGE; +} diff --git a/x-pack/plugins/profiling/public/utils/get_flamegraph_model/index.ts b/x-pack/plugins/profiling/public/utils/get_flamegraph_model/index.ts new file mode 100644 index 000000000000..0b779c8ccac8 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/get_flamegraph_model/index.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ColumnarViewModel } from '@elastic/charts'; +import d3 from 'd3'; +import { uniqueId } from 'lodash'; +import { ElasticFlameGraph, FlameGraphComparisonMode, rgbToRGBA } from '../../../common/flamegraph'; +import { getInterpolationValue } from './get_interpolation_value'; + +const nullColumnarViewModel = { + label: [], + value: new Float64Array(), + color: new Float32Array(), + position0: new Float32Array(), + position1: new Float32Array(), + size0: new Float32Array(), + size1: new Float32Array(), +}; + +export function getFlamegraphModel({ + primaryFlamegraph, + comparisonFlamegraph, + colorSuccess, + colorDanger, + colorNeutral, + comparisonMode, +}: { + primaryFlamegraph?: ElasticFlameGraph; + comparisonFlamegraph?: ElasticFlameGraph; + colorSuccess: string; + colorDanger: string; + colorNeutral: string; + comparisonMode: FlameGraphComparisonMode; +}) { + const comparisonNodesById: Record< + string, + { Value: number; CountInclusive: number; CountExclusive: number } + > = {}; + + if (!primaryFlamegraph || !primaryFlamegraph.Label || primaryFlamegraph.Label.length === 0) { + return { key: uniqueId(), viewModel: nullColumnarViewModel, comparisonNodesById }; + } + + let colors: number[] | undefined = primaryFlamegraph.Color; + + if (comparisonFlamegraph) { + colors = []; + + comparisonFlamegraph.ID.forEach((nodeID, index) => { + comparisonNodesById[nodeID] = { + Value: comparisonFlamegraph.Value[index], + CountInclusive: comparisonFlamegraph.CountInclusive[index], + CountExclusive: comparisonFlamegraph.CountExclusive[index], + }; + }); + + const positiveChangeInterpolator = d3.interpolateRgb(colorNeutral, colorDanger); + + const negativeChangeInterpolator = d3.interpolateRgb(colorNeutral, colorSuccess); + + const comparisonExclusive: number[] = []; + const comparisonInclusive: number[] = []; + + primaryFlamegraph.ID.forEach((nodeID, index) => { + const countInclusive = primaryFlamegraph.CountInclusive[index]; + const countExclusive = primaryFlamegraph.CountExclusive[index]; + + const comparisonNode = comparisonNodesById[nodeID]; + + comparisonExclusive![index] = comparisonNode?.CountExclusive; + comparisonInclusive![index] = comparisonNode?.CountInclusive; + + const [foreground, background] = + comparisonMode === FlameGraphComparisonMode.Absolute + ? [countInclusive, comparisonNode?.CountInclusive] + : [countExclusive, comparisonNode?.CountExclusive]; + + const interpolationValue = getInterpolationValue(foreground, background); + + const nodeColor = + interpolationValue >= 0 + ? positiveChangeInterpolator(interpolationValue) + : negativeChangeInterpolator(Math.abs(interpolationValue)); + + colors!.push(...rgbToRGBA(Number(nodeColor.replace('#', '0x')))); + }); + } + + const value = new Float64Array(primaryFlamegraph.Value); + const position = new Float32Array(primaryFlamegraph.Position); + const size = new Float32Array(primaryFlamegraph.Size); + const color = new Float32Array(colors); + + return { + key: uniqueId(), + viewModel: { + label: primaryFlamegraph.Label, + value, + color, + position0: position, + position1: position, + size0: size, + size1: size, + } as ColumnarViewModel, + comparisonNodesById, + }; +} diff --git a/x-pack/plugins/profiling/public/utils/get_next_time_range/index.test.ts b/x-pack/plugins/profiling/public/utils/get_next_time_range/index.test.ts new file mode 100644 index 000000000000..92d2c5457874 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/get_next_time_range/index.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import datemath from '@elastic/datemath'; +import moment from 'moment'; +import { getNextTimeRange } from '.'; + +describe('getNextTimeRange', () => { + beforeEach(() => { + jest.restoreAllMocks(); + }); + describe('getDateRange', () => { + describe('when rangeFrom and rangeTo are not changed', () => { + it('returns the previous state', () => { + expect( + getNextTimeRange({ + state: { + rangeFrom: 'now-1m', + rangeTo: 'now', + start: '1970-01-01T00:00:00.000Z', + end: '1971-01-01T00:00:00.000Z', + }, + rangeFrom: 'now-1m', + rangeTo: 'now', + }) + ).toEqual({ + start: '1970-01-01T00:00:00.000Z', + end: '1971-01-01T00:00:00.000Z', + }); + }); + }); + + describe('when rangeFrom or rangeTo are falsy', () => { + it('returns the previous state', () => { + // Disable console warning about not receiving a valid date for rangeFrom + jest.spyOn(console, 'warn').mockImplementationOnce(() => {}); + + expect( + getNextTimeRange({ + state: { + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }, + rangeFrom: '', + rangeTo: 'now', + }) + ).toEqual({ + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }); + }); + }); + + describe('when the start or end are invalid', () => { + it('returns the previous state', () => { + const endDate = moment('2021-06-04T18:03:24.211Z'); + jest.spyOn(datemath, 'parse').mockReturnValueOnce(undefined).mockReturnValueOnce(endDate); + + expect( + getNextTimeRange({ + state: { + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }, + rangeFrom: 'nope', + rangeTo: 'now', + }) + ).toEqual({ + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }); + }); + }); + + describe('when rangeFrom or rangeTo have changed', () => { + it('returns new state', () => { + jest.spyOn(Date, 'now').mockReturnValue(moment(0).unix()); + + expect( + getNextTimeRange({ + state: { + rangeFrom: 'now-1m', + rangeTo: 'now', + start: '1972-01-01T00:00:00.000Z', + end: '1973-01-01T00:00:00.000Z', + }, + rangeFrom: 'now-2m', + rangeTo: 'now', + }) + ).toEqual({ + start: '1969-12-31T23:58:00.000Z', + end: '1970-01-01T00:00:00.000Z', + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/profiling/public/utils/get_next_time_range/index.ts b/x-pack/plugins/profiling/public/utils/get_next_time_range/index.ts new file mode 100644 index 000000000000..27414d2d9427 --- /dev/null +++ b/x-pack/plugins/profiling/public/utils/get_next_time_range/index.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import datemath from '@elastic/datemath'; + +function getParsedDate(rawDate?: string, options = {}) { + if (rawDate) { + const parsed = datemath.parse(rawDate, options); + if (parsed && parsed.isValid()) { + return parsed.toDate(); + } + } +} + +export function getNextTimeRange({ + state = {}, + rangeFrom, + rangeTo, +}: { + state?: { rangeFrom?: string; rangeTo?: string; start?: string; end?: string }; + rangeFrom?: string; + rangeTo?: string; +}) { + // If the previous state had the same range, just return that instead of calculating a new range. + if (state.rangeFrom === rangeFrom && state.rangeTo === rangeTo) { + return { + start: state.start, + end: state.end, + }; + } + const start = getParsedDate(rangeFrom); + const end = getParsedDate(rangeTo, { roundUp: true }); + + // `getParsedDate` will return undefined for invalid or empty dates. We return + // the previous state if either date is undefined. + if (!start || !end) { + return { + start: state.start, + end: state.end, + }; + } + + return { + start: start.toISOString(), + end: end.toISOString(), + }; +} diff --git a/x-pack/plugins/profiling/scripts/export_from_testing.sh b/x-pack/plugins/profiling/scripts/export_from_testing.sh new file mode 100755 index 000000000000..84c76415e335 --- /dev/null +++ b/x-pack/plugins/profiling/scripts/export_from_testing.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +# Exit on use of undefined variables and on command failures. +set -eu + +die() { + if [[ "$@" ]]; then + echo -e "$@\n" >&2 + fi + exit 1 +} + +usage() { + die "Usage: $0 [options]\n" \ + " --dateFrom Start of data, date format (e.g. \"2022-02-28T00:00:00Z\") or a unix timestamp\n" \ + " --dateTo End of data\n" \ + "\n" \ + "The data will be exported into the current working directory.\n" \ + "\n" \ + "You need to put the credentials into the ELASTIC_TESTING_CREDENTIALS env var (...=\"user:pw\")" +} + +get_timestamp() { + local date_input=$1 + + if [[ "${date_input}" =~ ^[0-9]+$ ]]; then + echo "${date_input}" + else + echo $(date +%s --date="${date_input}") + fi +} + +export_events() { + local index=$1 + local from=$2 + local to=$3 + + docker run --rm -ti --net=host -v "$PWD:/data" -w /data -e "NODE_OPTIONS=--max_old_space_size=8192" \ + elasticdump/elasticsearch-dump:latest \ + --input="https://""${ELASTIC_TESTING_CREDENTIALS}""@profiling-8-5-rc.es.us-west2.gcp.elastic-cloud.com/${index}" \ + --output="${index}-data_${from}_${to}.json.gz" \ + --type=data --fsCompress --noRefresh --limit=100000 --support-big-int \ + --searchBody=' +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "service.name": "922" + } + }, + { + "range": { + "@timestamp": { + "gte": "'"$from"'", + "lt": "'"$to"'", + "format": "epoch_second" + } + } + } + ] + } + } +}' +} + +export_index() { + local index=$1 + + docker run --rm -ti --net=host -v "$PWD:/data" -w /data -e "NODE_OPTIONS=--max_old_space_size=8192" \ + elasticdump/elasticsearch-dump:latest \ + --input="https://""${ELASTIC_TESTING_CREDENTIALS}""@profiling-8-5-rc.es.us-west2.gcp.elastic-cloud.com/${index}" \ + --output="${index}-data.json.gz" \ + --type=data --fsCompress --noRefresh --limit=100000 --support-big-int +} + +if [[ "$ELASTIC_TESTING_CREDENTIALS" == "" ]]; then + usage +fi + +date_from=0 +date_to=$(date +%s --date="now") + +while [[ $# -gt 0 ]]; do + case $1 in + --dateFrom) + date_from=$(get_timestamp "$2") + shift 2 + ;; + --dateTo) + date_to=$(get_timestamp "$2") + shift 2 + ;; + *) + usage + ;; + esac +done + +[[ "$date_to" -le "$date_from" ]] && die "Invalid time range" + +# Pull latest docker image +docker pull elasticdump/elasticsearch-dump:latest + +# Export full events table +export_events "profiling-events-all" "$date_from" "$date_to" + +# Export down-sampled tables +for ((i = 1; i <= 11; i++)); do + export_events "profiling-events-5pow$(printf '%02d' $i)" "$date_from" "$date_to" +done + +# We need all stacktraces, stackframes and executables as 'LastSeen' may be outside [date_from, date_to]. +export_index "profiling-stacktraces" +export_index "profiling-stackframes" +export_index "profiling-executables" + +echo -e ' + Import the data with ... + or upload to S3 with + for i in profiling-*; do aws s3 cp $i s3://oblt-profiling-es-snapshot/; done +' diff --git a/x-pack/plugins/profiling/scripts/import_from_testing.sh b/x-pack/plugins/profiling/scripts/import_from_testing.sh new file mode 100755 index 000000000000..f5848632f199 --- /dev/null +++ b/x-pack/plugins/profiling/scripts/import_from_testing.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Exit on use of undefined variables and on command failures. +set -eu + +die() { + if [[ "$@" ]]; then + echo -e "$@\n" >&2 + fi + exit 1 +} + +usage() { + die "Usage: $0 \n" \ + "\n" \ + "The data files are imported into the local ES instance (localhost:9200).\n" \ + "The naming convention is that the first part (before -data...) is the index name.\n" + "\n" \ + "You need to put the credentials into the ELASTIC_LOCAL_CREDENTIALS env var (...=\"user:pw\")" +} + +import() { + local index=$1 + local path=$2 + local dir=${path%$file} + + if [[ "$dir" == "" ]]; then + dir="." + fi + + pushd "$dir" + docker run --rm -ti --net=host -v "$PWD:/data" elasticdump/elasticsearch-dump:latest \ + --input="/data/$file" \ + --output="http://admin:changeme@localhost:9200/$index" \ + --type=data --fsCompress --noRefresh --support-big-int --limit=10000 + popd +} + +while [[ $# -gt 0 ]]; do + path=$1 + file=${1##*/} + index=${file%-data*} + + import "$index" "$path" + shift +done diff --git a/x-pack/plugins/profiling/server/feature.ts b/x-pack/plugins/profiling/server/feature.ts new file mode 100644 index 000000000000..86e9e70b55a9 --- /dev/null +++ b/x-pack/plugins/profiling/server/feature.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server'; + +export const PROFILING_SERVER_FEATURE_ID = 'profiling'; + +export const PROFILING_FEATURE = { + id: PROFILING_SERVER_FEATURE_ID, + name: i18n.translate('xpack.profiling.featureRegistry.profilingFeatureName', { + defaultMessage: 'Profiling', + }), + order: 1200, + category: DEFAULT_APP_CATEGORIES.observability, + app: ['kibana'], + catalogue: [], + // see x-pack/plugins/features/common/feature_kibana_privileges.ts + privileges: { + all: { + app: ['kibana'], + catalogue: [], + savedObject: { + all: [], + read: [], + }, + ui: ['show'], + }, + read: { + app: ['kibana'], + catalogue: [], + savedObject: { + all: [], + read: [], + }, + ui: [], + }, + }, +}; diff --git a/x-pack/plugins/profiling/server/index.ts b/x-pack/plugins/profiling/server/index.ts new file mode 100644 index 000000000000..26858b770b73 --- /dev/null +++ b/x-pack/plugins/profiling/server/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server'; +import { ProfilingPlugin } from './plugin'; + +const configSchema = schema.object({ + enabled: schema.boolean({ defaultValue: false }), +}); + +type ProfilingConfig = TypeOf; + +// plugin config +export const config: PluginConfigDescriptor = { + schema: configSchema, +}; + +// This exports static code and TypeScript types, +// as well as, Kibana Platform `plugin()` initializer. + +export function plugin(initializerContext: PluginInitializerContext) { + return new ProfilingPlugin(initializerContext); +} + +export type { ProfilingPluginSetup, ProfilingPluginStart } from './types'; diff --git a/x-pack/plugins/profiling/server/plugin.ts b/x-pack/plugins/profiling/server/plugin.ts new file mode 100644 index 000000000000..40071b54c0c4 --- /dev/null +++ b/x-pack/plugins/profiling/server/plugin.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server'; + +import { PROFILING_FEATURE } from './feature'; +import { registerRoutes } from './routes'; +import { + ProfilingPluginSetup, + ProfilingPluginSetupDeps, + ProfilingPluginStart, + ProfilingPluginStartDeps, + ProfilingRequestHandlerContext, +} from './types'; + +export class ProfilingPlugin + implements + Plugin< + ProfilingPluginSetup, + ProfilingPluginStart, + ProfilingPluginSetupDeps, + ProfilingPluginStartDeps + > +{ + private readonly logger: Logger; + + constructor(initializerContext: PluginInitializerContext) { + this.logger = initializerContext.logger.get(); + } + + public setup(core: CoreSetup, deps: ProfilingPluginSetupDeps) { + this.logger.debug('profiling: Setup'); + const router = core.http.createRouter(); + + deps.features.registerKibanaFeature(PROFILING_FEATURE); + + core.getStartServices().then(([_, depsStart]) => { + registerRoutes({ + router, + logger: this.logger!, + dependencies: { + start: depsStart, + setup: deps, + }, + }); + }); + + return {}; + } + + public start(core: CoreStart) { + this.logger.debug('profiling: Started'); + return {}; + } + + public stop() {} +} diff --git a/x-pack/plugins/profiling/server/routes/compat.ts b/x-pack/plugins/profiling/server/routes/compat.ts new file mode 100644 index 000000000000..6969e84bff6e --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/compat.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Code that works around incompatibilities between different +// versions of Kibana / ES. +// Currently, we work with 8.1 and 8.3 and thus this code only needs +// to address the incompatibilities between those two versions. + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { ProfilingRequestHandlerContext } from '../types'; + +export async function getClient( + context: ProfilingRequestHandlerContext +): Promise { + return (await context.core).elasticsearch.client.asCurrentUser; +} diff --git a/x-pack/plugins/profiling/server/routes/downsampling.ts b/x-pack/plugins/profiling/server/routes/downsampling.ts new file mode 100644 index 000000000000..e1280dd0903c --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/downsampling.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import seedrandom from 'seedrandom'; +import { StackTraceID } from '../../common/profiling'; +import { ProfilingESClient } from '../utils/create_profiling_es_client'; +import { ProjectTimeQuery } from './query'; + +export interface DownsampledEventsIndex { + name: string; + sampleRate: number; +} + +function getFullDownsampledIndex(index: string, pow: number, factor: number): string { + const downsampledIndexPrefix = index.replaceAll('-all', '') + '-' + factor + 'pow'; + return downsampledIndexPrefix + pow.toString().padStart(2, '0'); +} + +// Return the index that has between targetSampleSize..targetSampleSize*samplingFactor entries. +// The starting point is the number of entries from the profiling-events-5pow index. +// +// More details on how the down-sampling works can be found at the write path +// https://github.com/elastic/prodfiler/blob/bdcc2711c6cd7e89d63b58a17329fb9fdbabe008/pf-elastic-collector/elastic.go +export function getSampledTraceEventsIndex( + index: string, + targetSampleSize: number, + sampleCountFromInitialExp: number, + initialExp: number +): DownsampledEventsIndex { + const maxExp = 11; + const samplingFactor = 5; + const fullEventsIndex: DownsampledEventsIndex = { name: index, sampleRate: 1 }; + + if (sampleCountFromInitialExp === 0) { + // Take the shortcut to the full events index. + return fullEventsIndex; + } + + let pow = Math.floor( + initialExp - + Math.log((targetSampleSize * samplingFactor) / sampleCountFromInitialExp) / Math.log(5) + + 1 + ); + + if (pow < 1) { + return fullEventsIndex; + } + + if (pow > maxExp) { + pow = maxExp; + } + + return { + name: getFullDownsampledIndex(index, pow, samplingFactor), + sampleRate: 1 / samplingFactor ** pow, + }; +} + +export async function findDownsampledIndex({ + logger, + client, + index, + filter, + sampleSize, +}: { + logger: Logger; + client: ProfilingESClient; + index: string; + filter: ProjectTimeQuery; + sampleSize: number; +}): Promise { + // Start with counting the results in the index down-sampled by 5^6. + // That is in the middle of our down-sampled indexes. + const initialExp = 6; + let sampleCountFromInitialExp = 0; + try { + const resp = await client.search('find_downsampled_index', { + index: getFullDownsampledIndex(index, initialExp, 5), + body: { + query: filter, + size: 0, + track_total_hits: true, + }, + }); + sampleCountFromInitialExp = resp.hits.total.value; + } catch (e) { + logger.info(e.message); + } + + logger.info('sampleCountFromPow6 ' + sampleCountFromInitialExp); + return getSampledTraceEventsIndex(index, sampleSize, sampleCountFromInitialExp, initialExp); +} + +export function downsampleEventsRandomly( + stackTraceEvents: Map, + p: number, + seed: string +): number { + let totalCount = 0; + + // Make the RNG predictable to get reproducible results. + const random = seedrandom(seed); + + for (const [id, count] of stackTraceEvents) { + let newCount = 0; + for (let i = 0; i < count; i++) { + if (random() < p) { + newCount++; + } + } + if (newCount) { + stackTraceEvents.set(id, newCount); + totalCount += newCount; + } else { + stackTraceEvents.delete(id); + } + } + + return totalCount; +} diff --git a/x-pack/plugins/profiling/server/routes/flamechart.test.ts b/x-pack/plugins/profiling/server/routes/flamechart.test.ts new file mode 100644 index 000000000000..e4e527082aad --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/flamechart.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DownsampledEventsIndex, getSampledTraceEventsIndex } from './downsampling'; + +describe('Using down-sampled indexes', () => { + test('getSampledTraceEventsIndex', () => { + const targetSampleSize = 20000; + const initialExp = 6; + const tests: Array<{ + sampleCountFromPow6: number; + expected: DownsampledEventsIndex; + }> = [ + { + // stay with the input downsampled index + sampleCountFromPow6: targetSampleSize, + expected: { name: 'profiling-events-5pow06', sampleRate: 1 / 5 ** 6 }, + }, + { + // stay with the input downsampled index + sampleCountFromPow6: targetSampleSize * 5 - 1, + expected: { name: 'profiling-events-5pow06', sampleRate: 1 / 5 ** 6 }, + }, + { + // go down one downsampling step + sampleCountFromPow6: targetSampleSize * 5, + expected: { name: 'profiling-events-5pow07', sampleRate: 1 / 5 ** 7 }, + }, + { + // go up one downsampling step + sampleCountFromPow6: targetSampleSize - 1, + expected: { name: 'profiling-events-5pow05', sampleRate: 1 / 5 ** 5 }, + }, + { + // go to the full events index + sampleCountFromPow6: 0, + expected: { name: 'profiling-events-all', sampleRate: 1 }, + }, + { + // go to the most downsampled index + sampleCountFromPow6: targetSampleSize * 5 ** 8, + expected: { name: 'profiling-events-5pow11', sampleRate: 1 / 5 ** 11 }, + }, + ]; + + for (const t of tests) { + expect( + getSampledTraceEventsIndex( + 'profiling-events-all', + targetSampleSize, + t.sampleCountFromPow6, + initialExp + ) + ).toEqual(t.expected); + } + }); +}); diff --git a/x-pack/plugins/profiling/server/routes/flamechart.ts b/x-pack/plugins/profiling/server/routes/flamechart.ts new file mode 100644 index 000000000000..4c27d87ea39e --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/flamechart.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { RouteRegisterParameters } from '.'; +import { getRoutePaths } from '../../common'; +import { FlameGraph } from '../../common/flamegraph'; +import { createProfilingEsClient } from '../utils/create_profiling_es_client'; +import { withProfilingSpan } from '../utils/with_profiling_span'; +import { getClient } from './compat'; +import { getExecutablesAndStackTraces } from './get_executables_and_stacktraces'; +import { createCommonFilter } from './query'; + +export function registerFlameChartSearchRoute({ router, logger }: RouteRegisterParameters) { + const paths = getRoutePaths(); + router.get( + { + path: paths.Flamechart, + validate: { + query: schema.object({ + timeFrom: schema.number(), + timeTo: schema.number(), + kuery: schema.string(), + }), + }, + }, + async (context, request, response) => { + const { timeFrom, timeTo, kuery } = request.query; + const targetSampleSize = 20000; // minimum number of samples to get statistically sound results + + try { + const esClient = await getClient(context); + const filter = createCommonFilter({ + timeFrom, + timeTo, + kuery, + }); + + const { stackTraces, executables, stackFrames, eventsIndex, totalCount, stackTraceEvents } = + await getExecutablesAndStackTraces({ + logger, + client: createProfilingEsClient({ request, esClient }), + filter, + sampleSize: targetSampleSize, + }); + + const flamegraph = await withProfilingSpan('collect_flamegraph', async () => { + return new FlameGraph({ + sampleRate: eventsIndex.sampleRate, + totalCount, + events: stackTraceEvents, + stackTraces, + stackFrames, + executables, + totalSeconds: timeTo - timeFrom, + }).toElastic(); + }); + + logger.info('returning payload response to client'); + + return response.ok({ + body: flamegraph, + }); + } catch (e) { + logger.error(e); + return response.customError({ + statusCode: e.statusCode ?? 500, + body: { + message: e.message, + }, + }); + } + } + ); +} diff --git a/x-pack/plugins/profiling/server/routes/frames.ts b/x-pack/plugins/profiling/server/routes/frames.ts new file mode 100644 index 000000000000..4a0ce745c724 --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/frames.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { Logger } from '@kbn/logging'; +import { RouteRegisterParameters } from '.'; +import { getRoutePaths } from '../../common'; +import { + createStackFrameMetadata, + Executable, + StackFrame, + StackFrameMetadata, +} from '../../common/profiling'; +import { createProfilingEsClient, ProfilingESClient } from '../utils/create_profiling_es_client'; +import { mgetStackFrames, mgetExecutables } from './stacktrace'; + +async function getFrameInformation({ + frameID, + executableID, + logger, + client, +}: { + frameID: string; + executableID: string; + logger: Logger; + client: ProfilingESClient; +}): Promise { + const [stackFrames, executables] = await Promise.all([ + mgetStackFrames({ + logger, + client, + stackFrameIDs: new Set([frameID]), + }), + mgetExecutables({ + logger, + client, + executableIDs: new Set([executableID]), + }), + ]); + + const frame = Array.from(stackFrames.values())[0] as StackFrame | undefined; + const executable = Array.from(executables.values())[0] as Executable | undefined; + + if (frame) { + return createStackFrameMetadata({ + FrameID: frameID, + FileID: executableID, + SourceFilename: frame.FileName, + FunctionName: frame.FunctionName, + ExeFileName: executable?.FileName, + }); + } +} + +export function registerFrameInformationRoute(params: RouteRegisterParameters) { + const { logger, router } = params; + + const routePaths = getRoutePaths(); + + router.get( + { + path: routePaths.FrameInformation, + validate: { + query: schema.object({ + frameID: schema.string(), + executableID: schema.string(), + }), + }, + }, + async (context, request, response) => { + const { frameID, executableID } = request.query; + + const client = createProfilingEsClient({ + request, + esClient: (await context.core).elasticsearch.client.asCurrentUser, + }); + + try { + const frame = await getFrameInformation({ + frameID, + executableID, + logger, + client, + }); + + return response.ok({ body: frame }); + } catch (error: any) { + logger.error(error); + return response.custom({ + statusCode: error.statusCode ?? 500, + body: { + message: error.message ?? 'An internal server error occured', + }, + }); + } + } + ); +} diff --git a/x-pack/plugins/profiling/server/routes/functions.ts b/x-pack/plugins/profiling/server/routes/functions.ts new file mode 100644 index 000000000000..dffc56e8aee7 --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/functions.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import { RouteRegisterParameters } from '.'; +import { getRoutePaths } from '../../common'; +import { createTopNFunctions } from '../../common/functions'; +import { createProfilingEsClient } from '../utils/create_profiling_es_client'; +import { withProfilingSpan } from '../utils/with_profiling_span'; +import { getClient } from './compat'; +import { getExecutablesAndStackTraces } from './get_executables_and_stacktraces'; +import { createCommonFilter } from './query'; + +const querySchema = schema.object({ + timeFrom: schema.number(), + timeTo: schema.number(), + startIndex: schema.number(), + endIndex: schema.number(), + kuery: schema.string(), +}); + +type QuerySchemaType = TypeOf; + +export function registerTopNFunctionsSearchRoute({ router, logger }: RouteRegisterParameters) { + const paths = getRoutePaths(); + router.get( + { + path: paths.TopNFunctions, + validate: { + query: querySchema, + }, + }, + async (context, request, response) => { + try { + const { timeFrom, timeTo, startIndex, endIndex, kuery }: QuerySchemaType = request.query; + + const targetSampleSize = 20000; // minimum number of samples to get statistically sound results + const esClient = await getClient(context); + const filter = createCommonFilter({ + timeFrom, + timeTo, + kuery, + }); + + const { stackFrames, stackTraceEvents, stackTraces, executables } = + await getExecutablesAndStackTraces({ + client: createProfilingEsClient({ request, esClient }), + filter, + logger, + sampleSize: targetSampleSize, + }); + + const topNFunctions = await withProfilingSpan('collect_topn_functions', async () => { + return createTopNFunctions( + stackTraceEvents, + stackTraces, + stackFrames, + executables, + startIndex, + endIndex + ); + }); + + logger.info('returning payload response to client'); + + return response.ok({ + body: topNFunctions, + }); + } catch (e) { + logger.error(e); + return response.customError({ + statusCode: e.statusCode ?? 500, + body: { + message: e.message, + }, + }); + } + } + ); +} diff --git a/x-pack/plugins/profiling/server/routes/get_executables_and_stacktraces.ts b/x-pack/plugins/profiling/server/routes/get_executables_and_stacktraces.ts new file mode 100644 index 000000000000..9e5773ad9aa9 --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/get_executables_and_stacktraces.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from '@kbn/logging'; +import { INDEX_EVENTS } from '../../common'; +import { ProfilingESClient } from '../utils/create_profiling_es_client'; +import { withProfilingSpan } from '../utils/with_profiling_span'; +import { downsampleEventsRandomly, findDownsampledIndex } from './downsampling'; +import { logExecutionLatency } from './logger'; +import { ProjectTimeQuery } from './query'; +import { + mgetExecutables, + mgetStackFrames, + mgetStackTraces, + searchEventsGroupByStackTrace, +} from './stacktrace'; + +export async function getExecutablesAndStackTraces({ + logger, + client, + filter, + sampleSize, +}: { + logger: Logger; + client: ProfilingESClient; + filter: ProjectTimeQuery; + sampleSize: number; +}) { + return withProfilingSpan('get_executables_and_stack_traces', async () => { + const eventsIndex = await findDownsampledIndex({ + logger, + client, + index: INDEX_EVENTS, + filter, + sampleSize, + }); + + const { totalCount, stackTraceEvents } = await searchEventsGroupByStackTrace({ + logger, + client, + index: eventsIndex, + filter, + }); + + // Manual downsampling if totalCount exceeds sampleSize by 10%. + let p = 1.0; + if (totalCount > sampleSize * 1.1) { + p = sampleSize / totalCount; + logger.info('downsampling events with p=' + p); + await logExecutionLatency(logger, 'downsampling events', async () => { + const downsampledTotalCount = downsampleEventsRandomly( + stackTraceEvents, + p, + filter.toString() + ); + logger.info('downsampled total count: ' + downsampledTotalCount); + }); + logger.info('unique downsampled stacktraces: ' + stackTraceEvents.size); + } + + // Adjust the sample counts from down-sampled to fully sampled. + // Be aware that downsampling drops entries from stackTraceEvents, so that + // the sum of the upscaled count values is less that totalCount. + for (const [id, count] of stackTraceEvents) { + stackTraceEvents.set(id, Math.floor(count / (eventsIndex.sampleRate * p))); + } + + const { stackTraces, stackFrameDocIDs, executableDocIDs } = await mgetStackTraces({ + logger, + client, + events: stackTraceEvents, + }); + + return withProfilingSpan('get_stackframes_and_executables', () => + Promise.all([ + mgetStackFrames({ logger, client, stackFrameIDs: stackFrameDocIDs }), + mgetExecutables({ logger, client, executableIDs: executableDocIDs }), + ]) + ).then(([stackFrames, executables]) => { + return { + stackTraces, + executables, + stackFrames, + stackTraceEvents, + totalCount, + eventsIndex, + }; + }); + }); +} diff --git a/x-pack/plugins/profiling/server/routes/index.ts b/x-pack/plugins/profiling/server/routes/index.ts new file mode 100644 index 000000000000..6e44bf690958 --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/index.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter, Logger } from '@kbn/core/server'; +import { + ProfilingPluginSetupDeps, + ProfilingPluginStartDeps, + ProfilingRequestHandlerContext, +} from '../types'; + +import { registerFlameChartSearchRoute } from './flamechart'; +import { registerFrameInformationRoute } from './frames'; +import { registerTopNFunctionsSearchRoute } from './functions'; + +import { + registerTraceEventsTopNContainersSearchRoute, + registerTraceEventsTopNDeploymentsSearchRoute, + registerTraceEventsTopNHostsSearchRoute, + registerTraceEventsTopNStackTracesSearchRoute, + registerTraceEventsTopNThreadsSearchRoute, +} from './topn'; + +export interface RouteRegisterParameters { + router: IRouter; + logger: Logger; + dependencies: { + start: ProfilingPluginStartDeps; + setup: ProfilingPluginSetupDeps; + }; +} + +export function registerRoutes(params: RouteRegisterParameters) { + registerFlameChartSearchRoute(params); + registerTopNFunctionsSearchRoute(params); + registerTraceEventsTopNContainersSearchRoute(params); + registerTraceEventsTopNDeploymentsSearchRoute(params); + registerTraceEventsTopNHostsSearchRoute(params); + registerTraceEventsTopNStackTracesSearchRoute(params); + registerTraceEventsTopNThreadsSearchRoute(params); + registerFrameInformationRoute(params); +} diff --git a/x-pack/plugins/profiling/server/routes/logger.ts b/x-pack/plugins/profiling/server/routes/logger.ts new file mode 100644 index 000000000000..8d4c82962fae --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/logger.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; + +export async function logExecutionLatency( + logger: Logger, + activity: string, + func: () => Promise +): Promise { + const start = new Date().getTime(); + return await func().then((res) => { + logger.info(activity + ' took ' + (new Date().getTime() - start) + 'ms'); + return res; + }); +} diff --git a/x-pack/plugins/profiling/server/routes/query.ts b/x-pack/plugins/profiling/server/routes/query.ts new file mode 100644 index 000000000000..f8a776ee68ce --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/query.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslBoolQuery } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { kqlQuery } from '@kbn/observability-plugin/server'; +import { ProfilingESField } from '../../common/elasticsearch'; + +export interface ProjectTimeQuery { + bool: QueryDslBoolQuery; +} + +export function createCommonFilter({ + kuery, + timeFrom, + timeTo, +}: { + kuery: string; + timeFrom: number; + timeTo: number; +}): ProjectTimeQuery { + return { + bool: { + filter: [ + ...kqlQuery(kuery), + { + range: { + [ProfilingESField.Timestamp]: { + gte: String(timeFrom), + lt: String(timeTo), + format: 'epoch_second', + boost: 1.0, + }, + }, + }, + ], + }, + }; +} diff --git a/x-pack/plugins/profiling/server/routes/stacktrace.test.ts b/x-pack/plugins/profiling/server/routes/stacktrace.test.ts new file mode 100644 index 000000000000..54a9f7e15fc6 --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/stacktrace.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createStackFrameID, StackTrace } from '../../common/profiling'; +import { + decodeStackTrace, + EncodedStackTrace, + runLengthDecode, + runLengthEncode, +} from './stacktrace'; + +enum fileID { + A = 'aQpJmTLWydNvOapSFZOwKg', + B = 'hz_u-HGyrN6qeIk6UIJeCA', + C = 'AJ8qrcXSoJbl_haPhlc4og', + D = 'lHZiv7a58px6Gumcpo-6yA', + E = 'fkbxUTZgljnk71ZMnqJnyA', + F = 'gnEsgxvvEODj6iFYMQWYlA', +} + +enum addressOrLine { + A = 515512, + B = 26278522, + C = 6712518, + D = 105806025, + E = 111, + F = 106182663, + G = 100965370, +} + +const frameID: Record = { + A: createStackFrameID(fileID.A, addressOrLine.A), + B: createStackFrameID(fileID.B, addressOrLine.B), + C: createStackFrameID(fileID.C, addressOrLine.C), + D: createStackFrameID(fileID.D, addressOrLine.D), + E: createStackFrameID(fileID.E, addressOrLine.E), + F: createStackFrameID(fileID.F, addressOrLine.F), + G: createStackFrameID(fileID.F, addressOrLine.G), +}; + +const frameTypeA = [0, 0, 0]; +const frameTypeB = [8, 8, 8, 8]; + +describe('Stack trace operations', () => { + test('decodeStackTrace', () => { + const tests: Array<{ + original: EncodedStackTrace; + expected: StackTrace; + }> = [ + { + original: { + Stacktrace: { + frame: { + ids: frameID.A + frameID.B + frameID.C, + types: runLengthEncode(frameTypeA).toString('base64url'), + }, + }, + } as EncodedStackTrace, + expected: { + FrameIDs: [frameID.A, frameID.B, frameID.C], + FileIDs: [fileID.A, fileID.B, fileID.C], + AddressOrLines: [addressOrLine.A, addressOrLine.B, addressOrLine.C], + Types: frameTypeA, + } as StackTrace, + }, + { + original: { + Stacktrace: { + frame: { + ids: frameID.D + frameID.E + frameID.F + frameID.G, + types: runLengthEncode(frameTypeB).toString('base64url'), + }, + }, + } as EncodedStackTrace, + expected: { + FrameIDs: [frameID.D, frameID.E, frameID.F, frameID.G], + FileIDs: [fileID.D, fileID.E, fileID.F, fileID.F], + AddressOrLines: [addressOrLine.D, addressOrLine.E, addressOrLine.F, addressOrLine.G], + Types: frameTypeB, + } as StackTrace, + }, + ]; + + for (const t of tests) { + expect(decodeStackTrace(t.original)).toEqual(t.expected); + } + }); + + test('run length is fully reversible', () => { + const tests: number[][] = [[], [0], [0, 1, 2, 3], [0, 1, 1, 2, 2, 2, 3, 3, 3, 3]]; + + for (const t of tests) { + expect(runLengthDecode(runLengthEncode(t))).toEqual(t); + } + }); + + test('runLengthDecodeReverse with optional parameter', () => { + const tests: Array<{ + bytes: Buffer; + expected: number[]; + }> = [ + { + bytes: Buffer.from([0x5, 0x0, 0x2, 0x2]), + expected: [0, 0, 0, 0, 0, 2, 2], + }, + { + bytes: Buffer.from([0x1, 0x8]), + expected: [8], + }, + ]; + + for (const t of tests) { + expect(runLengthDecode(t.bytes, t.expected.length)).toEqual(t.expected); + } + }); + + test('runLengthDecodeReverse without optional parameter', () => { + const tests: Array<{ + bytes: Buffer; + expected: number[]; + }> = [ + { + bytes: Buffer.from([0x5, 0x0, 0x2, 0x2]), + expected: [0, 0, 0, 0, 0, 2, 2], + }, + { + bytes: Buffer.from([0x1, 0x8]), + expected: [8], + }, + ]; + + for (const t of tests) { + expect(runLengthDecode(t.bytes)).toEqual(t.expected); + } + }); +}); diff --git a/x-pack/plugins/profiling/server/routes/stacktrace.ts b/x-pack/plugins/profiling/server/routes/stacktrace.ts new file mode 100644 index 000000000000..ecf51313695f --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/stacktrace.ts @@ -0,0 +1,417 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { chunk } from 'lodash'; +import LRUCache from 'lru-cache'; +import { INDEX_EXECUTABLES, INDEX_FRAMES, INDEX_TRACES } from '../../common'; +import { + DedotObject, + PickFlattened, + ProfilingESField, + ProfilingExecutable, + ProfilingStackFrame, + ProfilingStackTrace, +} from '../../common/elasticsearch'; +import { + Executable, + FileID, + StackFrame, + StackFrameID, + StackTrace, + StackTraceID, +} from '../../common/profiling'; +import { ProfilingESClient } from '../utils/create_profiling_es_client'; +import { withProfilingSpan } from '../utils/with_profiling_span'; +import { DownsampledEventsIndex } from './downsampling'; +import { logExecutionLatency } from './logger'; +import { ProjectTimeQuery } from './query'; + +const traceLRU = new LRUCache({ max: 20000 }); + +const BASE64_FRAME_ID_LENGTH = 32; + +export type EncodedStackTrace = DedotObject<{ + // This field is a base64-encoded byte string. The string represents a + // serialized list of frame IDs in which the order of frames are + // reversed to allow for prefix compression (leaf frame last). Each + // frame ID is composed of two concatenated values: a 16-byte file ID + // and an 8-byte address or line number (depending on the context of + // the downstream reader). + // + // Frame ID #1 Frame ID #2 + // +----------------+--------+----------------+--------+---- + // | File ID | Addr | File ID | Addr | + // +----------------+--------+----------------+--------+---- + [ProfilingESField.StacktraceFrameIDs]: string; + + // This field is a run-length encoding of a list of uint8s. The order is + // reversed from the original input. + [ProfilingESField.StacktraceFrameTypes]: string; +}>; + +// runLengthEncode run-length encodes the input array. +// +// The input is a list of uint8s. The output is a binary stream of +// 2-byte pairs (first byte is the length and the second byte is the +// binary representation of the object) in reverse order. +// +// E.g. uint8 array [0, 0, 0, 0, 0, 2, 2, 2] is converted into the byte +// array [5, 0, 3, 2]. +export function runLengthEncode(input: number[]): Buffer { + const output: number[] = []; + + if (input.length === 0) { + return Buffer.from(output); + } + + let count = 0; + let current = input[0]; + + for (let i = 1; i < input.length; i++) { + const next = input[i]; + + if (next === current && count < 255) { + count++; + continue; + } + + output.push(count + 1, current); + + count = 0; + current = next; + } + + output.push(count + 1, current); + + return Buffer.from(output); +} + +// runLengthDecode decodes a run-length encoding for the input array. +// +// The input is a binary stream of 2-byte pairs (first byte is the length and the +// second byte is the binary representation of the object). The output is a list of +// uint8s. +// +// E.g. byte array [5, 0, 3, 2] is converted into an uint8 array like +// [0, 0, 0, 0, 0, 2, 2, 2]. +export function runLengthDecode(input: Buffer, outputSize?: number): number[] { + let size; + + if (typeof outputSize === 'undefined') { + size = 0; + for (let i = 0; i < input.length; i += 2) { + size += input[i]; + } + } else { + size = outputSize; + } + + const output: number[] = new Array(size); + + let idx = 0; + for (let i = 0; i < input.length; i += 2) { + for (let j = 0; j < input[i]; j++) { + output[idx] = input[i + 1]; + idx++; + } + } + + return output; +} + +// decodeStackTrace unpacks an encoded stack trace from Elasticsearch +export function decodeStackTrace(input: EncodedStackTrace): StackTrace { + const inputFrameIDs = input.Stacktrace.frame.ids; + const inputFrameTypes = input.Stacktrace.frame.types; + const countsFrameIDs = inputFrameIDs.length / BASE64_FRAME_ID_LENGTH; + + const fileIDs: string[] = new Array(countsFrameIDs); + const frameIDs: string[] = new Array(countsFrameIDs); + const addressOrLines: number[] = new Array(countsFrameIDs); + + // Step 1: Convert the base64-encoded frameID list into two separate + // lists (frame IDs and file IDs), both of which are also base64-encoded. + // + // To get the frame ID, we grab the next 32 bytes. + // + // To get the file ID, we grab the first 22 bytes of the frame ID. + // However, since the file ID is base64-encoded using 21.33 bytes + // (16 * 4 / 3), then the 22 bytes have an extra 4 bits from the + // address (see diagram in definition of EncodedStackTrace). + for (let i = 0; i < countsFrameIDs; i++) { + const pos = i * BASE64_FRAME_ID_LENGTH; + const frameID = inputFrameIDs.slice(pos, pos + BASE64_FRAME_ID_LENGTH); + const buf = Buffer.from(frameID, 'base64url'); + + fileIDs[i] = buf.toString('base64url', 0, 16); + addressOrLines[i] = Number(buf.readBigUInt64BE(16)); + frameIDs[i] = frameID; + } + + // Step 2: Convert the run-length byte encoding into a list of uint8s. + const types = Buffer.from(inputFrameTypes, 'base64url'); + const typeIDs = runLengthDecode(types, countsFrameIDs); + + return { + AddressOrLines: addressOrLines, + FileIDs: fileIDs, + FrameIDs: frameIDs, + Types: typeIDs, + } as StackTrace; +} + +export async function searchEventsGroupByStackTrace({ + logger, + client, + index, + filter, +}: { + logger: Logger; + client: ProfilingESClient; + index: DownsampledEventsIndex; + filter: ProjectTimeQuery; +}) { + const resEvents = await client.search('get_events_group_by_stack_trace', { + index: index.name, + track_total_hits: false, + query: filter, + aggs: { + group_by: { + terms: { + // 'size' should be max 100k, but might be slightly more. Better be on the safe side. + size: 150000, + field: ProfilingESField.StacktraceID, + // 'execution_hint: map' skips the slow building of ordinals that we don't need. + // Especially with high cardinality fields, this makes aggregations really slow. + // E.g. it reduces the latency from 70s to 0.7s on our 8.1. MVP cluster (as of 28.04.2022). + execution_hint: 'map', + }, + aggs: { + count: { + sum: { + field: ProfilingESField.StacktraceCount, + }, + }, + }, + }, + total_count: { + sum: { + field: ProfilingESField.StacktraceCount, + }, + }, + }, + pre_filter_shard_size: 1, + filter_path: + 'aggregations.group_by.buckets.key,aggregations.group_by.buckets.count,aggregations.total_count,_shards.failures', + }); + + const totalCount = resEvents.aggregations?.total_count.value ?? 0; + const stackTraceEvents = new Map(); + + resEvents.aggregations?.group_by?.buckets.forEach((item) => { + const traceid: StackTraceID = String(item.key); + stackTraceEvents.set(traceid, item.count.value ?? 0); + }); + + logger.info('events total count: ' + totalCount); + logger.info('unique stacktraces: ' + stackTraceEvents.size); + + return { totalCount, stackTraceEvents }; +} + +export async function mgetStackTraces({ + logger, + client, + events, + concurrency = 1, +}: { + logger: Logger; + client: ProfilingESClient; + events: Map; + concurrency?: number; +}) { + const stackTraceIDs = [...events.keys()]; + const chunkSize = Math.floor(events.size / concurrency); + let chunks = chunk(stackTraceIDs, chunkSize); + + if (chunks.length !== concurrency) { + // The last array element contains the remainder, just drop it as irrelevant. + chunks = chunks.slice(0, concurrency); + } + + const stackResponses = await withProfilingSpan('mget_stacktraces', () => + Promise.all( + chunks.map((ids) => { + return client.mget< + PickFlattened< + ProfilingStackTrace, + ProfilingESField.StacktraceFrameIDs | ProfilingESField.StacktraceFrameTypes + > + >('mget_stacktraces_chunk', { + index: INDEX_TRACES, + ids, + realtime: true, + _source_includes: [ + ProfilingESField.StacktraceFrameIDs, + ProfilingESField.StacktraceFrameTypes, + ], + }); + }) + ) + ); + + let totalFrames = 0; + const stackTraces = new Map(); + const stackFrameDocIDs = new Set(); + const executableDocIDs = new Set(); + + await logExecutionLatency(logger, 'processing data', async () => { + // flatMap() is significantly slower than an explicit for loop + for (const res of stackResponses) { + for (const trace of res.docs) { + if ('error' in trace) { + continue; + } + // Sometimes we don't find the trace. + // This is due to ES delays writing (data is not immediately seen after write). + // Also, ES doesn't know about transactions. + if (trace.found) { + const traceid = trace._id as StackTraceID; + let stackTrace = traceLRU.get(traceid) as StackTrace; + if (!stackTrace) { + stackTrace = decodeStackTrace(trace._source as EncodedStackTrace); + traceLRU.set(traceid, stackTrace); + } + + totalFrames += stackTrace.FrameIDs.length; + stackTraces.set(traceid, stackTrace); + for (const frameID of stackTrace.FrameIDs) { + stackFrameDocIDs.add(frameID); + } + for (const fileID of stackTrace.FileIDs) { + executableDocIDs.add(fileID); + } + } + } + } + }); + + if (stackTraces.size !== 0) { + logger.info('Average size of stacktrace: ' + totalFrames / stackTraces.size); + } + + if (stackTraces.size < events.size) { + logger.info( + 'failed to find ' + (events.size - stackTraces.size) + ' stacktraces (todo: find out why)' + ); + } + + return { stackTraces, stackFrameDocIDs, executableDocIDs }; +} + +export async function mgetStackFrames({ + logger, + client, + stackFrameIDs, +}: { + logger: Logger; + client: ProfilingESClient; + stackFrameIDs: Set; +}): Promise> { + const stackFrames = new Map(); + + if (stackFrameIDs.size === 0) { + return stackFrames; + } + + const resStackFrames = await client.mget('mget_stackframes', { + index: INDEX_FRAMES, + ids: [...stackFrameIDs], + realtime: true, + }); + + // Create a lookup map StackFrameID -> StackFrame. + let framesFound = 0; + await logExecutionLatency(logger, 'processing data', async () => { + const docs = resStackFrames.docs; + for (const frame of docs) { + if ('error' in frame) { + continue; + } + if (frame.found) { + stackFrames.set(frame._id, { + FileName: frame._source!.Stackframe.file?.name, + FunctionName: frame._source!.Stackframe.function?.name, + FunctionOffset: frame._source!.Stackframe.function?.offset, + LineNumber: frame._source!.Stackframe.line?.number, + SourceType: frame._source!.Stackframe.source?.type, + }); + framesFound++; + } else { + stackFrames.set(frame._id, { + FileName: '', + FunctionName: '', + FunctionOffset: 0, + LineNumber: 0, + SourceType: 0, + }); + } + } + }); + + logger.info('found ' + framesFound + ' / ' + stackFrameIDs.size + ' frames'); + + return stackFrames; +} + +export async function mgetExecutables({ + logger, + client, + executableIDs, +}: { + logger: Logger; + client: ProfilingESClient; + executableIDs: Set; +}): Promise> { + const executables = new Map(); + + if (executableIDs.size === 0) { + return executables; + } + + const resExecutables = await client.mget('mget_executables', { + index: INDEX_EXECUTABLES, + ids: [...executableIDs], + _source_includes: [ProfilingESField.ExecutableFileName], + }); + + // Create a lookup map StackFrameID -> StackFrame. + let exeFound = 0; + await logExecutionLatency(logger, 'processing data', async () => { + const docs = resExecutables.docs; + for (const exe of docs) { + if ('error' in exe) { + continue; + } + if (exe.found) { + executables.set(exe._id, { + FileName: exe._source!.Executable.file.name, + }); + exeFound++; + } else { + executables.set(exe._id, { + FileName: '', + }); + } + } + }); + + logger.info('found ' + exeFound + ' / ' + executableIDs.size + ' executables'); + + return executables; +} diff --git a/x-pack/plugins/profiling/server/routes/topn.test.ts b/x-pack/plugins/profiling/server/routes/topn.test.ts new file mode 100644 index 000000000000..2185db03d4ca --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/topn.test.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AggregationsAggregationContainer } from '@elastic/elasticsearch/lib/api/types'; +import { coreMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { ProfilingESField } from '../../common/elasticsearch'; +import { ProfilingESClient } from '../utils/create_profiling_es_client'; +import { topNElasticSearchQuery } from './topn'; + +const anyQuery = 'any::query'; +const smallestInterval = '1s'; +const testAgg = { aggs: { test: {} } }; + +jest.mock('./query', () => ({ + createCommonFilter: ({}: {}) => { + return anyQuery; + }, + findFixedIntervalForBucketsPerTimeRange: (from: number, to: number, buckets: number): string => { + return smallestInterval; + }, + aggregateByFieldAndTimestamp: ( + searchField: string, + interval: string + ): AggregationsAggregationContainer => { + return testAgg; + }, +})); + +describe('TopN data from Elasticsearch', () => { + const context = coreMock.createRequestHandlerContext(); + const client: ProfilingESClient = { + search: jest.fn( + (operationName, request) => + context.elasticsearch.client.asCurrentUser.search(request) as Promise + ), + mget: jest.fn( + (operationName, request) => + context.elasticsearch.client.asCurrentUser.search(request) as Promise + ), + }; + const logger = loggerMock.create(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when fetching Stack Traces', () => { + it('should search first then skip mget', async () => { + await topNElasticSearchQuery({ + client, + logger, + timeFrom: 456, + timeTo: 789, + searchField: ProfilingESField.StacktraceID, + highCardinality: false, + kuery: '', + }); + + // Calls to mget are skipped since data doesn't exist + expect(client.search).toHaveBeenCalledTimes(2); + expect(client.mget).toHaveBeenCalledTimes(0); + }); + }); +}); diff --git a/x-pack/plugins/profiling/server/routes/topn.ts b/x-pack/plugins/profiling/server/routes/topn.ts new file mode 100644 index 000000000000..eb031ef025b0 --- /dev/null +++ b/x-pack/plugins/profiling/server/routes/topn.ts @@ -0,0 +1,260 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import type { IRouter, Logger } from '@kbn/core/server'; +import { RouteRegisterParameters } from '.'; +import { fromMapToRecord, getRoutePaths, INDEX_EVENTS } from '../../common'; +import { ProfilingESField } from '../../common/elasticsearch'; +import { computeBucketWidthFromTimeRangeAndBucketCount } from '../../common/histogram'; +import { groupStackFrameMetadataByStackTrace, StackTraceID } from '../../common/profiling'; +import { getFieldNameForTopNType, TopNType } from '../../common/stack_traces'; +import { createTopNSamples, getTopNAggregationRequest, TopNResponse } from '../../common/topn'; +import { ProfilingRequestHandlerContext } from '../types'; +import { createProfilingEsClient, ProfilingESClient } from '../utils/create_profiling_es_client'; +import { withProfilingSpan } from '../utils/with_profiling_span'; +import { getClient } from './compat'; +import { findDownsampledIndex } from './downsampling'; +import { createCommonFilter } from './query'; +import { mgetExecutables, mgetStackFrames, mgetStackTraces } from './stacktrace'; + +export async function topNElasticSearchQuery({ + client, + logger, + timeFrom, + timeTo, + searchField, + highCardinality, + kuery, +}: { + client: ProfilingESClient; + logger: Logger; + timeFrom: number; + timeTo: number; + searchField: string; + highCardinality: boolean; + kuery: string; +}): Promise { + const filter = createCommonFilter({ timeFrom, timeTo, kuery }); + const targetSampleSize = 20000; // minimum number of samples to get statistically sound results + + const bucketWidth = computeBucketWidthFromTimeRangeAndBucketCount(timeFrom, timeTo, 50); + + const eventsIndex = await findDownsampledIndex({ + logger, + client, + index: INDEX_EVENTS, + filter, + sampleSize: targetSampleSize, + }); + + const resEvents = await client.search('get_topn_histogram', { + index: eventsIndex.name, + size: 0, + query: filter, + aggs: getTopNAggregationRequest({ + searchField, + highCardinality, + fixedInterval: `${bucketWidth}s`, + }), + // Adrien and Dario found out this is a work-around for some bug in 8.1. + // It reduces the query time by avoiding unneeded searches. + pre_filter_shard_size: 1, + }); + + const { aggregations } = resEvents; + + if (!aggregations) { + return { + TotalCount: 0, + TopN: [], + Metadata: {}, + }; + } + + // Creating top N samples requires the time range and bucket width to + // be in milliseconds, not seconds + const topN = createTopNSamples(aggregations, timeFrom * 1000, timeTo * 1000, bucketWidth * 1000); + + for (let i = 0; i < topN.length; i++) { + topN[i].Count = (topN[i].Count ?? 0) / eventsIndex.sampleRate; + } + + let totalSampledStackTraces = aggregations.total_count.value ?? 0; + logger.info('total sampled stacktraces: ' + totalSampledStackTraces); + totalSampledStackTraces = Math.floor(totalSampledStackTraces / eventsIndex.sampleRate); + + if (searchField !== ProfilingESField.StacktraceID) { + return { + TotalCount: totalSampledStackTraces, + TopN: topN, + Metadata: {}, + }; + } + + const stackTraceEvents = new Map(); + const groupByBuckets = aggregations.group_by.buckets ?? []; + let totalAggregatedStackTraces = 0; + + for (let i = 0; i < groupByBuckets.length; i++) { + const stackTraceID = String(groupByBuckets[i].key); + const count = Math.floor((groupByBuckets[i].count.value ?? 0) / eventsIndex.sampleRate); + totalAggregatedStackTraces += count; + stackTraceEvents.set(stackTraceID, count); + } + + logger.info('total aggregated stacktraces: ' + totalAggregatedStackTraces); + logger.info('unique aggregated stacktraces: ' + stackTraceEvents.size); + + const { stackTraces, stackFrameDocIDs, executableDocIDs } = await mgetStackTraces({ + logger, + client, + events: stackTraceEvents, + }); + + const [stackFrames, executables] = await withProfilingSpan( + 'get_stackframes_and_executables', + () => { + return Promise.all([ + mgetStackFrames({ logger, client, stackFrameIDs: stackFrameDocIDs }), + mgetExecutables({ logger, client, executableIDs: executableDocIDs }), + ]); + } + ); + + const metadata = await withProfilingSpan('collect_stackframe_metadata', async () => { + return fromMapToRecord( + groupStackFrameMetadataByStackTrace(stackTraces, stackFrames, executables) + ); + }); + + logger.info('returning payload response to client'); + + return { + TotalCount: totalSampledStackTraces, + TopN: topN, + Metadata: metadata, + }; +} + +export function queryTopNCommon( + router: IRouter, + logger: Logger, + pathName: string, + searchField: string, + highCardinality: boolean +) { + router.get( + { + path: pathName, + validate: { + query: schema.object({ + timeFrom: schema.number(), + timeTo: schema.number(), + kuery: schema.string(), + }), + }, + }, + async (context, request, response) => { + const { timeFrom, timeTo, kuery } = request.query; + const client = await getClient(context); + + try { + return response.ok({ + body: await topNElasticSearchQuery({ + client: createProfilingEsClient({ request, esClient: client }), + logger, + timeFrom, + timeTo, + searchField, + highCardinality, + kuery, + }), + }); + } catch (e) { + logger.error(e); + + return response.customError({ + statusCode: e.statusCode ?? 500, + body: { + message: 'Profiling TopN request failed: ' + e.message + '; full error ' + e.toString(), + }, + }); + } + } + ); +} + +export function registerTraceEventsTopNContainersSearchRoute({ + router, + logger, +}: RouteRegisterParameters) { + const paths = getRoutePaths(); + return queryTopNCommon( + router, + logger, + paths.TopNContainers, + getFieldNameForTopNType(TopNType.Containers), + false + ); +} + +export function registerTraceEventsTopNDeploymentsSearchRoute({ + router, + logger, +}: RouteRegisterParameters) { + const paths = getRoutePaths(); + return queryTopNCommon( + router, + logger, + paths.TopNDeployments, + getFieldNameForTopNType(TopNType.Deployments), + false + ); +} + +export function registerTraceEventsTopNHostsSearchRoute({ + router, + logger, +}: RouteRegisterParameters) { + const paths = getRoutePaths(); + return queryTopNCommon( + router, + logger, + paths.TopNHosts, + getFieldNameForTopNType(TopNType.Hosts), + false + ); +} + +export function registerTraceEventsTopNStackTracesSearchRoute({ + router, + logger, +}: RouteRegisterParameters) { + const paths = getRoutePaths(); + return queryTopNCommon( + router, + logger, + paths.TopNTraces, + getFieldNameForTopNType(TopNType.Traces), + false + ); +} + +export function registerTraceEventsTopNThreadsSearchRoute({ + router, + logger, +}: RouteRegisterParameters) { + const paths = getRoutePaths(); + return queryTopNCommon( + router, + logger, + paths.TopNThreads, + getFieldNameForTopNType(TopNType.Threads), + true + ); +} diff --git a/x-pack/plugins/profiling/server/types.ts b/x-pack/plugins/profiling/server/types.ts new file mode 100644 index 000000000000..8432085ef102 --- /dev/null +++ b/x-pack/plugins/profiling/server/types.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RequestHandlerContext } from '@kbn/core/server'; +import { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server'; +import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; + +export interface ProfilingPluginSetupDeps { + observability: ObservabilityPluginSetup; + features: FeaturesPluginSetup; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface ProfilingPluginStartDeps {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface ProfilingPluginSetup {} +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface ProfilingPluginStart {} + +export type ProfilingRequestHandlerContext = RequestHandlerContext; diff --git a/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts b/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts new file mode 100644 index 000000000000..a7b985a567a4 --- /dev/null +++ b/x-pack/plugins/profiling/server/utils/create_profiling_es_client.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import { ESSearchRequest, InferSearchResponseOf } from '@kbn/core/types/elasticsearch'; +import type { KibanaRequest } from '@kbn/core/server'; +import { unwrapEsResponse } from '@kbn/observability-plugin/server'; +import { MgetRequest, MgetResponse } from '@elastic/elasticsearch/lib/api/types'; +import { ProfilingESEvent } from '../../common/elasticsearch'; +import { withProfilingSpan } from './with_profiling_span'; + +export function cancelEsRequestOnAbort>( + promise: T, + request: KibanaRequest, + controller: AbortController +): T { + const subscription = request.events.aborted$.subscribe(() => { + controller.abort(); + }); + + return promise.finally(() => subscription.unsubscribe()) as T; +} + +export interface ProfilingESClient { + search( + operationName: string, + searchRequest: TSearchRequest + ): Promise>; + mget( + operationName: string, + mgetRequest: MgetRequest + ): Promise>; +} + +export function createProfilingEsClient({ + request, + esClient, +}: { + request: KibanaRequest; + esClient: ElasticsearchClient; +}): ProfilingESClient { + return { + search( + operationName: string, + searchRequest: TSearchRequest + ): Promise> { + const controller = new AbortController(); + + const promise = withProfilingSpan(operationName, () => { + return cancelEsRequestOnAbort( + esClient.search(searchRequest, { + signal: controller.signal, + meta: true, + }) as unknown as Promise<{ + body: InferSearchResponseOf; + }>, + request, + controller + ); + }); + + return unwrapEsResponse(promise); + }, + mget( + operationName: string, + mgetRequest: MgetRequest + ): Promise> { + const controller = new AbortController(); + + const promise = withProfilingSpan(operationName, () => { + return cancelEsRequestOnAbort( + esClient.mget(mgetRequest, { + signal: controller.signal, + meta: true, + }), + request, + controller + ); + }); + + return unwrapEsResponse(promise); + }, + }; +} diff --git a/x-pack/plugins/profiling/server/utils/with_profiling_span.ts b/x-pack/plugins/profiling/server/utils/with_profiling_span.ts new file mode 100644 index 000000000000..6d366799780e --- /dev/null +++ b/x-pack/plugins/profiling/server/utils/with_profiling_span.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { withSpan, SpanOptions, parseSpanOptions } from '@kbn/apm-utils'; + +export function withProfilingSpan( + optionsOrName: SpanOptions | string, + cb: () => Promise +): Promise { + const options = parseSpanOptions(optionsOrName); + + const optionsWithDefaults = { + ...(options.intercept ? {} : { type: 'plugin:profiling' }), + ...options, + labels: { + plugin: 'profiling', + ...options.labels, + }, + }; + + return withSpan(optionsWithDefaults, cb); +} diff --git a/x-pack/plugins/profiling/tsconfig.json b/x-pack/plugins/profiling/tsconfig.json new file mode 100644 index 000000000000..5b8daabf46cb --- /dev/null +++ b/x-pack/plugins/profiling/tsconfig.json @@ -0,0 +1,45 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true, + }, + "include": [ + // add all the folders containing files to be compiled + "index.ts", + "common/**/*.ts", + "public/**/*.ts", + "public/**/*.tsx", + "server/**/*.ts" + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/data/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_react/tsconfig.json" }, + { "path": "../../../src/plugins/navigation/tsconfig.json" }, + { "path": "../../../src/plugins/share/tsconfig.json" }, + { "path": "../observability/tsconfig.json" }, + // add references to other TypeScript projects the plugin depends on + + // requiredPlugins from ./kibana.json + // { "path": "../licensing/tsconfig.json" }, + // { "path": "../../../src/plugins/data/tsconfig.json" }, + // { "path": "../encrypted_saved_objects/tsconfig.json" }, + + // optionalPlugins from ./kibana.json + // { "path": "../security/tsconfig.json" }, + // { "path": "../features/tsconfig.json" }, + // { "path": "../cloud/tsconfig.json" }, + // { "path": "../../../src/plugins/usage_collection/tsconfig.json" }, + // { "path": "../../../src/plugins/home/tsconfig.json" }, + + // requiredBundles from ./kibana.json + // { "path": "../../../src/plugins/kibana_react/tsconfig.json" }, + // { "path": "../../../src/plugins/es_ui_shared/tsconfig.json" }, + // { "path": "../infra/tsconfig.json" }, + // { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, + ] +} diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index cf0222e6f1fa..bc908da4b43e 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -17,13 +17,10 @@ export const allowedExperimentalValues = Object.freeze({ excludePoliciesInFilterEnabled: false, kubernetesEnabled: true, disableIsolationUIPendingStatuses: false, - riskyHostsEnabled: false, - riskyUsersEnabled: false, pendingActionResponsesWithAck: true, policyListEnabled: true, policyResponseInFleetEnabled: true, threatIntelligenceEnabled: false, - entityAnalyticsDashboardEnabled: false, /** * This is used for enabling the end-to-end tests for the security_solution telemetry. diff --git a/x-pack/plugins/security_solution/public/app/deep_links/index.ts b/x-pack/plugins/security_solution/public/app/deep_links/index.ts index 6ac3a0aa7a3f..99db764285ab 100644 --- a/x-pack/plugins/security_solution/public/app/deep_links/index.ts +++ b/x-pack/plugins/security_solution/public/app/deep_links/index.ts @@ -173,7 +173,6 @@ export const securitySolutionsDeepLinks: SecuritySolutionDeepLink[] = [ title: ENTITY_ANALYTICS, path: ENTITY_ANALYTICS_PATH, features: [FEATURE.general], - experimentalKey: 'entityAnalyticsDashboardEnabled', isPremium: true, keywords: [ i18n.translate('xpack.securitySolution.search.entityAnalytics', { @@ -296,7 +295,6 @@ export const securitySolutionsDeepLinks: SecuritySolutionDeepLink[] = [ defaultMessage: 'Host risk', }), path: `${HOSTS_PATH}/hostRisk`, - experimentalKey: 'riskyHostsEnabled', }, { id: SecurityPageName.sessions, @@ -386,7 +384,6 @@ export const securitySolutionsDeepLinks: SecuritySolutionDeepLink[] = [ defaultMessage: 'User risk', }), path: `${USERS_PATH}/userRisk`, - experimentalKey: 'riskyUsersEnabled', }, { id: SecurityPageName.usersEvents, diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.test.tsx index 0c6cf454e73e..2320b85113c3 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.test.tsx @@ -9,9 +9,9 @@ import React from 'react'; import { render } from '@testing-library/react'; import { TestProviders } from '../../../mock'; -import { NO_HOST_RISK_DATA_DESCRIPTION } from './translations'; import { HostRiskSummary } from './host_risk_summary'; import { RiskSeverity } from '../../../../../common/search_strategy'; +import { getEmptyValue } from '../../empty_value'; describe('HostRiskSummary', () => { it('renders host risk data', () => { @@ -60,36 +60,7 @@ describe('HostRiskSummary', () => { expect(getByTestId('loading')).toBeInTheDocument(); }); - it('renders no host data message when module is diabled', () => { - const hostRisk = { - loading: false, - isModuleEnabled: false, - result: [ - { - '@timestamp': '1641902530', - host: { - name: 'test-host-name', - risk: { - multipliers: [], - calculated_score_norm: 9999, - calculated_level: RiskSeverity.low, - rule_risks: [], - }, - }, - }, - ], - }; - - const { getByText } = render( - - - - ); - - expect(getByText(NO_HOST_RISK_DATA_DESCRIPTION)).toBeInTheDocument(); - }); - - it('renders no host data message when there is no host data', () => { + it('renders empty value when there is no host data', () => { const hostRisk = { loading: false, isModuleEnabled: true, @@ -102,6 +73,6 @@ describe('HostRiskSummary', () => { ); - expect(getByText(NO_HOST_RISK_DATA_DESCRIPTION)).toBeInTheDocument(); + expect(getByText(getEmptyValue())).toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx index 970656933b93..536d77c8c831 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/host_risk_summary.tsx @@ -6,12 +6,13 @@ */ import React from 'react'; -import { EuiLoadingSpinner, EuiPanel, EuiSpacer, EuiLink, EuiText } from '@elastic/eui'; +import { EuiLoadingSpinner, EuiPanel, EuiLink } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import * as i18n from './translations'; import { EnrichedDataRow, ThreatSummaryPanelHeader } from './threat_summary_view'; import { RiskScore } from '../../severity/common'; import type { HostRisk } from '../../../../risk_score/containers'; +import { getEmptyValue } from '../../empty_value'; import { RISKY_HOSTS_DOC_LINK } from '../../../../../common/constants'; const HostRiskSummaryComponent: React.FC<{ @@ -41,24 +42,19 @@ const HostRiskSummaryComponent: React.FC<{ {hostRisk.loading && } - {!hostRisk.loading && (!hostRisk.isModuleEnabled || hostRisk.result?.length === 0) && ( - <> - - - {i18n.NO_HOST_RISK_DATA_DESCRIPTION} - - - )} - - {hostRisk.isModuleEnabled && hostRisk.result && hostRisk.result.length > 0 && ( + {!hostRisk.loading && ( <> + hostRisk.result && hostRisk.result.length > 0 ? ( + + ) : ( + getEmptyValue() + ) } /> diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.test.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.test.tsx index b87c96c34632..23e1da2dde8c 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.test.tsx @@ -27,6 +27,14 @@ jest.mock('../../../lib/kibana', () => ({ jest.mock('../table/action_cell', () => ({ ActionCell: () => <> })); jest.mock('../table/field_name_cell'); +const RISK_SCORE_DATA_ROWS = 2; + +const EMPTY_RISK_SCORE = { + loading: false, + isModuleEnabled: true, + result: [], +}; + describe('ThreatSummaryView', () => { const eventId = '5d1d53da502f56aacc14c3cb5c669363d102b31f99822e5d369d4804ed370a31'; const timelineId = 'detections-page'; @@ -38,6 +46,7 @@ describe('ThreatSummaryView', () => { buildEventEnrichmentMock({ 'matched.id': ['test.id'], 'matched.field': ['test.field'] }), buildEventEnrichmentMock({ 'matched.id': ['other.id'], 'matched.field': ['other.field'] }), ]; + const { getByText, getAllByTestId } = render( { enrichments={enrichments} eventId={eventId} timelineId={timelineId} - hostRisk={null} + hostRisk={EMPTY_RISK_SCORE} + userRisk={EMPTY_RISK_SCORE} /> ); expect(getByText('Enriched with Threat Intelligence')).toBeInTheDocument(); - expect(getAllByTestId('EnrichedDataRow')).toHaveLength(enrichments.length); + expect(getAllByTestId('EnrichedDataRow')).toHaveLength( + enrichments.length + RISK_SCORE_DATA_ROWS + ); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx index a7a371e60cf9..866d7ae0a6c3 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/cti_details/threat_summary_view.tsx @@ -26,8 +26,9 @@ import type { TimelineEventsDetailsItem, } from '../../../../../common/search_strategy'; import { HostRiskSummary } from './host_risk_summary'; +import { UserRiskSummary } from './user_risk_summary'; import { EnrichmentSummary } from './enrichment_summary'; -import type { HostRisk } from '../../../../risk_score/containers'; +import type { HostRisk, UserRisk } from '../../../../risk_score/containers'; const UppercaseEuiTitle = styled(EuiTitle)` text-transform: uppercase; @@ -125,7 +126,8 @@ const ThreatSummaryViewComponent: React.FC<{ enrichments: CtiEnrichment[]; eventId: string; timelineId: string; - hostRisk: HostRisk | null; + hostRisk: HostRisk; + userRisk: UserRisk; isDraggable?: boolean; isReadOnly?: boolean; }> = ({ @@ -135,13 +137,10 @@ const ThreatSummaryViewComponent: React.FC<{ eventId, timelineId, hostRisk, + userRisk, isDraggable, isReadOnly, }) => { - if (!hostRisk && enrichments.length === 0) { - return null; - } - return ( <> @@ -152,11 +151,13 @@ const ThreatSummaryViewComponent: React.FC<{ - {hostRisk && ( - - - - )} + + + + + + + = ({ userRisk }) => ( + <> + + + + + ), + }} + /> + } + /> + + {userRisk.loading && } + + {!userRisk.loading && ( + <> + 0 ? ( + + ) : ( + getEmptyValue() + ) + } + /> + + )} + + +); + +export const UserRiskSummary = React.memo(UserRiskSummaryComponent); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx index 11ca04b1224d..92f4c57964e1 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx @@ -40,8 +40,8 @@ import { EnrichmentRangePicker } from './cti_details/enrichment_range_picker'; import { Reason } from './reason'; import { InvestigationGuideView } from './investigation_guide_view'; import { Overview } from './overview'; -import type { HostRisk } from '../../../risk_score/containers'; import { Insights } from './insights/insights'; +import { useRiskScoreData } from './use_risk_score_data'; type EventViewTab = EuiTabbedContentTab; @@ -67,7 +67,6 @@ interface Props { rawEventData: object | undefined; timelineTabType: TimelineTabs | 'flyout'; timelineId: string; - hostRisk: HostRisk | null; handleOnEventClosed: () => void; isReadOnly?: boolean; } @@ -111,7 +110,6 @@ const EventDetailsComponent: React.FC = ({ rawEventData, timelineId, timelineTabType, - hostRisk, handleOnEventClosed, isReadOnly, }) => { @@ -148,6 +146,8 @@ const EventDetailsComponent: React.FC = ({ const enrichmentCount = allEnrichments.length; + const { hostRisk, userRisk, isLicenseValid } = useRiskScoreData(data); + const summaryTab: EventViewTab | undefined = useMemo( () => isAlert @@ -193,10 +193,11 @@ const EventDetailsComponent: React.FC = ({ isReadOnly={isReadOnly} /> - {(enrichmentCount > 0 || hostRisk) && ( + {enrichmentCount > 0 && isLicenseValid && ( = ({ goToTableTab, handleOnEventClosed, isReadOnly, + userRisk, + isLicenseValid, ] ); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/use_risk_score_data.test.ts b/x-pack/plugins/security_solution/public/common/components/event_details/use_risk_score_data.test.ts new file mode 100644 index 000000000000..4dee4aeb3008 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/event_details/use_risk_score_data.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { TestProviders } from '../../mock'; +import { ONLY_FIRST_ITEM_PAGINATION, useRiskScoreData } from './use_risk_score_data'; +import { useUserRiskScore, useHostRiskScore } from '../../../risk_score/containers'; +import { useBasicDataFromDetailsData } from '../../../timelines/components/side_panel/event_details/helpers'; + +const mockUseUserRiskScore = useUserRiskScore as jest.Mock; +const mockUseHostRiskScore = useHostRiskScore as jest.Mock; +const mockUseBasicDataFromDetailsData = useBasicDataFromDetailsData as jest.Mock; +jest.mock('../../../risk_score/containers'); +jest.mock('../../../timelines/components/side_panel/event_details/helpers'); +const defaultResult = { + data: [], + inspect: {}, + isInspected: false, + isLicenseValid: true, + isModuleEnabled: true, + refetch: () => {}, + totalCount: 0, +}; +const defaultRisk = { + loading: false, + isModuleEnabled: true, + result: [], +}; + +const defaultArgs = [ + { + field: 'host.name', + isObjectArray: false, + }, +]; + +describe('useRiskScoreData', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockUseUserRiskScore.mockReturnValue([false, defaultResult]); + mockUseHostRiskScore.mockReturnValue([false, defaultResult]); + mockUseBasicDataFromDetailsData.mockReturnValue({ + hostName: 'host', + userName: 'user', + }); + }); + test('returns expected default values', () => { + const { result } = renderHook(() => useRiskScoreData(defaultArgs), { + wrapper: TestProviders, + }); + expect(result.current).toEqual({ + hostRisk: defaultRisk, + userRisk: defaultRisk, + isLicenseValid: true, + }); + }); + + test('builds filter query for risk score hooks', () => { + renderHook(() => useRiskScoreData(defaultArgs), { + wrapper: TestProviders, + }); + expect(mockUseUserRiskScore).toHaveBeenCalledWith({ + filterQuery: { terms: { 'user.name': ['user'] } }, + pagination: ONLY_FIRST_ITEM_PAGINATION, + skip: false, + }); + expect(mockUseHostRiskScore).toHaveBeenCalledWith({ + filterQuery: { terms: { 'host.name': ['host'] } }, + pagination: ONLY_FIRST_ITEM_PAGINATION, + skip: false, + }); + }); + + test('skips risk score hooks with no entity name', () => { + mockUseBasicDataFromDetailsData.mockReturnValue({ hostName: undefined, userName: undefined }); + renderHook(() => useRiskScoreData(defaultArgs), { + wrapper: TestProviders, + }); + expect(mockUseUserRiskScore).toHaveBeenCalledWith({ + filterQuery: undefined, + pagination: ONLY_FIRST_ITEM_PAGINATION, + skip: true, + }); + expect(mockUseHostRiskScore).toHaveBeenCalledWith({ + filterQuery: undefined, + pagination: ONLY_FIRST_ITEM_PAGINATION, + skip: true, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/use_risk_score_data.ts b/x-pack/plugins/security_solution/public/common/components/event_details/use_risk_score_data.ts new file mode 100644 index 000000000000..eb462671afd9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/components/event_details/use_risk_score_data.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useMemo } from 'react'; +import { useBasicDataFromDetailsData } from '../../../timelines/components/side_panel/event_details/helpers'; +import type { TimelineEventsDetailsItem } from '../../../../common/search_strategy'; +import { buildHostNamesFilter, buildUserNamesFilter } from '../../../../common/search_strategy'; +import type { HostRisk, UserRisk } from '../../../risk_score/containers'; +import { useUserRiskScore, useHostRiskScore } from '../../../risk_score/containers'; + +export const ONLY_FIRST_ITEM_PAGINATION = { + cursorStart: 0, + querySize: 1, +}; + +export const useRiskScoreData = (data: TimelineEventsDetailsItem[]) => { + const { hostName, userName } = useBasicDataFromDetailsData(data); + + const hostNameFilterQuery = useMemo( + () => (hostName ? buildHostNamesFilter([hostName]) : undefined), + [hostName] + ); + + const [ + hostRiskLoading, + { + data: hostRiskData, + isLicenseValid: isHostLicenseValid, + isModuleEnabled: isHostRiskModuleEnabled, + }, + ] = useHostRiskScore({ + filterQuery: hostNameFilterQuery, + pagination: ONLY_FIRST_ITEM_PAGINATION, + skip: !hostNameFilterQuery, + }); + + const hostRisk: HostRisk = useMemo( + () => ({ + loading: hostRiskLoading, + isModuleEnabled: isHostRiskModuleEnabled, + result: hostRiskData, + }), + [hostRiskData, hostRiskLoading, isHostRiskModuleEnabled] + ); + + const userNameFilterQuery = useMemo( + () => (userName ? buildUserNamesFilter([userName]) : undefined), + [userName] + ); + + const [ + userRiskLoading, + { + data: userRiskData, + isLicenseValid: isUserLicenseValid, + isModuleEnabled: isUserRiskModuleEnabled, + }, + ] = useUserRiskScore({ + filterQuery: userNameFilterQuery, + pagination: ONLY_FIRST_ITEM_PAGINATION, + skip: !userNameFilterQuery, + }); + + const userRisk: UserRisk = useMemo( + () => ({ + loading: userRiskLoading, + isModuleEnabled: isUserRiskModuleEnabled, + result: userRiskData, + }), + [userRiskLoading, isUserRiskModuleEnabled, userRiskData] + ); + + return { userRisk, hostRisk, isLicenseValid: isHostLicenseValid && isUserLicenseValid }; +}; diff --git a/x-pack/plugins/security_solution/public/common/components/header_page/title.tsx b/x-pack/plugins/security_solution/public/common/components/header_page/title.tsx index 0353d4f76c1c..0f54c6f579a9 100644 --- a/x-pack/plugins/security_solution/public/common/components/header_page/title.tsx +++ b/x-pack/plugins/security_solution/public/common/components/header_page/title.tsx @@ -59,6 +59,7 @@ const TitleComponent: React.FC = ({ draggableArguments, title, badgeOptio label={badgeOptions.text} tooltipContent={badgeOptions.tooltip} tooltipPosition="bottom" + size={badgeOptions.size} /> ) : ( diff --git a/x-pack/plugins/security_solution/public/common/components/header_page/types.ts b/x-pack/plugins/security_solution/public/common/components/header_page/types.ts index 358a8118e5cc..698178480045 100644 --- a/x-pack/plugins/security_solution/public/common/components/header_page/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/header_page/types.ts @@ -6,6 +6,7 @@ */ import type { EuiBadgeProps } from '@elastic/eui'; +import type { BetaBadgeSize } from '@elastic/eui/src/components/badge/beta_badge/beta_badge'; import type React from 'react'; export type TitleProp = string | React.ReactNode; @@ -19,4 +20,5 @@ export interface BadgeOptions { text: React.ReactNode; tooltip?: React.ReactNode; color?: EuiBadgeProps['color']; + size?: BetaBadgeSize; } diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/nav_item_beta_badge.tsx b/x-pack/plugins/security_solution/public/common/components/navigation/nav_item_beta_badge.tsx index ddccbee1546d..bef5ba4d8362 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/nav_item_beta_badge.tsx +++ b/x-pack/plugins/security_solution/public/common/components/navigation/nav_item_beta_badge.tsx @@ -10,12 +10,12 @@ import { css } from '@emotion/react'; import { EuiBetaBadge, useEuiTheme } from '@elastic/eui'; import { BETA } from '../../translations'; -export const NavItemBetaBadge = () => { +export const NavItemBetaBadge = ({ text }: { text?: string }) => { const { euiTheme } = useEuiTheme(); return ( ...(link.landingImage != null ? { image: link.landingImage } : {}), ...(link.skipUrlState != null ? { skipUrlState: link.skipUrlState } : {}), ...(link.isBeta != null ? { isBeta: link.isBeta } : {}), + ...(link.betaOptions != null ? { betaOptions: link.betaOptions } : {}), ...(link.links && link.links.length ? { links: formatNavLinkItems(link.links), diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/security_side_nav/security_side_nav.tsx b/x-pack/plugins/security_solution/public/common/components/navigation/security_side_nav/security_side_nav.tsx index 4b7c58ce2ba0..ce239c34eae4 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/security_side_nav/security_side_nav.tsx +++ b/x-pack/plugins/security_solution/public/common/components/navigation/security_side_nav/security_side_nav.tsx @@ -78,6 +78,7 @@ const useFormatSideNavItem = (): FormatSideNavItems => { label: current.title, description: current.description, isBeta: current.isBeta, + betaOptions: current.betaOptions, ...getSecuritySolutionLinkProps({ deepLinkId: current.id }), }); } diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/solution_grouped_nav_panel.tsx b/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/solution_grouped_nav_panel.tsx index 4516f103f916..2c7dbf407498 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/solution_grouped_nav_panel.tsx +++ b/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/solution_grouped_nav_panel.tsx @@ -156,7 +156,7 @@ const SolutionNavPanelCategories: React.FC = ({ const SolutionNavPanelItems: React.FC = ({ items, onClose }) => ( <> - {items.map(({ id, href, onClick, label, description, isBeta }) => ( + {items.map(({ id, href, onClick, label, description, isBeta, betaOptions }) => ( = ({ items, on }} > {label} - {isBeta && } + {isBeta && } {description} diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/types.ts b/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/types.ts index f9e186930e24..f47e1755cc79 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/navigation/solution_grouped_nav/types.ts @@ -18,6 +18,9 @@ export interface DefaultSideNavItem { items?: DefaultSideNavItem[]; categories?: LinkCategories; isBeta?: boolean; + betaOptions?: { + text: string; + }; } export interface CustomSideNavItem { diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/index.tsx b/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/index.tsx index d364a5526a22..abc4350eb83f 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/index.tsx @@ -23,6 +23,7 @@ const TabNavigationItemComponent = ({ name, isSelected, isBeta, + betaOptions, }: TabNavigationItemProps) => { const { getAppUrl, navigateTo } = useNavigation(); @@ -47,7 +48,7 @@ const TabNavigationItemComponent = ({ isSelected={isSelected} href={appHref} onClick={handleClick} - append={isBeta && } + append={isBeta && } > {name} @@ -92,6 +93,7 @@ export const TabNavigationComponent: React.FC = ({ navTabs, disabled={tab.disabled} isSelected={isSelected} isBeta={tab.isBeta} + betaOptions={tab.betaOptions} /> ); }), diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/types.ts b/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/types.ts index fa8d0a0e3fa1..2e687d09f290 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/navigation/tab_navigation/types.ts @@ -21,4 +21,7 @@ export interface TabNavigationItemProps { name: string; isSelected: boolean; isBeta?: boolean; + betaOptions?: { + text: string; + }; } diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/types.ts b/x-pack/plugins/security_solution/public/common/components/navigation/types.ts index 5a4c346be2e1..84341f532116 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/navigation/types.ts @@ -62,6 +62,9 @@ export interface NavTab { urlKey?: UrlStateType; pageId?: SecurityPageName; isBeta?: boolean; + betaOptions?: { + text: string; + }; } export const securityNavKeys = [ SecurityPageName.alerts, @@ -113,4 +116,7 @@ export interface NavLinkItem { title: string; skipUrlState?: boolean; isBeta?: boolean; + betaOptions?: { + text: string; + }; } diff --git a/x-pack/plugins/security_solution/public/common/components/navigation/use_security_solution_navigation/use_navigation_items.tsx b/x-pack/plugins/security_solution/public/common/components/navigation/use_security_solution_navigation/use_navigation_items.tsx index 2a8d977760cb..83a8e97f0a65 100644 --- a/x-pack/plugins/security_solution/public/common/components/navigation/use_security_solution_navigation/use_navigation_items.tsx +++ b/x-pack/plugins/security_solution/public/common/components/navigation/use_security_solution_navigation/use_navigation_items.tsx @@ -77,9 +77,7 @@ function usePrimaryNavigationItemsToDisplay(navTabs: Record) { const isPolicyListEnabled = useIsExperimentalFeatureEnabled('policyListEnabled'); const mlCapabilities = useMlCapabilities(); const hasMlPermissions = hasMlLicense(mlCapabilities) && hasMlUserPermissions(mlCapabilities); - const isEntityAnalyticsDashboardEnabled = useIsExperimentalFeatureEnabled( - 'entityAnalyticsDashboardEnabled' - ); + const uiCapabilities = useKibana().services.application.capabilities; return useMemo( () => @@ -99,9 +97,7 @@ function usePrimaryNavigationItemsToDisplay(navTabs: Record) { ...(navTabs[SecurityPageName.kubernetes] != null ? [navTabs[SecurityPageName.kubernetes]] : []), - ...(isEntityAnalyticsDashboardEnabled && hasMlPermissions - ? [navTabs[SecurityPageName.entityAnalytics]] - : []), + ...(hasMlPermissions ? [navTabs[SecurityPageName.entityAnalytics]] : []), ], }, { @@ -163,7 +159,6 @@ function usePrimaryNavigationItemsToDisplay(navTabs: Record) { [ uiCapabilities.siem.show, navTabs, - isEntityAnalyticsDashboardEnabled, hasCasesReadPermissions, canSeeHostIsolationExceptions, isPolicyListEnabled, diff --git a/x-pack/plugins/security_solution/public/common/links/types.ts b/x-pack/plugins/security_solution/public/common/links/types.ts index 210a5f4a5975..f9a2c5776262 100644 --- a/x-pack/plugins/security_solution/public/common/links/types.ts +++ b/x-pack/plugins/security_solution/public/common/links/types.ts @@ -84,6 +84,12 @@ export interface LinkItem { * Displays the "Beta" badge. Defaults to false. */ isBeta?: boolean; + /** + * Customize the "Beta" badge content. + */ + betaOptions?: { + text: string; + }; /** * Icon that is displayed on menu navigation landing page. * Only required for pages that are displayed inside a landing page. diff --git a/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.test.tsx b/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.test.tsx index a5ebfadd071e..1da0711f21aa 100644 --- a/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.test.tsx +++ b/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.test.tsx @@ -24,7 +24,6 @@ import { HostsTableType } from '../../store/model'; import { HostsTable } from '.'; import { mockData } from './mock'; import { render } from '@testing-library/react'; -import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; jest.mock('../../../common/lib/kibana'); @@ -45,7 +44,11 @@ jest.mock('../../../common/components/query_bar', () => ({ jest.mock('../../../common/components/link_to'); -jest.mock('../../../common/hooks/use_experimental_features'); +const mockUseMlCapabilities = jest.fn(); + +jest.mock('../../../common/components/ml/hooks/use_ml_capabilities', () => ({ + useMlCapabilities: () => mockUseMlCapabilities(), +})); describe('Hosts Table', () => { const loadPage = jest.fn(); @@ -81,8 +84,8 @@ describe('Hosts Table', () => { expect(wrapper.find('HostsTable')).toMatchSnapshot(); }); - test('it renders "Host Risk classfication" column when "riskyHostsEnabled" feature flag is enabled', () => { - (useIsExperimentalFeatureEnabled as jest.Mock).mockReturnValue(true); + test('it renders "Host Risk classfication" column when "isPlatinumOrTrialLicense" is truthy', () => { + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: true }); const { queryByTestId } = render( @@ -104,8 +107,8 @@ describe('Hosts Table', () => { expect(queryByTestId('tableHeaderCell_node.risk_4')).toBeInTheDocument(); }); - test("it doesn't renders 'Host Risk classfication' column when 'riskyHostsEnabled' feature flag is disabled", () => { - (useIsExperimentalFeatureEnabled as jest.Mock).mockReturnValue(false); + test("it doesn't renders 'Host Risk classfication' column when 'isPlatinumOrTrialLicense' is falsy", () => { + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: false }); const { queryByTestId } = render( diff --git a/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.tsx b/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.tsx index 6b701b7f7a8e..4aeb80bbdeec 100644 --- a/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/components/hosts_table/index.tsx @@ -27,10 +27,10 @@ import type { import { HostsFields } from '../../../../common/search_strategy/security_solution/hosts'; import type { Direction, RiskSeverity } from '../../../../common/search_strategy'; import type { HostEcs, OsEcs } from '../../../../common/ecs/host'; -import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; import { SecurityPageName } from '../../../../common/constants'; import { HostsTableType } from '../../store/model'; import { useNavigateTo } from '../../../common/lib/kibana/hooks'; +import { useMlCapabilities } from '../../../common/components/ml/hooks/use_ml_capabilities'; const tableType = hostsModel.HostsTableType.hosts; @@ -132,7 +132,7 @@ const HostsTableComponent: React.FC = ({ }, [direction, sortField, type, dispatch] ); - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; const dispatchSeverityUpdate = useCallback( (s: RiskSeverity) => { @@ -151,8 +151,8 @@ const HostsTableComponent: React.FC = ({ ); const hostsColumns = useMemo( - () => getHostsColumns(riskyHostsFeatureEnabled, dispatchSeverityUpdate), - [dispatchSeverityUpdate, riskyHostsFeatureEnabled] + () => getHostsColumns(isPlatinumOrTrialLicense, dispatchSeverityUpdate), + [dispatchSeverityUpdate, isPlatinumOrTrialLicense] ); const sorting = useMemo(() => getSorting(sortField, direction), [sortField, direction]); diff --git a/x-pack/plugins/security_solution/public/hosts/components/kpi_hosts/index.tsx b/x-pack/plugins/security_solution/public/hosts/components/kpi_hosts/index.tsx index f7c9352f3a95..ea7e12b75a5f 100644 --- a/x-pack/plugins/security_solution/public/hosts/components/kpi_hosts/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/components/kpi_hosts/index.tsx @@ -18,11 +18,11 @@ import { RISKY_HOSTS_DOC_LINK } from '../../../../common/constants'; export const HostsKpiComponent = React.memo( ({ filterQuery, from, indexNames, to, setQuery, skip, updateDateRange }) => { - const [_, { isModuleEnabled }] = useHostRiskScore(); + const [loading, { isLicenseValid, isModuleEnabled }] = useHostRiskScore(); return ( <> - {isModuleEnabled === false && ( + {isLicenseValid && !isModuleEnabled && !loading && ( <> = ({ detailName, hostDeta dispatch(setHostDetailsTablesActivePageToZero()); }, [dispatch, detailName]); - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; return ( <> @@ -206,7 +205,7 @@ const HostDetailsComponent: React.FC = ({ detailName, hostDeta diff --git a/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.test.tsx b/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.test.tsx index 442f16dc444f..3dcebfc19706 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.test.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.test.tsx @@ -52,21 +52,31 @@ describe('navTabsHostDetails', () => { expect(tabs).toHaveProperty(HostsTableType.risk); }); - test('it should display Beta badge for sessions tab only', () => { + test('it should display Beta badge for sessions tab', () => { const tabs = navTabsHostDetails({ hasMlUserPermissions: false, isRiskyHostsEnabled: true, hostName: mockHostName, }); - Object.values(tabs).forEach((item) => { - const tab = item as TabNavigationItemProps; + const sessionsTab = Object.values(tabs).find( + (item) => item.id === HostsTableType.sessions + ); - if (tab.id === HostsTableType.sessions) { - expect(tab.isBeta).toEqual(true); - } else { - expect(tab.isBeta).toEqual(undefined); - } + expect(sessionsTab?.isBeta).toEqual(true); + }); + + test('it should display Beta badge for risk tab', () => { + const tabs = navTabsHostDetails({ + hasMlUserPermissions: false, + isRiskyHostsEnabled: true, + hostName: mockHostName, }); + + const riskTab = Object.values(tabs).find( + (item) => item.id === HostsTableType.risk + ); + + expect(riskTab?.isBeta).toEqual(true); }); }); diff --git a/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.tsx b/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.tsx index d070b19035df..7113131e3c3f 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/details/nav_tabs.tsx @@ -10,6 +10,7 @@ import * as i18n from '../translations'; import type { HostDetailsNavTab } from './types'; import { HostsTableType } from '../../store/model'; import { HOSTS_PATH } from '../../../../common/constants'; +import { TECHNICAL_PREVIEW } from '../../../overview/pages/translations'; const getTabsOnHostDetailsUrl = (hostName: string, tabName: HostsTableType) => `${HOSTS_PATH}/name/${hostName}/${tabName}`; @@ -55,6 +56,10 @@ export const navTabsHostDetails = ({ name: i18n.NAVIGATION_HOST_RISK_TITLE, href: getTabsOnHostDetailsUrl(hostName, HostsTableType.risk), disabled: false, + isBeta: true, + betaOptions: { + text: TECHNICAL_PREVIEW, + }, }, [HostsTableType.sessions]: { id: HostsTableType.sessions, diff --git a/x-pack/plugins/security_solution/public/hosts/pages/hosts.tsx b/x-pack/plugins/security_solution/public/hosts/pages/hosts.tsx index 9d6687a771f3..f853502c2266 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/hosts.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/hosts.tsx @@ -55,8 +55,6 @@ import { useSourcererDataView } from '../../common/containers/sourcerer'; import { useDeepEqualSelector, useShallowEqualSelector } from '../../common/hooks/use_selector'; import { useInvalidFilterQuery } from '../../common/hooks/use_invalid_filter_query'; import { ID } from '../containers/hosts'; -import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; - import { LandingPageComponent } from '../../common/components/landing_page'; import { hostNameExistsFilter } from '../../common/components/visualization_actions/utils'; @@ -146,8 +144,6 @@ const HostsComponent = () => { [indexPattern, query, tabsFilters, uiSettings] ); - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); - useInvalidFilterQuery({ id: ID, filterQuery, kqlError, query, startDate: from, endDate: to }); const onSkipFocusBeforeEventsTable = useCallback(() => { @@ -208,7 +204,7 @@ const HostsComponent = () => { diff --git a/x-pack/plugins/security_solution/public/hosts/pages/nav_tabs.tsx b/x-pack/plugins/security_solution/public/hosts/pages/nav_tabs.tsx index e8ec705bd84d..ba278ad7fdb5 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/nav_tabs.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/nav_tabs.tsx @@ -10,6 +10,7 @@ import * as i18n from './translations'; import { HostsTableType } from '../store/model'; import type { HostsNavTab } from './navigation/types'; import { HOSTS_PATH } from '../../../common/constants'; +import { TECHNICAL_PREVIEW } from '../../overview/pages/translations'; const getTabsOnHostsUrl = (tabName: HostsTableType) => `${HOSTS_PATH}/${tabName}`; @@ -51,6 +52,10 @@ export const navTabsHosts = ({ name: i18n.NAVIGATION_HOST_RISK_TITLE, href: getTabsOnHostsUrl(HostsTableType.risk), disabled: false, + isBeta: true, + betaOptions: { + text: TECHNICAL_PREVIEW, + }, }, [HostsTableType.sessions]: { id: HostsTableType.sessions, diff --git a/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_icons.tsx b/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_icons.tsx index b82ecc9215a0..7ccd58117b3b 100644 --- a/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_icons.tsx +++ b/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_icons.tsx @@ -36,7 +36,7 @@ const StyledEuiTitle = styled(EuiTitle)` export const LandingLinksIcons: React.FC = ({ items }) => ( - {items.map(({ title, description, id, icon, isBeta }) => ( + {items.map(({ title, description, id, icon, isBeta, betaOptions }) => ( = ({ items }) {isBeta && ( - + )} diff --git a/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_images.tsx b/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_images.tsx index b5479e0e6c5a..ff7c791c01e3 100644 --- a/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_images.tsx +++ b/x-pack/plugins/security_solution/public/landing_pages/components/landing_links_images.tsx @@ -57,7 +57,7 @@ const SecuritySolutionLink = withSecuritySolutionLink(Link); export const LandingLinksImages: React.FC = ({ items }) => ( - {items.map(({ title, description, image, id, isBeta }) => ( + {items.map(({ title, description, image, id, isBeta, betaOptions }) => ( {/* Empty onClick is to force hover style on `EuiPanel` */} @@ -78,7 +78,7 @@ export const LandingLinksImages: React.FC = ({ items }) => ( {title} - {isBeta && } + {isBeta && } @@ -114,7 +114,7 @@ const SecuritySolutionCard = withSecuritySolutionLink(PrimaryTitleCard); export const LandingImageCards: React.FC = React.memo(({ items }) => ( - {items.map(({ id, image, title, description, isBeta }) => ( + {items.map(({ id, image, title, description, isBeta, betaOptions }) => ( = React.memo(({ ite {title} - {isBeta && } + {isBeta && } } diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.test.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.test.tsx index feed7baf8819..c7e7eb411d4f 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.test.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.test.tsx @@ -24,8 +24,8 @@ const mockSeverityCount: SeverityCount = { [RiskSeverity.critical]: 99, }; -jest.mock('../../../../common/hooks/use_experimental_features', () => ({ - useIsExperimentalFeatureEnabled: () => true, +jest.mock('../../../../common/components/ml/hooks/use_ml_capabilities', () => ({ + useMlCapabilities: () => ({ isPlatinumOrTrialLicense: true, capabilities: {} }), })); jest.mock('../../../../risk_score/containers', () => { diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.tsx index 4ee2bab00f1d..f94c7d52f6fc 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/header/index.tsx @@ -9,6 +9,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui'; import styled from 'styled-components'; import { useDispatch } from 'react-redux'; import { sum } from 'lodash/fp'; +import { ML_PAGES, useMlHref } from '@kbn/ml-plugin/public'; import { useHostRiskScoreKpi, useUserRiskScoreKpi } from '../../../../risk_score/containers'; import { LinkAnchor, useGetSecuritySolutionLinkProps } from '../../../../common/components/links'; import { Direction, RiskScoreFields, RiskSeverity } from '../../../../../common/search_strategy'; @@ -20,9 +21,10 @@ import { hostsActions } from '../../../../hosts/store'; import { usersActions } from '../../../../users/store'; import { getTabsOnUsersUrl } from '../../../../common/components/link_to/redirect_to_users'; import { UsersTableType } from '../../../../users/store/model'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; import { useNotableAnomaliesSearch } from '../../../../common/components/ml/anomaly/use_anomalies_search'; import { useGlobalTime } from '../../../../common/containers/use_global_time'; +import { useKibana } from '../../../../common/lib/kibana'; +import { useMlCapabilities } from '../../../../common/components/ml/hooks/use_ml_capabilities'; const StyledEuiTitle = styled(EuiTitle)` color: ${({ theme: { eui } }) => eui.euiColorVis9}; @@ -35,8 +37,11 @@ export const EntityAnalyticsHeader = () => { const { data } = useNotableAnomaliesSearch({ skip: false, from, to }); const dispatch = useDispatch(); const getSecuritySolutionLinkProps = useGetSecuritySolutionLinkProps(); - const riskyUsersFeatureEnabled = useIsExperimentalFeatureEnabled('riskyUsersEnabled'); - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; + + const { + services: { ml, http }, + } = useKibana(); const [goToHostRiskTabFilterdByCritical, hostRiskTabUrl] = useMemo(() => { const { onClick, href } = getSecuritySolutionLinkProps({ @@ -85,13 +90,17 @@ export const EntityAnalyticsHeader = () => { const totalAnomalies = useMemo(() => sum(data.map(({ count }) => count)), [data]); + const jobsUrl = useMlHref(ml, http.basePath.get(), { + page: ML_PAGES.ANOMALY_DETECTION_JOBS_MANAGE, + }); + return ( - {riskyHostsFeatureEnabled && ( + {isPlatinumOrTrialLicense && ( - + {hostsSeverityCount[RiskSeverity.critical]} @@ -108,10 +117,10 @@ export const EntityAnalyticsHeader = () => { )} - {riskyUsersFeatureEnabled && ( + {isPlatinumOrTrialLicense && ( - + {usersSeverityCount[RiskSeverity.critical]} @@ -131,12 +140,16 @@ export const EntityAnalyticsHeader = () => { - + {totalAnomalies} - {i18n.ANOMALIES} + + + {i18n.ANOMALIES} + + diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.test.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.test.tsx index 156527bd11e1..4f4edd279a56 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.test.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.test.tsx @@ -11,6 +11,7 @@ import { TestProviders } from '../../../../common/mock'; import { EntityAnalyticsHostRiskScores } from '.'; import { RiskSeverity } from '../../../../../common/search_strategy'; import type { SeverityCount } from '../../../../common/components/severity/types'; +import { useHostRiskScore, useHostRiskScoreKpi } from '../../../../risk_score/containers'; const mockSeverityCount: SeverityCount = { [RiskSeverity.low]: 1, @@ -20,10 +21,6 @@ const mockSeverityCount: SeverityCount = { [RiskSeverity.critical]: 1, }; -jest.mock('../../../../common/hooks/use_experimental_features', () => ({ - useIsExperimentalFeatureEnabled: () => true, -})); - const mockUseQueryToggle = jest .fn() .mockReturnValue({ toggleStatus: false, setToggleStatus: jest.fn() }); @@ -32,30 +29,26 @@ jest.mock('../../../../common/containers/query_toggle', () => { useQueryToggle: () => mockUseQueryToggle(), }; }); - -const mockUseHostRiskScore = jest - .fn() - .mockReturnValue([ - false, - { data: undefined, inspect: null, refetch: () => {}, isModuleEnabled: true }, - ]); -jest.mock('../../../../risk_score/containers', () => { - return { - useHostRiskScoreKpi: () => ({ severityCount: mockSeverityCount, loading: false }), - useHostRiskScore: (params: unknown) => mockUseHostRiskScore(params), - }; -}); +const defaultProps = { + data: undefined, + inspect: null, + refetch: () => {}, + isModuleEnabled: true, + isLicenseValid: true, +}; +const mockUseHostRiskScore = useHostRiskScore as jest.Mock; +const mockUseHostRiskScoreKpi = useHostRiskScoreKpi as jest.Mock; +jest.mock('../../../../risk_score/containers'); describe('EntityAnalyticsHostRiskScores', () => { beforeEach(() => { jest.clearAllMocks(); + mockUseHostRiskScoreKpi.mockReturnValue({ severityCount: mockSeverityCount, loading: false }); + mockUseHostRiskScore.mockReturnValue([false, defaultProps]); }); it('renders enable button when module is disable', () => { - mockUseHostRiskScore.mockReturnValue([ - false, - { data: undefined, inspect: null, refetch: () => {}, isModuleEnabled: false }, - ]); + mockUseHostRiskScore.mockReturnValue([false, { ...defaultProps, isModuleEnabled: false }]); const { getByTestId } = render( @@ -66,10 +59,6 @@ describe('EntityAnalyticsHostRiskScores', () => { }); it("doesn't render enable button when module is enable", () => { - mockUseHostRiskScore.mockReturnValue([ - false, - { data: undefined, inspect: null, refetch: () => {}, isModuleEnabled: true }, - ]); const { queryByTestId } = render( diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.tsx index fa3cda0921c8..69881a128715 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/host_risk_score/index.tsx @@ -40,7 +40,6 @@ import { useCheckSignalIndex } from '../../../../detections/containers/detection import { RiskScoreDonutChart } from '../common/risk_score_donut_chart'; import { BasicTableWithoutBorderBottom } from '../common/basic_table_without_border_bottom'; import { useEnableHostRiskFromUrl } from '../../../../common/hooks/use_enable_host_risk_from_url'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; const TABLE_QUERY_ID = 'hostRiskDashboardTable'; @@ -56,7 +55,6 @@ export const EntityAnalyticsHostRiskScores = () => { const [selectedSeverity, setSelectedSeverity] = useState([]); const getSecuritySolutionLinkProps = useGetSecuritySolutionLinkProps(); const dispatch = useDispatch(); - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); const severityFilter = useMemo(() => { const [filter] = generateSeverityFilter(selectedSeverity, RiskScoreEntity.host); @@ -69,14 +67,15 @@ export const EntityAnalyticsHostRiskScores = () => { skip: !toggleStatus, }); - const [isTableLoading, { data, inspect, refetch, isModuleEnabled }] = useHostRiskScore({ - filterQuery: severityFilter, - skip: !toggleStatus, - pagination: { - cursorStart: 0, - querySize: 5, - }, - }); + const [isTableLoading, { data, inspect, refetch, isLicenseValid, isModuleEnabled }] = + useHostRiskScore({ + filterQuery: severityFilter, + skip: !toggleStatus, + pagination: { + cursorStart: 0, + querySize: 5, + }, + }); useQueryInspector({ queryId: TABLE_QUERY_ID, @@ -124,7 +123,7 @@ export const EntityAnalyticsHostRiskScores = () => { ); }, []); - if (!riskyHostsFeatureEnabled) { + if (!isLicenseValid) { return null; } diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.test.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.test.tsx index 030dbe51d74f..6ddfd912e832 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.test.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.test.tsx @@ -11,6 +11,7 @@ import { TestProviders } from '../../../../common/mock'; import { EntityAnalyticsUserRiskScores } from '.'; import { RiskSeverity } from '../../../../../common/search_strategy'; import type { SeverityCount } from '../../../../common/components/severity/types'; +import { useUserRiskScore, useUserRiskScoreKpi } from '../../../../risk_score/containers'; const mockSeverityCount: SeverityCount = { [RiskSeverity.low]: 1, @@ -20,10 +21,6 @@ const mockSeverityCount: SeverityCount = { [RiskSeverity.critical]: 1, }; -jest.mock('../../../../common/hooks/use_experimental_features', () => ({ - useIsExperimentalFeatureEnabled: () => true, -})); - const mockUseQueryToggle = jest .fn() .mockReturnValue({ toggleStatus: false, setToggleStatus: jest.fn() }); @@ -33,29 +30,27 @@ jest.mock('../../../../common/containers/query_toggle', () => { }; }); -const mockUseUserRiskScore = jest - .fn() - .mockReturnValue([ - false, - { data: undefined, inspect: null, refetch: () => {}, isModuleEnabled: true }, - ]); -jest.mock('../../../../risk_score/containers', () => { - return { - useUserRiskScoreKpi: () => ({ severityCount: mockSeverityCount, loading: false }), - useUserRiskScore: (params: unknown) => mockUseUserRiskScore(params), - }; -}); +const defaultProps = { + data: undefined, + inspect: null, + refetch: () => {}, + isModuleEnabled: true, + isLicenseValid: true, +}; + +const mockUseUserRiskScore = useUserRiskScore as jest.Mock; +const mockUseUserRiskScoreKpi = useUserRiskScoreKpi as jest.Mock; +jest.mock('../../../../risk_score/containers'); describe('EntityAnalyticsUserRiskScores', () => { beforeEach(() => { jest.clearAllMocks(); + mockUseUserRiskScoreKpi.mockReturnValue({ severityCount: mockSeverityCount, loading: false }); + mockUseUserRiskScore.mockReturnValue([false, defaultProps]); }); it('renders enable button when module is disable', () => { - mockUseUserRiskScore.mockReturnValue([ - false, - { data: undefined, inspect: null, refetch: () => {}, isModuleEnabled: false }, - ]); + mockUseUserRiskScore.mockReturnValue([false, { ...defaultProps, isModuleEnabled: false }]); const { getByTestId } = render( @@ -66,10 +61,6 @@ describe('EntityAnalyticsUserRiskScores', () => { }); it("doesn't render enable button when module is enable", () => { - mockUseUserRiskScore.mockReturnValue([ - false, - { data: undefined, inspect: null, refetch: () => {}, isModuleEnabled: true }, - ]); const { queryByTestId } = render( diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.tsx index 68ed1082f4c0..e37235b9b1df 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/user_risk_score/index.tsx @@ -37,7 +37,6 @@ import { getTabsOnUsersUrl } from '../../../../common/components/link_to/redirec import { RISKY_USERS_DOC_LINK } from '../../../../users/components/constants'; import { RiskScoreDonutChart } from '../common/risk_score_donut_chart'; import { BasicTableWithoutBorderBottom } from '../common/basic_table_without_border_bottom'; -import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; const TABLE_QUERY_ID = 'userRiskDashboardTable'; @@ -53,7 +52,6 @@ export const EntityAnalyticsUserRiskScores = () => { const [selectedSeverity, setSelectedSeverity] = useState([]); const getSecuritySolutionLinkProps = useGetSecuritySolutionLinkProps(); const dispatch = useDispatch(); - const riskyUsersFeatureEnabled = useIsExperimentalFeatureEnabled('riskyUsersEnabled'); const severityFilter = useMemo(() => { const [filter] = generateSeverityFilter(selectedSeverity, RiskScoreEntity.user); @@ -66,14 +64,15 @@ export const EntityAnalyticsUserRiskScores = () => { skip: !toggleStatus, }); - const [isTableLoading, { data, inspect, refetch, isModuleEnabled }] = useUserRiskScore({ - filterQuery: severityFilter, - skip: !toggleStatus, - pagination: { - cursorStart: 0, - querySize: 5, - }, - }); + const [isTableLoading, { data, inspect, refetch, isLicenseValid, isModuleEnabled }] = + useUserRiskScore({ + filterQuery: severityFilter, + skip: !toggleStatus, + pagination: { + cursorStart: 0, + querySize: 5, + }, + }); useQueryInspector({ queryId: TABLE_QUERY_ID, @@ -120,7 +119,7 @@ export const EntityAnalyticsUserRiskScores = () => { ); }, []); - if (!riskyUsersFeatureEnabled) { + if (!isLicenseValid) { return null; } diff --git a/x-pack/plugins/security_solution/public/overview/components/host_overview/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/overview/components/host_overview/__snapshots__/index.test.tsx.snap index 540a36debf28..a79c7abde9cc 100644 --- a/x-pack/plugins/security_solution/public/overview/components/host_overview/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/overview/components/host_overview/__snapshots__/index.test.tsx.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Host Summary Component rendering it renders the default Host Summary 1`] = ` +exports[`Host Summary Component it renders the default Host Summary 1`] = ` `; -exports[`Host Summary Component rendering it renders the panel view Host Summary 1`] = ` +exports[`Host Summary Component it renders the panel view Host Summary 1`] = ` ({ - useHostRiskScore: jest.fn().mockReturnValue([ - true, - { - data: undefined, - isModuleEnabled: false, - }, - ]), -})); +const defaultProps = { + data: undefined, + inspect: null, + refetch: () => {}, + isModuleEnabled: true, + isLicenseValid: true, +}; + +jest.mock('../../../risk_score/containers/all'); + +const mockUseHostRiskScore = useHostRiskScore as jest.Mock; describe('Host Summary Component', () => { - describe('rendering', () => { - const mockProps = { - anomaliesData: mockAnomalies, - data: mockData.Hosts.edges[0].node, - endDate: '2019-06-18T06:00:00.000Z', - id: 'hostOverview', - indexNames: [], - isInDetailsSidePanel: false, - isLoadingAnomaliesData: false, - loading: false, - narrowDateRange: jest.fn(), - startDate: '2019-06-15T06:00:00.000Z', - hostName: 'testHostName', - }; + const mockProps = { + anomaliesData: mockAnomalies, + data: mockData.Hosts.edges[0].node, + endDate: '2019-06-18T06:00:00.000Z', + id: 'hostOverview', + indexNames: [], + isInDetailsSidePanel: false, + isLoadingAnomaliesData: false, + loading: false, + narrowDateRange: jest.fn(), + startDate: '2019-06-15T06:00:00.000Z', + hostName: 'testHostName', + }; - test('it renders the default Host Summary', () => { - const wrapper = shallow( - - - - ); + beforeEach(() => { + jest.clearAllMocks(); + mockUseHostRiskScore.mockReturnValue([true, { ...defaultProps, isModuleEnabled: false }]); + }); - expect(wrapper.find('HostOverview')).toMatchSnapshot(); - }); + test('it renders the default Host Summary', () => { + const wrapper = shallow( + + + + ); - test('it renders the panel view Host Summary', () => { - const panelViewProps = { - ...mockProps, - isInDetailsSidePanel: true, - }; + expect(wrapper.find('HostOverview')).toMatchSnapshot(); + }); - const wrapper = shallow( - - - - ); + test('it renders the panel view Host Summary', () => { + const panelViewProps = { + ...mockProps, + isInDetailsSidePanel: true, + }; - expect(wrapper.find('HostOverview')).toMatchSnapshot(); - }); + const wrapper = shallow( + + + + ); - test('it renders host risk score and level', () => { - const panelViewProps = { - ...mockProps, - isInDetailsSidePanel: true, - }; - const risk = 'very high host risk'; - const riskScore = 9999999; + expect(wrapper.find('HostOverview')).toMatchSnapshot(); + }); - (useHostRiskScore as jest.Mock).mockReturnValue([ - false, - { - data: [ - { - host: { - name: 'testHostmame', - risk: { - rule_risks: [], - calculated_score_norm: riskScore, - calculated_level: risk, - }, + test('it renders host risk score and level', () => { + const panelViewProps = { + ...mockProps, + isInDetailsSidePanel: true, + }; + const risk = 'very high host risk'; + const riskScore = 9999999; + mockUseHostRiskScore.mockReturnValue([ + false, + { + ...defaultProps, + data: [ + { + host: { + name: 'testHostmame', + risk: { + rule_risks: [], + calculated_score_norm: riskScore, + calculated_level: risk, }, }, - ], - isModuleEnabled: true, - }, - ]); + }, + ], + }, + ]); - const { getByTestId } = render( - - - - ); + const { getByTestId } = render( + + + + ); - expect(getByTestId('host-risk-overview')).toHaveTextContent(risk); - expect(getByTestId('host-risk-overview')).toHaveTextContent(riskScore.toString()); - }); + expect(getByTestId('host-risk-overview')).toHaveTextContent(risk); + expect(getByTestId('host-risk-overview')).toHaveTextContent(riskScore.toString()); }); }); diff --git a/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx b/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx index d3a1f601445f..83b6243c4e04 100644 --- a/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx @@ -83,7 +83,7 @@ export const HostOverview = React.memo( [hostName] ); - const [_, { data: hostRisk, isModuleEnabled }] = useHostRiskScore({ + const [_, { data: hostRisk, isLicenseValid }] = useHostRiskScore({ filterQuery, skip: hostName == null, }); @@ -101,39 +101,32 @@ export const HostOverview = React.memo( ); const [hostRiskScore, hostRiskLevel] = useMemo(() => { - if (isModuleEnabled) { - const hostRiskData = hostRisk && hostRisk.length > 0 ? hostRisk[0] : undefined; - return [ - { - title: i18n.HOST_RISK_SCORE, - description: ( - <> - {hostRiskData - ? Math.round(hostRiskData.host.risk.calculated_score_norm) - : getEmptyTagValue()} - - ), - }, - - { - title: i18n.HOST_RISK_CLASSIFICATION, - description: ( - <> - {hostRiskData ? ( - - ) : ( - getEmptyTagValue() - )} - - ), - }, - ]; - } - return [undefined, undefined]; - }, [hostRisk, isModuleEnabled]); + const hostRiskData = hostRisk && hostRisk.length > 0 ? hostRisk[0] : undefined; + return [ + { + title: i18n.HOST_RISK_SCORE, + description: ( + <> + {hostRiskData + ? Math.round(hostRiskData.host.risk.calculated_score_norm) + : getEmptyTagValue()} + + ), + }, + { + title: i18n.HOST_RISK_CLASSIFICATION, + description: ( + <> + {hostRiskData ? ( + + ) : ( + getEmptyTagValue() + )} + + ), + }, + ]; + }, [hostRisk]); const column: DescriptionList[] = useMemo( () => [ @@ -273,7 +266,7 @@ export const HostOverview = React.memo( )} - {hostRiskScore && hostRiskLevel && ( + {isLicenseValid && ( `; -exports[`User Summary Component rendering it renders the panel view User Summary 1`] = ` +exports[`User Summary Component it renders the panel view User Summary 1`] = ` ({ - useUserRiskScore: jest.fn().mockReturnValue([ - true, - { - data: undefined, - isModuleEnabled: false, - }, - ]), -})); +const defaultProps = { + data: undefined, + inspect: null, + refetch: () => {}, + isModuleEnabled: true, + isLicenseValid: true, +}; + +jest.mock('../../../risk_score/containers/all'); + +const mockUseUserRiskScore = useUserRiskScore as jest.Mock; describe('User Summary Component', () => { - describe('rendering', () => { - const mockProps: UserSummaryProps = { - anomaliesData: mockAnomalies, - data: { - user: { - id: ['aa7ca589f1b8220002f2fc61c64cfbf1'], - name: ['username'], - domain: ['domain'], - }, - host: { - ip: ['10.142.0.7', 'fe80::4001:aff:fe8e:7'], - os: { - family: ['debian'], - name: ['Debian GNU/Linux'], - }, + const mockProps: UserSummaryProps = { + anomaliesData: mockAnomalies, + data: { + user: { + id: ['aa7ca589f1b8220002f2fc61c64cfbf1'], + name: ['username'], + domain: ['domain'], + }, + host: { + ip: ['10.142.0.7', 'fe80::4001:aff:fe8e:7'], + os: { + family: ['debian'], + name: ['Debian GNU/Linux'], }, }, - endDate: '2019-06-18T06:00:00.000Z', - id: 'userOverview', - isInDetailsSidePanel: false, - isLoadingAnomaliesData: false, - loading: false, - narrowDateRange: jest.fn(), - startDate: '2019-06-15T06:00:00.000Z', - userName: 'testUserName', - indexPatterns: [], + }, + endDate: '2019-06-18T06:00:00.000Z', + id: 'userOverview', + isInDetailsSidePanel: false, + isLoadingAnomaliesData: false, + loading: false, + narrowDateRange: jest.fn(), + startDate: '2019-06-15T06:00:00.000Z', + userName: 'testUserName', + indexPatterns: [], + }; + + beforeEach(() => { + jest.clearAllMocks(); + mockUseUserRiskScore.mockReturnValue([true, { ...defaultProps, isModuleEnabled: false }]); + }); + + test('it renders the default User Summary', () => { + const wrapper = shallow( + + + + ); + + expect(wrapper.find('UserOverview')).toMatchSnapshot(); + }); + + test('it renders the panel view User Summary', () => { + const panelViewProps = { + ...mockProps, + isInDetailsSidePanel: true, + }; + + const wrapper = shallow( + + + + ); + + expect(wrapper.find('UserOverview')).toMatchSnapshot(); + }); + + test('it renders user risk score and level', () => { + const panelViewProps = { + ...mockProps, + isInDetailsSidePanel: true, }; + const risk = 'very high hos risk'; + const riskScore = 9999999; - test('it renders the default User Summary', () => { - const wrapper = shallow( - - - - ); - - expect(wrapper.find('UserOverview')).toMatchSnapshot(); - }); - - test('it renders the panel view User Summary', () => { - const panelViewProps = { - ...mockProps, - isInDetailsSidePanel: true, - }; - - const wrapper = shallow( - - - - ); - - expect(wrapper.find('UserOverview')).toMatchSnapshot(); - }); - - test('it renders user risk score and level', () => { - const panelViewProps = { - ...mockProps, - isInDetailsSidePanel: true, - }; - const risk = 'very high hos risk'; - const riskScore = 9999999; - - (useUserRiskScore as jest.Mock).mockReturnValue([ - false, - { - data: [ - { - user: { - name: 'testUsermame', - risk: { - rule_risks: [], - calculated_level: risk, - calculated_score_norm: riskScore, - }, + mockUseUserRiskScore.mockReturnValue([ + false, + { + ...defaultProps, + data: [ + { + user: { + name: 'testUsermame', + risk: { + rule_risks: [], + calculated_level: risk, + calculated_score_norm: riskScore, }, }, - ], - isModuleEnabled: true, - }, - ]); + }, + ], + }, + ]); - const { getByTestId } = render( - - - - ); + const { getByTestId } = render( + + + + ); - expect(getByTestId('user-risk-overview')).toHaveTextContent(risk); - expect(getByTestId('user-risk-overview')).toHaveTextContent(riskScore.toString()); - }); + expect(getByTestId('user-risk-overview')).toHaveTextContent(risk); + expect(getByTestId('user-risk-overview')).toHaveTextContent(riskScore.toString()); }); }); diff --git a/x-pack/plugins/security_solution/public/overview/components/user_overview/index.tsx b/x-pack/plugins/security_solution/public/overview/components/user_overview/index.tsx index 6c5f4a952e9a..32a488f896ab 100644 --- a/x-pack/plugins/security_solution/public/overview/components/user_overview/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/user_overview/index.tsx @@ -10,7 +10,6 @@ import { euiLightVars as lightTheme, euiDarkVars as darkTheme } from '@kbn/ui-th import { getOr } from 'lodash/fp'; import React, { useCallback, useMemo } from 'react'; import styled from 'styled-components'; -import type { RiskSeverity } from '../../../../common/search_strategy'; import { buildUserNamesFilter } from '../../../../common/search_strategy'; import { DEFAULT_DARK_MODE } from '../../../../common/constants'; import type { DescriptionList } from '../../../../common/utility_types'; @@ -81,7 +80,7 @@ export const UserOverview = React.memo( () => (userName ? buildUserNamesFilter([userName]) : undefined), [userName] ); - const [_, { data: userRisk, isModuleEnabled }] = useUserRiskScore({ + const [_, { data: userRisk, isLicenseValid }] = useUserRiskScore({ filterQuery, skip: userName == null, }); @@ -91,7 +90,7 @@ export const UserOverview = React.memo( ), @@ -99,38 +98,32 @@ export const UserOverview = React.memo( ); const [userRiskScore, userRiskLevel] = useMemo(() => { - if (isModuleEnabled) { - const userRiskData = userRisk && userRisk.length > 0 ? userRisk[0] : undefined; - return [ - { - title: i18n.USER_RISK_SCORE, - description: ( - <> - {userRiskData - ? Math.round(userRiskData.user.risk.calculated_score_norm) - : getEmptyTagValue()} - - ), - }, - { - title: i18n.USER_RISK_CLASSIFICATION, - description: ( - <> - {userRiskData ? ( - - ) : ( - getEmptyTagValue() - )} - - ), - }, - ]; - } - return [undefined, undefined]; - }, [userRisk, isModuleEnabled]); + const userRiskData = userRisk && userRisk.length > 0 ? userRisk[0] : undefined; + return [ + { + title: i18n.USER_RISK_SCORE, + description: ( + <> + {userRiskData + ? Math.round(userRiskData.user.risk.calculated_score_norm) + : getEmptyTagValue()} + + ), + }, + { + title: i18n.USER_RISK_CLASSIFICATION, + description: ( + <> + {userRiskData ? ( + + ) : ( + getEmptyTagValue() + )} + + ), + }, + ]; + }, [userRisk]); const column = useMemo( () => [ @@ -255,7 +248,7 @@ export const UserOverview = React.memo( )} - {userRiskScore && userRiskLevel && ( + {isLicenseValid && ( { const { indicesExist, loading: isSourcererLoading, indexPattern } = useSourcererDataView(); + return ( <> {indicesExist ? ( <> - + ({ useSpaceId: jest.fn().mockReturnValue('default'), })); -jest.mock('../../../common/hooks/use_experimental_features', () => ({ - useIsExperimentalFeatureEnabled: jest.fn(), -})); - jest.mock('../../../common/hooks/use_app_toasts'); +const mockUseMlCapabilities = jest.fn(); +jest.mock('../../../common/components/ml/hooks/use_ml_capabilities', () => ({ + useMlCapabilities: () => mockUseMlCapabilities(), +})); + const mockUseSearchStrategy = useSearchStrategy as jest.Mock; -const mockUseIsExperimentalFeatureEnabled = useIsExperimentalFeatureEnabled as jest.Mock; const mockSearch = jest.fn(); const mockRefetch = jest.fn(); @@ -43,7 +42,7 @@ let appToastsMock: jest.Mocked>; (useAppToasts as jest.Mock).mockReturnValue(appToastsMock); }); test('does not search if feature is not enabled', () => { - mockUseIsExperimentalFeatureEnabled.mockReturnValue(false); + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: false }); mockUseSearchStrategy.mockReturnValue({ loading: false, result: { @@ -65,6 +64,7 @@ let appToastsMock: jest.Mocked>; data: undefined, inspect: {}, isInspected: false, + isLicenseValid: false, isModuleEnabled: false, refetch: mockRefetch, totalCount: 0, @@ -73,7 +73,7 @@ let appToastsMock: jest.Mocked>; }); test('if query skipped and feature is enabled, isModuleEnabled should be true', () => { - mockUseIsExperimentalFeatureEnabled.mockReturnValue(true); + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: true }); mockUseSearchStrategy.mockReturnValue({ loading: false, result: { @@ -94,6 +94,7 @@ let appToastsMock: jest.Mocked>; data: undefined, inspect: {}, isInspected: false, + isLicenseValid: true, isModuleEnabled: true, refetch: mockRefetch, totalCount: 0, @@ -102,7 +103,7 @@ let appToastsMock: jest.Mocked>; }); test('handle index not found error', () => { - mockUseIsExperimentalFeatureEnabled.mockReturnValue(true); + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: true }); mockUseSearchStrategy.mockReturnValue({ loading: false, @@ -130,6 +131,7 @@ let appToastsMock: jest.Mocked>; data: undefined, inspect: {}, isInspected: false, + isLicenseValid: true, isModuleEnabled: false, refetch: mockRefetch, totalCount: 0, @@ -138,7 +140,7 @@ let appToastsMock: jest.Mocked>; }); test('show error toast', () => { - mockUseIsExperimentalFeatureEnabled.mockReturnValue(true); + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: true }); const error = new Error(); mockUseSearchStrategy.mockReturnValue({ @@ -161,7 +163,7 @@ let appToastsMock: jest.Mocked>; }); test('runs search if feature is enabled', () => { - mockUseIsExperimentalFeatureEnabled.mockReturnValue(true); + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: true }); mockUseSearchStrategy.mockReturnValue({ loading: false, result: { @@ -187,7 +189,7 @@ let appToastsMock: jest.Mocked>; }); test('return result', async () => { - mockUseIsExperimentalFeatureEnabled.mockReturnValue(true); + mockUseMlCapabilities.mockReturnValue({ isPlatinumOrTrialLicense: true }); mockUseSearchStrategy.mockReturnValue({ loading: false, result: { @@ -209,6 +211,7 @@ let appToastsMock: jest.Mocked>; data: [], inspect: {}, isInspected: false, + isLicenseValid: true, isModuleEnabled: true, refetch: mockRefetch, totalCount: 0, diff --git a/x-pack/plugins/security_solution/public/risk_score/containers/all/index.tsx b/x-pack/plugins/security_solution/public/risk_score/containers/all/index.tsx index ef58555629c8..49605e5c227d 100644 --- a/x-pack/plugins/security_solution/public/risk_score/containers/all/index.tsx +++ b/x-pack/plugins/security_solution/public/risk_score/containers/all/index.tsx @@ -20,10 +20,10 @@ import * as i18n from './translations'; import type { InspectResponse } from '../../../types'; import { useAppToasts } from '../../../common/hooks/use_app_toasts'; import { isIndexNotFoundError } from '../../../common/utils/exceptions'; -import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; import type { inputsModel } from '../../../common/store'; import { useSpaceId } from '../../../common/hooks/use_space_id'; import { useSearchStrategy } from '../../../common/containers/use_search_strategy'; +import { useMlCapabilities } from '../../../common/components/ml/hooks/use_ml_capabilities'; export interface RiskScoreState { data: undefined | StrategyResponseType['data']; @@ -31,7 +31,8 @@ export interface RiskScoreState extends UseRiskScoreParams { defaultIndex: string | undefined; factoryQueryType: T; - featureEnabled: boolean; } export const initialResult: Omit< @@ -66,7 +66,6 @@ export const useHostRiskScore = (params?: UseRiskScoreParams) => { const { timerange, onlyLatest, filterQuery, sort, skip = false, pagination } = params ?? {}; const spaceId = useSpaceId(); const defaultIndex = spaceId ? getHostRiskIndex(spaceId, onlyLatest) : undefined; - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); return useRiskScore({ timerange, @@ -75,7 +74,6 @@ export const useHostRiskScore = (params?: UseRiskScoreParams) => { sort, skip, pagination, - featureEnabled: riskyHostsFeatureEnabled, defaultIndex, factoryQueryType: RiskQueries.hostsRiskScore, }); @@ -86,7 +84,6 @@ export const useUserRiskScore = (params?: UseRiskScoreParams) => { const spaceId = useSpaceId(); const defaultIndex = spaceId ? getUserRiskIndex(spaceId, onlyLatest) : undefined; - const riskyUsersFeatureEnabled = useIsExperimentalFeatureEnabled('riskyUsersEnabled'); return useRiskScore({ timerange, onlyLatest, @@ -94,7 +91,6 @@ export const useUserRiskScore = (params?: UseRiskScoreParams) => { sort, skip, pagination, - featureEnabled: riskyUsersFeatureEnabled, defaultIndex, factoryQueryType: RiskQueries.usersRiskScore, }); @@ -106,7 +102,6 @@ const useRiskScore = ): [boolean, RiskScoreState] => { @@ -127,6 +122,7 @@ const useRiskScore = ({ @@ -134,10 +130,13 @@ const useRiskScore = { - if (!skip && riskScoreRequest != null && featureEnabled) { + if (!skip && riskScoreRequest != null && isPlatinumOrTrialLicense) { search(riskScoreRequest); } - }, [featureEnabled, riskScoreRequest, search, skip]); + }, [isPlatinumOrTrialLicense, riskScoreRequest, search, skip]); return [loading, riskScoreResponse]; }; diff --git a/x-pack/plugins/security_solution/public/risk_score/containers/index.ts b/x-pack/plugins/security_solution/public/risk_score/containers/index.ts index 323a6d26acb3..605bbd7d0d6b 100644 --- a/x-pack/plugins/security_solution/public/risk_score/containers/index.ts +++ b/x-pack/plugins/security_solution/public/risk_score/containers/index.ts @@ -5,7 +5,10 @@ * 2.0. */ -import type { HostRiskScore } from '../../../common/search_strategy/security_solution/risk_score'; +import type { + HostRiskScore, + UserRiskScore, +} from '../../../common/search_strategy/security_solution/risk_score'; export * from './all'; export * from './kpi'; @@ -24,6 +27,12 @@ export const enum HostRiskScoreQueryId { export interface HostRisk { loading: boolean; - isModuleEnabled?: boolean; + isModuleEnabled: boolean; result?: HostRiskScore[]; } + +export interface UserRisk { + loading: boolean; + isModuleEnabled: boolean; + result?: UserRiskScore[]; +} diff --git a/x-pack/plugins/security_solution/public/risk_score/containers/kpi/index.tsx b/x-pack/plugins/security_solution/public/risk_score/containers/kpi/index.tsx index 6d4ee5c73c5f..2c137a8bf1e0 100644 --- a/x-pack/plugins/security_solution/public/risk_score/containers/kpi/index.tsx +++ b/x-pack/plugins/security_solution/public/risk_score/containers/kpi/index.tsx @@ -28,9 +28,9 @@ import { import { useKibana } from '../../../common/lib/kibana'; import { isIndexNotFoundError } from '../../../common/utils/exceptions'; import type { ESTermQuery } from '../../../../common/typed_json'; -import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; import type { SeverityCount } from '../../../common/components/severity/types'; import { useSpaceId } from '../../../common/hooks/use_space_id'; +import { useMlCapabilities } from '../../../common/components/ml/hooks/use_ml_capabilities'; type GetHostRiskScoreProps = KpiRiskScoreRequestOptions & { data: DataPublicPluginStart; @@ -93,14 +93,14 @@ export const useUserRiskScoreKpi = ({ }: UseUserRiskScoreKpiProps): RiskScoreKpi => { const spaceId = useSpaceId(); const defaultIndex = spaceId ? getUserRiskIndex(spaceId) : undefined; - const riskyUsersFeatureEnabled = useIsExperimentalFeatureEnabled('riskyUsersEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; return useRiskScoreKpi({ filterQuery, skip, defaultIndex, entity: RiskScoreEntity.user, - featureEnabled: riskyUsersFeatureEnabled, + featureEnabled: isPlatinumOrTrialLicense, }); }; @@ -110,14 +110,14 @@ export const useHostRiskScoreKpi = ({ }: UseHostRiskScoreKpiProps): RiskScoreKpi => { const spaceId = useSpaceId(); const defaultIndex = spaceId ? getHostRiskIndex(spaceId) : undefined; - const riskyHostsFeatureEnabled = useIsExperimentalFeatureEnabled('riskyHostsEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; return useRiskScoreKpi({ filterQuery, skip, defaultIndex, entity: RiskScoreEntity.host, - featureEnabled: riskyHostsFeatureEnabled, + featureEnabled: isPlatinumOrTrialLicense, }); }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx index 7ec4e5c2c942..305eef7ef219 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/expandable_event.tsx @@ -24,7 +24,6 @@ import { EventDetails } from '../../../../common/components/event_details/event_ import type { TimelineEventsDetailsItem } from '../../../../../common/search_strategy/timeline'; import * as i18n from './translations'; import { PreferenceFormattedDate } from '../../../../common/components/formatted_date'; -import type { HostRisk } from '../../../../risk_score/containers'; export type HandleOnEventClosed = () => void; interface Props { @@ -38,7 +37,6 @@ interface Props { rawEventData: object | undefined; timelineTabType: TimelineTabs | 'flyout'; timelineId: string; - hostRisk: HostRisk | null; handleOnEventClosed: HandleOnEventClosed; isReadOnly?: boolean; } @@ -107,7 +105,6 @@ export const ExpandableEvent = React.memo( isDraggable, loading, detailsData, - hostRisk, rawEventData, handleOnEventClosed, isReadOnly, @@ -133,7 +130,6 @@ export const ExpandableEvent = React.memo( rawEventData={rawEventData} timelineId={timelineId} timelineTabType={timelineTabType} - hostRisk={hostRisk} handleOnEventClosed={handleOnEventClosed} isReadOnly={isReadOnly} /> diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/body.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/body.tsx index 58d09a645001..25d7371c2930 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/body.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/body.tsx @@ -16,7 +16,6 @@ import type { } from '../../../../../../common/search_strategy'; import type { HandleOnEventClosed } from '../expandable_event'; import { ExpandableEvent } from '../expandable_event'; -import type { HostRisk } from '../../../../../risk_score/containers'; const StyledEuiFlyoutBody = styled(EuiFlyoutBody)` .euiFlyoutBody__overflow { @@ -40,7 +39,6 @@ interface FlyoutBodyComponentProps { handleIsolationActionSuccess: () => void; handleOnEventClosed: HandleOnEventClosed; hostName: string; - hostRisk: HostRisk | null; isAlert: boolean; isDraggable?: boolean; isReadOnly?: boolean; @@ -61,7 +59,6 @@ const FlyoutBodyComponent = ({ handleIsolationActionSuccess, handleOnEventClosed, hostName, - hostRisk, isAlert, isDraggable, isReadOnly, @@ -100,7 +97,6 @@ const FlyoutBodyComponent = ({ rawEventData={rawEventData} timelineId={timelineId} timelineTabType="flyout" - hostRisk={hostRisk} handleOnEventClosed={handleOnEventClosed} isReadOnly={isReadOnly} /> diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/index.tsx index 22e4a8c568d1..0704bee217b1 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/flyout/index.tsx @@ -9,9 +9,6 @@ import { EntityType, TimelineId } from '@kbn/timelines-plugin/common'; import { noop } from 'lodash/fp'; import React, { useCallback, useMemo, useState } from 'react'; -import { buildHostNamesFilter } from '../../../../../../common/search_strategy'; -import type { HostRisk } from '../../../../../risk_score/containers'; -import { useHostRiskScore } from '../../../../../risk_score/containers'; import { useHostIsolationTools } from '../use_host_isolation_tools'; import { FlyoutHeaderContent } from './header'; import { FlyoutBody } from './body'; @@ -45,35 +42,6 @@ export const useToGetInternalFlyout = () => { const { alertId, isAlert, hostName, ruleName, timestamp } = useBasicDataFromDetailsData(detailsData); - const filterQuery = useMemo( - () => (hostName ? buildHostNamesFilter([hostName]) : undefined), - [hostName] - ); - - const pagination = useMemo( - () => ({ - cursorStart: 0, - querySize: 1, - }), - [] - ); - const [hostRiskLoading, { data, isModuleEnabled }] = useHostRiskScore({ - filterQuery, - pagination, - }); - - const hostRisk: HostRisk | null = useMemo( - () => - data - ? { - loading: hostRiskLoading, - isModuleEnabled, - result: data, - } - : null, - [data, hostRiskLoading, isModuleEnabled] - ); - const { isolateAction, isHostIsolationPanelOpen, @@ -99,7 +67,6 @@ export const useToGetInternalFlyout = () => { detailsData={detailsData} event={{ eventId: localAlert._id, indexName: localAlert._index }} hostName={hostName ?? ''} - hostRisk={hostRisk} handleIsolationActionSuccess={handleIsolationActionSuccess} handleOnEventClosed={noop} isAlert={isAlert} @@ -121,7 +88,6 @@ export const useToGetInternalFlyout = () => { detailsData, handleIsolationActionSuccess, hostName, - hostRisk, isAlert, isHostIsolationPanelOpen, isIsolateActionSuccessBannerVisible, diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/helpers.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/helpers.tsx index 065ac297ee46..6bee08902747 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/helpers.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/helpers.tsx @@ -16,6 +16,7 @@ export interface GetBasicDataFromDetailsData { agentId?: string; isAlert: boolean; hostName: string; + userName: string; ruleName: string; timestamp: string; } @@ -42,6 +43,11 @@ export const useBasicDataFromDetailsData = ( [data] ); + const userName = useMemo( + () => getFieldValue({ category: 'user', field: 'user.name' }, data), + [data] + ); + const timestamp = useMemo( () => getFieldValue({ category: 'base', field: '@timestamp' }, data), [data] @@ -53,10 +59,11 @@ export const useBasicDataFromDetailsData = ( agentId, isAlert, hostName, + userName, ruleName, timestamp, }), - [agentId, alertId, hostName, isAlert, ruleName, timestamp] + [agentId, alertId, hostName, isAlert, ruleName, timestamp, userName] ); }; diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.test.tsx index 7044ee548c3a..30713af95df5 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.test.tsx @@ -103,6 +103,13 @@ jest.mock('../../../../risk_score/containers', () => { isModuleEnabled: false, }, ]), + useUserRiskScore: jest.fn().mockReturnValue([ + true, + { + data: undefined, + isModuleEnabled: false, + }, + ]), }; }); diff --git a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx index 00397ea43e59..11a20a0c64f0 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/side_panel/event_details/index.tsx @@ -15,9 +15,6 @@ import type { BrowserFields } from '../../../../common/containers/source'; import { ExpandableEvent, ExpandableEventTitle } from './expandable_event'; import { useTimelineEventsDetails } from '../../../containers/details'; import type { TimelineTabs } from '../../../../../common/types/timeline'; -import { buildHostNamesFilter } from '../../../../../common/search_strategy'; -import type { HostRisk } from '../../../../risk_score/containers'; -import { useHostRiskScore } from '../../../../risk_score/containers'; import { useHostIsolationTools } from './use_host_isolation_tools'; import { FlyoutBody, FlyoutHeader, FlyoutFooter } from './flyout'; import { useBasicDataFromDetailsData, getAlertIndexAlias } from './helpers'; @@ -79,34 +76,6 @@ const EventDetailsPanelComponent: React.FC = ({ const { alertId, isAlert, hostName, ruleName, timestamp } = useBasicDataFromDetailsData(detailsData); - const filterQuery = useMemo( - () => (hostName ? buildHostNamesFilter([hostName]) : undefined), - [hostName] - ); - - const pagination = useMemo( - () => ({ - cursorStart: 0, - querySize: 1, - }), - [] - ); - - const [hostRiskLoading, { data, isModuleEnabled }] = useHostRiskScore({ - filterQuery, - pagination, - }); - - const hostRisk: HostRisk | null = useMemo(() => { - return data - ? { - loading: hostRiskLoading, - isModuleEnabled, - result: data, - } - : null; - }, [data, hostRiskLoading, isModuleEnabled]); - const header = useMemo( () => isFlyoutView || isHostIsolationPanelOpen ? ( @@ -149,7 +118,6 @@ const EventDetailsPanelComponent: React.FC = ({ detailsData={detailsData} event={expandedEvent} hostName={hostName} - hostRisk={hostRisk} handleIsolationActionSuccess={handleIsolationActionSuccess} handleOnEventClosed={handleOnEventClosed} isAlert={isAlert} @@ -198,7 +166,6 @@ const EventDetailsPanelComponent: React.FC = ({ rawEventData={rawEventData} timelineId={timelineId} timelineTabType={tabType} - hostRisk={hostRisk} handleOnEventClosed={handleOnEventClosed} /> @@ -212,7 +179,6 @@ const EventDetailsPanelComponent: React.FC = ({ handleIsolationActionSuccess, handleOnEventClosed, hostName, - hostRisk, isAlert, isDraggable, isFlyoutView, diff --git a/x-pack/plugins/security_solution/public/users/components/kpi_users/index.tsx b/x-pack/plugins/security_solution/public/users/components/kpi_users/index.tsx index 3c97f0ad49b4..277403c5292f 100644 --- a/x-pack/plugins/security_solution/public/users/components/kpi_users/index.tsx +++ b/x-pack/plugins/security_solution/public/users/components/kpi_users/index.tsx @@ -19,11 +19,11 @@ import { RISKY_USERS_DOC_LINK } from '../constants'; export const UsersKpiComponent = React.memo( ({ filterQuery, from, indexNames, to, setQuery, skip, updateDateRange }) => { - const [_, { isModuleEnabled }] = useUserRiskScore(); + const [loading, { isLicenseValid, isModuleEnabled }] = useUserRiskScore(); return ( <> - {isModuleEnabled === false && ( + {isLicenseValid && !isModuleEnabled && !loading && ( <> = ({ @@ -61,7 +60,7 @@ const UsersDetailsComponent: React.FC = ({ usersDetailsPagePath, }) => { const dispatch = useDispatch(); - const riskyUsersFeatureEnabled = useIsExperimentalFeatureEnabled('riskyUsersEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; const getTimeline = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); const graphEventId = useShallowEqualSelector( (state) => (getTimeline(state, TimelineId.hostsPageEvents) ?? timelineDefaults).graphEventId @@ -183,7 +182,7 @@ const UsersDetailsComponent: React.FC = ({ navTabs={navTabsUsersDetails( detailName, hasMlUserPermissions(capabilities), - riskyUsersFeatureEnabled + isPlatinumOrTrialLicense )} /> diff --git a/x-pack/plugins/security_solution/public/users/pages/details/nav_tabs.tsx b/x-pack/plugins/security_solution/public/users/pages/details/nav_tabs.tsx index 90bc856e0d05..588372272e54 100644 --- a/x-pack/plugins/security_solution/public/users/pages/details/nav_tabs.tsx +++ b/x-pack/plugins/security_solution/public/users/pages/details/nav_tabs.tsx @@ -10,6 +10,7 @@ import * as i18n from '../translations'; import type { UsersDetailsNavTab } from './types'; import { UsersTableType } from '../../store/model'; import { USERS_PATH } from '../../../../common/constants'; +import { TECHNICAL_PREVIEW } from '../../../overview/pages/translations'; const getTabsOnUsersDetailsUrl = (userName: string, tabName: UsersTableType) => `${USERS_PATH}/name/${userName}/${tabName}`; @@ -45,6 +46,10 @@ export const navTabsUsersDetails = ( name: i18n.NAVIGATION_RISK_TITLE, href: getTabsOnUsersDetailsUrl(userName, UsersTableType.risk), disabled: false, + isBeta: true, + betaOptions: { + text: TECHNICAL_PREVIEW, + }, }, }; diff --git a/x-pack/plugins/security_solution/public/users/pages/nav_tabs.tsx b/x-pack/plugins/security_solution/public/users/pages/nav_tabs.tsx index 0e5218090f16..801074b797bf 100644 --- a/x-pack/plugins/security_solution/public/users/pages/nav_tabs.tsx +++ b/x-pack/plugins/security_solution/public/users/pages/nav_tabs.tsx @@ -10,6 +10,7 @@ import * as i18n from './translations'; import { UsersTableType } from '../store/model'; import type { UsersNavTab } from './navigation/types'; import { USERS_PATH } from '../../../common/constants'; +import { TECHNICAL_PREVIEW } from '../../overview/pages/translations'; const getTabsOnUsersUrl = (tabName: UsersTableType) => `${USERS_PATH}/${tabName}`; @@ -49,6 +50,10 @@ export const navTabsUsers = ( name: i18n.NAVIGATION_RISK_TITLE, href: getTabsOnUsersUrl(UsersTableType.risk), disabled: false, + isBeta: true, + betaOptions: { + text: TECHNICAL_PREVIEW, + }, }, }; diff --git a/x-pack/plugins/security_solution/public/users/pages/users.tsx b/x-pack/plugins/security_solution/public/users/pages/users.tsx index 94345fa24f37..d3140a330da6 100644 --- a/x-pack/plugins/security_solution/public/users/pages/users.tsx +++ b/x-pack/plugins/security_solution/public/users/pages/users.tsx @@ -50,7 +50,6 @@ import { generateSeverityFilter } from '../../hosts/store/helpers'; import { UsersTableType } from '../store/model'; import { hasMlUserPermissions } from '../../../common/machine_learning/has_ml_user_permissions'; import { useMlCapabilities } from '../../common/components/ml/hooks/use_ml_capabilities'; -import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; import { LandingPageComponent } from '../../common/components/landing_page'; import { userNameExistsFilter } from './details/helpers'; @@ -169,10 +168,10 @@ const UsersComponent = () => { ); const capabilities = useMlCapabilities(); - const riskyUsersFeatureEnabled = useIsExperimentalFeatureEnabled('riskyUsersEnabled'); + const isPlatinumOrTrialLicense = useMlCapabilities().isPlatinumOrTrialLicense; const navTabs = useMemo( - () => navTabsUsers(hasMlUserPermissions(capabilities), riskyUsersFeatureEnabled), - [capabilities, riskyUsersFeatureEnabled] + () => navTabsUsers(hasMlUserPermissions(capabilities), isPlatinumOrTrialLicense), + [capabilities, isPlatinumOrTrialLicense] ); return ( diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.test.ts index d21bc53de178..4208ea44937b 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.test.ts @@ -28,15 +28,11 @@ class IndexNotFoundException extends Error { } } -const mockDeps = (riskyHostsEnabled = true) => ({ +const mockDeps = () => ({ ...defaultMockDeps, spaceId: 'test-space', endpointContext: { ...defaultMockDeps.endpointContext, - experimentalFeatures: { - ...defaultMockDeps.endpointContext.experimentalFeatures, - riskyHostsEnabled, - }, }, }); @@ -70,7 +66,11 @@ describe('allHosts search strategy', () => { describe('parse', () => { test('should parse data correctly', async () => { - const result = await allHosts.parse(mockOptions, mockSearchStrategyResponse, mockDeps(false)); + const mockedDeps = mockDeps(); + // @ts-expect-error incomplete type + mockedDeps.esClient.asCurrentUser.search.mockResponse({ hits: { hits: [] } }); + + const result = await allHosts.parse(mockOptions, mockSearchStrategyResponse, mockDeps()); expect(result).toMatchObject(formattedSearchStrategyResponse); }); @@ -131,35 +131,6 @@ describe('allHosts search strategy', () => { }); }); - test('should not enhance data when feature flag is disabled', async () => { - const risk = 'TEST_RISK_SCORE'; - const hostName: string = get( - 'aggregations.host_data.buckets[0].key', - mockSearchStrategyResponse.rawResponse - ); - const mockedDeps = mockDeps(false); - - mockedDeps.esClient.asCurrentUser.search.mockResponse({ - hits: { - hits: [ - // @ts-expect-error incomplete type - { - _source: { - risk, - host: { - name: hostName, - }, - }, - }, - ], - }, - }); - - const result = await allHosts.parse(mockOptions, mockSearchStrategyResponse, mockedDeps); - - expect(result.edges[0].node.risk).toBeUndefined(); - }); - test("should not enhance data when index doesn't exist", async () => { const mockedDeps = mockDeps(); mockedDeps.esClient.asCurrentUser.search.mockImplementation(() => { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts index cecfc60fbbae..c84717b6e1a8 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/all/index.ts @@ -62,10 +62,9 @@ export const allHosts: SecuritySolutionFactory = { const hostNames = edges.map((edge) => getOr('', 'node.host.name[0]', edge)); - const enhancedEdges = - deps?.spaceId && deps?.endpointContext.experimentalFeatures.riskyHostsEnabled - ? await enhanceEdges(edges, hostNames, deps.spaceId, deps.esClient) - : edges; + const enhancedEdges = deps?.spaceId + ? await enhanceEdges(edges, hostNames, deps.spaceId, deps.esClient) + : edges; return { ...response, diff --git a/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.test.ts b/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.test.ts index d6ae8024b92f..3be60d3f3a31 100644 --- a/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.test.ts @@ -847,6 +847,63 @@ describe('estimateCapacity', () => { }, }); }); + + test("estimates minutes_to_drain_overdue even if there isn't any overdue task", async () => { + expect( + estimateCapacity( + logger, + mockStats( + { max_workers: 10, poll_interval: 3000 }, + { + overdue: undefined, + owner_ids: 1, + overdue_non_recurring: 0, + capacity_requirements: { + per_minute: 60, + per_hour: 0, + per_day: 0, + }, + }, + { + execution: { + duration: {}, + duration_by_persistence: { + ephemeral: { + p50: 400, + p90: 500, + p95: 1200, + p99: 1500, + }, + non_recurring: { + p50: 400, + p90: 500, + p95: 1200, + p99: 1500, + }, + recurring: { + p50: 400, + p90: 500, + p95: 1200, + p99: 1500, + }, + }, + // no non-recurring executions in the system in recent history + persistence: { + ephemeral: 0, + non_recurring: 0, + recurring: 100, + }, + result_frequency_percent_as_number: {}, + }, + } + ) + ).value.observed + ).toMatchObject({ + observed_kibana_instances: 1, + minutes_to_drain_overdue: 0, + max_throughput_per_minute: 200, + }); + }); }); function mockStats( diff --git a/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.ts b/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.ts index 9b48c84da531..88feea306050 100644 --- a/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.ts +++ b/x-pack/plugins/task_manager/server/monitoring/capacity_estimation.ts @@ -199,7 +199,7 @@ export function estimateCapacity( max_throughput_per_minute_per_kibana: capacityPerMinutePerKibana, max_throughput_per_minute: assumedCapacityAvailablePerMinute, minutes_to_drain_overdue: - overdue / (assumedKibanaInstances * averageCapacityUsedByPersistedTasksPerKibana), + overdue ?? 0 / (assumedKibanaInstances * averageCapacityUsedByPersistedTasksPerKibana), avg_recurring_required_throughput_per_minute: averageRecurringRequiredPerMinute, avg_recurring_required_throughput_per_minute_per_kibana: assumedAverageRecurringRequiredThroughputPerMinutePerKibana, diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 516824f72afe..210b5da64217 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -25576,11 +25576,9 @@ "xpack.securitySolution.administration.os.windows": "Windows", "xpack.securitySolution.alertDetails.enrichmentQueryEndDate": "Date de fin", "xpack.securitySolution.alertDetails.enrichmentQueryStartDate": "Date de début", - "xpack.securitySolution.alertDetails.hostRiskClassification": "Classification de risque de l'hôte", "xpack.securitySolution.alertDetails.investigationTimeQueryTitle": "Enrichissement avec la Threat Intelligence", "xpack.securitySolution.alertDetails.noEnrichmentsFoundDescription": "Nous n'avons pas trouvé de Threat Intelligence correspondant à l'une de vos règles de correspondance d'indicateur ou à un enrichissement pour cette alerte.", "xpack.securitySolution.alertDetails.noInvestigationEnrichmentsDescription": "Nous n'avons pas trouvé de valeur de champ comportant des informations supplémentaires disponibles depuis les sources de Threat Intelligence dans lesquelles nous avons lancé la recherche sur les 30 derniers jours par défaut.", - "xpack.securitySolution.alertDetails.noRiskDataDescription": "Aucune donnée de risque de l’hôte n’a été détectée pour cette alerte.", "xpack.securitySolution.alertDetails.overview": "Aperçu", "xpack.securitySolution.alertDetails.overview.enrichedDataTitle": "Données enrichies", "xpack.securitySolution.alertDetails.overview.highlightedFields": "Champs en surbrillance", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 7c9157abe6c5..278b153606d8 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -25553,11 +25553,9 @@ "xpack.securitySolution.administration.os.windows": "Windows", "xpack.securitySolution.alertDetails.enrichmentQueryEndDate": "終了日", "xpack.securitySolution.alertDetails.enrichmentQueryStartDate": "開始日", - "xpack.securitySolution.alertDetails.hostRiskClassification": "ホストリスク分類", "xpack.securitySolution.alertDetails.investigationTimeQueryTitle": "Threat Intelligenceで拡張", "xpack.securitySolution.alertDetails.noEnrichmentsFoundDescription": "指標一致ルールのいずれかまたはこのアラートの拡張と一致する脅威インテリジェンスが見つかりませんでした。", "xpack.securitySolution.alertDetails.noInvestigationEnrichmentsDescription": "デフォルトで過去30日間に検索した脅威インテリジェンスソースから使用可能な追加情報がフィールド値にはないことがわかりました。", - "xpack.securitySolution.alertDetails.noRiskDataDescription": "このアラートのホストリスクデータが見つかりません", "xpack.securitySolution.alertDetails.overview": "概要", "xpack.securitySolution.alertDetails.overview.enrichedDataTitle": "強化されたデータ", "xpack.securitySolution.alertDetails.overview.highlightedFields": "ハイライトされたフィールド", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index fe7acd6cc4e3..a3791dbadf47 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -25584,11 +25584,9 @@ "xpack.securitySolution.administration.os.windows": "Windows", "xpack.securitySolution.alertDetails.enrichmentQueryEndDate": "结束日期", "xpack.securitySolution.alertDetails.enrichmentQueryStartDate": "开始日期", - "xpack.securitySolution.alertDetails.hostRiskClassification": "主机风险分类", "xpack.securitySolution.alertDetails.investigationTimeQueryTitle": "使用威胁情报扩充", "xpack.securitySolution.alertDetails.noEnrichmentsFoundDescription": "我们未找到匹配任何指标匹配规则的威胁情报或此告警的任何扩充。", "xpack.securitySolution.alertDetails.noInvestigationEnrichmentsDescription": "我们未发现字段值具有在过去 30 天中我们默认搜索的威胁情报源提供的其他信息。", - "xpack.securitySolution.alertDetails.noRiskDataDescription": "未找到此告警的主机风险数据", "xpack.securitySolution.alertDetails.overview": "概览", "xpack.securitySolution.alertDetails.overview.enrichedDataTitle": "扩充数据", "xpack.securitySolution.alertDetails.overview.highlightedFields": "突出显示的字段", diff --git a/x-pack/test/accessibility/apps/lens.ts b/x-pack/test/accessibility/apps/lens.ts index a48476f5ce5d..4029cf778755 100644 --- a/x-pack/test/accessibility/apps/lens.ts +++ b/x-pack/test/accessibility/apps/lens.ts @@ -160,6 +160,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await a11y.testAppSnapshot(); }); + it('lens XY chart with reference line layer', async () => { + await PageObjects.lens.createLayer('referenceLine'); + await a11y.testAppSnapshot(); + }); + + it('lens XY chart with annotations layer', async () => { + await PageObjects.lens.createLayer('annotations'); + await a11y.testAppSnapshot(); + }); + it('saves lens chart', async () => { await PageObjects.lens.save(lensChartName); await a11y.testAppSnapshot(); diff --git a/x-pack/test/apm_api_integration/tests/metrics_charts/serverless.spec.ts b/x-pack/test/apm_api_integration/tests/metrics_charts/serverless.spec.ts new file mode 100644 index 000000000000..f08e3c644c8c --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/metrics_charts/serverless.spec.ts @@ -0,0 +1,431 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; +import { apm, timerange } from '@kbn/apm-synthtrace'; +import expect from '@kbn/expect'; +import { meanBy, sumBy } from 'lodash'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const apmApiClient = getService('apmApiClient'); + const synthtraceEsClient = getService('synthtraceEsClient'); + + const start = new Date('2021-01-01T00:00:00.000Z').getTime(); + const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; + + async function callApi(serviceName: string, agentName: string) { + return await apmApiClient.readUser({ + endpoint: `GET /internal/apm/services/{serviceName}/metrics/charts`, + params: { + path: { serviceName }, + query: { + environment: 'test', + agentName, + kuery: '', + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + serviceRuntimeName: 'aws_lambda', + }, + }, + }); + } + + registry.when( + 'Serverless metrics charts when data is loaded', + { config: 'basic', archives: [] }, + () => { + const MEMORY_TOTAL = 536870912; // 0.5gb; + const MEMORY_FREE = 94371840; // ~0.08 gb; + const BILLED_DURATION_MS = 4000; + const FAAS_TIMEOUT_MS = 10000; + const COLD_START_DURATION_PYTHON = 4000; + const COLD_START_DURATION_NODE = 0; + const FAAS_DURATION = 4000; + const TRANSACTION_DURATION = 1000; + + const numberOfTransactionsCreated = 15; + const numberOfPythonInstances = 2; + + before(async () => { + const cloudFields = { + 'cloud.provider': 'aws', + 'cloud.service.name': 'lambda', + 'cloud.region': 'us-west-2', + }; + + const instanceLambdaPython = apm + .serverlessFunction({ + serviceName: 'lambda-python', + environment: 'test', + agentName: 'python', + functionName: 'fn-lambda-python', + }) + .instance({ instanceName: 'instance python', ...cloudFields }); + + const instanceLambdaPython2 = apm + .serverlessFunction({ + serviceName: 'lambda-python', + environment: 'test', + agentName: 'python', + functionName: 'fn-lambda-python-2', + }) + .instance({ instanceName: 'instance python 2', ...cloudFields }); + + const instanceLambdaNode = apm + .serverlessFunction({ + serviceName: 'lambda-node', + environment: 'test', + agentName: 'nodejs', + functionName: 'fn-lambda-node', + }) + .instance({ instanceName: 'instance node', ...cloudFields }); + + const systemMemory = { + free: MEMORY_FREE, + total: MEMORY_TOTAL, + }; + + const transactionsEvents = timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => [ + instanceLambdaPython + .invocation() + .billedDuration(BILLED_DURATION_MS) + .coldStart(true) + .coldStartDuration(COLD_START_DURATION_PYTHON) + .faasDuration(FAAS_DURATION) + .faasTimeout(FAAS_TIMEOUT_MS) + .memory(systemMemory) + .timestamp(timestamp) + .duration(TRANSACTION_DURATION) + .success(), + instanceLambdaPython2 + .invocation() + .billedDuration(BILLED_DURATION_MS) + .coldStart(true) + .coldStartDuration(COLD_START_DURATION_PYTHON) + .faasDuration(FAAS_DURATION) + .faasTimeout(FAAS_TIMEOUT_MS) + .memory(systemMemory) + .timestamp(timestamp) + .duration(TRANSACTION_DURATION) + .success(), + instanceLambdaNode + .invocation() + .billedDuration(BILLED_DURATION_MS) + .coldStart(false) + .coldStartDuration(COLD_START_DURATION_NODE) + .faasDuration(FAAS_DURATION) + .faasTimeout(FAAS_TIMEOUT_MS) + .memory(systemMemory) + .timestamp(timestamp) + .duration(TRANSACTION_DURATION) + .success(), + ]); + + await synthtraceEsClient.index(transactionsEvents); + }); + + after(() => synthtraceEsClient.clean()); + + describe('python', () => { + let metrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/charts'>; + before(async () => { + const { status, body } = await callApi('lambda-python', 'python'); + + expect(status).to.be(200); + metrics = body; + }); + + it('returns all metrics chart', () => { + expect(metrics.charts.length).to.be.greaterThan(0); + expect(metrics.charts.map(({ title }) => title).sort()).to.eql([ + 'Active instances', + 'Avg. Duration', + 'Cold start', + 'Cold start duration', + 'Compute usage', + 'System memory usage', + ]); + }); + + describe('Avg. Duration', () => { + const transactionDurationInMicroSeconds = TRANSACTION_DURATION * 1000; + [ + { title: 'Billed Duration', expectedValue: BILLED_DURATION_MS }, + { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const avgDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'avg_duration'; + }); + const series = avgDurationMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Cold start duration', () => { + let coldStartDurationMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_duration'; + }); + }); + it('returns correct overall value', () => { + expect(coldStartDurationMetric?.series[0].overallValue).to.equal( + COLD_START_DURATION_PYTHON + ); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + coldStartDurationMetric?.series[0]?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.equal(COLD_START_DURATION_PYTHON); + }); + }); + + describe('Cold start count', () => { + let coldStartCountMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartCountMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_count'; + }); + }); + + it('returns correct overall value', () => { + expect(coldStartCountMetric?.series[0].overallValue).to.equal( + numberOfTransactionsCreated * numberOfPythonInstances + ); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + coldStartCountMetric?.series[0]?.data.filter((item) => item.y !== null), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated * numberOfPythonInstances); + }); + }); + + describe('memory usage', () => { + const expectedFreeMemory = 1 - MEMORY_FREE / MEMORY_TOTAL; + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const memoryUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'memory_usage_chart'; + }); + const series = memoryUsageMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Compute usage', () => { + const GBSeconds = 1024 * 1024 * 1024 * 1000; + const expectedValue = (MEMORY_TOTAL * BILLED_DURATION_MS) / GBSeconds; + let computeUsageMetric: typeof metrics['charts'][0] | undefined; + before(() => { + computeUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'compute_usage'; + }); + }); + it('returns correct overall value', () => { + expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(meanValue).to.equal(expectedValue); + }); + }); + + describe('Active instances', () => { + let activeInstancesMetric: typeof metrics['charts'][0] | undefined; + before(() => { + activeInstancesMetric = metrics.charts.find((chart) => { + return chart.key === 'active_instances'; + }); + }); + it('returns correct overall value', () => { + expect(activeInstancesMetric?.series[0].overallValue).to.equal(numberOfPythonInstances); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + activeInstancesMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated * numberOfPythonInstances); + }); + }); + }); + + describe('nodejs', () => { + let metrics: APIReturnType<'GET /internal/apm/services/{serviceName}/metrics/charts'>; + before(async () => { + const { status, body } = await callApi('lambda-node', 'nodejs'); + expect(status).to.be(200); + metrics = body; + }); + + it('returns all metrics chart', () => { + expect(metrics.charts.length).to.be.greaterThan(0); + expect(metrics.charts.map(({ title }) => title).sort()).to.eql([ + 'Active instances', + 'Avg. Duration', + 'Cold start', + 'Cold start duration', + 'Compute usage', + 'System memory usage', + ]); + }); + describe('Avg. Duration', () => { + const transactionDurationInMicroSeconds = TRANSACTION_DURATION * 1000; + [ + { title: 'Billed Duration', expectedValue: BILLED_DURATION_MS }, + { title: 'Transaction Duration', expectedValue: transactionDurationInMicroSeconds }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const avgDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'avg_duration'; + }); + const series = avgDurationMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Cold start duration', () => { + let coldStartDurationMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartDurationMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_duration'; + }); + }); + + it('returns 0 overall value', () => { + expect(coldStartDurationMetric?.series[0].overallValue).to.equal( + COLD_START_DURATION_NODE + ); + }); + + it('returns 0 mean value', () => { + const meanValue = meanBy( + coldStartDurationMetric?.series[0]?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.equal(COLD_START_DURATION_NODE); + }); + }); + + describe('Cold start count', () => { + let coldStartCountMetric: typeof metrics['charts'][0] | undefined; + before(() => { + coldStartCountMetric = metrics.charts.find((chart) => { + return chart.key === 'cold_start_count'; + }); + }); + + it('does not return cold start count', () => { + expect(coldStartCountMetric?.series).to.be.empty(); + }); + }); + + describe('memory usage', () => { + const expectedFreeMemory = 1 - MEMORY_FREE / MEMORY_TOTAL; + [ + { title: 'Max', expectedValue: expectedFreeMemory }, + { title: 'Average', expectedValue: expectedFreeMemory }, + ].map(({ title, expectedValue }) => + it(`returns correct ${title} value`, () => { + const memoryUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'memory_usage_chart'; + }); + const series = memoryUsageMetric?.series.find((item) => item.title === title); + expect(series?.overallValue).to.eql(expectedValue); + const meanValue = meanBy( + series?.data.filter((item) => item.y !== null), + 'y' + ); + expect(meanValue).to.eql(expectedValue); + }) + ); + }); + + describe('Compute usage', () => { + const GBSeconds = 1024 * 1024 * 1024 * 1000; + const expectedValue = (MEMORY_TOTAL * BILLED_DURATION_MS) / GBSeconds; + let computeUsageMetric: typeof metrics['charts'][0] | undefined; + before(() => { + computeUsageMetric = metrics.charts.find((chart) => { + return chart.key === 'compute_usage'; + }); + }); + it('returns correct overall value', () => { + expect(computeUsageMetric?.series[0].overallValue).to.equal(expectedValue); + }); + + it('returns correct mean value', () => { + const meanValue = meanBy( + computeUsageMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(meanValue).to.equal(expectedValue); + }); + }); + + describe('Active instances', () => { + let activeInstancesMetric: typeof metrics['charts'][0] | undefined; + before(() => { + activeInstancesMetric = metrics.charts.find((chart) => { + return chart.key === 'active_instances'; + }); + }); + it('returns correct overall value', () => { + // there's only one node instance + expect(activeInstancesMetric?.series[0].overallValue).to.equal(1); + }); + + it('returns correct sum value', () => { + const sumValue = sumBy( + activeInstancesMetric?.series[0]?.data.filter((item) => item.y !== 0), + 'y' + ); + expect(sumValue).to.equal(numberOfTransactionsCreated); + }); + }); + }); + } + ); +} diff --git a/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts b/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts index 865b095da534..f459ca8123e4 100644 --- a/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts +++ b/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts @@ -130,7 +130,8 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { }); } - describe('explain log rate spikes', function () { + // Failing: See https://github.com/elastic/kibana/issues/140848 + describe.skip('explain log rate spikes', function () { this.tags(['aiops']); before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); diff --git a/x-pack/test/functional/apps/lens/group2/index.ts b/x-pack/test/functional/apps/lens/group2/index.ts index f63fc0ecebca..477ad2f8561d 100644 --- a/x-pack/test/functional/apps/lens/group2/index.ts +++ b/x-pack/test/functional/apps/lens/group2/index.ts @@ -76,5 +76,6 @@ export default ({ getService, loadTestFile, getPageObjects }: FtrProviderContext loadTestFile(require.resolve('./epoch_millis')); loadTestFile(require.resolve('./show_underlying_data')); loadTestFile(require.resolve('./show_underlying_data_dashboard')); + loadTestFile(require.resolve('./tsdb')); }); }; diff --git a/x-pack/test/functional/apps/lens/group2/tsdb.ts b/x-pack/test/functional/apps/lens/group2/tsdb.ts new file mode 100644 index 000000000000..7a43fc47471a --- /dev/null +++ b/x-pack/test/functional/apps/lens/group2/tsdb.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const queryBar = getService('queryBar'); + const PageObjects = getPageObjects(['common', 'timePicker', 'lens', 'dashboard']); + const testSubjects = getService('testSubjects'); + const retry = getService('retry'); + const es = getService('es'); + const log = getService('log'); + const indexPatterns = getService('indexPatterns'); + const kibanaServer = getService('kibanaServer'); + const esArchiver = getService('esArchiver'); + + describe('lens tsdb', function () { + const dataViewTitle = 'sample-01'; + const rollupDataViewTitle = 'sample-01,sample-01-rollup'; + const fromTime = 'Jun 17, 2022 @ 00:00:00.000'; + const toTime = 'Jun 23, 2022 @ 00:00:00.000'; + const testArchive = 'test/functional/fixtures/es_archiver/search/downsampled'; + const testIndex = 'sample-01'; + const testRollupIndex = 'sample-01-rollup'; + + before(async () => { + // create rollup data + log.info(`loading ${testIndex} index...`); + await esArchiver.loadIfNeeded(testArchive); + log.info(`add write block to ${testIndex} index...`); + await es.indices.addBlock({ index: testIndex, block: 'write' }); + try { + log.info(`rolling up ${testIndex} index...`); + // es client currently does not have method for downsample + await es.transport.request({ + method: 'POST', + path: '/sample-01/_downsample/sample-01-rollup', + body: { fixed_interval: '1h' }, + }); + } catch (err) { + log.info(`ignoring resource_already_exists_exception...`); + if (!err.message.match(/resource_already_exists_exception/)) { + throw err; + } + } + + log.info(`creating ${rollupDataViewTitle} data view...`); + await indexPatterns.create( + { + title: rollupDataViewTitle, + timeFieldName: '@timestamp', + }, + { override: true } + ); + await indexPatterns.create( + { + title: dataViewTitle, + timeFieldName: '@timestamp', + }, + { override: true } + ); + await kibanaServer.uiSettings.update({ + 'dateFormat:tz': 'UTC', + defaultIndex: '0ae0bc7a-e4ca-405c-ab67-f2b5913f2a51', + 'timepicker:timeDefaults': '{ "from": "now-1y", "to": "now" }', + }); + }); + + after(async () => { + await kibanaServer.savedObjects.cleanStandardList(); + await kibanaServer.uiSettings.replace({}); + await es.indices.delete({ index: [testIndex, testRollupIndex] }); + }); + + describe('for regular metric', () => { + it('defaults to median for non-rolled up metric', async () => { + await PageObjects.common.navigateToApp('lens'); + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + await PageObjects.lens.switchDataPanelIndexPattern(dataViewTitle); + await PageObjects.lens.waitForField('kubernetes.container.memory.available.bytes'); + await PageObjects.lens.dragFieldToWorkspace( + 'kubernetes.container.memory.available.bytes', + 'xyVisChart' + ); + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + 'Median of kubernetes.container.memory.available.bytes' + ); + }); + + it('does not show a warning', async () => { + await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel'); + await testSubjects.missingOrFail('median-partial-warning'); + await PageObjects.lens.assertNoEditorWarning(); + await PageObjects.lens.closeDimensionEditor(); + }); + }); + + describe('for rolled up metric', () => { + it('defaults to average for rolled up metric', async () => { + await PageObjects.lens.switchDataPanelIndexPattern(rollupDataViewTitle); + await PageObjects.lens.removeLayer(); + await PageObjects.lens.waitForField('kubernetes.container.memory.available.bytes'); + await PageObjects.lens.dragFieldToWorkspace( + 'kubernetes.container.memory.available.bytes', + 'xyVisChart' + ); + expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( + 'Average of kubernetes.container.memory.available.bytes' + ); + }); + it('shows warnings in editor when using median', async () => { + await PageObjects.lens.openDimensionEditor('lnsXY_yDimensionPanel'); + await testSubjects.existOrFail('median-partial-warning'); + await testSubjects.click('lns-indexPatternDimension-median'); + await PageObjects.lens.waitForVisualization('xyVisChart'); + await PageObjects.lens.assertEditorWarning( + '"Median of kubernetes.container.memory.available.bytes" does not work for all indices in the selected data view because it\'s using a function which is not supported on rolled up data. Please edit the visualization to use another function or change the time range.' + ); + }); + it('shows warnings in dashboards as well', async () => { + await PageObjects.lens.save('New', false, false, false, 'new'); + + await PageObjects.dashboard.waitForRenderComplete(); + await PageObjects.lens.assertInlineWarning( + '"Median of kubernetes.container.memory.available.bytes" does not work for all indices in the selected data view because it\'s using a function which is not supported on rolled up data. Please edit the visualization to use another function or change the time range.' + ); + }); + it('still shows other warnings as toast', async () => { + await es.indices.delete({ index: [testRollupIndex] }); + // index a document which will produce a shard failure because a string field doesn't support median + await es.create({ + id: '1', + index: testRollupIndex, + document: { + 'kubernetes.container.memory.available.bytes': 'fsdfdsf', + '@timestamp': '2022-06-20', + }, + wait_for_active_shards: 1, + }); + await retry.try(async () => { + await queryBar.clickQuerySubmitButton(); + expect( + await (await testSubjects.find('euiToastHeader__title', 1000)).getVisibleText() + ).to.equal('1 of 3 shards failed'); + }); + // as the rollup index is gone, there is no inline warning left + await PageObjects.lens.assertNoInlineWarning(); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/lens/group3/annotations.ts b/x-pack/test/functional/apps/lens/group3/annotations.ts index e139677737b4..e090385dfc91 100644 --- a/x-pack/test/functional/apps/lens/group3/annotations.ts +++ b/x-pack/test/functional/apps/lens/group3/annotations.ts @@ -78,5 +78,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.closeDimensionEditor(); await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); }); + + it('should add query annotation layer and allow edition', async () => { + await PageObjects.lens.removeLayer(1); + await PageObjects.lens.createLayer('annotations'); + + expect((await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length).to.eql(2); + expect( + await ( + await testSubjects.find('lnsXY_xAnnotationsPanel > lns-dimensionTrigger') + ).getVisibleText() + ).to.eql('Event'); + await testSubjects.click('lnsXY_xAnnotationsPanel > lns-dimensionTrigger'); + await testSubjects.click('lnsXY_annotation_query'); + await PageObjects.lens.configureQueryAnnotation({ + queryString: '*', + timeField: 'utc_time', + textDecoration: { type: 'name' }, + extraFields: ['clientip'], + }); + await PageObjects.lens.closeDimensionEditor(); + + await testSubjects.existOrFail('xyVisGroupedAnnotationIcon'); + }); }); } diff --git a/x-pack/test/functional/apps/lens/group3/error_handling.ts b/x-pack/test/functional/apps/lens/group3/error_handling.ts index 794547fb96f0..4d63e6d12306 100644 --- a/x-pack/test/functional/apps/lens/group3/error_handling.ts +++ b/x-pack/test/functional/apps/lens/group3/error_handling.ts @@ -64,6 +64,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.waitForMissingDataViewWarningDisappear(); await PageObjects.lens.waitForEmptyWorkspace(); }); + + it('works fine when the dataViews is missing for referenceLines and annotations', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName( + 'lnsXYWithReferenceLinesAndAnnotationsWithNonExistingDataView' + ); + await PageObjects.lens.clickVisualizeListItemTitle( + 'lnsXYWithReferenceLinesAndAnnotationsWithNonExistingDataView' + ); + await PageObjects.lens.waitForMissingDataViewWarning(); + }); }); }); } diff --git a/x-pack/test/functional/fixtures/kbn_archiver/lens/errors.json b/x-pack/test/functional/fixtures/kbn_archiver/lens/errors.json index 9ecc14164d86..49cfc4715ad4 100644 --- a/x-pack/test/functional/fixtures/kbn_archiver/lens/errors.json +++ b/x-pack/test/functional/fixtures/kbn_archiver/lens/errors.json @@ -75,4 +75,199 @@ "type": "lens", "updated_at": "2021-10-19T13:41:04.038Z", "version": "WzU2NjEsMl0=" +} + +{ + "attributes": { + "description": "", + "state": { + "datasourceStates": { + "indexpattern": { + "layers": { + "3c85b7f0-3227-43e7-88ac-9416c6311ebc": { + "columns": { + "951ad12c-8fae-4e81-964d-84827e387515": { + "label": "order_date", + "dataType": "date", + "operationType": "date_histogram", + "sourceField": "order_date", + "isBucketed": true, + "scale": "interval", + "params": { + "interval": "auto", + "includeEmptyRows": true, + "dropPartials": false + } + }, + "e311d921-2525-4fb1-9716-94fe787ad623": { + "label": "Count of records", + "dataType": "number", + "operationType": "count", + "isBucketed": false, + "scale": "ratio", + "sourceField": "___records___", + "params": { + "emptyAsNull": true + } + } + }, + "columnOrder": [ + "951ad12c-8fae-4e81-964d-84827e387515", + "e311d921-2525-4fb1-9716-94fe787ad623" + ], + "incompleteColumns": {} + }, + "5321ae4b-8f8a-4300-a9bc-ec7245e2cb0f": { + "columns": { + "735cacfd-52af-4ff9-afa5-0e14c1b7c7fd": { + "label": "Static value: 127.5", + "dataType": "number", + "operationType": "static_value", + "isStaticValue": true, + "isBucketed": false, + "scale": "ratio", + "params": { + "value": "127.5" + }, + "references": [] + } + }, + "columnOrder": [ + "735cacfd-52af-4ff9-afa5-0e14c1b7c7fd" + ], + "incompleteColumns": {} + } + } + } + + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "legend": { + "isVisible": true, + "position": "right" + }, + "valueLabels": "hide", + "fittingFunction": "None", + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "preferredSeriesType": "bar_stacked", + "layers": [ + { + "layerId": "3c85b7f0-3227-43e7-88ac-9416c6311ebc", + "accessors": [ + "e311d921-2525-4fb1-9716-94fe787ad623" + ], + "position": "top", + "seriesType": "bar_stacked", + "showGridlines": false, + "layerType": "data", + "xAccessor": "951ad12c-8fae-4e81-964d-84827e387515" + }, + { + "layerId": "5321ae4b-8f8a-4300-a9bc-ec7245e2cb0f", + "layerType": "referenceLine", + "accessors": [ + "735cacfd-52af-4ff9-afa5-0e14c1b7c7fd" + ], + "yConfig": [ + { + "forAccessor": "735cacfd-52af-4ff9-afa5-0e14c1b7c7fd", + "axisMode": "left" + } + ] + }, + { + "layerId": "396c620c-1b6b-4754-a8fa-7f0e830a825c", + "layerType": "annotations", + "annotations": [ + { + "label": "Event", + "type": "manual", + "key": { + "type": "point_in_time", + "timestamp": "2022-07-25T22:00:00.000Z" + }, + "icon": "triangle", + "id": "13354257-3cd4-46b5-9462-d3fbbab6a433" + }, + { + "type": "query", + "id": "06539b10-c487-4aba-bf21-761c014c8d60", + "label": "Event", + "key": { + "type": "point_in_time" + }, + "icon": "triangle", + "textVisibility": true, + "textField": "customer_gender", + "filter": { + "type": "kibana_query", + "query": "*", + "language": "kuery" + }, + "extraFields": [ + "category.keyword" + ] + } + ] + } + ] + } + }, + "title": "lnsXYWithReferenceLinesAndAnnotationsWithNonExistingDataView", + "visualizationType": "lnsXY" + }, + "coreMigrationVersion": "8.0.0", + "id": "3454af30-30e2-11ec-8dbc-f13e30d4f8ac1", + "migrationVersion": { + "lens": "8.0.0" + }, + "references": [ + { + "id": "nonExistingDataView", + "name": "indexpattern-datasource-current-indexpattern", + "type": "index-pattern" + }, + { + "id": "nonExistingDataView", + "name": "indexpattern-datasource-layer-3c85b7f0-3227-43e7-88ac-9416c6311ebc", + "type": "index-pattern" + }, + { + "id": "nonExistingDataView", + "name": "indexpattern-datasource-layer-5321ae4b-8f8a-4300-a9bc-ec7245e2cb0f", + "type": "index-pattern" + }, + { + "id": "nonExistingDataView", + "name": "xy-visualization-layer-396c620c-1b6b-4754-a8fa-7f0e830a825c", + "type": "index-pattern" + } + ], + "type": "lens", + "updated_at": "2021-10-19T13:41:04.038Z", + "version": "WzU2NjEsMl0=" } \ No newline at end of file diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index f45b8ad9c22d..abcf958e8712 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -104,6 +104,55 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); }, + async selectOptionFromComboBox(testTargetId: string, name: string) { + const target = await testSubjects.find(testTargetId, 1000); + await comboBox.openOptionsList(target); + await comboBox.setElement(target, name); + }, + + async configureQueryAnnotation(opts: { + queryString: string; + timeField: string; + textDecoration?: { type: 'none' | 'name' | 'field'; textField?: string }; + extraFields?: string[]; + }) { + // type * in the query editor + const queryInput = await testSubjects.find('lnsXY-annotation-query-based-query-input'); + await queryInput.type(opts.queryString); + await testSubjects.click('indexPattern-filters-existingFilterTrigger'); + await this.selectOptionFromComboBox( + 'lnsXY-annotation-query-based-field-picker', + opts.timeField + ); + if (opts.textDecoration) { + await testSubjects.click(`lnsXY_textVisibility_${opts.textDecoration.type}`); + if (opts.textDecoration.textField) { + await this.selectOptionFromComboBox( + 'lnsXY-annotation-query-based-text-decoration-field-picker', + opts.textDecoration.textField + ); + } + } + if (opts.extraFields) { + for (const field of opts.extraFields) { + await this.addFieldToTooltip(field); + } + } + }, + + async addFieldToTooltip(fieldName: string) { + const lastIndex = ( + await find.allByCssSelector('[data-test-subj^="lnsXY-annotation-tooltip-field-picker"]') + ).length; + await retry.try(async () => { + await testSubjects.click('lnsXY-annotation-tooltip-add_field'); + await this.selectOptionFromComboBox( + `lnsXY-annotation-tooltip-field-picker--${lastIndex}`, + fieldName + ); + }); + }, + /** * Changes the specified dimension to the specified operation and (optinally) field. * @@ -150,9 +199,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); } if (opts.field) { - const target = await testSubjects.find('indexPattern-dimension-field'); - await comboBox.openOptionsList(target); - await comboBox.setElement(target, opts.field); + await this.selectOptionFromComboBox('indexPattern-dimension-field', opts.field); } if (opts.formula) { @@ -188,15 +235,17 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont isPreviousIncompatible?: boolean; }) { if (opts.operation) { - const target = await testSubjects.find('indexPattern-subFunction-selection-row'); - await comboBox.openOptionsList(target); - await comboBox.setElement(target, opts.operation); + await this.selectOptionFromComboBox( + 'indexPattern-subFunction-selection-row', + opts.operation + ); } if (opts.field) { - const target = await testSubjects.find('indexPattern-reference-field-selection-row'); - await comboBox.openOptionsList(target); - await comboBox.setElement(target, opts.field); + await this.selectOptionFromComboBox( + 'indexPattern-reference-field-selection-row', + opts.field + ); } }, @@ -507,8 +556,8 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont // closes the dimension editor flyout async closeDimensionEditor() { await retry.try(async () => { - await testSubjects.click('lns-indexPattern-dimensionContainerBack'); - await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerBack'); + await testSubjects.click('lns-indexPattern-dimensionContainerClose'); + await testSubjects.missingOrFail('lns-indexPattern-dimensionContainerClose'); }); }, @@ -588,10 +637,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont ).length; await retry.try(async () => { await testSubjects.click('indexPattern-terms-add-field'); - // count the number of defined terms - const target = await testSubjects.find(`indexPattern-dimension-field-${lastIndex}`, 1000); - await comboBox.openOptionsList(target); - await comboBox.setElement(target, field); + await this.selectOptionFromComboBox(`indexPattern-dimension-field-${lastIndex}`, field); }); }, @@ -608,7 +654,6 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }); // count the number of defined terms const target = await testSubjects.find(`indexPattern-dimension-field-${lastIndex}`); - // await comboBox.openOptionsList(target); for (const field of fields) { await comboBox.setCustom(`indexPattern-dimension-field-${lastIndex}`, field); await comboBox.openOptionsList(target); @@ -661,9 +706,7 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.setValue('column-label-edit', label, { clearWithKeyboard: true }); }, async editDimensionFormat(format: string) { - const formatInput = await testSubjects.find('indexPattern-dimension-format'); - await comboBox.openOptionsList(formatInput); - await comboBox.setElement(formatInput, format); + await this.selectOptionFromComboBox('indexPattern-dimension-format', format); }, async editDimensionColor(color: string) { const colorPickerInput = await testSubjects.find('~indexPattern-dimension-colorPicker'); @@ -812,17 +855,15 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont * Returns the number of layers visible in the chart configuration */ async getLayerCount() { - const elements = await testSubjects.findAll('lnsLayerRemove'); - return elements.length; + return (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)).length; }, /** * Adds a new layer to the chart, fails if the chart does not support new layers */ - async createLayer(layerType: string = 'data') { + async createLayer(layerType: 'data' | 'referenceLine' | 'annotations' = 'data') { await testSubjects.click('lnsLayerAddButton'); - const layerCount = (await find.allByCssSelector(`[data-test-subj^="lns-layerPanel-"]`)) - .length; + const layerCount = await this.getLayerCount(); await retry.waitFor('check for layer type support', async () => { const fasterChecks = await Promise.all([ @@ -1257,9 +1298,9 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont }, /** resets visualization/layer or removes a layer */ - async removeLayer() { + async removeLayer(index: number = 0) { await retry.try(async () => { - await testSubjects.click('lnsLayerRemove'); + await testSubjects.click(`lnsLayerRemove--${index}`); if (await testSubjects.exists('lnsLayerRemoveModal')) { await testSubjects.exists('lnsLayerRemoveConfirmButton'); await testSubjects.click('lnsLayerRemoveConfirmButton'); @@ -1462,5 +1503,49 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await testSubjects.waitForEnabled(applyButtonSelector); await testSubjects.click(applyButtonSelector); }, + + async assertNoInlineWarning() { + await testSubjects.missingOrFail('chart-inline-warning'); + }, + + async assertNoEditorWarning() { + await testSubjects.missingOrFail('lens-editor-warning'); + }, + + async assertInlineWarning(warningText: string) { + await testSubjects.click('chart-inline-warning-button'); + await testSubjects.existOrFail('chart-inline-warning'); + const warnings = await testSubjects.findAll('chart-inline-warning'); + let found = false; + for (const warning of warnings) { + const text = await warning.getVisibleText(); + log.info(text); + if (text === warningText) { + found = true; + } + } + await testSubjects.click('chart-inline-warning-button'); + if (!found) { + throw new Error(`Warning with text "${warningText}" not found`); + } + }, + + async assertEditorWarning(warningText: string) { + await testSubjects.click('lens-editor-warning-button'); + await testSubjects.existOrFail('lens-editor-warning'); + const warnings = await testSubjects.findAll('lens-editor-warning'); + let found = false; + for (const warning of warnings) { + const text = await warning.getVisibleText(); + log.info(text); + if (text === warningText) { + found = true; + } + } + await testSubjects.click('lens-editor-warning-button'); + if (!found) { + throw new Error(`Warning with text "${warningText}" not found`); + } + }, }); } diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index cb6ffd7d79f3..8d638789b61f 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -49,9 +49,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.alerting.rules.minimumScheduleInterval.value=1s', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ - 'riskyHostsEnabled', - 'riskyUsersEnabled', - 'entityAnalyticsDashboardEnabled', 'threatIntelligenceEnabled', ])}`, `--home.disableWelcomeScreen=true`, diff --git a/x-pack/test/threat_intelligence_cypress/config.ts b/x-pack/test/threat_intelligence_cypress/config.ts index 43d80103c5a6..8d638789b61f 100644 --- a/x-pack/test/threat_intelligence_cypress/config.ts +++ b/x-pack/test/threat_intelligence_cypress/config.ts @@ -49,8 +49,6 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.alerting.rules.minimumScheduleInterval.value=1s', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ - 'riskyHostsEnabled', - 'riskyUsersEnabled', 'threatIntelligenceEnabled', ])}`, `--home.disableWelcomeScreen=true`, diff --git a/x-pack/test/usage_collection/test_suites/application_usage/index.ts b/x-pack/test/usage_collection/test_suites/application_usage/index.ts index 754ae98997c1..9311e554832e 100644 --- a/x-pack/test/usage_collection/test_suites/application_usage/index.ts +++ b/x-pack/test/usage_collection/test_suites/application_usage/index.ts @@ -24,7 +24,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { ); } try { - expect(Object.keys(applicationUsageSchema).sort()).to.eql(appIds.sort()); + const enabledAppIds = Object.keys(applicationUsageSchema).filter( + // Profiling is currently disabled by default as it's in closed beta + (appId) => appId !== 'profiling' + ); + expect(enabledAppIds.sort()).to.eql(appIds.sort()); } catch (err) { err.message = `Application Usage's schema is not up-to-date with the actual registered apps. Please update it at src/plugins/kibana_usage_collection/server/collectors/application_usage/schema.ts.\n${err.message}`; throw err; diff --git a/yarn.lock b/yarn.lock index eb8b7fbdf36d..325401de9406 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6202,6 +6202,11 @@ resolved "https://registry.yarnpkg.com/@types/d3/-/d3-3.5.43.tgz#e9b4992817e0b6c5efaa7d6e5bb2cee4d73eab58" integrity sha512-t9ZmXOcpVxywRw86YtIC54g7M9puRh8hFedRvVfHKf5YyOP6pSxA0TvpXpfseXSCInoW4P7bggTrSDiUOs4g5w== +"@types/dagre@^0.7.47": + version "0.7.47" + resolved "https://registry.yarnpkg.com/@types/dagre/-/dagre-0.7.47.tgz#1d1b89e1fac36aaf5ef5ed6274bb123073095ac5" + integrity sha512-oX+3aRf7L6Cqq1MvbWmmD7FpAU/T8URwFFuHBagAiyHILn3i+RNZ35/tvyq28de+lZGY3W19BxJ7FeITQDO7aA== + "@types/dedent@^0.7.0": version "0.7.0" resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050" @@ -6294,6 +6299,11 @@ dependencies: "@types/jquery" "*" +"@types/fnv-plus@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@types/fnv-plus/-/fnv-plus-1.3.0.tgz#0f43f0b7e7b4b24de3a1cab69bfa009508f4c084" + integrity sha512-ijls8MsO6Q9JUSd5w1v4y2ijM6S4D/nmOyI/FwcepvrZfym0wZhLdYGFD5TJID7tga0O3I7SmtK69RzpSJ1Fcw== + "@types/fs-extra@^8.0.0": version "8.1.1" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-8.1.1.tgz#1e49f22d09aa46e19b51c0b013cb63d0d923a068" @@ -15002,6 +15012,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +fnv-plus@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/fnv-plus/-/fnv-plus-1.3.1.tgz#c34cb4572565434acb08ba257e4044ce2b006d67" + integrity sha512-Gz1EvfOneuFfk4yG458dJ3TLJ7gV19q3OM/vVvvHf7eT02Hm1DleB4edsia6ahbKgAYxO9gvyQ1ioWZR+a00Yw== + focus-lock@^0.11.2: version "0.11.2" resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-0.11.2.tgz#aeef3caf1cea757797ac8afdebaec8fd9ab243ed"