From 02ac5fc90f08615d877befc05f9675838ade77b4 Mon Sep 17 00:00:00 2001 From: Pablo Machado Date: Wed, 13 Nov 2024 11:22:44 +0100 Subject: [PATCH 01/53] [SecuritySolution] Improve asset criticality bulk error when entities are duplicated (#199651) ## Summary * Improve asset criticality bulk error when entities are duplicated * It also fixes the server errors line to be '1' based. ![Screenshot 2024-11-11 at 11 46 31](https://github.com/user-attachments/assets/3fbf35fb-cd27-417a-bf53-41a197d1bbe9) ### Performance Test parameters: file with +33k lines and ~1 MB size. * Before 6.24 seconds * After 6.46 seconds Execution time Increased ~0.22 seconds ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../asset_criticality_data_client.test.ts | 82 +++++++++++++++++++ .../asset_criticality_data_client.ts | 29 +++++-- .../asset_criticality/routes/upload_csv.ts | 1 + .../asset_criticality_csv_upload.ts | 18 ++-- 4 files changed, 116 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts index 75ec685a4756e..0907af8fa92d5 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.test.ts @@ -6,10 +6,12 @@ */ import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { Readable } from 'stream'; import { AssetCriticalityDataClient } from './asset_criticality_data_client'; import { createOrUpdateIndex } from '../utils/create_or_update_index'; import { auditLoggerMock } from '@kbn/security-plugin/server/audit/mocks'; import type { AssetCriticalityUpsert } from '../../../../common/entity_analytics/asset_criticality/types'; +import type { ElasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; type MockInternalEsClient = ReturnType< typeof elasticsearchServiceMock.createScopedClusterClient @@ -264,4 +266,84 @@ describe('AssetCriticalityDataClient', () => { ); }); }); + + describe('#bulkUpsertFromStream()', () => { + let esClientMock: MockInternalEsClient; + let loggerMock: ReturnType; + let subject: AssetCriticalityDataClient; + + beforeEach(() => { + esClientMock = elasticsearchServiceMock.createScopedClusterClient().asInternalUser; + + esClientMock.helpers.bulk = mockEsBulk(); + loggerMock = loggingSystemMock.createLogger(); + subject = new AssetCriticalityDataClient({ + esClient: esClientMock, + logger: loggerMock, + namespace: 'default', + auditLogger: mockAuditLogger, + }); + }); + + it('returns valid stats', async () => { + const recordsStream = [ + { idField: 'host.name', idValue: 'host1', criticalityLevel: 'high_impact' }, + ]; + + const result = await subject.bulkUpsertFromStream({ + recordsStream: Readable.from(recordsStream), + retries: 3, + flushBytes: 1_000, + }); + + expect(result).toEqual({ + errors: [], + stats: { + failed: 0, + successful: 1, + total: 1, + }, + }); + }); + + it('returns error for duplicated entities', async () => { + const recordsStream = [ + { idField: 'host.name', idValue: 'host1', criticalityLevel: 'high_impact' }, + { idField: 'host.name', idValue: 'host1', criticalityLevel: 'high_impact' }, + ]; + + const result = await subject.bulkUpsertFromStream({ + recordsStream: Readable.from(recordsStream), + retries: 3, + flushBytes: 1_000, + streamIndexStart: 9, + }); + + expect(result).toEqual({ + errors: [ + { + index: 10, + message: 'Duplicated entity', + }, + ], + stats: { + failed: 1, + successful: 1, + total: 2, + }, + }); + }); + }); }); + +const mockEsBulk = () => + jest.fn().mockImplementation(async ({ datasource }) => { + let count = 0; + for await (const _ of datasource) { + count++; + } + return { + failed: 0, + successful: count, + }; + }) as unknown as ElasticsearchClientMock['helpers']['bulk']; diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts index b957030f2c8e5..b4d7e2f492eb8 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/asset_criticality_data_client.ts @@ -35,6 +35,10 @@ type AssetCriticalityIdParts = Pick[0], 'flushBytes' | 'retries'>; type StoredAssetCriticalityRecord = { @@ -236,6 +240,7 @@ export class AssetCriticalityDataClient { * @param recordsStream a stream of records to upsert, records may also be an error e.g if there was an error parsing * @param flushBytes how big elasticsearch bulk requests should be before they are sent * @param retries the number of times to retry a failed bulk request + * @param streamIndexStart By default the errors are zero-indexed. You can change it by setting this param to a value like `1`. It could be useful for file upload. * @returns an object containing the number of records updated, created, errored, and the total number of records processed * @throws an error if the stream emits an error * @remarks @@ -248,6 +253,7 @@ export class AssetCriticalityDataClient { recordsStream, flushBytes, retries, + streamIndexStart = 0, }: BulkUpsertFromStreamOptions): Promise => { const errors: BulkUpsertAssetCriticalityRecordsResponse['errors'] = []; const stats: BulkUpsertAssetCriticalityRecordsResponse['stats'] = { @@ -256,10 +262,13 @@ export class AssetCriticalityDataClient { total: 0, }; - let streamIndex = 0; + let streamIndex = streamIndexStart; const recordGenerator = async function* () { + const processedEntities = new Set(); + for await (const untypedRecord of recordsStream) { const record = untypedRecord as unknown as AssetCriticalityUpsert | Error; + stats.total++; if (record instanceof Error) { stats.failed++; @@ -268,10 +277,20 @@ export class AssetCriticalityDataClient { index: streamIndex, }); } else { - yield { - record, - index: streamIndex, - }; + const entityKey = `${record.idField}-${record.idValue}`; + if (processedEntities.has(entityKey)) { + errors.push({ + message: 'Duplicated entity', + index: streamIndex, + }); + stats.failed++; + } else { + processedEntities.add(entityKey); + yield { + record, + index: streamIndex, + }; + } } streamIndex++; } diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/routes/upload_csv.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/routes/upload_csv.ts index dccf24d161054..22dba4bf321c7 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/routes/upload_csv.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/asset_criticality/routes/upload_csv.ts @@ -102,6 +102,7 @@ export const assetCriticalityPublicCSVUploadRoute = ( recordsStream, retries: errorRetries, flushBytes: maxBulkRequestBodySizeBytes, + streamIndexStart: 1, // It is the first line number }); const end = new Date(); diff --git a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts index 496cde9a79e13..737458f75a516 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/asset_criticality_csv_upload.ts @@ -102,36 +102,36 @@ export default ({ getService }: FtrProviderContext) => { expect(body.errors).toEqual([ { - index: 0, + index: 1, message: 'Invalid criticality level "invalid_criticality", expected one of extreme_impact, high_impact, medium_impact, low_impact', }, { - index: 1, + index: 2, message: 'Invalid entity type "invalid_entity", expected host or user', }, { - index: 2, + index: 3, message: 'Missing identifier', }, { - index: 3, + index: 4, message: 'Missing criticality level', }, { - index: 4, + index: 5, message: 'Missing entity type', }, { - index: 5, + index: 6, message: 'Expected 3 columns, got 2', }, { - index: 6, + index: 7, message: 'Expected 3 columns, got 4', }, { - index: 7, + index: 8, message: `Identifier is too long, expected less than 1000 characters, got 1001`, }, ]); @@ -154,7 +154,7 @@ export default ({ getService }: FtrProviderContext) => { expect(body.errors).toEqual([ { - index: 1, + index: 2, message: 'Invalid criticality level "invalid_criticality", expected one of extreme_impact, high_impact, medium_impact, low_impact', }, From 411573bded37b3e338d6709ddfadf2c95987b0f5 Mon Sep 17 00:00:00 2001 From: Irene Blanco Date: Wed, 13 Nov 2024 11:40:06 +0100 Subject: [PATCH 02/53] [APM] Migrate latency API tests to be deployment-agnostic (#199802) ### How to test Closes https://github.com/elastic/kibana/issues/198978 Part of https://github.com/elastic/kibana/issues/193245 This PR contains the changes to migrate `latency` test folder to deployment-agnostic testing strategy. ### How to test - Serverless ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM" ``` It's recommended to be run against [MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki) - Stateful ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM" ``` ## Checks - [ ] (OPTIONAL, only if a test has been unskipped) Run flaky test suite - [x] local run for serverless - [x] local run for stateful - [x] MKI run for serverless --- .../apis/observability/apm/index.ts | 1 + .../apis/observability/apm/latency/index.ts | 15 +++++++++++++++ .../apm}/latency/service_apis.spec.ts | 18 ++++++++++++------ .../apm}/latency/service_maps.spec.ts | 19 +++++++++++++------ 4 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/index.ts rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/latency/service_apis.spec.ts (94%) rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/latency/service_maps.spec.ts (90%) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts index 8cde7cb77bca8..fc98e85850bd8 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts @@ -22,6 +22,7 @@ export default function apmApiIntegrationTests({ loadTestFile(require.resolve('./correlations')); loadTestFile(require.resolve('./entities')); loadTestFile(require.resolve('./cold_start')); + loadTestFile(require.resolve('./latency')); loadTestFile(require.resolve('./infrastructure')); }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/index.ts new file mode 100644 index 0000000000000..0b9a71293f687 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) { + describe('latency', () => { + loadTestFile(require.resolve('./service_apis.spec.ts')); + loadTestFile(require.resolve('./service_maps.spec.ts')); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/service_apis.spec.ts similarity index 94% rename from x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/service_apis.spec.ts index 35ef23c8a4430..dee5f27b7a61d 100644 --- a/x-pack/test/apm_api_integration/tests/latency/service_apis.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/service_apis.spec.ts @@ -12,12 +12,12 @@ import { isFiniteNumber } from '@kbn/apm-plugin/common/utils/is_finite_number'; import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); const serviceName = 'synth-go'; const start = new Date('2021-01-01T00:00:00.000Z').getTime(); @@ -156,7 +156,13 @@ export default function ApiTest({ getService }: FtrProviderContext) { let latencyTransactionValues: Awaited>; // FLAKY: https://github.com/elastic/kibana/issues/177387 - registry.when('Services APIs', { config: 'basic', archives: [] }, () => { + describe('Services APIs', () => { + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + }); + describe('when data is loaded ', () => { const GO_PROD_RATE = 80; const GO_DEV_RATE = 20; diff --git a/x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/service_maps.spec.ts similarity index 90% rename from x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/service_maps.spec.ts index 298fde675bc4a..fa088e4f12dc9 100644 --- a/x-pack/test/apm_api_integration/tests/latency/service_maps.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/latency/service_maps.spec.ts @@ -10,12 +10,12 @@ import { meanBy } from 'lodash'; import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; import { ProcessorEvent } from '@kbn/observability-plugin/common'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); const serviceName = 'synth-go'; const start = new Date('2021-01-01T00:00:00.000Z').getTime(); @@ -73,7 +73,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { let latencyMetricValues: Awaited>; let latencyTransactionValues: Awaited>; - registry.when('Service Maps APIs', { config: 'trial', archives: [] }, () => { + + describe('Service Maps APIs', () => { + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + }); + describe('when data is loaded ', () => { const GO_PROD_RATE = 80; const GO_DEV_RATE = 20; From fb71f4e027a955f82eed930fd87fb10c311905e0 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Wed, 13 Nov 2024 11:44:59 +0100 Subject: [PATCH 03/53] [Security Solution] Adds missing configuration to the cypress tests (#199959) ## Summary Adds missing configuration to the cypress tests. --- .../cypress/cypress_ci_serverless_qa.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts index c2ad97b6ddb05..2e76bcf87bf1f 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts @@ -8,6 +8,7 @@ import { defineCypressConfig } from '@kbn/cypress-config'; import { esArchiver } from './support/es_archiver'; import { samlAuthentication } from './support/saml_auth'; +import { esClient } from './support/es_client'; // eslint-disable-next-line import/no-default-export export default defineCypressConfig({ @@ -56,6 +57,7 @@ export default defineCypressConfig({ return launchOptions; }); samlAuthentication(on, config); + esClient(on, config); process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // eslint-disable-next-line @typescript-eslint/no-var-requires require('@cypress/grep/src/plugin')(config); From b86dc8102ad29656ac159f73a374f980897a2c88 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 13 Nov 2024 03:52:59 -0700 Subject: [PATCH 04/53] [Streams] Introducing the new Streams plugin (#198713) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR introduces the new experimental "Streams" plugin into the Kibana project. The Streams project aims to simplify workflows around dealing with messy logs in Elasticsearch. Our current offering is either extremely opinionated with integrations or leaves the user alone with the high flexibility of Elasticsearch concepts like index templates, component templates and so on, which make it challenging to configure everything correctly for good performance and controlling search speed and cost. ### Scope of PR - Provides an API for the user to "enable" the streams framework which creates the "root" entity `logs` with all the backing Elasticsearch assets - Provides an API for the user to "fork" a stream - Provides an API for the user to "read" a stream and all of it's Elasticsearch assets. - Provides an API for the user to upsert a stream (and implicitly child streams that are mentioned) - Part of this API is placing grok and disscect processing steps as well as fields to the mapping - Implements the Stream Naming Schema (SNS) which uses dots to express the index patterns and stream IDs. Example: `logs.nginx.errors` - The APIs will fully manage the `index_template`, `component_template`, and `ingest_pipelines`. ### Out of scope - Integration tests (coming in a follow-up) ### Reviewer Notes - I haven't implemented tests beyond a unit test for converting the filter conditions to Painless. I wanted to get a PR up so we can start iterating on the interface and functionality before we invest in testing. - You might need to add `server.versioned.versionResolution: oldest` to your `config/kibana.dev.yaml` to play with the requests below in the Kibana "Dev console". ### Example API Calls Enable the root stream (and set the mapping for the internal `.streams` index) ``` POST kbn:/api/streams/_enable ``` Read the root entity "logs" ``` GET kbn:/api/streams/logs ``` Fork the "root" entity "logs" and create "logs.nginx" based on a condition ``` POST kbn:/api/streams/logs/_fork { "stream": { "id": "logs.nginx", "children": [], "processing": [], "fields": [], }, "condition": { "field": "log.logger", "operator": "eq", "value": "nginx_proxy" } } ``` Fork the entity "logs.nginx" and create "logs.nginx.errors" based on a condition ``` POST kbn:/api/streams/logs.nginx/_fork { "stream": { "id": "logs.nginx.error", "children": [], "processing": [], "fields": [], }, "condition": { "or": [ { "field": "log.level", "operator": "eq", "value": "error" }, { "field": "log.level", "operator": "eq", "value": "ERROR" } ] } } ``` Set some processing on a stream and map the generated field ``` PUT kbn:/api/streams/logs.nginx { "children": [], "processing": [ { "config": { "type": "grok", "patterns": ["^%{IP:ip} – –"], "field": "message" } } ], "fields": [ { "name": "ip", "type": "ip" } ], } } ``` Field definitions are checked for both descendants and ancestors for incompatibilities to ensure they stay additive. If children are defined in the `PUT /api/streams/` API, sub-streams are created implicitly. If a stream is `PUT`, it's added to the parent as well with a condition that is never true (can be edited subsequently). `POST /api/streams/_resync` can be used to re-sync all streams from their meta data in case the Elasticsearch objects got messed up by some external change - not sure whether we want to keep that. Follow-ups * API integration tests * Check read permissions on data streams to determine whether a user is allowed to read certain streams --------- Co-authored-by: Joe Reuter Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 + docs/developer/plugin-list.asciidoc | 4 + package.json | 1 + packages/kbn-optimizer/limits.yml | 1 + tsconfig.base.json | 2 + x-pack/plugins/streams/README.md | 3 + x-pack/plugins/streams/common/config.ts | 30 ++ x-pack/plugins/streams/common/constants.ts | 9 + x-pack/plugins/streams/common/types.ts | 91 ++++++ x-pack/plugins/streams/jest.config.js | 15 + x-pack/plugins/streams/kibana.jsonc | 28 ++ x-pack/plugins/streams/public/index.ts | 13 + x-pack/plugins/streams/public/plugin.ts | 32 ++ x-pack/plugins/streams/public/types.ts | 16 + x-pack/plugins/streams/server/index.ts | 19 ++ .../component_templates/generate_layer.ts | 43 +++ .../streams/component_templates/logs_layer.ts | 23 ++ .../manage_component_templates.ts | 47 +++ .../lib/streams/component_templates/name.ts | 10 + .../data_streams/manage_data_streams.ts | 93 ++++++ .../errors/component_template_not_found.ts | 13 + .../streams/errors/definition_id_invalid.ts | 13 + .../streams/errors/definition_not_found.ts | 13 + .../streams/errors/fork_condition_missing.ts | 13 + .../lib/streams/errors/id_conflict_error.ts | 13 + .../server/lib/streams/errors/index.ts | 15 + .../errors/index_template_not_found.ts | 13 + .../errors/ingest_pipeline_not_found.ts | 13 + .../lib/streams/errors/malformed_children.ts | 13 + .../lib/streams/errors/malformed_fields.ts | 13 + .../lib/streams/errors/malformed_stream_id.ts | 13 + .../lib/streams/errors/permission_denied.ts | 13 + .../lib/streams/errors/security_exception.ts | 13 + .../helpers/condition_to_painless.test.ts | 133 ++++++++ .../streams/helpers/condition_to_painless.ts | 83 +++++ .../server/lib/streams/helpers/hierarchy.ts | 35 +++ .../server/lib/streams/helpers/retry.ts | 58 ++++ .../generate_index_template.ts | 42 +++ .../index_templates/manage_index_templates.ts | 44 +++ .../lib/streams/index_templates/name.ts | 10 + .../generate_ingest_pipeline.ts | 42 +++ .../generate_reroute_pipeline.ts | 34 +++ .../ingest_pipelines/logs_default_pipeline.ts | 23 ++ .../manage_ingest_pipelines.ts | 48 +++ .../lib/streams/ingest_pipelines/name.ts | 14 + .../lib/streams/internal_stream_mapping.ts | 35 +++ .../lib/streams/root_stream_definition.ts | 32 ++ .../streams/server/lib/streams/stream_crud.ts | 286 ++++++++++++++++++ x-pack/plugins/streams/server/plugin.ts | 92 ++++++ .../server/routes/create_server_route.ts | 11 + x-pack/plugins/streams/server/routes/index.ts | 26 ++ .../streams/server/routes/streams/delete.ts | 109 +++++++ .../streams/server/routes/streams/edit.ts | 171 +++++++++++ .../streams/server/routes/streams/enable.ts | 48 +++ .../streams/server/routes/streams/fork.ts | 112 +++++++ .../streams/server/routes/streams/list.ts | 70 +++++ .../streams/server/routes/streams/read.ts | 60 ++++ .../streams/server/routes/streams/resync.ts | 47 +++ x-pack/plugins/streams/server/routes/types.ts | 22 ++ x-pack/plugins/streams/server/types.ts | 45 +++ x-pack/plugins/streams/tsconfig.json | 31 ++ yarn.lock | 4 + 62 files changed, 2419 insertions(+) create mode 100644 x-pack/plugins/streams/README.md create mode 100644 x-pack/plugins/streams/common/config.ts create mode 100644 x-pack/plugins/streams/common/constants.ts create mode 100644 x-pack/plugins/streams/common/types.ts create mode 100644 x-pack/plugins/streams/jest.config.js create mode 100644 x-pack/plugins/streams/kibana.jsonc create mode 100644 x-pack/plugins/streams/public/index.ts create mode 100644 x-pack/plugins/streams/public/plugin.ts create mode 100644 x-pack/plugins/streams/public/types.ts create mode 100644 x-pack/plugins/streams/server/index.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/component_templates/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/index.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/helpers/retry.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/index_templates/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts create mode 100644 x-pack/plugins/streams/server/lib/streams/stream_crud.ts create mode 100644 x-pack/plugins/streams/server/plugin.ts create mode 100644 x-pack/plugins/streams/server/routes/create_server_route.ts create mode 100644 x-pack/plugins/streams/server/routes/index.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/delete.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/edit.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/enable.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/fork.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/list.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/read.ts create mode 100644 x-pack/plugins/streams/server/routes/streams/resync.ts create mode 100644 x-pack/plugins/streams/server/routes/types.ts create mode 100644 x-pack/plugins/streams/server/types.ts create mode 100644 x-pack/plugins/streams/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f0d509e283b2a..52af5b22dcaf2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -967,6 +967,7 @@ x-pack/plugins/snapshot_restore @elastic/kibana-management x-pack/plugins/spaces @elastic/kibana-security x-pack/plugins/stack_alerts @elastic/response-ops x-pack/plugins/stack_connectors @elastic/response-ops +x-pack/plugins/streams @simianhacker @flash1293 @dgieselaar x-pack/plugins/task_manager @elastic/response-ops x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index b6ba24df78976..71ab26400f496 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -897,6 +897,10 @@ routes, etc. |The stack_connectors plugin provides connector types shipped with Kibana, built on top of the framework provided in the actions plugin. +|{kib-repo}blob/{branch}/x-pack/plugins/streams/README.md[streams] +|This plugin provides an interface to manage streams + + |{kib-repo}blob/{branch}/x-pack/plugins/observability_solution/synthetics/README.md[synthetics] |The purpose of this plugin is to provide users of Heartbeat more visibility of what's happening in their infrastructure. diff --git a/package.json b/package.json index 48e377c9a6237..e2491ea76b89b 100644 --- a/package.json +++ b/package.json @@ -929,6 +929,7 @@ "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a", "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b", "@kbn/std": "link:packages/kbn-std", + "@kbn/streams-plugin": "link:x-pack/plugins/streams", "@kbn/synthetics-plugin": "link:x-pack/plugins/observability_solution/synthetics", "@kbn/synthetics-private-location": "link:x-pack/packages/kbn-synthetics-private-location", "@kbn/task-manager-fixture-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture", diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index aca3a1a0c3c99..79145938fa109 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -159,6 +159,7 @@ pageLoadAssetSize: spaces: 57868 stackAlerts: 58316 stackConnectors: 67227 + streams: 16742 synthetics: 55971 telemetry: 51957 telemetryManagementSection: 38586 diff --git a/tsconfig.base.json b/tsconfig.base.json index a525823e98e9d..e7a64097448ad 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1828,6 +1828,8 @@ "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"], "@kbn/storybook": ["packages/kbn-storybook"], "@kbn/storybook/*": ["packages/kbn-storybook/*"], + "@kbn/streams-plugin": ["x-pack/plugins/streams"], + "@kbn/streams-plugin/*": ["x-pack/plugins/streams/*"], "@kbn/synthetics-e2e": ["x-pack/plugins/observability_solution/synthetics/e2e"], "@kbn/synthetics-e2e/*": ["x-pack/plugins/observability_solution/synthetics/e2e/*"], "@kbn/synthetics-plugin": ["x-pack/plugins/observability_solution/synthetics"], diff --git a/x-pack/plugins/streams/README.md b/x-pack/plugins/streams/README.md new file mode 100644 index 0000000000000..9a3539a33535d --- /dev/null +++ b/x-pack/plugins/streams/README.md @@ -0,0 +1,3 @@ +# Streams Plugin + +This plugin provides an interface to manage streams \ No newline at end of file diff --git a/x-pack/plugins/streams/common/config.ts b/x-pack/plugins/streams/common/config.ts new file mode 100644 index 0000000000000..3371b1b6d8cbc --- /dev/null +++ b/x-pack/plugins/streams/common/config.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf } from '@kbn/config-schema'; + +export const configSchema = schema.object({}); + +export type StreamsConfig = TypeOf; + +/** + * The following map is passed to the server plugin setup under the + * exposeToBrowser: option, and controls which of the above config + * keys are allow-listed to be available in the browser config. + * + * NOTE: anything exposed here will be visible in the UI dev tools, + * and therefore MUST NOT be anything that is sensitive information! + */ +export const exposeToBrowserConfig = {} as const; + +type ValidKeys = keyof { + [K in keyof typeof exposeToBrowserConfig as (typeof exposeToBrowserConfig)[K] extends true + ? K + : never]: true; +}; + +export type StreamsPublicConfig = Pick; diff --git a/x-pack/plugins/streams/common/constants.ts b/x-pack/plugins/streams/common/constants.ts new file mode 100644 index 0000000000000..d7595990ded6e --- /dev/null +++ b/x-pack/plugins/streams/common/constants.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const ASSET_VERSION = 1; +export const STREAMS_INDEX = '.kibana_streams'; diff --git a/x-pack/plugins/streams/common/types.ts b/x-pack/plugins/streams/common/types.ts new file mode 100644 index 0000000000000..6cdb2f923f6f4 --- /dev/null +++ b/x-pack/plugins/streams/common/types.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; + +const stringOrNumberOrBoolean = z.union([z.string(), z.number(), z.boolean()]); + +export const filterConditionSchema = z.object({ + field: z.string(), + operator: z.enum(['eq', 'neq', 'lt', 'lte', 'gt', 'gte', 'contains', 'startsWith', 'endsWith']), + value: stringOrNumberOrBoolean, +}); + +export type FilterCondition = z.infer; + +export interface AndCondition { + and: Condition[]; +} + +export interface RerouteOrCondition { + or: Condition[]; +} + +export type Condition = FilterCondition | AndCondition | RerouteOrCondition | undefined; + +export const conditionSchema: z.ZodType = z.lazy(() => + z.union([ + filterConditionSchema, + z.object({ and: z.array(conditionSchema) }), + z.object({ or: z.array(conditionSchema) }), + ]) +); + +export const grokProcessingDefinitionSchema = z.object({ + type: z.literal('grok'), + field: z.string(), + patterns: z.array(z.string()), + pattern_definitions: z.optional(z.record(z.string())), +}); + +export const dissectProcessingDefinitionSchema = z.object({ + type: z.literal('dissect'), + field: z.string(), + pattern: z.string(), +}); + +export const processingDefinitionSchema = z.object({ + condition: z.optional(conditionSchema), + config: z.discriminatedUnion('type', [ + grokProcessingDefinitionSchema, + dissectProcessingDefinitionSchema, + ]), +}); + +export type ProcessingDefinition = z.infer; + +export const fieldDefinitionSchema = z.object({ + name: z.string(), + type: z.enum(['keyword', 'match_only_text', 'long', 'double', 'date', 'boolean', 'ip']), +}); + +export type FieldDefinition = z.infer; + +export const streamWithoutIdDefinitonSchema = z.object({ + processing: z.array(processingDefinitionSchema).default([]), + fields: z.array(fieldDefinitionSchema).default([]), + children: z + .array( + z.object({ + id: z.string(), + condition: conditionSchema, + }) + ) + .default([]), +}); + +export type StreamWithoutIdDefinition = z.infer; + +export const streamDefinitonSchema = streamWithoutIdDefinitonSchema.extend({ + id: z.string(), +}); + +export type StreamDefinition = z.infer; + +export const streamDefinitonWithoutChildrenSchema = streamDefinitonSchema.omit({ children: true }); + +export type StreamWithoutChildrenDefinition = z.infer; diff --git a/x-pack/plugins/streams/jest.config.js b/x-pack/plugins/streams/jest.config.js new file mode 100644 index 0000000000000..43d4fd28da9b5 --- /dev/null +++ b/x-pack/plugins/streams/jest.config.js @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/streams'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/streams', + coverageReporters: ['text', 'html'], + collectCoverageFrom: ['/x-pack/plugins/streams/{common,public,server}/**/*.{js,ts,tsx}'], +}; diff --git a/x-pack/plugins/streams/kibana.jsonc b/x-pack/plugins/streams/kibana.jsonc new file mode 100644 index 0000000000000..06c37ed245cf1 --- /dev/null +++ b/x-pack/plugins/streams/kibana.jsonc @@ -0,0 +1,28 @@ +{ + "type": "plugin", + "id": "@kbn/streams-plugin", + "owner": "@simianhacker @flash1293 @dgieselaar", + "description": "A manager for Streams", + "group": "observability", + "visibility": "private", + "plugin": { + "id": "streams", + "configPath": ["xpack", "streams"], + "browser": true, + "server": true, + "requiredPlugins": [ + "data", + "security", + "encryptedSavedObjects", + "usageCollection", + "licensing", + "taskManager" + ], + "optionalPlugins": [ + "cloud", + "serverless" + ], + "requiredBundles": [ + ] + } +} diff --git a/x-pack/plugins/streams/public/index.ts b/x-pack/plugins/streams/public/index.ts new file mode 100644 index 0000000000000..5b83ea1d297d3 --- /dev/null +++ b/x-pack/plugins/streams/public/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializer, PluginInitializerContext } from '@kbn/core/public'; +import { Plugin } from './plugin'; + +export const plugin: PluginInitializer<{}, {}> = (context: PluginInitializerContext) => { + return new Plugin(context); +}; diff --git a/x-pack/plugins/streams/public/plugin.ts b/x-pack/plugins/streams/public/plugin.ts new file mode 100644 index 0000000000000..f35d18e06ff70 --- /dev/null +++ b/x-pack/plugins/streams/public/plugin.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreSetup, CoreStart, PluginInitializerContext } from '@kbn/core/public'; +import { Logger } from '@kbn/logging'; + +import type { StreamsPublicConfig } from '../common/config'; +import { StreamsPluginClass, StreamsPluginSetup, StreamsPluginStart } from './types'; + +export class Plugin implements StreamsPluginClass { + public config: StreamsPublicConfig; + public logger: Logger; + + constructor(context: PluginInitializerContext<{}>) { + this.config = context.config.get(); + this.logger = context.logger.get(); + } + + setup(core: CoreSetup, pluginSetup: StreamsPluginSetup) { + return {}; + } + + start(core: CoreStart) { + return {}; + } + + stop() {} +} diff --git a/x-pack/plugins/streams/public/types.ts b/x-pack/plugins/streams/public/types.ts new file mode 100644 index 0000000000000..61e5fa94098f0 --- /dev/null +++ b/x-pack/plugins/streams/public/types.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Plugin as PluginClass } from '@kbn/core/public'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginSetup {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginStart {} + +export type StreamsPluginClass = PluginClass<{}, {}, StreamsPluginSetup, StreamsPluginStart>; diff --git a/x-pack/plugins/streams/server/index.ts b/x-pack/plugins/streams/server/index.ts new file mode 100644 index 0000000000000..bd8aee304ad15 --- /dev/null +++ b/x-pack/plugins/streams/server/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PluginInitializerContext } from '@kbn/core-plugins-server'; +import { StreamsConfig } from '../common/config'; +import { StreamsPluginSetup, StreamsPluginStart, config } from './plugin'; +import { StreamsRouteRepository } from './routes'; + +export type { StreamsConfig, StreamsPluginSetup, StreamsPluginStart, StreamsRouteRepository }; +export { config }; + +export const plugin = async (context: PluginInitializerContext) => { + const { StreamsPlugin } = await import('./plugin'); + return new StreamsPlugin(context); +}; diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts new file mode 100644 index 0000000000000..82c89c9ab9171 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/generate_layer.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ClusterPutComponentTemplateRequest, + MappingProperty, +} from '@elastic/elasticsearch/lib/api/types'; +import { StreamDefinition } from '../../../../common/types'; +import { ASSET_VERSION } from '../../../../common/constants'; +import { logsSettings } from './logs_layer'; +import { isRoot } from '../helpers/hierarchy'; +import { getComponentTemplateName } from './name'; + +export function generateLayer( + id: string, + definition: StreamDefinition +): ClusterPutComponentTemplateRequest { + const properties: Record = {}; + definition.fields.forEach((field) => { + properties[field.name] = { + type: field.type, + }; + }); + return { + name: getComponentTemplateName(id), + template: { + settings: isRoot(definition.id) ? logsSettings : {}, + mappings: { + subobjects: false, + properties, + }, + }, + version: ASSET_VERSION, + _meta: { + managed: true, + description: `Default settings for the ${id} stream`, + }, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts new file mode 100644 index 0000000000000..6b41d04131c56 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/logs_layer.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types'; + +export const logsSettings: IndicesIndexSettings = { + index: { + lifecycle: { + name: 'logs', + }, + codec: 'best_compression', + mapping: { + total_fields: { + ignore_dynamic_beyond_limit: true, + }, + ignore_malformed: true, + }, + }, +}; diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts new file mode 100644 index 0000000000000..a7d707a4ce42a --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/manage_component_templates.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DeleteComponentOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface ComponentManagementOptions { + esClient: ElasticsearchClient; + component: ClusterPutComponentTemplateRequest; + logger: Logger; +} + +export async function deleteComponent({ esClient, name, logger }: DeleteComponentOptions) { + try { + await retryTransientEsErrors( + () => esClient.cluster.deleteComponentTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting component template: ${error.message}`); + throw error; + } +} + +export async function upsertComponent({ esClient, component, logger }: ComponentManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(component), { + logger, + }); + logger.debug(() => `Installed component template: ${JSON.stringify(component)}`); + } catch (error: any) { + logger.error(`Error updating component template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts b/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts new file mode 100644 index 0000000000000..6ea05b9a53b28 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/component_templates/name.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getComponentTemplateName(id: string) { + return `${id}@stream.layer`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts new file mode 100644 index 0000000000000..812739db56c73 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DataStreamManagementOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface DeleteDataStreamOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +interface RolloverDataStreamOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertDataStream({ esClient, name, logger }: DataStreamManagementOptions) { + const dataStreamExists = await esClient.indices.exists({ index: name }); + if (dataStreamExists) { + return; + } + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + logger.debug(() => `Installed data stream: ${name}`); + } catch (error: any) { + logger.error(`Error creating data stream: ${error.message}`); + throw error; + } +} + +export async function deleteDataStream({ esClient, name, logger }: DeleteDataStreamOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteDataStream({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting data stream: ${error.message}`); + throw error; + } +} + +export async function rolloverDataStreamIfNecessary({ + esClient, + name, + logger, +}: RolloverDataStreamOptions) { + const dataStreams = await esClient.indices.getDataStream({ name: `${name},${name}.*` }); + for (const dataStream of dataStreams.data_streams) { + const currentMappings = + Object.values( + await esClient.indices.getMapping({ + index: dataStream.indices.at(-1)?.index_name, + }) + )[0].mappings.properties || {}; + const simulatedIndex = await esClient.indices.simulateIndexTemplate({ name: dataStream.name }); + const simulatedMappings = simulatedIndex.template.mappings.properties || {}; + + // check whether the same fields and same types are listed (don't check for other mapping attributes) + const isDifferent = + Object.values(simulatedMappings).length !== Object.values(currentMappings).length || + Object.entries(simulatedMappings || {}).some(([fieldName, { type }]) => { + const currentType = currentMappings[fieldName]?.type; + return currentType !== type; + }); + + if (!isDifferent) { + continue; + } + + try { + await retryTransientEsErrors(() => esClient.indices.rollover({ alias: dataStream.name }), { + logger, + }); + logger.debug(() => `Rolled over data stream: ${dataStream.name}`); + } catch (error: any) { + logger.error(`Error rolling over data stream: ${error.message}`); + throw error; + } + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts new file mode 100644 index 0000000000000..a7e9cebf98507 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/component_template_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ComponentTemplateNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'ComponentTemplateNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts b/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts new file mode 100644 index 0000000000000..817e8f67bf25d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/definition_id_invalid.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DefinitionIdInvalid extends Error { + constructor(message: string) { + super(message); + this.name = 'DefinitionIdInvalid'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts new file mode 100644 index 0000000000000..f7e60193baa5f --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/definition_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class DefinitionNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'DefinitionNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts b/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts new file mode 100644 index 0000000000000..713751dbe4363 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/fork_condition_missing.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class ForkConditionMissing extends Error { + constructor(message: string) { + super(message); + this.name = 'ForkConditionMissing'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts new file mode 100644 index 0000000000000..a24c7357379fa --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/id_conflict_error.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IdConflict extends Error { + constructor(message: string) { + super(message); + this.name = 'IdConflict'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/index.ts b/x-pack/plugins/streams/server/lib/streams/errors/index.ts new file mode 100644 index 0000000000000..73842ef3018fe --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './definition_id_invalid'; +export * from './definition_not_found'; +export * from './id_conflict_error'; +export * from './permission_denied'; +export * from './security_exception'; +export * from './index_template_not_found'; +export * from './fork_condition_missing'; +export * from './component_template_not_found'; diff --git a/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts new file mode 100644 index 0000000000000..4f4735dd15fa1 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/index_template_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IndexTemplateNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'IndexTemplateNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts b/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts new file mode 100644 index 0000000000000..8bf9bbd4933ce --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/ingest_pipeline_not_found.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class IngestPipelineNotFound extends Error { + constructor(message: string) { + super(message); + this.name = 'IngestPipelineNotFound'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts new file mode 100644 index 0000000000000..699c4cdd5b1ef --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_children.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedChildren extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedChildren'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts new file mode 100644 index 0000000000000..b8f7ac1392610 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_fields.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedFields extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedFields'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts b/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts new file mode 100644 index 0000000000000..2f988204c74b0 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/malformed_stream_id.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class MalformedStreamId extends Error { + constructor(message: string) { + super(message); + this.name = 'MalformedStreamId'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts b/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts new file mode 100644 index 0000000000000..f0133e28063ca --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/permission_denied.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class PermissionDenied extends Error { + constructor(message: string) { + super(message); + this.name = 'PermissionDenied'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts b/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts new file mode 100644 index 0000000000000..0b4ae450c2530 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/errors/security_exception.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class SecurityException extends Error { + constructor(message: string) { + super(message); + this.name = 'SecurityException'; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts new file mode 100644 index 0000000000000..aab7f27f12d14 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { conditionToPainless } from './condition_to_painless'; + +const operatorConditionAndResutls = [ + { + condition: { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + result: 'ctx.log?.logger == "nginx_proxy"', + }, + { + condition: { field: 'log.logger', operator: 'neq' as const, value: 'nginx_proxy' }, + result: 'ctx.log?.logger != "nginx_proxy"', + }, + { + condition: { field: 'http.response.status_code', operator: 'lt' as const, value: 500 }, + result: 'ctx.http?.response?.status_code < 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'lte' as const, value: 500 }, + result: 'ctx.http?.response?.status_code <= 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'gt' as const, value: 500 }, + result: 'ctx.http?.response?.status_code > 500', + }, + { + condition: { field: 'http.response.status_code', operator: 'gte' as const, value: 500 }, + result: 'ctx.http?.response?.status_code >= 500', + }, + { + condition: { field: 'log.logger', operator: 'startsWith' as const, value: 'nginx' }, + result: 'ctx.log?.logger.startsWith("nginx")', + }, + { + condition: { field: 'log.logger', operator: 'endsWith' as const, value: 'proxy' }, + result: 'ctx.log?.logger.endsWith("proxy")', + }, + { + condition: { field: 'log.logger', operator: 'contains' as const, value: 'proxy' }, + result: 'ctx.log?.logger.contains("proxy")', + }, +]; + +describe('conditionToPainless', () => { + describe('operators', () => { + operatorConditionAndResutls.forEach((setup) => { + test(`${setup.condition.operator}`, () => { + expect(conditionToPainless(setup.condition)).toEqual(setup.result); + }); + }); + }); + + describe('and', () => { + test('simple', () => { + const condition = { + and: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + ], + }; + expect( + expect(conditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" && ctx.log?.level == "error"' + ) + ); + }); + }); + + describe('or', () => { + test('simple', () => { + const condition = { + or: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + ], + }; + expect( + expect(conditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" || ctx.log?.level == "error"' + ) + ); + }); + }); + + describe('nested', () => { + test('and with a filter and or with 2 filters', () => { + const condition = { + and: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { + or: [ + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + { field: 'log.level', operator: 'eq' as const, value: 'ERROR' }, + ], + }, + ], + }; + expect( + expect(conditionToPainless(condition)).toEqual( + 'ctx.log?.logger == "nginx_proxy" && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' + ) + ); + }); + test('and with 2 or with filters', () => { + const condition = { + and: [ + { + or: [ + { field: 'log.logger', operator: 'eq' as const, value: 'nginx_proxy' }, + { field: 'service.name', operator: 'eq' as const, value: 'nginx' }, + ], + }, + { + or: [ + { field: 'log.level', operator: 'eq' as const, value: 'error' }, + { field: 'log.level', operator: 'eq' as const, value: 'ERROR' }, + ], + }, + ], + }; + expect( + expect(conditionToPainless(condition)).toEqual( + '(ctx.log?.logger == "nginx_proxy" || ctx.service?.name == "nginx") && (ctx.log?.level == "error" || ctx.log?.level == "ERROR")' + ) + ); + }); + }); +}); diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts new file mode 100644 index 0000000000000..539ad3603535b --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/condition_to_painless.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isBoolean, isString } from 'lodash'; +import { + AndCondition, + Condition, + conditionSchema, + FilterCondition, + filterConditionSchema, + RerouteOrCondition, +} from '../../../../common/types'; + +function isFilterCondition(subject: any): subject is FilterCondition { + const result = filterConditionSchema.safeParse(subject); + return result.success; +} + +function isAndCondition(subject: any): subject is AndCondition { + const result = conditionSchema.safeParse(subject); + return result.success && subject.and != null; +} + +function isOrCondition(subject: any): subject is RerouteOrCondition { + const result = conditionSchema.safeParse(subject); + return result.success && subject.or != null; +} + +function safePainlessField(condition: FilterCondition) { + return `ctx.${condition.field.split('.').join('?.')}`; +} + +function encodeValue(value: string | number | boolean) { + if (isString(value)) { + return `"${value}"`; + } + if (isBoolean(value)) { + return value ? 'true' : 'false'; + } + return value; +} + +function toPainless(condition: FilterCondition) { + switch (condition.operator) { + case 'neq': + return `${safePainlessField(condition)} != ${encodeValue(condition.value)}`; + case 'lt': + return `${safePainlessField(condition)} < ${encodeValue(condition.value)}`; + case 'lte': + return `${safePainlessField(condition)} <= ${encodeValue(condition.value)}`; + case 'gt': + return `${safePainlessField(condition)} > ${encodeValue(condition.value)}`; + case 'gte': + return `${safePainlessField(condition)} >= ${encodeValue(condition.value)}`; + case 'startsWith': + return `${safePainlessField(condition)}.startsWith(${encodeValue(condition.value)})`; + case 'endsWith': + return `${safePainlessField(condition)}.endsWith(${encodeValue(condition.value)})`; + case 'contains': + return `${safePainlessField(condition)}.contains(${encodeValue(condition.value)})`; + default: + return `${safePainlessField(condition)} == ${encodeValue(condition.value)}`; + } +} + +export function conditionToPainless(condition: Condition, nested = false): string { + if (isFilterCondition(condition)) { + return toPainless(condition); + } + if (isAndCondition(condition)) { + const and = condition.and.map((filter) => conditionToPainless(filter, true)).join(' && '); + return nested ? `(${and})` : and; + } + if (isOrCondition(condition)) { + const or = condition.or.map((filter) => conditionToPainless(filter, true)).join(' || '); + return nested ? `(${or})` : or; + } + return 'false'; +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts new file mode 100644 index 0000000000000..6f1cd308f3c3d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/hierarchy.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../../common/types'; + +export function isDescendandOf(parent: StreamDefinition, child: StreamDefinition) { + return child.id.startsWith(parent.id); +} + +export function isChildOf(parent: StreamDefinition, child: StreamDefinition) { + return ( + isDescendandOf(parent, child) && child.id.split('.').length === parent.id.split('.').length + 1 + ); +} + +export function getParentId(id: string) { + const parts = id.split('.'); + if (parts.length === 1) { + return undefined; + } + return parts.slice(0, parts.length - 1).join('.'); +} + +export function isRoot(id: string) { + return id.split('.').length === 1; +} + +export function getAncestors(id: string) { + const parts = id.split('.'); + return parts.slice(0, parts.length - 1).map((_, index) => parts.slice(0, index + 1).join('.')); +} diff --git a/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts b/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts new file mode 100644 index 0000000000000..32604a22bf9be --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/helpers/retry.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setTimeout } from 'timers/promises'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import type { Logger } from '@kbn/logging'; +import { SecurityException } from '../errors'; + +const MAX_ATTEMPTS = 5; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: any) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +/** + * Retries any transient network or configuration issues encountered from Elasticsearch with an exponential backoff. + * Should only be used to wrap operations that are idempotent and can be safely executed more than once. + */ +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { logger, attempt = 0 }: { logger?: Logger; attempt?: number } = {} +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec = Math.min(Math.pow(2, retryCount), 64); // 2s, 4s, 8s, 16s, 32s, 64s, 64s, 64s ... + + logger?.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + await setTimeout(retryDelaySec * 1000); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + if (e.meta?.body?.error?.type === 'security_exception') { + throw new SecurityException(e.meta.body.error.reason); + } + + throw e; + } +}; diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts new file mode 100644 index 0000000000000..7a16534a618da --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/generate_index_template.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSET_VERSION } from '../../../../common/constants'; +import { getProcessingPipelineName } from '../ingest_pipelines/name'; +import { getIndexTemplateName } from './name'; + +export function generateIndexTemplate(id: string) { + const composedOf = id.split('.').reduce((acc, _, index, array) => { + const parent = array.slice(0, index + 1).join('.'); + return [...acc, `${parent}@stream.layer`]; + }, [] as string[]); + + return { + name: getIndexTemplateName(id), + index_patterns: [id], + composed_of: composedOf, + priority: 200, + version: ASSET_VERSION, + _meta: { + managed: true, + description: `The index template for ${id} stream`, + }, + data_stream: { + hidden: false, + }, + template: { + settings: { + index: { + default_pipeline: getProcessingPipelineName(id), + }, + }, + }, + allow_auto_create: true, + // ignore missing component templates to be more robust against out-of-order syncs + ignore_missing_component_templates: composedOf, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts new file mode 100644 index 0000000000000..9383e698b3436 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/manage_index_templates.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndicesPutIndexTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface TemplateManagementOptions { + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; + logger: Logger; +} + +interface DeleteTemplateOptions { + esClient: ElasticsearchClient; + name: string; + logger: Logger; +} + +export async function upsertTemplate({ esClient, template, logger }: TemplateManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(template)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} + +export async function deleteTemplate({ esClient, name, logger }: DeleteTemplateOptions) { + try { + await retryTransientEsErrors( + () => esClient.indices.deleteIndexTemplate({ name }, { ignore: [404] }), + { logger } + ); + } catch (error: any) { + logger.error(`Error deleting index template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts b/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts new file mode 100644 index 0000000000000..ec8ea5519a6b4 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/index_templates/name.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getIndexTemplateName(id: string) { + return `${id}@stream`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts new file mode 100644 index 0000000000000..eb09df8831304 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_ingest_pipeline.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../../common/types'; +import { ASSET_VERSION } from '../../../../common/constants'; +import { conditionToPainless } from '../helpers/condition_to_painless'; +import { logsDefaultPipelineProcessors } from './logs_default_pipeline'; +import { isRoot } from '../helpers/hierarchy'; +import { getProcessingPipelineName } from './name'; + +export function generateIngestPipeline(id: string, definition: StreamDefinition) { + return { + id: getProcessingPipelineName(id), + processors: [ + ...(isRoot(definition.id) ? logsDefaultPipelineProcessors : []), + ...definition.processing.map((processor) => { + const { type, ...config } = processor.config; + return { + [type]: { + ...config, + if: processor.condition ? conditionToPainless(processor.condition) : undefined, + }, + }; + }), + { + pipeline: { + name: `${id}@stream.reroutes`, + ignore_missing_pipeline: true, + }, + }, + ], + _meta: { + description: `Default pipeline for the ${id} stream`, + managed: true, + }, + version: ASSET_VERSION, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts new file mode 100644 index 0000000000000..9b46e0cf4ac92 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/generate_reroute_pipeline.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../../common/types'; +import { ASSET_VERSION } from '../../../../common/constants'; +import { conditionToPainless } from '../helpers/condition_to_painless'; +import { getReroutePipelineName } from './name'; + +interface GenerateReroutePipelineParams { + definition: StreamDefinition; +} + +export async function generateReroutePipeline({ definition }: GenerateReroutePipelineParams) { + return { + id: getReroutePipelineName(definition.id), + processors: definition.children.map((child) => { + return { + reroute: { + destination: child.id, + if: conditionToPainless(child.condition), + }, + }; + }), + _meta: { + description: `Reoute pipeline for the ${definition.id} stream`, + managed: true, + }, + version: ASSET_VERSION, + }; +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts new file mode 100644 index 0000000000000..762155ba5047c --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/logs_default_pipeline.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const logsDefaultPipelineProcessors = [ + { + set: { + description: "If '@timestamp' is missing, set it with the ingest timestamp", + field: '@timestamp', + override: false, + copy_from: '_ingest.timestamp', + }, + }, + { + pipeline: { + name: 'logs@json-pipeline', + ignore_missing_pipeline: true, + }, + }, +]; diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts new file mode 100644 index 0000000000000..467e2efb48f0d --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/manage_ingest_pipelines.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { IngestPutPipelineRequest } from '@elastic/elasticsearch/lib/api/types'; +import { retryTransientEsErrors } from '../helpers/retry'; + +interface DeletePipelineOptions { + esClient: ElasticsearchClient; + id: string; + logger: Logger; +} + +interface PipelineManagementOptions { + esClient: ElasticsearchClient; + pipeline: IngestPutPipelineRequest; + logger: Logger; +} + +export async function deleteIngestPipeline({ esClient, id, logger }: DeletePipelineOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.deletePipeline({ id }, { ignore: [404] }), { + logger, + }); + } catch (error: any) { + logger.error(`Error deleting ingest pipeline: ${error.message}`); + throw error; + } +} + +export async function upsertIngestPipeline({ + esClient, + pipeline, + logger, +}: PipelineManagementOptions) { + try { + await retryTransientEsErrors(() => esClient.ingest.putPipeline(pipeline), { logger }); + logger.debug(() => `Installed index template: ${JSON.stringify(pipeline)}`); + } catch (error: any) { + logger.error(`Error updating index template: ${error.message}`); + throw error; + } +} diff --git a/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts new file mode 100644 index 0000000000000..8d2a97ff3137f --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/ingest_pipelines/name.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getProcessingPipelineName(id: string) { + return `${id}@stream.processing`; +} + +export function getReroutePipelineName(id: string) { + return `${id}@stream.reroutes`; +} diff --git a/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts new file mode 100644 index 0000000000000..8e88eeef8cd84 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/internal_stream_mapping.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { STREAMS_INDEX } from '../../../common/constants'; + +export function createStreamsIndex(scopedClusterClient: IScopedClusterClient) { + return scopedClusterClient.asInternalUser.indices.create({ + index: STREAMS_INDEX, + mappings: { + dynamic: 'strict', + properties: { + processing: { + type: 'object', + enabled: false, + }, + fields: { + type: 'object', + enabled: false, + }, + children: { + type: 'object', + enabled: false, + }, + id: { + type: 'keyword', + }, + }, + }, + }); +} diff --git a/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts new file mode 100644 index 0000000000000..2b7deed877309 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/root_stream_definition.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { StreamDefinition } from '../../../common/types'; + +export const rootStreamDefinition: StreamDefinition = { + id: 'logs', + processing: [], + children: [], + fields: [ + { + name: '@timestamp', + type: 'date', + }, + { + name: 'message', + type: 'match_only_text', + }, + { + name: 'host.name', + type: 'keyword', + }, + { + name: 'log.level', + type: 'keyword', + }, + ], +}; diff --git a/x-pack/plugins/streams/server/lib/streams/stream_crud.ts b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts new file mode 100644 index 0000000000000..78a126905d9a4 --- /dev/null +++ b/x-pack/plugins/streams/server/lib/streams/stream_crud.ts @@ -0,0 +1,286 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { FieldDefinition, StreamDefinition } from '../../../common/types'; +import { STREAMS_INDEX } from '../../../common/constants'; +import { DefinitionNotFound } from './errors'; +import { deleteTemplate, upsertTemplate } from './index_templates/manage_index_templates'; +import { generateLayer } from './component_templates/generate_layer'; +import { generateIngestPipeline } from './ingest_pipelines/generate_ingest_pipeline'; +import { generateReroutePipeline } from './ingest_pipelines/generate_reroute_pipeline'; +import { generateIndexTemplate } from './index_templates/generate_index_template'; +import { deleteComponent, upsertComponent } from './component_templates/manage_component_templates'; +import { getIndexTemplateName } from './index_templates/name'; +import { getComponentTemplateName } from './component_templates/name'; +import { getProcessingPipelineName, getReroutePipelineName } from './ingest_pipelines/name'; +import { + deleteIngestPipeline, + upsertIngestPipeline, +} from './ingest_pipelines/manage_ingest_pipelines'; +import { getAncestors } from './helpers/hierarchy'; +import { MalformedFields } from './errors/malformed_fields'; +import { + deleteDataStream, + rolloverDataStreamIfNecessary, + upsertDataStream, +} from './data_streams/manage_data_streams'; + +interface BaseParams { + scopedClusterClient: IScopedClusterClient; +} + +interface BaseParamsWithDefinition extends BaseParams { + definition: StreamDefinition; +} + +interface DeleteStreamParams extends BaseParams { + id: string; + logger: Logger; +} + +export async function deleteStreamObjects({ id, scopedClusterClient, logger }: DeleteStreamParams) { + await deleteDataStream({ + esClient: scopedClusterClient.asCurrentUser, + name: id, + logger, + }); + await deleteTemplate({ + esClient: scopedClusterClient.asCurrentUser, + name: getIndexTemplateName(id), + logger, + }); + await deleteComponent({ + esClient: scopedClusterClient.asCurrentUser, + name: getComponentTemplateName(id), + logger, + }); + await deleteIngestPipeline({ + esClient: scopedClusterClient.asCurrentUser, + id: getProcessingPipelineName(id), + logger, + }); + await deleteIngestPipeline({ + esClient: scopedClusterClient.asCurrentUser, + id: getReroutePipelineName(id), + logger, + }); + await scopedClusterClient.asInternalUser.delete({ + id, + index: STREAMS_INDEX, + refresh: 'wait_for', + }); +} + +async function upsertInternalStream({ definition, scopedClusterClient }: BaseParamsWithDefinition) { + return scopedClusterClient.asInternalUser.index({ + id: definition.id, + index: STREAMS_INDEX, + document: definition, + refresh: 'wait_for', + }); +} + +type ListStreamsParams = BaseParams; + +export async function listStreams({ scopedClusterClient }: ListStreamsParams) { + const response = await scopedClusterClient.asInternalUser.search({ + index: STREAMS_INDEX, + size: 10000, + fields: ['id'], + _source: false, + sort: [{ id: 'asc' }], + }); + const definitions = response.hits.hits.map((hit) => hit.fields as { id: string[] }); + return definitions; +} + +interface ReadStreamParams extends BaseParams { + id: string; +} + +export async function readStream({ id, scopedClusterClient }: ReadStreamParams) { + try { + const response = await scopedClusterClient.asInternalUser.get({ + id, + index: STREAMS_INDEX, + }); + const definition = response._source as StreamDefinition; + return { + definition, + }; + } catch (e) { + if (e.meta?.statusCode === 404) { + throw new DefinitionNotFound(`Stream definition for ${id} not found.`); + } + throw e; + } +} + +interface ReadAncestorsParams extends BaseParams { + id: string; +} + +export async function readAncestors({ id, scopedClusterClient }: ReadAncestorsParams) { + const ancestorIds = getAncestors(id); + + return await Promise.all( + ancestorIds.map((ancestorId) => readStream({ scopedClusterClient, id: ancestorId })) + ); +} + +interface ReadDescendantsParams extends BaseParams { + id: string; +} + +export async function readDescendants({ id, scopedClusterClient }: ReadDescendantsParams) { + const response = await scopedClusterClient.asInternalUser.search({ + index: STREAMS_INDEX, + size: 10000, + body: { + query: { + bool: { + filter: { + prefix: { + id, + }, + }, + must_not: { + term: { + id, + }, + }, + }, + }, + }, + }); + return response.hits.hits.map((hit) => hit._source as StreamDefinition); +} + +export async function validateAncestorFields( + scopedClusterClient: IScopedClusterClient, + id: string, + fields: FieldDefinition[] +) { + const ancestors = await readAncestors({ + id, + scopedClusterClient, + }); + for (const ancestor of ancestors) { + for (const field of fields) { + if ( + ancestor.definition.fields.some( + (ancestorField) => ancestorField.type !== field.type && ancestorField.name === field.name + ) + ) { + throw new MalformedFields( + `Field ${field.name} is already defined with incompatible type in the parent stream ${ancestor.definition.id}` + ); + } + } + } +} + +export async function validateDescendantFields( + scopedClusterClient: IScopedClusterClient, + id: string, + fields: FieldDefinition[] +) { + const descendants = await readDescendants({ + id, + scopedClusterClient, + }); + for (const descendant of descendants) { + for (const field of fields) { + if ( + descendant.fields.some( + (descendantField) => + descendantField.type !== field.type && descendantField.name === field.name + ) + ) { + throw new MalformedFields( + `Field ${field.name} is already defined with incompatible type in the child stream ${descendant.id}` + ); + } + } + } +} + +export async function checkStreamExists({ id, scopedClusterClient }: ReadStreamParams) { + try { + await readStream({ id, scopedClusterClient }); + return true; + } catch (e) { + if (e instanceof DefinitionNotFound) { + return false; + } + throw e; + } +} + +interface SyncStreamParams { + scopedClusterClient: IScopedClusterClient; + definition: StreamDefinition; + rootDefinition?: StreamDefinition; + logger: Logger; +} + +export async function syncStream({ + scopedClusterClient, + definition, + rootDefinition, + logger, +}: SyncStreamParams) { + await upsertComponent({ + esClient: scopedClusterClient.asCurrentUser, + logger, + component: generateLayer(definition.id, definition), + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asCurrentUser, + logger, + pipeline: generateIngestPipeline(definition.id, definition), + }); + const reroutePipeline = await generateReroutePipeline({ + definition, + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asCurrentUser, + logger, + pipeline: reroutePipeline, + }); + await upsertTemplate({ + esClient: scopedClusterClient.asCurrentUser, + logger, + template: generateIndexTemplate(definition.id), + }); + if (rootDefinition) { + const parentReroutePipeline = await generateReroutePipeline({ + definition: rootDefinition, + }); + await upsertIngestPipeline({ + esClient: scopedClusterClient.asCurrentUser, + logger, + pipeline: parentReroutePipeline, + }); + } + await upsertDataStream({ + esClient: scopedClusterClient.asCurrentUser, + logger, + name: definition.id, + }); + await upsertInternalStream({ + scopedClusterClient, + definition, + }); + await rolloverDataStreamIfNecessary({ + esClient: scopedClusterClient.asCurrentUser, + name: definition.id, + logger, + }); +} diff --git a/x-pack/plugins/streams/server/plugin.ts b/x-pack/plugins/streams/server/plugin.ts new file mode 100644 index 0000000000000..ef070984803d5 --- /dev/null +++ b/x-pack/plugins/streams/server/plugin.ts @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + CoreSetup, + CoreStart, + KibanaRequest, + Logger, + Plugin, + PluginConfigDescriptor, + PluginInitializerContext, +} from '@kbn/core/server'; +import { registerRoutes } from '@kbn/server-route-repository'; +import { StreamsConfig, configSchema, exposeToBrowserConfig } from '../common/config'; +import { StreamsRouteRepository } from './routes'; +import { RouteDependencies } from './routes/types'; +import { + StreamsPluginSetupDependencies, + StreamsPluginStartDependencies, + StreamsServer, +} from './types'; + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginSetup {} +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface StreamsPluginStart {} + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToBrowser: exposeToBrowserConfig, +}; + +export class StreamsPlugin + implements + Plugin< + StreamsPluginSetup, + StreamsPluginStart, + StreamsPluginSetupDependencies, + StreamsPluginStartDependencies + > +{ + public config: StreamsConfig; + public logger: Logger; + public server?: StreamsServer; + + constructor(context: PluginInitializerContext) { + this.config = context.config.get(); + this.logger = context.logger.get(); + } + + public setup(core: CoreSetup, plugins: StreamsPluginSetupDependencies): StreamsPluginSetup { + this.server = { + config: this.config, + logger: this.logger, + } as StreamsServer; + + registerRoutes({ + repository: StreamsRouteRepository, + dependencies: { + server: this.server, + getScopedClients: async ({ request }: { request: KibanaRequest }) => { + const [coreStart] = await core.getStartServices(); + const scopedClusterClient = coreStart.elasticsearch.client.asScoped(request); + const soClient = coreStart.savedObjects.getScopedClient(request); + return { scopedClusterClient, soClient }; + }, + }, + core, + logger: this.logger, + }); + + return {}; + } + + public start(core: CoreStart, plugins: StreamsPluginStartDependencies): StreamsPluginStart { + if (this.server) { + this.server.core = core; + this.server.isServerless = core.elasticsearch.getCapabilities().serverless; + this.server.security = plugins.security; + this.server.encryptedSavedObjects = plugins.encryptedSavedObjects; + this.server.taskManager = plugins.taskManager; + } + + return {}; + } + + public stop() {} +} diff --git a/x-pack/plugins/streams/server/routes/create_server_route.ts b/x-pack/plugins/streams/server/routes/create_server_route.ts new file mode 100644 index 0000000000000..94d85a71c82bb --- /dev/null +++ b/x-pack/plugins/streams/server/routes/create_server_route.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createServerRouteFactory } from '@kbn/server-route-repository'; +import { StreamsRouteHandlerResources } from './types'; + +export const createServerRoute = createServerRouteFactory(); diff --git a/x-pack/plugins/streams/server/routes/index.ts b/x-pack/plugins/streams/server/routes/index.ts new file mode 100644 index 0000000000000..6fc734d3371b4 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { deleteStreamRoute } from './streams/delete'; +import { editStreamRoute } from './streams/edit'; +import { enableStreamsRoute } from './streams/enable'; +import { forkStreamsRoute } from './streams/fork'; +import { listStreamsRoute } from './streams/list'; +import { readStreamRoute } from './streams/read'; +import { resyncStreamsRoute } from './streams/resync'; + +export const StreamsRouteRepository = { + ...enableStreamsRoute, + ...resyncStreamsRoute, + ...forkStreamsRoute, + ...readStreamRoute, + ...editStreamRoute, + ...deleteStreamRoute, + ...listStreamsRoute, +}; + +export type StreamsRouteRepository = typeof StreamsRouteRepository; diff --git a/x-pack/plugins/streams/server/routes/streams/delete.ts b/x-pack/plugins/streams/server/routes/streams/delete.ts new file mode 100644 index 0000000000000..3820975dbe16a --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/delete.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { syncStream, readStream, deleteStreamObjects } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { getParentId } from '../../lib/streams/helpers/hierarchy'; + +export const deleteStreamRoute = createServerRoute({ + endpoint: 'DELETE /api/streams/{id} 2023-10-31', + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + const parentId = getParentId(params.path.id); + if (!parentId) { + throw new MalformedStreamId('Cannot delete root stream'); + } + + await updateParentStream(scopedClusterClient, params.path.id, parentId, logger); + + await deleteStream(scopedClusterClient, params.path.id, logger); + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +async function deleteStream(scopedClusterClient: IScopedClusterClient, id: string, logger: Logger) { + try { + const { definition } = await readStream({ scopedClusterClient, id }); + for (const child of definition.children) { + await deleteStream(scopedClusterClient, child.id, logger); + } + await deleteStreamObjects({ scopedClusterClient, id, logger }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + logger.debug(`Stream definition for ${id} not found.`); + } else { + throw e; + } + } +} + +async function updateParentStream( + scopedClusterClient: IScopedClusterClient, + id: string, + parentId: string, + logger: Logger +) { + const { definition: parentDefinition } = await readStream({ + scopedClusterClient, + id: parentId, + }); + + parentDefinition.children = parentDefinition.children.filter((child) => child.id !== id); + + await syncStream({ + scopedClusterClient, + definition: parentDefinition, + logger, + }); + return parentDefinition; +} diff --git a/x-pack/plugins/streams/server/routes/streams/edit.ts b/x-pack/plugins/streams/server/routes/streams/edit.ts new file mode 100644 index 0000000000000..b82b4d54044da --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/edit.ts @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { StreamDefinition, streamWithoutIdDefinitonSchema } from '../../../common/types'; +import { + syncStream, + readStream, + checkStreamExists, + validateAncestorFields, + validateDescendantFields, +} from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { getParentId } from '../../lib/streams/helpers/hierarchy'; +import { MalformedChildren } from '../../lib/streams/errors/malformed_children'; + +export const editStreamRoute = createServerRoute({ + endpoint: 'PUT /api/streams/{id} 2023-10-31', + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: streamWithoutIdDefinitonSchema, + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + + await validateStreamChildren(scopedClusterClient, params.path.id, params.body.children); + await validateAncestorFields(scopedClusterClient, params.path.id, params.body.fields); + await validateDescendantFields(scopedClusterClient, params.path.id, params.body.fields); + + const parentId = getParentId(params.path.id); + let parentDefinition: StreamDefinition | undefined; + + if (parentId) { + parentDefinition = await updateParentStream( + scopedClusterClient, + parentId, + params.path.id, + logger + ); + } + const streamDefinition = { ...params.body }; + + await syncStream({ + scopedClusterClient, + definition: { ...streamDefinition, id: params.path.id }, + rootDefinition: parentDefinition, + logger, + }); + + for (const child of streamDefinition.children) { + const streamExists = await checkStreamExists({ + scopedClusterClient, + id: child.id, + }); + if (streamExists) { + continue; + } + // create empty streams for each child if they don't exist + const childDefinition = { + id: child.id, + children: [], + fields: [], + processing: [], + }; + + await syncStream({ + scopedClusterClient, + definition: childDefinition, + logger, + }); + } + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +async function updateParentStream( + scopedClusterClient: IScopedClusterClient, + parentId: string, + id: string, + logger: Logger +) { + const { definition: parentDefinition } = await readStream({ + scopedClusterClient, + id: parentId, + }); + + if (!parentDefinition.children.some((child) => child.id === id)) { + // add the child to the parent stream with an empty condition for now + parentDefinition.children.push({ + id, + condition: undefined, + }); + + await syncStream({ + scopedClusterClient, + definition: parentDefinition, + logger, + }); + } + return parentDefinition; +} + +async function validateStreamChildren( + scopedClusterClient: IScopedClusterClient, + id: string, + children: Array<{ id: string }> +) { + try { + const { definition: oldDefinition } = await readStream({ + scopedClusterClient, + id, + }); + const oldChildren = oldDefinition.children.map((child) => child.id); + const newChildren = new Set(children.map((child) => child.id)); + if (oldChildren.some((child) => !newChildren.has(child))) { + throw new MalformedChildren( + 'Cannot remove children from a stream, please delete the stream instead' + ); + } + } catch (e) { + // Ignore if the stream does not exist, but re-throw if it's another error + if (!(e instanceof DefinitionNotFound)) { + throw e; + } + } +} diff --git a/x-pack/plugins/streams/server/routes/streams/enable.ts b/x-pack/plugins/streams/server/routes/streams/enable.ts new file mode 100644 index 0000000000000..27d8929b28e50 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/enable.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { SecurityException } from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { syncStream } from '../../lib/streams/stream_crud'; +import { rootStreamDefinition } from '../../lib/streams/root_stream_definition'; +import { createStreamsIndex } from '../../lib/streams/internal_stream_mapping'; + +export const enableStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_enable 2023-10-31', + params: z.object({}), + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + handler: async ({ request, response, logger, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + await createStreamsIndex(scopedClusterClient); + await syncStream({ + scopedClusterClient, + definition: rootStreamDefinition, + logger, + }); + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof SecurityException) { + return response.customError({ body: e, statusCode: 400 }); + } + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/fork.ts b/x-pack/plugins/streams/server/routes/streams/fork.ts new file mode 100644 index 0000000000000..44f4052878003 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/fork.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { + DefinitionNotFound, + ForkConditionMissing, + IndexTemplateNotFound, + SecurityException, +} from '../../lib/streams/errors'; +import { createServerRoute } from '../create_server_route'; +import { conditionSchema, streamDefinitonWithoutChildrenSchema } from '../../../common/types'; +import { syncStream, readStream, validateAncestorFields } from '../../lib/streams/stream_crud'; +import { MalformedStreamId } from '../../lib/streams/errors/malformed_stream_id'; +import { isChildOf } from '../../lib/streams/helpers/hierarchy'; + +export const forkStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/{id}/_fork 2023-10-31', + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + params: z.object({ + path: z.object({ + id: z.string(), + }), + body: z.object({ stream: streamDefinitonWithoutChildrenSchema, condition: conditionSchema }), + }), + handler: async ({ response, params, logger, request, getScopedClients }) => { + try { + if (!params.body.condition) { + throw new ForkConditionMissing('You must provide a condition to fork a stream'); + } + + const { scopedClusterClient } = await getScopedClients({ request }); + + const { definition: rootDefinition } = await readStream({ + scopedClusterClient, + id: params.path.id, + }); + + const childDefinition = { ...params.body.stream, children: [] }; + + // check whether root stream has a child of the given name already + if (rootDefinition.children.some((child) => child.id === childDefinition.id)) { + throw new MalformedStreamId( + `The stream with ID (${params.body.stream.id}) already exists as a child of the parent stream` + ); + } + + if (!isChildOf(rootDefinition, childDefinition)) { + throw new MalformedStreamId( + `The ID (${params.body.stream.id}) from the new stream must start with the parent's id (${rootDefinition.id}), followed by a dot and a name` + ); + } + + await validateAncestorFields( + scopedClusterClient, + params.body.stream.id, + params.body.stream.fields + ); + + rootDefinition.children.push({ + id: params.body.stream.id, + condition: params.body.condition, + }); + + await syncStream({ + scopedClusterClient, + definition: rootDefinition, + rootDefinition, + logger, + }); + + await syncStream({ + scopedClusterClient, + definition: childDefinition, + rootDefinition, + logger, + }); + + return response.ok({ body: { acknowledged: true } }); + } catch (e) { + if (e instanceof IndexTemplateNotFound || e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + if ( + e instanceof SecurityException || + e instanceof ForkConditionMissing || + e instanceof MalformedStreamId + ) { + return response.customError({ body: e, statusCode: 400 }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/list.ts b/x-pack/plugins/streams/server/routes/streams/list.ts new file mode 100644 index 0000000000000..2e4f13a89bb41 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/list.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { DefinitionNotFound } from '../../lib/streams/errors'; +import { listStreams } from '../../lib/streams/stream_crud'; + +export const listStreamsRoute = createServerRoute({ + endpoint: 'GET /api/streams 2023-10-31', + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + params: z.object({}), + handler: async ({ response, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + const definitions = await listStreams({ scopedClusterClient }); + + const trees = asTrees(definitions); + + return response.ok({ body: { streams: trees } }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); + +interface ListStreamDefinition { + id: string; + children: ListStreamDefinition[]; +} + +function asTrees(definitions: Array<{ id: string[] }>) { + const trees: ListStreamDefinition[] = []; + definitions.forEach((definition) => { + const path = definition.id[0].split('.'); + let currentTree = trees; + path.forEach((_id, index) => { + const partialPath = path.slice(0, index + 1).join('.'); + const existingNode = currentTree.find((node) => node.id === partialPath); + if (existingNode) { + currentTree = existingNode.children; + } else { + const newNode = { id: partialPath, children: [] }; + currentTree.push(newNode); + currentTree = newNode.children; + } + }); + }); + return trees; +} diff --git a/x-pack/plugins/streams/server/routes/streams/read.ts b/x-pack/plugins/streams/server/routes/streams/read.ts new file mode 100644 index 0000000000000..5ea2aaf5f2542 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/read.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { DefinitionNotFound } from '../../lib/streams/errors'; +import { readAncestors, readStream } from '../../lib/streams/stream_crud'; + +export const readStreamRoute = createServerRoute({ + endpoint: 'GET /api/streams/{id} 2023-10-31', + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + params: z.object({ + path: z.object({ id: z.string() }), + }), + handler: async ({ response, params, request, getScopedClients }) => { + try { + const { scopedClusterClient } = await getScopedClients({ request }); + const streamEntity = await readStream({ + scopedClusterClient, + id: params.path.id, + }); + + const ancestors = await readAncestors({ + id: streamEntity.definition.id, + scopedClusterClient, + }); + + const body = { + ...streamEntity.definition, + inheritedFields: ancestors.flatMap(({ definition: { id, fields } }) => + fields.map((field) => ({ ...field, from: id })) + ), + }; + + return response.ok({ body }); + } catch (e) { + if (e instanceof DefinitionNotFound) { + return response.notFound({ body: e }); + } + + return response.customError({ body: e, statusCode: 500 }); + } + }, +}); diff --git a/x-pack/plugins/streams/server/routes/streams/resync.ts b/x-pack/plugins/streams/server/routes/streams/resync.ts new file mode 100644 index 0000000000000..2365252ab00e6 --- /dev/null +++ b/x-pack/plugins/streams/server/routes/streams/resync.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { z } from '@kbn/zod'; +import { createServerRoute } from '../create_server_route'; +import { syncStream, readStream, listStreams } from '../../lib/streams/stream_crud'; + +export const resyncStreamsRoute = createServerRoute({ + endpoint: 'POST /api/streams/_resync 2023-10-31', + options: { + access: 'public', + availability: { + stability: 'experimental', + }, + security: { + authz: { + enabled: false, + reason: + 'This API delegates security to the currently logged in user and their Elasticsearch permissions.', + }, + }, + }, + params: z.object({}), + handler: async ({ response, logger, request, getScopedClients }) => { + const { scopedClusterClient } = await getScopedClients({ request }); + + const streams = await listStreams({ scopedClusterClient }); + + for (const stream of streams) { + const { definition } = await readStream({ + scopedClusterClient, + id: stream.id[0], + }); + await syncStream({ + scopedClusterClient, + definition, + logger, + }); + } + + return response.ok({}); + }, +}); diff --git a/x-pack/plugins/streams/server/routes/types.ts b/x-pack/plugins/streams/server/routes/types.ts new file mode 100644 index 0000000000000..d547d56c088cd --- /dev/null +++ b/x-pack/plugins/streams/server/routes/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from '@kbn/core-http-server'; +import { DefaultRouteHandlerResources } from '@kbn/server-route-repository'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { StreamsServer } from '../types'; + +export interface RouteDependencies { + server: StreamsServer; + getScopedClients: ({ request }: { request: KibanaRequest }) => Promise<{ + scopedClusterClient: IScopedClusterClient; + soClient: SavedObjectsClientContract; + }>; +} + +export type StreamsRouteHandlerResources = RouteDependencies & DefaultRouteHandlerResources; diff --git a/x-pack/plugins/streams/server/types.ts b/x-pack/plugins/streams/server/types.ts new file mode 100644 index 0000000000000..f119faa0ed010 --- /dev/null +++ b/x-pack/plugins/streams/server/types.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CoreStart, ElasticsearchClient, Logger } from '@kbn/core/server'; +import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import { + EncryptedSavedObjectsPluginSetup, + EncryptedSavedObjectsPluginStart, +} from '@kbn/encrypted-saved-objects-plugin/server'; +import { LicensingPluginStart } from '@kbn/licensing-plugin/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; +import { StreamsConfig } from '../common/config'; + +export interface StreamsServer { + core: CoreStart; + config: StreamsConfig; + logger: Logger; + security: SecurityPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; + isServerless: boolean; + taskManager: TaskManagerStartContract; +} + +export interface ElasticsearchAccessorOptions { + elasticsearchClient: ElasticsearchClient; +} + +export interface StreamsPluginSetupDependencies { + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; + taskManager: TaskManagerSetupContract; +} + +export interface StreamsPluginStartDependencies { + security: SecurityPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; + licensing: LicensingPluginStart; + taskManager: TaskManagerStartContract; +} diff --git a/x-pack/plugins/streams/tsconfig.json b/x-pack/plugins/streams/tsconfig.json new file mode 100644 index 0000000000000..c2fde35f9ca22 --- /dev/null +++ b/x-pack/plugins/streams/tsconfig.json @@ -0,0 +1,31 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "../../../typings/**/*", + "common/**/*", + "server/**/*", + "public/**/*", + "types/**/*" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/config-schema", + "@kbn/core", + "@kbn/logging", + "@kbn/core-plugins-server", + "@kbn/core-http-server", + "@kbn/security-plugin", + "@kbn/core-saved-objects-api-server", + "@kbn/core-elasticsearch-server", + "@kbn/task-manager-plugin", + "@kbn/server-route-repository", + "@kbn/zod", + "@kbn/encrypted-saved-objects-plugin", + "@kbn/licensing-plugin", + ] +} diff --git a/yarn.lock b/yarn.lock index 6deb91b67bed0..8e2333250dd7d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6910,6 +6910,10 @@ version "0.0.0" uid "" +"@kbn/streams-plugin@link:x-pack/plugins/streams": + version "0.0.0" + uid "" + "@kbn/synthetics-e2e@link:x-pack/plugins/observability_solution/synthetics/e2e": version "0.0.0" uid "" From 05a9b26d3c8c9932aa344e53182296996e707dd3 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 13 Nov 2024 22:11:07 +1100 Subject: [PATCH 05/53] Authorized route migration for routes owned by @elastic/kibana-presentation (#198193) ### Authz API migration for authorized routes This PR migrates `access:` tags used in route definitions to new security configuration. Please refer to the documentation for more information: [Authorization API](https://docs.elastic.dev/kibana-dev-docs/key-concepts/security-api-authorization) ### **Before migration:** Access control tags were defined in the `options` object of the route: ```ts router.get({ path: '/api/path', options: { tags: ['access:', 'access:'], }, ... }, handler); ``` ### **After migration:** Tags have been replaced with the more robust `security.authz.requiredPrivileges` field under `security`: ```ts router.get({ path: '/api/path', security: { authz: { requiredPrivileges: ['', ''], }, }, ... }, handler); ``` ### What to do next? 1. Review the changes in this PR. 2. You might need to update your tests to reflect the new security configuration: - If you have tests that rely on checking `access` tags. - If you have snapshot tests that include the route definition. - If you have FTR tests that rely on checking unauthorized error message. The error message changed to also include missing privileges. ## Any questions? If you have any questions or need help with API authorization, please reach out to the `@elastic/kibana-security` team. Co-authored-by: James Gowdy --- x-pack/plugins/file_upload/server/routes.ts | 24 +++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts index 8818e0a2e2dff..b39a63471c15c 100644 --- a/x-pack/plugins/file_upload/server/routes.ts +++ b/x-pack/plugins/file_upload/server/routes.ts @@ -109,12 +109,16 @@ export function fileUploadRoutes(coreSetup: CoreSetup, logge .post({ path: '/internal/file_upload/analyze_file', access: 'internal', + security: { + authz: { + requiredPrivileges: ['fileUpload:analyzeFile'], + }, + }, options: { body: { accepts: ['text/*', 'application/json'], maxBytes: MAX_FILE_SIZE_BYTES, }, - tags: ['access:fileUpload:analyzeFile'], }, }) .addVersion( @@ -260,8 +264,10 @@ export function fileUploadRoutes(coreSetup: CoreSetup, logge .post({ path: '/internal/file_upload/time_field_range', access: 'internal', - options: { - tags: ['access:fileUpload:analyzeFile'], + security: { + authz: { + requiredPrivileges: ['fileUpload:analyzeFile'], + }, }, }) .addVersion( @@ -313,8 +319,10 @@ export function fileUploadRoutes(coreSetup: CoreSetup, logge .post({ path: '/internal/file_upload/preview_index_time_range', access: 'internal', - options: { - tags: ['access:fileUpload:analyzeFile'], + security: { + authz: { + requiredPrivileges: ['fileUpload:analyzeFile'], + }, }, }) .addVersion( @@ -356,8 +364,12 @@ export function fileUploadRoutes(coreSetup: CoreSetup, logge .post({ path: '/internal/file_upload/preview_tika_contents', access: 'internal', + security: { + authz: { + requiredPrivileges: ['fileUpload:analyzeFile'], + }, + }, options: { - tags: ['access:fileUpload:analyzeFile'], body: { accepts: ['application/json'], maxBytes: MAX_TIKA_FILE_SIZE_BYTES, From d65d67bd012ae56cce5b1b75b3572120f71c471a Mon Sep 17 00:00:00 2001 From: Maryam Saeidi Date: Wed, 13 Nov 2024 12:13:40 +0100 Subject: [PATCH 06/53] Change the CODEOWNER of the alerting deployment agnostic tests (#199272) ## Summary This PR changes the CODEOWNER file of the alerting deployment agnostic tests. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 52af5b22dcaf2..1b9c828de461c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1232,7 +1232,7 @@ x-pack/test_serverless/**/test_suites/observability/ai_assistant @elastic/obs-ai /x-pack/test_serverless/**/test_suites/observability/custom_threshold_rule/ @elastic/obs-ux-management-team /x-pack/test_serverless/**/test_suites/observability/slos/ @elastic/obs-ux-management-team /x-pack/test_serverless/api_integration/test_suites/observability/es_query_rule @elastic/obs-ux-management-team -/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/burn_rate_rule @elastic/obs-ux-management-team +/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting @elastic/obs-ux-management-team /x-pack/test/api_integration/deployment_agnostic/services/alerting_api @elastic/obs-ux-management-team /x-pack/test/api_integration/deployment_agnostic/services/slo_api @elastic/obs-ux-management-team /x-pack/test_serverless/**/test_suites/observability/infra/ @elastic/obs-ux-infra_services-team From f1f6117f04ab18b3e67906b6ececae51fa5f0669 Mon Sep 17 00:00:00 2001 From: Julia Bardi <90178898+juliaElastic@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:41:40 +0100 Subject: [PATCH 07/53] [Fleet] added `eventIngestedEnabled` flag (#199733) ## Summary Closes https://github.com/elastic/integrations/issues/11491 Added a separate flag `xpack.fleet.eventIngestedEnabled` (false by default) to keep the `event.ingested` mapping even when `agentIdVerificationEnabled` is disabled (in serverless oblt projects) Created a new pipeline `.fleet_event_ingested_pipeline-1` to use when only `eventIngestedEnabled` is enabled, to skip the step of calculating `agent_id_status`. I couldn't change `.fleet_final_pipeline-1` because the pipeline steps have to be different based on the flags. ## To verify: Note: After changing the flags, the packages have to be reinstalled to see the changes in the index templates, tested with `elastic_agent` package. Also, the data streams should be rolled over to see the changes in the ingested data. ``` POST logs-elastic_agent-default/_rollover POST logs-elastic_agent.metricbeat-default/_rollover ``` ### Default behaviour unchanged (Agent id verification enabled, event.ingested flag disabled) - by default: no change in behaviour, both `event.ingested` and `event.agent_id_status` should be mapped image image ### Agent id verification disabled, event.ingested enabled - set in `kibana.yml` ``` xpack.fleet.agentIdVerificationEnabled: false xpack.fleet.eventIngestedEnabled: true ``` - verify that `event.ingested` is mapped, `event.agent_id_status` is not image image image image image ### Agent id verification disabled, event.ingested disabled - set in `kibana.yml` ``` xpack.fleet.agentIdVerificationEnabled: false xpack.fleet.eventIngestedEnabled: false # default ``` - verify that neither `event.ingested` and `event.agent_id_status` is mapped image ### Agent id verification enabled, event.ingested enabled - set in `kibana.yml` ``` xpack.fleet.agentIdVerificationEnabled: true # default xpack.fleet.eventIngestedEnabled: true ``` - both `event.ingested` and `event.agent_id_status` should be mapped image image ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .buildkite/ftr_platform_stateful_configs.yml | 1 + config/serverless.oblt.yml | 3 + x-pack/plugins/fleet/common/types/index.ts | 1 + .../fleet/public/mock/plugin_configuration.ts | 1 + x-pack/plugins/fleet/server/config.ts | 1 + .../fleet/server/constants/fleet_es_assets.ts | 93 ++++++++- .../plugins/fleet/server/constants/index.ts | 3 + x-pack/plugins/fleet/server/mocks/index.ts | 2 + .../elasticsearch/ingest_pipeline/install.ts | 36 ++++ .../elasticsearch/template/template.test.ts | 48 ++++- .../epm/elasticsearch/template/template.ts | 12 +- x-pack/plugins/fleet/server/services/setup.ts | 6 +- .../apis/event_ingested/index.js | 17 ++ .../apis/event_ingested/use_event_ingested.ts | 197 ++++++++++++++++++ .../config.event_ingested.ts | 30 +++ .../dataset_quality/degraded_field_flyout.ts | 8 +- 16 files changed, 439 insertions(+), 20 deletions(-) create mode 100644 x-pack/test/fleet_api_integration/apis/event_ingested/index.js create mode 100644 x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts create mode 100644 x-pack/test/fleet_api_integration/config.event_ingested.ts diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index b015b1c96c73a..3db1d194e59aa 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -183,6 +183,7 @@ enabled: - x-pack/test/fleet_api_integration/config.agent.ts - x-pack/test/fleet_api_integration/config.agent_policy.ts - x-pack/test/fleet_api_integration/config.epm.ts + - x-pack/test/fleet_api_integration/config.event_ingested.ts - x-pack/test/fleet_api_integration/config.fleet.ts - x-pack/test/fleet_api_integration/config.package_policy.ts - x-pack/test/fleet_api_integration/config.space_awareness.ts diff --git a/config/serverless.oblt.yml b/config/serverless.oblt.yml index 059094ac87cdd..55e7fec7a3d39 100644 --- a/config/serverless.oblt.yml +++ b/config/serverless.oblt.yml @@ -129,6 +129,9 @@ xpack.serverless.plugin.developer.projectSwitcher.currentType: 'observability' ## Disable adding the component template `.fleet_agent_id_verification-1` to every index template for each datastream for each integration xpack.fleet.agentIdVerificationEnabled: false +## Enable event.ingested separately because agentIdVerification is disabled +xpack.fleet.eventIngestedEnabled: true + ## Enable the capability for the observability feature ID in the serverless environment to take ownership of the rules. ## The value need to be a featureId observability Or stackAlerts Or siem xpack.alerting.rules.overwriteProducer: 'observability' diff --git a/x-pack/plugins/fleet/common/types/index.ts b/x-pack/plugins/fleet/common/types/index.ts index 647a8b917d0c0..f7ce99b7f6708 100644 --- a/x-pack/plugins/fleet/common/types/index.ts +++ b/x-pack/plugins/fleet/common/types/index.ts @@ -49,6 +49,7 @@ export interface FleetConfigType { packages?: PreconfiguredPackage[]; outputs?: PreconfiguredOutput[]; agentIdVerificationEnabled?: boolean; + eventIngestedEnabled?: boolean; enableExperimental?: string[]; packageVerification?: { gpgKeyPath?: string; diff --git a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts index 935561426d7c2..30c01b1dfeb43 100644 --- a/x-pack/plugins/fleet/public/mock/plugin_configuration.ts +++ b/x-pack/plugins/fleet/public/mock/plugin_configuration.ts @@ -13,6 +13,7 @@ export const createConfigurationMock = (): FleetConfigType => { registryUrl: '', registryProxyUrl: '', agentIdVerificationEnabled: true, + eventIngestedEnabled: false, agents: { enabled: true, elasticsearch: { diff --git a/x-pack/plugins/fleet/server/config.ts b/x-pack/plugins/fleet/server/config.ts index ab5e06ef03716..b4f41562fd3ec 100644 --- a/x-pack/plugins/fleet/server/config.ts +++ b/x-pack/plugins/fleet/server/config.ts @@ -170,6 +170,7 @@ export const config: PluginConfigDescriptor = { proxies: PreconfiguredFleetProxiesSchema, spaceSettings: PreconfiguredSpaceSettingsSchema, agentIdVerificationEnabled: schema.boolean({ defaultValue: true }), + eventIngestedEnabled: schema.boolean({ defaultValue: false }), setup: schema.maybe( schema.object({ agentPolicySchemaUpgradeBatchSize: schema.maybe(schema.number()), diff --git a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts index 55e6493c77891..621adc5b3b81c 100644 --- a/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts +++ b/x-pack/plugins/fleet/server/constants/fleet_es_assets.ts @@ -17,6 +17,8 @@ export const FLEET_AGENT_POLICIES_SCHEMA_VERSION = '1.1.1'; export const FLEET_FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; +export const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1'; + export const FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME = '.fleet_globals-1'; export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { @@ -46,6 +48,12 @@ export const FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT = { }; export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME = '.fleet_agent_id_verification-1'; +export const INGESTED_MAPPING = { + type: 'date', + format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', + ignore_malformed: false, +}; + export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { _meta: meta, template: { @@ -58,11 +66,7 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { properties: { event: { properties: { - ingested: { - type: 'date', - format: 'strict_date_time_no_millis||strict_date_optional_time||epoch_millis', - ignore_malformed: false, - }, + ingested: INGESTED_MAPPING, agent_id_status: { ignore_above: 1024, type: 'keyword', @@ -74,12 +78,38 @@ export const FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT = { }, }; +export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME = '.fleet_event_ingested-1'; + +export const FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT = { + _meta: meta, + template: { + settings: { + index: { + final_pipeline: FLEET_EVENT_INGESTED_PIPELINE_ID, + }, + }, + mappings: { + properties: { + event: { + properties: { + ingested: INGESTED_MAPPING, + }, + }, + }, + }, + }, +}; + export const FLEET_COMPONENT_TEMPLATES = [ { name: FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, body: FLEET_GLOBALS_COMPONENT_TEMPLATE_CONTENT }, { name: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, body: FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_CONTENT, }, + { + name: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + body: FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_CONTENT, + }, ]; export const STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS = `logs@settings`; @@ -96,6 +126,59 @@ export const STACK_COMPONENT_TEMPLATES = [ STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, ]; +export const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1; + +// If the content is updated you probably need to update the FLEET_EVENT_INGESTED_PIPELINE_VERSION too to allow upgrade of the pipeline +export const FLEET_EVENT_INGESTED_PIPELINE_CONTENT = `--- +version: ${FLEET_EVENT_INGESTED_PIPELINE_VERSION} +_meta: + managed_by: ${meta.managed_by} + managed: ${meta.managed} +description: > + Pipeline for processing all incoming Fleet Agent documents that adds event.ingested. +processors: + - script: + description: Add time when event was ingested (and remove sub-seconds to improve storage efficiency) + tag: truncate-subseconds-event-ingested + ignore_failure: true + source: |- + if (ctx?.event == null) { + ctx.event = [:]; + } + + ctx.event.ingested = metadata().now.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + - remove: + description: Remove any pre-existing untrusted values. + field: + - event.agent_id_status + - _security + ignore_missing: true + - remove: + description: Remove event.original unless the preserve_original_event tag is set + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - set_security_user: + field: _security + properties: + - authentication_type + - username + - realm + - api_key + - remove: + field: _security + ignore_missing: true +on_failure: + - remove: + field: _security + ignore_missing: true + ignore_failure: true + - append: + field: error.message + value: + - 'failed in Fleet agent event_ingested_pipeline: {{ _ingest.on_failure_message }}'`; + export const FLEET_FINAL_PIPELINE_VERSION = 4; // If the content is updated you probably need to update the FLEET_FINAL_PIPELINE_VERSION too to allow upgrade of the pipeline diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index fb7e27c8b0ef8..48de05c0b635d 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -111,6 +111,9 @@ export { FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_ID, + FLEET_EVENT_INGESTED_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_CONTENT, FLEET_INSTALL_FORMAT_VERSION, FLEET_AGENT_POLICIES_SCHEMA_VERSION, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index f032c1f7bb8c7..8d452b394dd18 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -81,6 +81,7 @@ export const createAppContextStartContractMock = ( agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + eventIngestedEnabled: false, ...configOverrides, }; @@ -120,6 +121,7 @@ export const createAppContextStartContractMock = ( agents: { enabled: true, elasticsearch: {} }, enabled: true, agentIdVerificationEnabled: true, + eventIngestedEnabled: false, }, config$, kibanaVersion: '8.99.0', // Fake version :) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 5a4672f67fe53..51162ac2c6335 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -20,6 +20,9 @@ import { FLEET_FINAL_PIPELINE_CONTENT, FLEET_FINAL_PIPELINE_ID, FLEET_FINAL_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_ID, + FLEET_EVENT_INGESTED_PIPELINE_VERSION, + FLEET_EVENT_INGESTED_PIPELINE_CONTENT, } from '../../../../constants'; import { getPipelineNameForDatastream } from '../../../../../common/services'; import type { ArchiveEntry, PackageInstallContext } from '../../../../../common/types'; @@ -302,6 +305,39 @@ export async function ensureFleetFinalPipelineIsInstalled( return { isCreated: false }; } +export async function ensureFleetEventIngestedPipelineIsInstalled( + esClient: ElasticsearchClient, + logger: Logger +) { + const esClientRequestOptions: TransportRequestOptions = { + ignore: [404], + }; + const res = await esClient.ingest.getPipeline( + { id: FLEET_EVENT_INGESTED_PIPELINE_ID }, + { ...esClientRequestOptions, meta: true } + ); + + const installedVersion = res?.body[FLEET_EVENT_INGESTED_PIPELINE_ID]?.version; + if ( + res.statusCode === 404 || + !installedVersion || + installedVersion < FLEET_EVENT_INGESTED_PIPELINE_VERSION + ) { + await installPipeline({ + esClient, + logger, + pipeline: { + nameForInstallation: FLEET_EVENT_INGESTED_PIPELINE_ID, + contentForInstallation: FLEET_EVENT_INGESTED_PIPELINE_CONTENT, + extension: 'yml', + }, + }); + return { isCreated: true }; + } + + return { isCreated: false }; +} + const isDirectory = ({ path }: ArchiveEntry) => path.endsWith('/'); const isDataStreamPipeline = (path: string, dataStreamDataset: string) => { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index c7d2e4eacb32a..c06d0cdbb6429 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -14,7 +14,11 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { errors } from '@elastic/elasticsearch'; -import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets'; +import { + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, +} from '../../../../constants/fleet_es_assets'; import { createAppContextStartContractMock } from '../../../../mocks'; import { appContextService } from '../../..'; @@ -22,7 +26,6 @@ import type { RegistryDataStream } from '../../../../types'; import { processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { - FLEET_COMPONENT_TEMPLATES, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, @@ -36,10 +39,6 @@ import { updateCurrentWriteIndices, } from './template'; -const FLEET_COMPONENT_TEMPLATES_NAMES = FLEET_COMPONENT_TEMPLATES.map( - (componentTemplate) => componentTemplate.name -); - // Add our own serialiser to just do JSON.stringify expect.addSnapshotSerializer({ print(val) { @@ -88,7 +87,8 @@ describe('EPM template', () => { STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, ...composedOfTemplates, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); @@ -108,7 +108,8 @@ describe('EPM template', () => { 'metrics@tsdb-settings', ...composedOfTemplates, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); @@ -138,6 +139,34 @@ describe('EPM template', () => { ]); }); + it('creates fleet event ingested component template if event ingested flag is enabled', () => { + appContextService.start( + createAppContextStartContractMock({ + agentIdVerificationEnabled: false, + eventIngestedEnabled: true, + }) + ); + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + templateIndexPattern: 'logs-*', + type: 'logs', + packageName: 'nginx', + composedOfTemplates, + templatePriority: 200, + mappings: { properties: [] }, + isIndexModeTimeSeries: false, + }); + expect(template.composed_of).toStrictEqual([ + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, + STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, + ...composedOfTemplates, + STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + ]); + }); + it('adds empty composed_of correctly', () => { const composedOfTemplates: string[] = []; @@ -154,7 +183,8 @@ describe('EPM template', () => { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, STACK_COMPONENT_TEMPLATE_LOGS_SETTINGS, STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, - ...FLEET_COMPONENT_TEMPLATES_NAMES, + FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, + FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME, ]); }); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3709975c57a5e..b9c0846f3e4f2 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -15,7 +15,10 @@ import type { import pMap from 'p-map'; import { isResponseError } from '@kbn/es-errors'; -import { STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS } from '../../../../constants/fleet_es_assets'; +import { + FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME, + STACK_COMPONENT_TEMPLATE_LOGS_MAPPINGS, +} from '../../../../constants/fleet_es_assets'; import type { Field, Fields } from '../../fields/field'; import type { @@ -27,6 +30,7 @@ import type { } from '../../../../types'; import { appContextService } from '../../..'; import { getRegistryDataStreamAssetBaseName } from '../../../../../common/services'; +import type { FleetConfigType } from '../../../../../common/types'; import { STACK_COMPONENT_TEMPLATE_ECS_MAPPINGS, FLEET_GLOBALS_COMPONENT_TEMPLATE_NAME, @@ -115,6 +119,9 @@ export function getTemplate({ const esBaseComponents = getBaseEsComponents(type, !!isIndexModeTimeSeries); + const isEventIngestedEnabled = (config?: FleetConfigType): boolean => + Boolean(!config?.agentIdVerificationEnabled && config?.eventIngestedEnabled); + template.composed_of = [ ...esBaseComponents, ...(template.composed_of || []), @@ -123,6 +130,9 @@ export function getTemplate({ ...(appContextService.getConfig()?.agentIdVerificationEnabled ? [FLEET_AGENT_ID_VERIFY_COMPONENT_TEMPLATE_NAME] : []), + ...(isEventIngestedEnabled(appContextService.getConfig()) + ? [FLEET_EVENT_INGESTED_COMPONENT_TEMPLATE_NAME] + : []), ]; template.ignore_missing_component_templates = template.composed_of.filter(isUserSettingsTemplate); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 0d6ec183531a4..ab882a013ebe3 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -36,7 +36,10 @@ import { downloadSourceService } from './download_source'; import { getRegistryUrl, settingsService } from '.'; import { awaitIfPending } from './setup_utils'; -import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install'; +import { + ensureFleetEventIngestedPipelineIsInstalled, + ensureFleetFinalPipelineIsInstalled, +} from './epm/elasticsearch/ingest_pipeline/install'; import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, reinstallPackageForInstallation } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; @@ -336,6 +339,7 @@ export async function ensureFleetGlobalEsAssets( const globalAssetsRes = await Promise.all([ ensureDefaultComponentTemplates(esClient, logger), // returns an array ensureFleetFinalPipelineIsInstalled(esClient, logger), + ensureFleetEventIngestedPipelineIsInstalled(esClient, logger), ]); const assetResults = globalAssetsRes.flat(); if (assetResults.some((asset) => asset.isCreated)) { diff --git a/x-pack/test/fleet_api_integration/apis/event_ingested/index.js b/x-pack/test/fleet_api_integration/apis/event_ingested/index.js new file mode 100644 index 0000000000000..c6c76ca423b5f --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/event_ingested/index.js @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setupTestUsers } from '../test_users'; + +export default function loadTests({ loadTestFile, getService }) { + describe('Event Ingested', () => { + before(async () => { + await setupTestUsers(getService('security')); + }); + loadTestFile(require.resolve('./use_event_ingested')); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts b/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts new file mode 100644 index 0000000000000..7badbedbd77ba --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/event_ingested/use_event_ingested.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { testUsers } from '../test_users'; + +const TEST_INDEX = 'logs-log.log-test'; + +const FLEET_EVENT_INGESTED_PIPELINE_ID = '.fleet_event_ingested_pipeline-1'; + +// TODO: Use test package or move to input package version github.com/elastic/kibana/issues/154243 +const LOG_INTEGRATION_VERSION = '1.1.2'; + +const FLEET_EVENT_INGESTED_PIPELINE_VERSION = 1; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const es = getService('es'); + const esArchiver = getService('esArchiver'); + const fleetAndAgents = getService('fleetAndAgents'); + + describe('fleet_event_ingested_pipeline', () => { + skipIfNoDockerRegistry(providerContext); + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + await fleetAndAgents.setup(); + // Use the custom log package to test the fleet final pipeline + await supertestWithoutAuth + .post(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + }); + + after(async () => { + await supertestWithoutAuth + .delete(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }) + .expect(200); + await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); + const res = await es.search({ + index: TEST_INDEX, + }); + + for (const hit of res.hits.hits) { + await es.delete({ + id: hit._id!, + index: hit._index, + }); + } + }); + + it('should correctly update the event ingested pipeline', async () => { + await es.ingest.putPipeline({ + id: FLEET_EVENT_INGESTED_PIPELINE_ID, + body: { + description: 'Test PIPELINE WITHOUT version', + processors: [ + { + set: { + field: 'my-keyword-field', + value: 'foo', + }, + }, + ], + }, + }); + await supertestWithoutAuth + .post(`/api/fleet/setup`) + .auth(testUsers.fleet_all_int_all.username, testUsers.fleet_all_int_all.password) + .set('kbn-xsrf', 'xxxx'); + const pipelineRes = await es.ingest.getPipeline({ id: FLEET_EVENT_INGESTED_PIPELINE_ID }); + expect(pipelineRes).to.have.property(FLEET_EVENT_INGESTED_PIPELINE_ID); + expect(pipelineRes[FLEET_EVENT_INGESTED_PIPELINE_ID].version).to.be(1); + }); + + it('should correctly setup the event ingested pipeline and apply to fleet managed index template', async () => { + const pipelineRes = await es.ingest.getPipeline({ id: FLEET_EVENT_INGESTED_PIPELINE_ID }); + expect(pipelineRes).to.have.property(FLEET_EVENT_INGESTED_PIPELINE_ID); + const res = await es.indices.getIndexTemplate({ name: 'logs-log.log' }); + expect(res.index_templates.length).to.be(FLEET_EVENT_INGESTED_PIPELINE_VERSION); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('ecs@mappings'); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain('.fleet_globals-1'); + expect(res.index_templates[0]?.index_template?.composed_of).to.contain( + '.fleet_event_ingested-1' + ); + }); + + it('all docs should contain event.ingested without sub-seconds', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + '@timestamp': '2020-01-01T09:09:00', + message: 'hello', + }, + }); + + const doc = await es.get({ + id: res._id, + index: res._index, + }); + // @ts-expect-error + const ingestTimestamp = doc._source.event.ingested; + + // 2021-06-30T12:06:28Z + expect(ingestTimestamp).to.match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/); + }); + + it('should remove agent_id_status', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + '@timestamp': '2020-01-01T09:09:00', + agent: { + id: 'agent1', + }, + event: { + agent_id_status: 'dummy', + }, + }, + }); + + const doc = await es.get({ + id: res._id, + index: res._index, + }); + // @ts-expect-error + const event = doc._source.event; + + expect(event.agent_id_status).to.be(undefined); + expect(event).to.have.property('ingested'); + }); + + it('removes event.original if preserve_original_event is not set', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + event: { + original: JSON.stringify({ foo: 'bar' }), + }, + '@timestamp': '2023-01-01T09:00:00', + tags: [], + agent: { + id: 'agent1', + }, + }, + }); + + const doc: any = await es.get({ + id: res._id, + index: res._index, + }); + + const event = doc._source.event; + + expect(event.original).to.be(undefined); + }); + + it('preserves event.original if preserve_original_event is set', async () => { + const res = await es.index({ + index: 'logs-log.log-test', + body: { + message: 'message-test-1', + event: { + original: JSON.stringify({ foo: 'bar' }), + }, + '@timestamp': '2023-01-01T09:00:00', + tags: ['preserve_original_event'], + agent: { + id: 'agent1', + }, + }, + }); + + const doc: any = await es.get({ + id: res._id, + index: res._index, + }); + + const event = doc._source.event; + + expect(event.original).to.eql(JSON.stringify({ foo: 'bar' })); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/config.event_ingested.ts b/x-pack/test/fleet_api_integration/config.event_ingested.ts new file mode 100644 index 0000000000000..cbdf4d501e1d2 --- /dev/null +++ b/x-pack/test/fleet_api_integration/config.event_ingested.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const baseFleetApiConfig = await readConfigFile(require.resolve('./config.base.ts')); + const serverArgs: string[] = [ + ...baseFleetApiConfig.get('kbnTestServer.serverArgs'), + // serverless oblt needs only event.ingested, without agent id verification + `--xpack.fleet.agentIdVerificationEnabled=false`, + `--xpack.fleet.eventIngestedEnabled=true`, + ]; + + return { + ...baseFleetApiConfig.getAll(), + kbnTestServer: { + ...baseFleetApiConfig.get('kbnTestServer'), + serverArgs, + }, + testFiles: [require.resolve('./apis/event_ingested')], + junit: { + reportName: 'X-Pack Event Ingested API Integration Tests', + }, + }; +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 9fb1f74cbae0e..42a5a095ed6c4 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -194,7 +194,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Set Limit of 42 await PageObjects.datasetQuality.setDataStreamSettings(nginxAccessDataStreamName, { - 'mapping.total_fields.limit': 43, + 'mapping.total_fields.limit': 42, }); await synthtrace.index([ @@ -262,13 +262,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Set Limit of 44 + // Set Limit of 43 await PageObjects.datasetQuality.setDataStreamSettings( PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ dataset: nginxAccessDatasetName, }) + '-000002', { - 'mapping.total_fields.limit': 44, + 'mapping.total_fields.limit': 43, } ); @@ -745,7 +745,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'disabled' ); - expect(currentFieldLimit).to.be(44); + expect(currentFieldLimit).to.be(43); expect(currentFieldLimitDisabledStatus).to.be('true'); // Should display new field limit From a15ab4d8557c371563eab8da6d5ef8c21b6bbbcb Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Wed, 13 Nov 2024 05:43:41 -0600 Subject: [PATCH 08/53] [index management] Index templates matching `a_fake_index_pattern_that_wont_match_any_indices` preview fix (#195174) ## Summary Preview functionality (Index template detail -> preview tab) on index templates matching `a_fake_index_pattern_that_wont_match_any_indices` (such as `*` or `a*`) would fail once saved. Now passing template name instead of content for saved templates which avoids annoyances with passing the `index_pattern`. Linked issue has a good set of steps for reproduction. Closes https://github.com/elastic/kibana/issues/189555 --- .../helpers/http_requests.ts | 7 ++ .../home/index_templates_tab.test.ts | 4 +- .../simulate_template/simulate_template.tsx | 19 ++--- .../template_details/tabs/tab_preview.tsx | 7 +- .../template_details_content.tsx | 2 +- .../public/application/services/api.ts | 18 ++++- .../api/templates/register_simulate_route.ts | 37 +++++++--- .../index_management/lib/templates.api.ts | 7 ++ .../management/index_management/templates.ts | 13 ++++ .../index_management/index_template_wizard.ts | 70 +++++++++++++++++++ 10 files changed, 156 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts index 8bd8672b8fbba..79daba4c73867 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/http_requests.ts @@ -147,6 +147,12 @@ const registerHttpRequestMockHelpers = ( const setSimulateTemplateResponse = (response?: HttpResponse, error?: ResponseError) => mockResponse('POST', `${API_BASE_PATH}/index_templates/simulate`, response, error); + const setSimulateTemplateByNameResponse = ( + name: string, + response?: HttpResponse, + error?: ResponseError + ) => mockResponse('POST', `${API_BASE_PATH}/index_templates/simulate/${name}`, response, error); + const setLoadComponentTemplatesResponse = (response?: HttpResponse, error?: ResponseError) => mockResponse('GET', `${API_BASE_PATH}/component_templates`, response, error); @@ -229,6 +235,7 @@ const registerHttpRequestMockHelpers = ( setLoadIndexStatsResponse, setUpdateIndexSettingsResponse, setSimulateTemplateResponse, + setSimulateTemplateByNameResponse, setLoadComponentTemplatesResponse, setLoadNodesPluginsResponse, setLoadTelemetryResponse, diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts index 615b8df18f905..ea536becfccac 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/index_templates_tab.test.ts @@ -617,7 +617,9 @@ describe('Index Templates tab', () => { const { find, actions, exists } = testBed; httpRequestsMockHelpers.setLoadTemplateResponse(templates[0].name, template); - httpRequestsMockHelpers.setSimulateTemplateResponse({ simulateTemplate: 'response' }); + httpRequestsMockHelpers.setSimulateTemplateByNameResponse(templates[0].name, { + simulateTemplate: 'response', + }); await actions.clickTemplateAt(0); diff --git a/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx b/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx index ed22baae580cc..fd1df7ba44697 100644 --- a/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx +++ b/x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx @@ -23,22 +23,25 @@ export interface Filters { } interface Props { - template: { [key: string]: any }; + template?: { [key: string]: any }; filters?: Filters; + templateName?: string; } -export const SimulateTemplate = React.memo(({ template, filters }: Props) => { +export const SimulateTemplate = React.memo(({ template, filters, templateName }: Props) => { const [templatePreview, setTemplatePreview] = useState('{}'); const updatePreview = useCallback(async () => { - if (!template || Object.keys(template).length === 0) { + if (!templateName && (!template || Object.keys(template).length === 0)) { return; } - const indexTemplate = serializeTemplate( - stripEmptyFields(template, { types: ['string'] }) as TemplateDeserialized - ); - const { data, error } = await simulateIndexTemplate(indexTemplate); + const indexTemplate = templateName + ? undefined + : serializeTemplate( + stripEmptyFields(template, { types: ['string'] }) as TemplateDeserialized + ); + const { data, error } = await simulateIndexTemplate({ template: indexTemplate, templateName }); let filteredTemplate = data; if (data) { @@ -67,7 +70,7 @@ export const SimulateTemplate = React.memo(({ template, filters }: Props) => { } setTemplatePreview(JSON.stringify(filteredTemplate ?? error, null, 2)); - }, [template, filters]); + }, [template, filters, templateName]); useEffect(() => { updatePreview(); diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx index 38f4a8b4f787b..02df1f6e1c682 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/tabs/tab_preview.tsx @@ -8,14 +8,13 @@ import React from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiText, EuiSpacer } from '@elastic/eui'; -import { TemplateDeserialized } from '../../../../../../../common'; import { SimulateTemplate } from '../../../../../components/index_templates'; interface Props { - templateDetails: TemplateDeserialized; + templateName: string; } -export const TabPreview = ({ templateDetails }: Props) => { +export const TabPreview = ({ templateName }: Props) => { return (
@@ -29,7 +28,7 @@ export const TabPreview = ({ templateDetails }: Props) => { - +
); }; diff --git a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx index d2156d1aa958e..75446c0c05f2d 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/template_list/template_details/template_details_content.tsx @@ -171,7 +171,7 @@ export const TemplateDetailsContent = ({ [SETTINGS_TAB_ID]: , [MAPPINGS_TAB_ID]: , [ALIASES_TAB_ID]: , - [PREVIEW_TAB_ID]: , + [PREVIEW_TAB_ID]: , }; const tabContent = tabToComponentMap[activeTab]; diff --git a/x-pack/plugins/index_management/public/application/services/api.ts b/x-pack/plugins/index_management/public/application/services/api.ts index 08baa49713573..9f03007014c4f 100644 --- a/x-pack/plugins/index_management/public/application/services/api.ts +++ b/x-pack/plugins/index_management/public/application/services/api.ts @@ -319,11 +319,23 @@ export async function updateTemplate(template: TemplateDeserialized) { return result; } -export function simulateIndexTemplate(template: { [key: string]: any }) { +export function simulateIndexTemplate({ + template, + templateName, +}: { + template?: { [key: string]: any }; + templateName?: string; +}) { + const path = templateName + ? `${API_BASE_PATH}/index_templates/simulate/${templateName}` + : `${API_BASE_PATH}/index_templates/simulate`; + + const body = templateName ? undefined : JSON.stringify(template); + return sendRequest({ - path: `${API_BASE_PATH}/index_templates/simulate`, + path, method: 'post', - body: JSON.stringify(template), + body, }).then((result) => { uiMetricService.trackMetric(METRIC_TYPE.COUNT, UIM_TEMPLATE_SIMULATE); return result; diff --git a/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts b/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts index 3dc6201f0831c..60e2cfbf8a53a 100644 --- a/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/templates/register_simulate_route.ts @@ -15,23 +15,38 @@ const bodySchema = schema.object({}, { unknowns: 'allow' }); export function registerSimulateRoute({ router, lib: { handleEsError } }: RouteDependencies) { router.post( { - path: addBasePath('/index_templates/simulate'), - validate: { body: bodySchema }, + path: addBasePath('/index_templates/simulate/{templateName?}'), + validate: { + body: schema.nullable(bodySchema), + params: schema.object({ templateName: schema.maybe(schema.string()) }), + }, }, async (context, request, response) => { const { client } = (await context.core).elasticsearch; const template = request.body as TypeOf; + // Until ES fixes a bug on their side we need to send a fake index pattern + // that won't match any indices. + // Issue: https://github.com/elastic/elasticsearch/issues/59152 + // eslint-disable-next-line @typescript-eslint/naming-convention + const index_patterns = ['a_fake_index_pattern_that_wont_match_any_indices']; + const templateName = request.params.templateName; + + const params: estypes.IndicesSimulateTemplateRequest = templateName + ? { + name: templateName, + body: { + index_patterns, + }, + } + : { + body: { + ...template, + index_patterns, + }, + }; try { - const templatePreview = await client.asCurrentUser.indices.simulateTemplate({ - body: { - ...template, - // Until ES fixes a bug on their side we need to send a fake index pattern - // that won't match any indices. - // Issue: https://github.com/elastic/elasticsearch/issues/59152 - index_patterns: ['a_fake_index_pattern_that_wont_match_any_indices'], - }, - } as estypes.IndicesSimulateTemplateRequest); + const templatePreview = await client.asCurrentUser.indices.simulateTemplate(params); return response.ok({ body: templatePreview }); } catch (error) { diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts index bae578e6c0490..21585d9f699ac 100644 --- a/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts +++ b/x-pack/test/api_integration/apis/management/index_management/lib/templates.api.ts @@ -53,6 +53,12 @@ export function templatesApi(getService: FtrProviderContext['getService']) { .set('kbn-xsrf', 'xxx') .send(payload); + const simulateTemplateByName = (name: string) => + supertest + .post(`${API_BASE_PATH}/index_templates/simulate/${name}`) + .set('kbn-xsrf', 'xxx') + .send(); + return { getAllTemplates, getOneTemplate, @@ -61,5 +67,6 @@ export function templatesApi(getService: FtrProviderContext['getService']) { deleteTemplates, cleanUpTemplates, simulateTemplate, + simulateTemplateByName, }; } diff --git a/x-pack/test/api_integration/apis/management/index_management/templates.ts b/x-pack/test/api_integration/apis/management/index_management/templates.ts index 66d6f34baa644..1fe7e022bfc9a 100644 --- a/x-pack/test/api_integration/apis/management/index_management/templates.ts +++ b/x-pack/test/api_integration/apis/management/index_management/templates.ts @@ -24,6 +24,7 @@ export default function ({ getService }: FtrProviderContext) { updateTemplate, cleanUpTemplates, simulateTemplate, + simulateTemplateByName, } = templatesApi(getService); describe('index templates', () => { @@ -452,6 +453,18 @@ export default function ({ getService }: FtrProviderContext) { const { body } = await simulateTemplate(payload).expect(200); expect(body.template).to.be.ok(); }); + + it('should simulate an index template by name', async () => { + const templateName = `template-${getRandomString()}`; + const payload = getTemplatePayload(templateName, [getRandomString()]); + + await createTemplate(payload).expect(200); + + await simulateTemplateByName(templateName).expect(200); + + // cleanup + await deleteTemplates([{ name: templateName }]); + }); }); }); } diff --git a/x-pack/test/functional/apps/index_management/index_template_wizard.ts b/x-pack/test/functional/apps/index_management/index_template_wizard.ts index cf6f1bf6a44a1..581a0b2761644 100644 --- a/x-pack/test/functional/apps/index_management/index_template_wizard.ts +++ b/x-pack/test/functional/apps/index_management/index_template_wizard.ts @@ -107,6 +107,76 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); }); + // https://github.com/elastic/kibana/pull/195174 + it('can preview index template that matches a_fake_index_pattern_that_wont_match_any_indices', async () => { + // Click Create Template button + await testSubjects.click('createTemplateButton'); + const pageTitleText = await testSubjects.getVisibleText('pageTitle'); + expect(pageTitleText).to.be('Create template'); + + const stepTitle1 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle1).to.be('Logistics'); + + // Fill out required fields + await testSubjects.setValue('nameField', 'a-star'); + await testSubjects.setValue('indexPatternsField', 'a*'); + await testSubjects.setValue('priorityField', '1000'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify empty prompt + const emptyPrompt = await testSubjects.exists('emptyPrompt'); + expect(emptyPrompt).to.be(true); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle2 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle2).to.be('Index settings (optional)'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle3 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle3).to.be('Mappings (optional)'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle4 = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle4).to.be('Aliases (optional)'); + + // Click Next button + await pageObjects.indexManagement.clickNextButton(); + + // Verify step title + const stepTitle = await testSubjects.getVisibleText('stepTitle'); + expect(stepTitle).to.be("Review details for 'a-star'"); + + // Verify that summary exists + const summaryTabContent = await testSubjects.exists('summaryTabContent'); + expect(summaryTabContent).to.be(true); + + // Verify that index mode is set to "Standard" + expect(await testSubjects.exists('indexModeTitle')).to.be(true); + expect(await testSubjects.getVisibleText('indexModeValue')).to.be('Standard'); + + // Click Create template + await pageObjects.indexManagement.clickNextButton(); + + // Click preview tab, we know its the last one + const tabs = await testSubjects.findAll('tab'); + await tabs[tabs.length - 1].click(); + const templatePreview = await testSubjects.getVisibleText('simulateTemplatePreview'); + expect(templatePreview).to.not.contain('error'); + + await testSubjects.click('closeDetailsButton'); + }); + describe('Mappings step', () => { beforeEach(async () => { await pageObjects.common.navigateToApp('indexManagement'); From 5dc30e775131e4768fdef48b8b31f735e7b26df2 Mon Sep 17 00:00:00 2001 From: mohamedhamed-ahmed Date: Wed, 13 Nov 2024 11:51:28 +0000 Subject: [PATCH 09/53] [Dataset Quality] Show dataset quality in details page (#199890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes https://github.com/elastic/kibana/issues/197800 ## 📝 Summary This PR add a new UI representation for the Dataset Quality in the Details page. ## 🎥 Demo https://github.com/user-attachments/assets/e0860fa3-cb82-44ef-bfb5-e8dcb00100f1 --- .../dataset_quality/table/columns.tsx | 5 ++++- .../overview/summary/index.tsx | 13 ++++++++++++- .../dataset_quality_indicator.tsx | 17 +++++++++++------ .../components/quality_indicator/indicator.tsx | 8 +++++--- .../public/hooks/use_overview_summary_panel.ts | 9 +++++++++ 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx index 99fdc25382bf2..14767f4acd8f5 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -276,7 +276,10 @@ export const getDatasetQualityTableColumns = ({ field: 'degradedDocs.percentage', sortable: true, render: (_, dataStreamStat: DataStreamStat) => ( - + ), width: '140px', }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index 752b224b6973a..897efb821ff64 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -19,6 +19,7 @@ import { overviewPanelTitleResources, } from '../../../../../common/translations'; import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel'; +import { DatasetQualityIndicator } from '../../../quality_indicator'; // Allow for lazy loading // eslint-disable-next-line import/no-default-export @@ -31,6 +32,7 @@ export default function Summary() { totalServicesCount, totalHostsCount, totalDegradedDocsCount, + quality, } = useOverviewSummaryPanel(); return ( @@ -59,7 +61,16 @@ export default function Summary() { isLoading={isSummaryPanelLoading} /> - + + } + > { - const { quality } = dataStreamStat; - const translatedQuality = i18n.translate('xpack.datasetQuality.datasetQualityIdicator', { defaultMessage: '{quality}', values: { quality: capitalize(quality) }, @@ -29,7 +29,12 @@ export const DatasetQualityIndicator = ({ return ( - + ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx index 137c558dfbdd7..49ff342446071 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/indicator.tsx @@ -7,16 +7,18 @@ import { EuiHealth, EuiText } from '@elastic/eui'; import React, { ReactNode } from 'react'; -import { QualityIndicators, InfoIndicators } from '../../../common/types'; +import type { QualityIndicators, InfoIndicators } from '../../../common/types'; export function QualityIndicator({ quality, description, isColoredDescription, + textSize = 's', }: { quality: QualityIndicators; description: string | ReactNode; isColoredDescription?: boolean; + textSize?: 'xs' | 's' | 'm'; }) { const qualityColors: Record = { poor: 'danger', @@ -25,8 +27,8 @@ export function QualityIndicator({ }; return ( - - + + {description} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts index 084210774f958..43cf6923075ee 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_overview_summary_panel.ts @@ -7,6 +7,7 @@ import { useSelector } from '@xstate/react'; import { formatNumber } from '@elastic/eui'; +import { mapPercentageToQuality } from '../../common/utils'; import { BYTE_NUMBER_FORMAT, MAX_HOSTS_METRIC_VALUE, NUMBER_FORMAT } from '../../common/constants'; import { useDatasetQualityDetailsContext } from '../components/dataset_quality_details/context'; @@ -54,6 +55,13 @@ export const useOverviewSummaryPanel = () => { NUMBER_FORMAT ); + const degradedPercentage = + Number(totalDocsCount) > 0 + ? (Number(totalDegradedDocsCount) / Number(totalDocsCount)) * 100 + : 0; + + const quality = mapPercentageToQuality(degradedPercentage); + return { totalDocsCount, sizeInBytes, @@ -62,6 +70,7 @@ export const useOverviewSummaryPanel = () => { totalHostsCount, isSummaryPanelLoading, totalDegradedDocsCount, + quality, }; }; From 5a138e392ff794561dc9a18d9a631be181543dca Mon Sep 17 00:00:00 2001 From: Rodney Norris Date: Wed, 13 Nov 2024 06:19:23 -0600 Subject: [PATCH 10/53] [Search] refactor: unify license check utils (#197675) --- .../common/utils/licensing.test.ts | 184 +++++++++++------- .../common/utils/licensing.ts | 34 +++- .../shared/licensing/licensing_logic.ts | 30 ++- 3 files changed, 159 insertions(+), 89 deletions(-) diff --git a/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts b/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts index 7b5fbc3088984..30b1b8cd4fe92 100644 --- a/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts +++ b/x-pack/plugins/enterprise_search/common/utils/licensing.test.ts @@ -5,92 +5,142 @@ * 2.0. */ -import type { ILicense } from '@kbn/licensing-plugin/public'; +import { licenseMock } from '@kbn/licensing-plugin/common/licensing.mock'; -import { hasEnterpriseLicense } from './licensing'; +import { License } from '@kbn/licensing-plugin/common/license'; + +import { + hasEnterpriseLicense, + hasGoldLicense, + hasPlatinumLicense, + isTrialLicense, +} from './licensing'; describe('licensing utils', () => { - const baseLicense: ILicense = { - isActive: true, - type: 'trial', - isAvailable: true, - signature: 'fake', - toJSON: jest.fn(), - getUnavailableReason: jest.fn().mockReturnValue(undefined), - hasAtLeast: jest.fn().mockReturnValue(false), - check: jest.fn().mockReturnValue({ state: 'valid' }), - getFeature: jest.fn().mockReturnValue({ isAvailable: false, isEnabled: false }), - }; - describe('hasEnterpriseLicense', () => { - let license: ILicense; - beforeEach(() => { - jest.resetAllMocks(); - license = { - ...baseLicense, - }; - }); - it('returns true for active enterprise license', () => { - license.type = 'enterprise'; + const basicLicense = licenseMock.createLicense(); + const basicExpiredLicense = licenseMock.createLicense({ license: { status: 'expired' } }); + const goldLicense = licenseMock.createLicense({ license: { type: 'gold' } }); + const goldLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'gold' }, + }); + const platinumLicense = licenseMock.createLicense({ license: { type: 'platinum' } }); + const platinumLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'platinum' }, + }); + const enterpriseLicense = licenseMock.createLicense({ license: { type: 'enterprise' } }); + const enterpriseLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'enterprise' }, + }); + const trialLicense = licenseMock.createLicense({ license: { type: 'trial' } }); + const trialLicenseExpired = licenseMock.createLicense({ + license: { status: 'expired', type: 'trial' }, + }); - expect(hasEnterpriseLicense(license)).toEqual(true); + const errorMessage = 'unavailable'; + const errorLicense = new License({ error: errorMessage, signature: '' }); + const unavailableLicense = new License({ signature: '' }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('hasEnterpriseLicense', () => { + it('returns true for active valid licenses', () => { + expect(hasEnterpriseLicense(enterpriseLicense)).toEqual(true); + expect(hasEnterpriseLicense(trialLicense)).toEqual(true); }); - it('returns true for active trial license', () => { - expect(hasEnterpriseLicense(license)).toEqual(true); + it('returns false for active invalid licenses', () => { + expect(hasEnterpriseLicense(basicLicense)).toEqual(false); + expect(hasEnterpriseLicense(goldLicense)).toEqual(false); + expect(hasEnterpriseLicense(platinumLicense)).toEqual(false); }); - it('returns false for active basic license', () => { - license.type = 'basic'; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for inactive licenses', () => { + expect(hasEnterpriseLicense(trialLicenseExpired)).toEqual(false); + expect(hasEnterpriseLicense(enterpriseLicenseExpired)).toEqual(false); + expect(hasEnterpriseLicense(basicExpiredLicense)).toEqual(false); }); - it('returns false for active gold license', () => { - license.type = 'gold'; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for unavailable license', () => { + expect(hasEnterpriseLicense(errorLicense)).toEqual(false); + expect(hasEnterpriseLicense(unavailableLicense)).toEqual(false); }); - it('returns false for active platinum license', () => { - license.type = 'platinum'; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for null license', () => { + expect(hasEnterpriseLicense(null)).toEqual(false); }); - it('returns false for inactive enterprise license', () => { - license.type = 'enterprise'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for undefined license', () => { + expect(hasEnterpriseLicense(undefined)).toEqual(false); }); - it('returns false for inactive trial license', () => { - license.isActive = false; + }); - expect(hasEnterpriseLicense(license)).toEqual(false); + describe('hasPlatinumLicense', () => { + it('returns true for valid active licenses', () => { + expect(hasPlatinumLicense(platinumLicense)).toEqual(true); + expect(hasPlatinumLicense(enterpriseLicense)).toEqual(true); + expect(hasPlatinumLicense(trialLicense)).toEqual(true); }); - it('returns false for inactive basic license', () => { - license.type = 'basic'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for invalid active licenses', () => { + expect(hasPlatinumLicense(goldLicense)).toEqual(false); + expect(hasPlatinumLicense(basicLicense)).toEqual(false); }); - it('returns false for inactive gold license', () => { - license.type = 'gold'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for inactive licenses', () => { + expect(hasPlatinumLicense(platinumLicenseExpired)).toEqual(false); + expect(hasPlatinumLicense(enterpriseLicenseExpired)).toEqual(false); + expect(hasPlatinumLicense(trialLicenseExpired)).toEqual(false); }); - it('returns false for inactive platinum license', () => { - license.type = 'platinum'; - license.isActive = false; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for bad licenses', () => { + expect(hasPlatinumLicense(errorLicense)).toEqual(false); + expect(hasPlatinumLicense(unavailableLicense)).toEqual(false); }); - it('returns false for active license is missing type', () => { - delete license.type; - - expect(hasEnterpriseLicense(license)).toEqual(false); + it('returns false for null license', () => { + expect(hasPlatinumLicense(null)).toEqual(false); + }); + it('returns false for undefined license', () => { + expect(hasPlatinumLicense(undefined)).toEqual(false); + }); + }); + describe('hasGoldLicense', () => { + it('returns true for valid active licenses', () => { + expect(hasGoldLicense(goldLicense)).toEqual(true); + expect(hasGoldLicense(platinumLicense)).toEqual(true); + expect(hasGoldLicense(enterpriseLicense)).toEqual(true); + expect(hasGoldLicense(trialLicense)).toEqual(true); + }); + it('returns false for invalid active licenses', () => { + expect(hasGoldLicense(basicLicense)).toEqual(false); + }); + it('returns false for inactive licenses', () => { + expect(hasGoldLicense(goldLicenseExpired)).toEqual(false); + expect(hasGoldLicense(platinumLicenseExpired)).toEqual(false); + expect(hasGoldLicense(enterpriseLicenseExpired)).toEqual(false); + expect(hasGoldLicense(trialLicenseExpired)).toEqual(false); + }); + it('returns false for bad licenses', () => { + expect(hasGoldLicense(errorLicense)).toEqual(false); + expect(hasGoldLicense(unavailableLicense)).toEqual(false); }); it('returns false for null license', () => { - expect(hasEnterpriseLicense(null)).toEqual(false); + expect(hasGoldLicense(null)).toEqual(false); }); it('returns false for undefined license', () => { - expect(hasEnterpriseLicense(undefined)).toEqual(false); + expect(hasGoldLicense(undefined)).toEqual(false); + }); + }); + describe('isTrialLicense', () => { + it('returns true for active trial license', () => { + expect(hasGoldLicense(trialLicense)).toEqual(true); + }); + it('returns false for non-trial license', () => { + expect(isTrialLicense(platinumLicense)).toEqual(false); + }); + it('returns false for invalid license', () => { + expect(isTrialLicense(trialLicenseExpired)).toEqual(false); + expect(isTrialLicense(errorLicense)).toEqual(false); + expect(isTrialLicense(unavailableLicense)).toEqual(false); + }); + it('returns false for null license', () => { + expect(isTrialLicense(null)).toEqual(false); + }); + it('returns false for undefined license', () => { + expect(isTrialLicense(undefined)).toEqual(false); }); }); }); diff --git a/x-pack/plugins/enterprise_search/common/utils/licensing.ts b/x-pack/plugins/enterprise_search/common/utils/licensing.ts index a78e603b3650d..6b6063516cca9 100644 --- a/x-pack/plugins/enterprise_search/common/utils/licensing.ts +++ b/x-pack/plugins/enterprise_search/common/utils/licensing.ts @@ -7,10 +7,38 @@ import type { ILicense } from '@kbn/licensing-plugin/public'; -/* hasEnterpriseLicense return if the given license is an active `enterprise` or `trial` license +/* hasEnterpriseLicense return if the given license is an active `enterprise` or greater license */ export function hasEnterpriseLicense(license: ILicense | null | undefined): boolean { if (license === undefined || license === null) return false; - const qualifyingLicenses = ['enterprise', 'trial']; - return license.isActive && qualifyingLicenses.includes(license?.type ?? ''); + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license.hasAtLeast('enterprise'); +} + +/* hasPlatinumLicense return if the given license is an active `platinum` or greater license + */ +export function hasPlatinumLicense(license: ILicense | null | undefined): boolean { + if (license === undefined || license === null) return false; + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license.hasAtLeast('platinum'); +} + +/* hasGoldLicense return if the given license is an active `gold` or greater license + */ +export function hasGoldLicense(license: ILicense | null | undefined): boolean { + if (license === undefined || license === null) return false; + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license.hasAtLeast('gold'); +} + +/* isTrialLicense returns if the given license is an active `trial` license + */ +export function isTrialLicense(license: ILicense | null | undefined): boolean { + if (license === undefined || license === null) return false; + if (!license.isAvailable) return false; + if (!license.isActive) return false; + return license?.type === 'trial'; } diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts b/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts index 7c83b446a67c8..736cf1c5c5d48 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/licensing/licensing_logic.ts @@ -10,6 +10,13 @@ import { Observable, Subscription } from 'rxjs'; import { ILicense } from '@kbn/licensing-plugin/public'; +import { + hasEnterpriseLicense, + hasGoldLicense, + hasPlatinumLicense, + isTrialLicense, +} from '../../../../common/utils/licensing'; + interface LicensingValues { license: ILicense | null; licenseSubscription: Subscription | null; @@ -50,29 +57,14 @@ export const LicensingLogic = kea [selectors.license], - (license) => { - const qualifyingLicenses = ['platinum', 'enterprise', 'trial']; - return license?.isActive && qualifyingLicenses.includes(license?.type); - }, + (license) => hasPlatinumLicense(license), ], hasEnterpriseLicense: [ (selectors) => [selectors.license], - (license) => { - const qualifyingLicenses = ['enterprise', 'trial']; - return license?.isActive && qualifyingLicenses.includes(license?.type); - }, - ], - hasGoldLicense: [ - (selectors) => [selectors.license], - (license) => { - const qualifyingLicenses = ['gold', 'platinum', 'enterprise', 'trial']; - return license?.isActive && qualifyingLicenses.includes(license?.type); - }, - ], - isTrial: [ - (selectors) => [selectors.license], - (license) => license?.isActive && license?.type === 'trial', + (license) => hasEnterpriseLicense(license), ], + hasGoldLicense: [(selectors) => [selectors.license], (license) => hasGoldLicense(license)], + isTrial: [(selectors) => [selectors.license], (license) => isTrialLicense(license)], }, events: ({ props, actions, values }) => ({ afterMount: () => { From b8836c5d6b356ea34563f986f651d1952d56e718 Mon Sep 17 00:00:00 2001 From: Konrad Szwarc Date: Wed, 13 Nov 2024 13:40:08 +0100 Subject: [PATCH 11/53] [EDR Workflows] Add DW Cypress and API tests to the Quality Gate. (#199742) This PR enables tests in the release pipeline that are 1) agentless (do not involve agent enrollment) and 2) have been stable over the past week. These tests are already running in the periodic pipeline, with addition of `@serverlessQA` tag they will be promoted to be executed in the release pipeline. Test eligibility breakdown - https://github.com/elastic/kibana/issues/187243#issuecomment-2470007062 CC @MadameSheema --- .../public/management/cypress/e2e/artifacts/blocklist.cy.ts | 2 +- .../public/management/cypress/e2e/policy/policy_list.cy.ts | 2 +- .../feature_access/api/agent_policy_settings_complete.cy.ts | 2 +- .../components/agent_policy_settings_complete.cy.ts | 2 +- .../components/agent_policy_settings_essentials.cy.ts | 2 +- .../components/policy_details_endpoint_essentials.cy.ts | 2 +- .../resolver/trial_license_complete_tier/entity.ts | 2 +- .../resolver/trial_license_complete_tier/entity_id.ts | 2 +- .../resolver/trial_license_complete_tier/events.ts | 2 +- .../edr_workflows/resolver/trial_license_complete_tier/tree.ts | 2 +- .../trial_license_complete_tier/agent_type_support.ts | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts index f0d3eb96e4581..0b010d2d60099 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/blocklist.cy.ts @@ -41,7 +41,7 @@ const { describe( 'Blocklist', { - tags: ['@ess', '@serverless'], + tags: ['@ess', '@serverless', '@serverlessQA'], }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts index fb258bc04efa8..ee6ff0951d11b 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/policy/policy_list.cy.ts @@ -16,7 +16,7 @@ import { createAgentPolicyTask, getEndpointIntegrationVersion } from '../../task describe( 'Policy List', { - tags: ['@ess', '@serverless'], + tags: ['@ess', '@serverless', '@serverlessQA'], env: { ftrConfig: { kbnServerArgs: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts index 4b2b4dbf369d2..68f1cd4e5ea63 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/api/agent_policy_settings_complete.cy.ts @@ -17,7 +17,7 @@ import { login } from '../../../../tasks/login'; describe( 'Agent policy settings API operations on Complete', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts index f099ac5da693b..34a09c490bcc3 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_complete.cy.ts @@ -17,7 +17,7 @@ import { describe( 'Agent Policy Settings - Complete', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts index 7fdd42ac196bd..8497806e176d2 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/agent_policy_settings_essentials.cy.ts @@ -17,7 +17,7 @@ import { describe( 'Agent Policy Settings - Essentials', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts index 5781fbea94880..09fd2cabd36e6 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/components/policy_details_endpoint_essentials.cy.ts @@ -13,7 +13,7 @@ import { APP_POLICIES_PATH } from '../../../../../../../common/constants'; describe( 'When displaying the Policy Details in Endpoint Essentials PLI', { - tags: ['@serverless'], + tags: ['@serverless', '@serverlessQA'], env: { ftrConfig: { productTypes: [ diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts index 680a439e326f5..141c9c0d864ef 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity.ts @@ -15,7 +15,7 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const utils = getService('securitySolutionUtils'); - describe('@ess @serverless Resolver tests for the entity route', function () { + describe('@ess @serverless @serverlessQA Resolver tests for the entity route', function () { let adminSupertest: TestAgent; before(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts index 05e7c72c98d4b..6ed414f6a0abd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/entity_id.ts @@ -38,7 +38,7 @@ export default function ({ getService }: FtrProviderContext) { } }; - describe('@ess @serverless Resolver handling of entity ids', function () { + describe('@ess @serverless @serverlessQA Resolver handling of entity ids', function () { let adminSupertest: TestAgent; before(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts index 91345c22ac82b..7e5848b5016a3 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/events.ts @@ -52,7 +52,7 @@ export default function ({ getService }: FtrProviderContext) { ancestryArraySize: 2, }; - describe('@ess @serverless event route', function () { + describe('@ess @serverless @serverlessQA event route', function () { let entityIDFilterArray: JsonObject[] | undefined; let entityIDFilter: string | undefined; let adminSupertest: TestAgent; diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts index 8ecfab9335a79..aa68fbc692ae5 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/tree.ts @@ -56,7 +56,7 @@ export default function ({ getService }: FtrProviderContext) { alwaysGenMaxChildrenPerNode: true, ancestryArraySize: 2, }; - describe('@ess @serverless Resolver tree', function () { + describe('@ess @serverless @serverlessQA Resolver tree', function () { let adminSupertest: TestAgent; before(async () => { diff --git a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts index c006ecb88ad84..9fbc8e3f15507 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/agent_type_support.ts @@ -10,7 +10,7 @@ import TestAgent from 'supertest/lib/agent'; import { FtrProviderContext } from '../../../../ftr_provider_context_edr_workflows'; export default function ({ getService }: FtrProviderContext) { - describe('@ess @serverless Response Actions support for sentinelOne agentType', function () { + describe('@ess @serverless @serverlessQA Response Actions support for sentinelOne agentType', function () { const utils = getService('securitySolutionUtils'); let adminSupertest: TestAgent; From 772b03c47a062bcc12b0de0b459cf2a3c32cd474 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 14 Nov 2024 00:06:47 +1100 Subject: [PATCH 12/53] Authorized route migration for routes owned by @elastic/ml-ui (#198190) ### Authz API migration for authorized routes This PR migrates `access:` tags used in route definitions to new security configuration. Please refer to the documentation for more information: [Authorization API](https://docs.elastic.dev/kibana-dev-docs/key-concepts/security-api-authorization) ### **Before migration:** Access control tags were defined in the `options` object of the route: ```ts router.get({ path: '/api/path', options: { tags: ['access:', 'access:'], }, ... }, handler); ``` ### **After migration:** Tags have been replaced with the more robust `security.authz.requiredPrivileges` field under `security`: ```ts router.get({ path: '/api/path', security: { authz: { requiredPrivileges: ['', ''], }, }, ... }, handler); ``` ### What to do next? 1. Review the changes in this PR. 2. You might need to update your tests to reflect the new security configuration: - If you have tests that rely on checking `access` tags. - If you have snapshot tests that include the route definition. - If you have FTR tests that rely on checking unauthorized error message. The error message changed to also include missing privileges. ## Any questions? If you have any questions or need help with API authorization, please reach out to the `@elastic/kibana-security` team. --------- Co-authored-by: James Gowdy --- .../plugins/data_visualizer/server/routes.ts | 12 +- x-pack/plugins/ml/server/routes/alerting.ts | 6 +- .../plugins/ml/server/routes/annotations.ts | 18 ++- .../ml/server/routes/anomaly_detectors.ts | 108 ++++++++----- x-pack/plugins/ml/server/routes/calendars.ts | 30 ++-- .../ml/server/routes/data_frame_analytics.ts | 96 +++++++---- .../ml/server/routes/data_visualizer.ts | 6 +- x-pack/plugins/ml/server/routes/datafeeds.ts | 60 ++++--- .../ml/server/routes/fields_service.ts | 12 +- x-pack/plugins/ml/server/routes/filters.ts | 36 +++-- .../ml/server/routes/inference_models.ts | 12 +- .../ml/server/routes/job_audit_messages.ts | 18 ++- .../plugins/ml/server/routes/job_service.ts | 150 ++++++++++++------ .../ml/server/routes/job_validation.ts | 30 ++-- x-pack/plugins/ml/server/routes/management.ts | 14 +- .../ml/server/routes/model_management.ts | 32 ++-- x-pack/plugins/ml/server/routes/modules.ts | 30 ++-- .../plugins/ml/server/routes/notifications.ts | 28 ++-- .../ml/server/routes/results_service.ts | 66 +++++--- .../plugins/ml/server/routes/saved_objects.ts | 94 ++++++----- x-pack/plugins/ml/server/routes/system.ts | 36 +++-- .../ml/server/routes/trained_models.ts | 108 ++++++++----- .../apis/ml/annotations/create_annotations.ts | 1 - .../apis/ml/annotations/delete_annotations.ts | 1 - .../apis/ml/annotations/get_annotations.ts | 1 - .../apis/ml/annotations/update_annotations.ts | 1 - .../apis/ml/anomaly_detectors/create.ts | 2 - .../apis/ml/anomaly_detectors/get.ts | 4 - .../apis/ml/calendars/create_calendars.ts | 2 - .../ml/data_frame_analytics/create_job.ts | 2 - .../apis/ml/data_frame_analytics/delete.ts | 2 - .../apis/ml/data_frame_analytics/evaluate.ts | 1 - .../apis/ml/data_frame_analytics/explain.ts | 2 - .../apis/ml/data_frame_analytics/get.ts | 5 - .../apis/ml/data_frame_analytics/start.ts | 2 - .../apis/ml/data_frame_analytics/stop.ts | 2 - .../apis/ml/data_frame_analytics/update.ts | 2 - .../apis/ml/data_frame_analytics/validate.ts | 2 - .../apis/ml/filters/create_filters.ts | 6 +- .../apis/ml/filters/get_filters.ts | 2 - .../apis/ml/filters/get_filters_stats.ts | 2 - .../ml/job_audit_messages/clear_messages.ts | 2 - .../get_job_audit_messages.ts | 1 - .../apis/ml/job_validation/cardinality.ts | 1 - .../apis/ml/job_validation/validate.ts | 1 - .../apis/ml/modules/setup_module.ts | 3 +- .../ml/results/get_anomalies_table_data.ts | 1 - .../apis/ml/results/get_categorizer_stats.ts | 2 - .../ml/results/get_datafeed_results_chart.ts | 1 - .../apis/ml/results/get_stopped_partitions.ts | 1 - .../apis/ml/system/has_privileges.ts | 26 +-- 51 files changed, 678 insertions(+), 405 deletions(-) diff --git a/x-pack/plugins/data_visualizer/server/routes.ts b/x-pack/plugins/data_visualizer/server/routes.ts index 8bede873dc7d9..9d213182ad049 100644 --- a/x-pack/plugins/data_visualizer/server/routes.ts +++ b/x-pack/plugins/data_visualizer/server/routes.ts @@ -25,8 +25,10 @@ export function routes(coreSetup: CoreSetup, logger: Logger) .post({ path: '/internal/data_visualizer/test_grok_pattern', access: 'internal', - options: { - tags: ['access:fileUpload:analyzeFile'], + security: { + authz: { + requiredPrivileges: ['fileUpload:analyzeFile'], + }, }, }) .addVersion( @@ -78,8 +80,10 @@ export function routes(coreSetup: CoreSetup, logger: Logger) .get({ path: '/internal/data_visualizer/inference_endpoints', access: 'internal', - options: { - tags: ['access:fileUpload:analyzeFile'], + security: { + authz: { + requiredPrivileges: ['fileUpload:analyzeFile'], + }, }, }) .addVersion( diff --git a/x-pack/plugins/ml/server/routes/alerting.ts b/x-pack/plugins/ml/server/routes/alerting.ts index ec4ec2b7c748d..4ca97423a2533 100644 --- a/x-pack/plugins/ml/server/routes/alerting.ts +++ b/x-pack/plugins/ml/server/routes/alerting.ts @@ -22,8 +22,10 @@ export function alertingRoutes( .post({ access: 'internal', path: `${ML_INTERNAL_BASE_PATH}/alerting/preview`, - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Previews an alerting condition', description: 'Returns a preview of the alerting condition', diff --git a/x-pack/plugins/ml/server/routes/annotations.ts b/x-pack/plugins/ml/server/routes/annotations.ts index 49528052c2bcc..3ccfdaa3a26e0 100644 --- a/x-pack/plugins/ml/server/routes/annotations.ts +++ b/x-pack/plugins/ml/server/routes/annotations.ts @@ -46,8 +46,10 @@ export function annotationRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/annotations`, access: 'internal', - options: { - tags: ['access:ml:canGetAnnotations'], + security: { + authz: { + requiredPrivileges: ['ml:canGetAnnotations'], + }, }, summary: 'Gets annotations', description: 'Gets annotations.', @@ -83,8 +85,10 @@ export function annotationRoutes( .put({ path: `${ML_INTERNAL_BASE_PATH}/annotations/index`, access: 'internal', - options: { - tags: ['access:ml:canCreateAnnotation'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateAnnotation'], + }, }, summary: 'Indexes annotation', description: 'Indexes the annotation.', @@ -127,8 +131,10 @@ export function annotationRoutes( .delete({ path: `${ML_INTERNAL_BASE_PATH}/annotations/delete/{annotationId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteAnnotation'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteAnnotation'], + }, }, summary: 'Deletes annotation', description: 'Deletes the specified annotation.', diff --git a/x-pack/plugins/ml/server/routes/anomaly_detectors.ts b/x-pack/plugins/ml/server/routes/anomaly_detectors.ts index 4c75b7a85556a..f9bd3f6661e4a 100644 --- a/x-pack/plugins/ml/server/routes/anomaly_detectors.ts +++ b/x-pack/plugins/ml/server/routes/anomaly_detectors.ts @@ -36,8 +36,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets anomaly detectors', description: 'Returns the list of anomaly detection jobs.', @@ -67,8 +69,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets anomaly detector by ID', description: 'Returns the anomaly detection job by ID', @@ -99,8 +103,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets anomaly detectors stats', description: 'Returns the anomaly detection jobs statistics.', @@ -126,8 +132,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets anomaly detector stats by ID', description: 'Returns the anomaly detection job statistics by ID', @@ -158,8 +166,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .put({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Creates an anomaly detection job', description: 'Creates an anomaly detection job.', @@ -205,8 +215,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/_update`, access: 'internal', - options: { - tags: ['access:ml:canUpdateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canUpdateJob'], + }, }, summary: 'Updates an anomaly detection job', description: 'Updates certain properties of an anomaly detection job.', @@ -242,8 +254,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/_open`, access: 'internal', - options: { - tags: ['access:ml:canOpenJob'], + security: { + authz: { + requiredPrivileges: ['ml:canOpenJob'], + }, }, summary: 'Opens an anomaly detection job', description: 'Opens an anomaly detection job.', @@ -274,8 +288,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/_close`, access: 'internal', - options: { - tags: ['access:ml:canCloseJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCloseJob'], + }, }, summary: 'Closes an anomaly detection job', description: 'Closes an anomaly detection job.', @@ -313,8 +329,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .delete({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteJob'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteJob'], + }, }, summary: 'Deletes an anomaly detection job', description: 'Deletes specified anomaly detection job.', @@ -353,8 +371,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .delete({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/_forecast/{forecastId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteForecast'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteForecast'], + }, }, summary: 'Deletes specified forecast for specified job', description: 'Deletes a specified forecast for the specified anomaly detection job.', @@ -388,8 +408,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/_forecast`, access: 'internal', - options: { - tags: ['access:ml:canForecastJob'], + security: { + authz: { + requiredPrivileges: ['ml:canForecastJob'], + }, }, summary: 'Creates forecast for specified job', description: @@ -427,8 +449,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/results/buckets/{timestamp?}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets bucket scores', description: @@ -470,8 +494,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/results/overall_buckets`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get overall buckets', description: @@ -510,8 +536,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/results/categories/{categoryId}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get categories', description: 'Retrieves the categories results for the specified job ID and category ID.', @@ -544,8 +572,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/model_snapshots`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get model snapshots by job ID', description: 'Returns the model snapshots for the specified job ID', @@ -577,8 +607,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/model_snapshots/{snapshotId}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get model snapshots by id', description: 'Returns the model snapshots for the specified job ID and snapshot ID', @@ -611,8 +643,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/model_snapshots/{snapshotId}/_update`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Updates model snapshot by snapshot ID', description: 'Updates the model snapshot for the specified snapshot ID', @@ -647,8 +681,10 @@ export function jobRoutes({ router, routeGuard }: RouteInitialization) { .delete({ path: `${ML_INTERNAL_BASE_PATH}/anomaly_detectors/{jobId}/model_snapshots/{snapshotId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Deletes model snapshots by snapshot ID', description: 'Deletes the model snapshot for the specified snapshot ID', diff --git a/x-pack/plugins/ml/server/routes/calendars.ts b/x-pack/plugins/ml/server/routes/calendars.ts index 9ca93a78a51a3..263854bd8b6b8 100644 --- a/x-pack/plugins/ml/server/routes/calendars.ts +++ b/x-pack/plugins/ml/server/routes/calendars.ts @@ -48,8 +48,10 @@ export function calendars({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/calendars`, access: 'internal', - options: { - tags: ['access:ml:canGetCalendars'], + security: { + authz: { + requiredPrivileges: ['ml:canGetCalendars'], + }, }, summary: 'Gets calendars', description: 'Gets calendars - size limit has been explicitly set to 10000', @@ -76,8 +78,10 @@ export function calendars({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/calendars/{calendarIds}`, access: 'internal', - options: { - tags: ['access:ml:canGetCalendars'], + security: { + authz: { + requiredPrivileges: ['ml:canGetCalendars'], + }, }, summary: 'Gets a calendar', description: 'Gets a calendar by id', @@ -115,8 +119,10 @@ export function calendars({ router, routeGuard }: RouteInitialization) { .put({ path: `${ML_INTERNAL_BASE_PATH}/calendars`, access: 'internal', - options: { - tags: ['access:ml:canCreateCalendar'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateCalendar'], + }, }, summary: 'Creates a calendar', description: 'Creates a calendar', @@ -149,8 +155,10 @@ export function calendars({ router, routeGuard }: RouteInitialization) { .put({ path: `${ML_INTERNAL_BASE_PATH}/calendars/{calendarId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateCalendar'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateCalendar'], + }, }, summary: 'Updates a calendar', description: 'Updates a calendar', @@ -185,8 +193,10 @@ export function calendars({ router, routeGuard }: RouteInitialization) { .delete({ path: `${ML_INTERNAL_BASE_PATH}/calendars/{calendarId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteCalendar'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteCalendar'], + }, }, summary: 'Deletes a calendar', description: 'Deletes a calendar', diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts index 0229f9e9bba5b..007361f97af4a 100644 --- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts @@ -119,8 +119,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Gets data frame analytics', description: 'Returns the list of data frame analytics jobs.', @@ -153,8 +155,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Gets data frame analytics by id', description: 'Returns the data frame analytics job by id.', @@ -191,8 +195,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Gets data frame analytics stats', description: 'Returns the data frame analytics job statistics.', @@ -218,8 +224,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Gets data frame analytics stats by id', description: 'Returns the data frame analytics job statistics by id.', @@ -252,8 +260,10 @@ export function dataFrameAnalyticsRoutes( .put({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateDataFrameAnalytics'], + }, }, summary: 'Updates data frame analytics job', description: @@ -329,8 +339,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/_evaluate`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Evaluates the data frame analytics', description: 'Evaluates the data frame analytics for an annotated index.', @@ -366,8 +378,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/_explain`, access: 'internal', - options: { - tags: ['access:ml:canCreateDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateDataFrameAnalytics'], + }, }, summary: 'Explains a data frame analytics job config', description: @@ -403,8 +417,10 @@ export function dataFrameAnalyticsRoutes( .delete({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteDataFrameAnalytics'], + }, }, summary: 'Deletes data frame analytics job', description: 'Deletes specified data frame analytics job.', @@ -506,8 +522,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}/_start`, access: 'internal', - options: { - tags: ['access:ml:canStartStopDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopDataFrameAnalytics'], + }, }, summary: 'Starts specified analytics job', description: 'Starts a data frame analytics job.', @@ -540,8 +558,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}/_stop`, access: 'internal', - options: { - tags: ['access:ml:canStartStopDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopDataFrameAnalytics'], + }, }, summary: 'Stops specified analytics job', description: 'Stops a data frame analytics job.', @@ -576,8 +596,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}/_update`, access: 'internal', - options: { - tags: ['access:ml:canCreateDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateDataFrameAnalytics'], + }, }, summary: 'Updates specified analytics job', description: 'Updates a data frame analytics job.', @@ -615,8 +637,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/{analyticsId}/messages`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Gets data frame analytics messages', description: 'Returns the list of audit messages for data frame analytics jobs.', @@ -649,8 +673,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/jobs_exist`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Checks if jobs exist', description: @@ -700,8 +726,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/map/{analyticsId}`, access: 'internal', - options: { - tags: ['access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDataFrameAnalytics'], + }, }, summary: 'Gets a data frame analytics jobs map', description: 'Returns map of objects leading up to analytics job.', @@ -761,8 +789,10 @@ export function dataFrameAnalyticsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/new_job_caps/{indexPattern}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get fields for a pattern of indices used for analytics', description: 'Returns the fields for a pattern of indices used for analytics.', @@ -809,8 +839,10 @@ export function dataFrameAnalyticsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/data_frame/analytics/validate`, access: 'internal', - options: { - tags: ['access:ml:canCreateDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateDataFrameAnalytics'], + }, }, summary: 'Validates the data frame analytics job config', description: 'Validates the data frame analytics job config.', diff --git a/x-pack/plugins/ml/server/routes/data_visualizer.ts b/x-pack/plugins/ml/server/routes/data_visualizer.ts index 32a782a2acd69..abdf79e6106da 100644 --- a/x-pack/plugins/ml/server/routes/data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/data_visualizer.ts @@ -38,8 +38,10 @@ export function dataVisualizerRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/data_visualizer/get_field_histograms/{indexPattern}`, access: 'internal', - options: { - tags: ['access:ml:canGetFieldInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFieldInfo'], + }, }, summary: 'Gets histograms for fields', description: 'Returns the histograms on a list fields in the specified index pattern.', diff --git a/x-pack/plugins/ml/server/routes/datafeeds.ts b/x-pack/plugins/ml/server/routes/datafeeds.ts index a8fbc8c2ceac5..8939471ef5624 100644 --- a/x-pack/plugins/ml/server/routes/datafeeds.ts +++ b/x-pack/plugins/ml/server/routes/datafeeds.ts @@ -25,8 +25,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds`, access: 'internal', - options: { - tags: ['access:ml:canGetDatafeeds'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDatafeeds'], + }, }, summary: 'Gets all datafeeds', description: 'Retrieves configuration information for datafeeds.', @@ -52,8 +54,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}`, access: 'internal', - options: { - tags: ['access:ml:canGetDatafeeds'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDatafeeds'], + }, }, summary: 'Get datafeed for given datafeed id', description: 'Retrieves configuration information for a datafeed.', @@ -85,8 +89,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetDatafeeds'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDatafeeds'], + }, }, summary: 'Gets stats for all datafeeds', description: 'Retrieves usage information for datafeeds.', @@ -112,8 +118,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetDatafeeds'], + security: { + authz: { + requiredPrivileges: ['ml:canGetDatafeeds'], + }, }, summary: 'Get datafeed stats for given datafeed id', description: 'Retrieves usage information for a datafeed.', @@ -147,8 +155,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .put({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateDatafeed'], + }, }, summary: 'Creates a datafeed', description: 'Instantiates a datafeed.', @@ -188,8 +198,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}/_update`, access: 'internal', - options: { - tags: ['access:ml:canUpdateDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canUpdateDatafeed'], + }, }, summary: 'Updates a datafeed', description: 'Updates certain properties of a datafeed.', @@ -229,8 +241,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .delete({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteDatafeed'], + }, }, summary: 'Deletes a datafeed', description: 'Deletes an existing datafeed.', @@ -270,8 +284,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}/_start`, access: 'internal', - options: { - tags: ['access:ml:canStartStopDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopDatafeed'], + }, }, summary: 'Starts a datafeed', description: 'Starts one or more datafeeds', @@ -312,8 +328,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}/_stop`, access: 'internal', - options: { - tags: ['access:ml:canStartStopDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopDatafeed'], + }, }, summary: 'Stops a datafeed', description: 'Stops one or more datafeeds', @@ -348,8 +366,10 @@ export function dataFeedRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/datafeeds/{datafeedId}/_preview`, access: 'internal', - options: { - tags: ['access:ml:canPreviewDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canPreviewDatafeed'], + }, }, summary: 'Previews a datafeed', description: 'Previews a datafeed', diff --git a/x-pack/plugins/ml/server/routes/fields_service.ts b/x-pack/plugins/ml/server/routes/fields_service.ts index ae4bfa6110a3e..a86f1d2c01cdc 100644 --- a/x-pack/plugins/ml/server/routes/fields_service.ts +++ b/x-pack/plugins/ml/server/routes/fields_service.ts @@ -37,8 +37,10 @@ export function fieldsService({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/fields_service/field_cardinality`, access: 'internal', - options: { - tags: ['access:ml:canGetFieldInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFieldInfo'], + }, }, summary: 'Gets cardinality of fields', description: @@ -76,8 +78,10 @@ export function fieldsService({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/fields_service/time_field_range`, access: 'internal', - options: { - tags: ['access:ml:canGetFieldInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFieldInfo'], + }, }, summary: 'Get time field range', description: diff --git a/x-pack/plugins/ml/server/routes/filters.ts b/x-pack/plugins/ml/server/routes/filters.ts index c654bbf0e2bae..c2b7ad5a9acb2 100644 --- a/x-pack/plugins/ml/server/routes/filters.ts +++ b/x-pack/plugins/ml/server/routes/filters.ts @@ -50,8 +50,10 @@ export function filtersRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/filters`, access: 'internal', - options: { - tags: ['access:ml:canGetFilters'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFilters'], + }, }, summary: 'Gets filters', description: @@ -79,8 +81,10 @@ export function filtersRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/filters/{filterId}`, access: 'internal', - options: { - tags: ['access:ml:canGetFilters'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFilters'], + }, }, summary: 'Gets filter by ID', description: 'Retrieves the filter with the specified ID.', @@ -108,8 +112,10 @@ export function filtersRoutes({ router, routeGuard }: RouteInitialization) { .put({ path: `${ML_INTERNAL_BASE_PATH}/filters`, access: 'internal', - options: { - tags: ['access:ml:canCreateFilter'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateFilter'], + }, }, summary: 'Creates a filter', description: 'Instantiates a filter, for use by custom rules in anomaly detection.', @@ -139,8 +145,10 @@ export function filtersRoutes({ router, routeGuard }: RouteInitialization) { .put({ path: `${ML_INTERNAL_BASE_PATH}/filters/{filterId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateFilter'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateFilter'], + }, }, summary: 'Updates a filter', description: 'Updates the description of a filter, adds items or removes items.', @@ -174,8 +182,10 @@ export function filtersRoutes({ router, routeGuard }: RouteInitialization) { .delete({ path: `${ML_INTERNAL_BASE_PATH}/filters/{filterId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteFilter'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteFilter'], + }, }, summary: 'Deletes a filter', description: 'Deletes the filter with the specified ID.', @@ -207,8 +217,10 @@ export function filtersRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/filters/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetFilters'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFilters'], + }, }, summary: 'Gets filters stats', description: diff --git a/x-pack/plugins/ml/server/routes/inference_models.ts b/x-pack/plugins/ml/server/routes/inference_models.ts index d51645a365c62..866398ac56ce9 100644 --- a/x-pack/plugins/ml/server/routes/inference_models.ts +++ b/x-pack/plugins/ml/server/routes/inference_models.ts @@ -26,8 +26,10 @@ export function inferenceModelRoutes( .put({ path: `${ML_INTERNAL_BASE_PATH}/_inference/{taskType}/{inferenceId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateInferenceEndpoint'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateInferenceEndpoint'], + }, }, summary: 'Create an inference endpoint', description: 'Create an inference endpoint', @@ -67,8 +69,10 @@ export function inferenceModelRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/_inference/all`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get all inference endpoints', description: 'Get all inference endpoints', diff --git a/x-pack/plugins/ml/server/routes/job_audit_messages.ts b/x-pack/plugins/ml/server/routes/job_audit_messages.ts index 4cc23555f71b5..09e432b925afb 100644 --- a/x-pack/plugins/ml/server/routes/job_audit_messages.ts +++ b/x-pack/plugins/ml/server/routes/job_audit_messages.ts @@ -23,8 +23,10 @@ export function jobAuditMessagesRoutes({ router, routeGuard }: RouteInitializati .get({ path: `${ML_INTERNAL_BASE_PATH}/job_audit_messages/messages/{jobId}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets audit messages', description: 'Retrieves the audit messages for the specified job ID.', @@ -66,8 +68,10 @@ export function jobAuditMessagesRoutes({ router, routeGuard }: RouteInitializati .get({ path: `${ML_INTERNAL_BASE_PATH}/job_audit_messages/messages`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Gets all audit messages', description: 'Retrieves all audit messages.', @@ -102,8 +106,10 @@ export function jobAuditMessagesRoutes({ router, routeGuard }: RouteInitializati .put({ path: `${ML_INTERNAL_BASE_PATH}/job_audit_messages/clear_messages`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Clear messages', description: 'Clear the job audit messages.', diff --git a/x-pack/plugins/ml/server/routes/job_service.ts b/x-pack/plugins/ml/server/routes/job_service.ts index 37d4bf134004c..3814d36bc3a6c 100644 --- a/x-pack/plugins/ml/server/routes/job_service.ts +++ b/x-pack/plugins/ml/server/routes/job_service.ts @@ -43,8 +43,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/force_start_datafeeds`, access: 'internal', - options: { - tags: ['access:ml:canStartStopDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopDatafeed'], + }, }, summary: 'Starts datafeeds', description: 'Starts one or more datafeeds.', @@ -77,8 +79,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/stop_datafeeds`, access: 'internal', - options: { - tags: ['access:ml:canStartStopDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopDatafeed'], + }, }, summary: 'Stops datafeeds', description: 'Stops one or more datafeeds.', @@ -111,8 +115,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/delete_jobs`, access: 'internal', - options: { - tags: ['access:ml:canDeleteJob'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteJob'], + }, }, summary: 'Deletes jobs', description: 'Deletes an existing anomaly detection job.', @@ -149,8 +155,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/close_jobs`, access: 'internal', - options: { - tags: ['access:ml:canCloseJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCloseJob'], + }, }, summary: 'Closes jobs', description: 'Closes one or more anomaly detection jobs.', @@ -183,8 +191,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/reset_jobs`, access: 'internal', - options: { - tags: ['access:ml:canResetJob'], + security: { + authz: { + requiredPrivileges: ['ml:canResetJob'], + }, }, summary: 'Resets jobs', description: 'Resets one or more anomaly detection jobs.', @@ -217,8 +227,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/force_stop_and_close_job`, access: 'internal', - options: { - tags: ['access:ml:canCloseJob', 'access:ml:canStartStopDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canCloseJob', 'ml:canStartStopDatafeed'], + }, }, summary: 'Force stops and closes job', description: @@ -252,8 +264,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/jobs_summary`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Jobs summary', description: @@ -286,8 +300,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/jobs/jobs_with_geo`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Jobs with geo', description: @@ -322,8 +338,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/jobs_with_time_range`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Jobs with time range', description: "Creates a list of jobs with data about the job's time range.", @@ -351,8 +369,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/job_for_cloning`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get job for cloning', description: 'Get the job configuration with auto generated fields excluded for cloning', @@ -385,8 +405,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/jobs`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Create jobs list', description: 'Creates a list of jobs.', @@ -424,8 +446,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/jobs/groups`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get all groups', description: 'Returns array of group objects with job ids listed for each group.', @@ -453,8 +477,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/update_groups`, access: 'internal', - options: { - tags: ['access:ml:canUpdateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canUpdateJob'], + }, }, summary: 'Update job groups', description: 'Updates the groups property of an anomaly detection job.', @@ -487,8 +513,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/jobs/blocking_jobs_tasks`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get blocking job tasks', description: 'Gets the ids of deleting, resetting or reverting anomaly detection jobs.', @@ -516,8 +544,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/jobs_exist`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Check if jobs exist', description: @@ -551,8 +581,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/jobs/new_job_caps/{indexPattern}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get new job capabilities', description: 'Retrieve the capabilities of fields for indices', @@ -591,8 +623,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/new_job_line_chart`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Get job line chart data', description: 'Returns line chart data for anomaly detection job', @@ -650,8 +684,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/new_job_population_chart`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Get job population chart data', description: 'Returns population chart data for anomaly detection job', @@ -707,8 +743,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .get({ path: `${ML_INTERNAL_BASE_PATH}/jobs/all_jobs_and_group_ids`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get all job and group IDs', description: 'Returns a list of all job IDs and all group IDs', @@ -736,8 +774,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/look_back_progress`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Get lookback progress', description: 'Returns current progress of anomaly detection job', @@ -770,8 +810,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/categorization_field_validation`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Get categorization field examples', description: 'Returns examples of categorization field', @@ -827,8 +869,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/top_categories`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get top categories', description: 'Returns list of top categories', @@ -870,8 +914,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/datafeed_preview`, access: 'internal', - options: { - tags: ['access:ml:canPreviewDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canPreviewDatafeed'], + }, }, summary: 'Get datafeed preview', description: 'Returns a preview of the datafeed search', @@ -918,8 +964,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/revert_model_snapshot`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob', 'access:ml:canStartStopDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob', 'ml:canStartStopDatafeed'], + }, }, summary: 'Revert model snapshot', description: @@ -961,8 +1009,10 @@ export function jobServiceRoutes({ router, routeGuard }: RouteInitialization) { .post({ path: `${ML_INTERNAL_BASE_PATH}/jobs/bulk_create`, access: 'internal', - options: { - tags: ['access:ml:canPreviewDatafeed'], + security: { + authz: { + requiredPrivileges: ['ml:canPreviewDatafeed'], + }, }, summary: 'Bulk create jobs and datafeeds', description: 'Bulk create jobs and datafeeds.', diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index 0418ccc57e2b3..c66e6aa5d3bfe 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -67,8 +67,10 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit .post({ path: `${ML_INTERNAL_BASE_PATH}/validate/estimate_bucket_span`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Estimates bucket span', description: @@ -112,8 +114,10 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit .post({ path: `${ML_INTERNAL_BASE_PATH}/validate/calculate_model_memory_limit`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Calculates model memory limit', description: 'Calls _estimate_model_memory endpoint to retrieve model memory estimation.', @@ -144,8 +148,10 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit .post({ path: `${ML_INTERNAL_BASE_PATH}/validate/cardinality`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Validates cardinality', description: 'Validates cardinality for the given job configuration.', @@ -177,8 +183,10 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit .post({ path: `${ML_INTERNAL_BASE_PATH}/validate/job`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Validates job', description: 'Validates the given job configuration.', @@ -215,8 +223,10 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit .post({ path: `${ML_INTERNAL_BASE_PATH}/validate/datafeed_preview`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Validates datafeed preview', description: 'Validates that the datafeed preview runs successfully and produces results.', diff --git a/x-pack/plugins/ml/server/routes/management.ts b/x-pack/plugins/ml/server/routes/management.ts index 422e5e0944aad..9d81aa06602c1 100644 --- a/x-pack/plugins/ml/server/routes/management.ts +++ b/x-pack/plugins/ml/server/routes/management.ts @@ -29,12 +29,14 @@ export function managementRoutes({ router, routeGuard, getEnabledFeatures }: Rou .get({ path: `${ML_INTERNAL_BASE_PATH}/management/list/{listType}`, access: 'internal', - options: { - tags: [ - 'access:ml:canCreateJob', - 'access:ml:canCreateDataFrameAnalytics', - 'access:ml:canCreateTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canCreateJob', + 'ml:canCreateDataFrameAnalytics', + 'ml:canCreateTrainedModels', + ], + }, }, summary: 'Gets management list', description: diff --git a/x-pack/plugins/ml/server/routes/model_management.ts b/x-pack/plugins/ml/server/routes/model_management.ts index d568b0f3ed91a..7db10ca17ff15 100644 --- a/x-pack/plugins/ml/server/routes/model_management.ts +++ b/x-pack/plugins/ml/server/routes/model_management.ts @@ -29,13 +29,15 @@ export function modelManagementRoutes({ .get({ path: `${ML_INTERNAL_BASE_PATH}/model_management/nodes_overview`, access: 'internal', - options: { - tags: [ - 'access:ml:canViewMlNodes', - 'access:ml:canGetDataFrameAnalytics', - 'access:ml:canGetJobs', - 'access:ml:canGetTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canViewMlNodes', + 'ml:canGetDataFrameAnalytics', + 'ml:canGetJobs', + 'ml:canGetTrainedModels', + ], + }, }, summary: 'Get node overview about the models allocation', description: 'Retrieves the list of ML nodes with memory breakdown and allocated models info', @@ -62,13 +64,15 @@ export function modelManagementRoutes({ .get({ path: `${ML_INTERNAL_BASE_PATH}/model_management/memory_usage`, access: 'internal', - options: { - tags: [ - 'access:ml:canViewMlNodes', - 'access:ml:canGetDataFrameAnalytics', - 'access:ml:canGetJobs', - 'access:ml:canGetTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canViewMlNodes', + 'ml:canGetDataFrameAnalytics', + 'ml:canGetJobs', + 'ml:canGetTrainedModels', + ], + }, }, summary: 'Get memory usage for jobs and trained models', description: 'Retrieves the memory usage for jobs and trained models', diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts index a4eefdf08cf66..5a0385d1ce7de 100644 --- a/x-pack/plugins/ml/server/routes/modules.ts +++ b/x-pack/plugins/ml/server/routes/modules.ts @@ -35,8 +35,10 @@ export function dataRecognizer( .get({ path: `${ML_INTERNAL_BASE_PATH}/modules/recognize/{indexPatternTitle}`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Recognize index pattern', description: @@ -96,8 +98,10 @@ export function dataRecognizer( .get({ path: `${ML_INTERNAL_BASE_PATH}/modules/recognize_by_module/{moduleId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Recognize module', description: @@ -152,8 +156,10 @@ export function dataRecognizer( .get({ path: `${ML_INTERNAL_BASE_PATH}/modules/get_module/{moduleId?}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get module', description: @@ -224,8 +230,10 @@ export function dataRecognizer( .post({ path: `${ML_INTERNAL_BASE_PATH}/modules/setup/{moduleId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob'], + }, }, summary: 'Setup module', description: @@ -323,8 +331,10 @@ export function dataRecognizer( .get({ path: `${ML_INTERNAL_BASE_PATH}/modules/jobs_exist/{moduleId}`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Check if module jobs exist', description: `Check whether the jobs in the module with the specified ID exist in the current list of jobs. The check runs a test to see if any of the jobs in existence have an ID which ends with the ID of each job in the module. This is done as a prefix may be supplied in the setup endpoint which is added to the start of the ID of every job in the module.`, diff --git a/x-pack/plugins/ml/server/routes/notifications.ts b/x-pack/plugins/ml/server/routes/notifications.ts index 02e7694834b73..13ece4a031d06 100644 --- a/x-pack/plugins/ml/server/routes/notifications.ts +++ b/x-pack/plugins/ml/server/routes/notifications.ts @@ -23,12 +23,14 @@ export function notificationsRoutes({ .get({ path: `${ML_INTERNAL_BASE_PATH}/notifications`, access: 'internal', - options: { - tags: [ - 'access:ml:canGetJobs', - 'access:ml:canGetDataFrameAnalytics', - 'access:ml:canGetTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canGetJobs', + 'ml:canGetDataFrameAnalytics', + 'ml:canGetTrainedModels', + ], + }, }, summary: 'Get notifications', description: 'Retrieves notifications based on provided criteria.', @@ -67,12 +69,14 @@ export function notificationsRoutes({ .get({ path: `${ML_INTERNAL_BASE_PATH}/notifications/count`, access: 'internal', - options: { - tags: [ - 'access:ml:canGetJobs', - 'access:ml:canGetDataFrameAnalytics', - 'access:ml:canGetTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canGetJobs', + 'ml:canGetDataFrameAnalytics', + 'ml:canGetTrainedModels', + ], + }, }, summary: 'Get notification counts', description: 'Counts notifications by level.', diff --git a/x-pack/plugins/ml/server/routes/results_service.ts b/x-pack/plugins/ml/server/routes/results_service.ts index e558c1f94a300..fb0e73789c240 100644 --- a/x-pack/plugins/ml/server/routes/results_service.ts +++ b/x-pack/plugins/ml/server/routes/results_service.ts @@ -110,8 +110,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/anomalies_table_data`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get anomalies records for table display', description: @@ -143,8 +145,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/category_definition`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get category definition', description: 'Returns the definition of the category with the specified ID and job ID.', @@ -175,8 +179,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/max_anomaly_score`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get the maximum anomaly_score', description: @@ -208,8 +214,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/category_examples`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get category examples', description: @@ -241,8 +249,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/partition_fields_values`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get partition fields values', description: @@ -274,8 +284,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/anomaly_search`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Run a search on the anomaly results index', description: @@ -307,8 +319,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .get({ path: `${ML_INTERNAL_BASE_PATH}/results/{jobId}/categorizer_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get categorizer statistics', description: 'Returns the categorizer statistics for the specified job ID.', @@ -339,8 +353,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/category_stopped_partitions`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get partitions that have stopped being categorized', description: @@ -371,8 +387,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/datafeed_results_chart`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get datafeed results chart data', description: 'Returns datafeed results chart data', @@ -404,8 +422,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/anomaly_charts`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get data for anomaly charts', description: 'Returns anomaly charts data', @@ -437,8 +457,10 @@ export function resultsServiceRoutes({ router, routeGuard }: RouteInitialization .post({ path: `${ML_INTERNAL_BASE_PATH}/results/anomaly_records`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'Get anomaly records for criteria', description: 'Returns anomaly records', diff --git a/x-pack/plugins/ml/server/routes/saved_objects.ts b/x-pack/plugins/ml/server/routes/saved_objects.ts index 74eb14ef68f8a..437f0a80eb1d7 100644 --- a/x-pack/plugins/ml/server/routes/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/saved_objects.ts @@ -32,8 +32,10 @@ export function savedObjectsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/status`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs', 'access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs', 'ml:canGetTrainedModels'], + }, }, summary: 'Get job and trained model saved object status', description: @@ -63,13 +65,17 @@ export function savedObjectsRoutes( path: `${ML_EXTERNAL_BASE_PATH}/saved_objects/sync`, access: 'public', summary: 'Synchronize machine learning saved objects', + security: { + authz: { + requiredPrivileges: [ + 'ml:canCreateJob', + 'ml:canCreateDataFrameAnalytics', + 'ml:canCreateTrainedModels', + ], + }, + }, options: { - tags: [ - 'access:ml:canCreateJob', - 'access:ml:canCreateDataFrameAnalytics', - 'access:ml:canCreateTrainedModels', - 'oas-tag:machine learning', - ], + tags: ['oas-tag:machine learning'], }, description: 'Synchronizes Kibana saved objects for machine learning jobs and trained models. This API runs automatically when you start Kibana and periodically thereafter.', @@ -104,12 +110,14 @@ export function savedObjectsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/initialize`, access: 'internal', - options: { - tags: [ - 'access:ml:canCreateJob', - 'access:ml:canCreateDataFrameAnalytics', - 'access:ml:canCreateTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canCreateJob', + 'ml:canCreateDataFrameAnalytics', + 'ml:canCreateTrainedModels', + ], + }, }, summary: 'Create saved objects for all job and trained models', description: @@ -145,12 +153,14 @@ export function savedObjectsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/sync_check`, access: 'internal', - options: { - tags: [ - 'access:ml:canGetJobs', - 'access:ml:canGetDataFrameAnalytics', - 'access:ml:canGetTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canGetJobs', + 'ml:canGetDataFrameAnalytics', + 'ml:canGetTrainedModels', + ], + }, }, summary: 'Check whether job and trained model saved objects need synchronizing', description: 'Check whether job and trained model saved objects need synchronizing.', @@ -185,8 +195,10 @@ export function savedObjectsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/update_jobs_spaces`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob', 'access:ml:canCreateDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob', 'ml:canCreateDataFrameAnalytics'], + }, }, summary: 'Update what spaces jobs are assigned to', description: 'Update a list of jobs to add and/or remove them from given spaces.', @@ -224,8 +236,10 @@ export function savedObjectsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/update_trained_models_spaces`, access: 'internal', - options: { - tags: ['access:ml:canCreateTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateTrainedModels'], + }, }, summary: 'Update what spaces trained models are assigned to', description: 'Update a list of trained models to add and/or remove them from given spaces.', @@ -262,8 +276,10 @@ export function savedObjectsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/remove_item_from_current_space`, access: 'internal', - options: { - tags: ['access:ml:canCreateJob', 'access:ml:canCreateDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateJob', 'ml:canCreateDataFrameAnalytics'], + }, }, summary: 'Remove jobs or trained models from the current space', description: 'Remove a list of jobs or trained models from the current space.', @@ -326,8 +342,10 @@ export function savedObjectsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/jobs_spaces`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs', 'access:ml:canGetDataFrameAnalytics'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs', 'ml:canGetDataFrameAnalytics'], + }, }, summary: 'Get all jobs and their spaces', description: 'List all jobs and their spaces.', @@ -355,8 +373,10 @@ export function savedObjectsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/trained_models_spaces`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get all trained models and their spaces', description: 'List all trained models and their spaces.', @@ -384,12 +404,14 @@ export function savedObjectsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/saved_objects/can_delete_ml_space_aware_item/{jobType}`, access: 'internal', - options: { - tags: [ - 'access:ml:canGetJobs', - 'access:ml:canGetDataFrameAnalytics', - 'access:ml:canGetTrainedModels', - ], + security: { + authz: { + requiredPrivileges: [ + 'ml:canGetJobs', + 'ml:canGetDataFrameAnalytics', + 'ml:canGetTrainedModels', + ], + }, }, summary: 'Check whether user can delete a job or trained model', description: `Check the user's ability to delete jobs or trained models. Returns whether they are able to fully delete the job or trained model and whether they are able to remove it from the current space. Note, this is only for enabling UI controls. A user calling endpoints directly will still be able to delete or remove the job or trained model from a space.`, diff --git a/x-pack/plugins/ml/server/routes/system.ts b/x-pack/plugins/ml/server/routes/system.ts index b6765c4b5f16c..d4127a7428397 100644 --- a/x-pack/plugins/ml/server/routes/system.ts +++ b/x-pack/plugins/ml/server/routes/system.ts @@ -27,8 +27,10 @@ export function systemRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/_has_privileges`, access: 'internal', - options: { - tags: ['access:ml:canGetMlInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetMlInfo'], + }, }, summary: 'Check privileges', description: 'Checks if the user has required privileges', @@ -136,8 +138,10 @@ export function systemRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/ml_node_count`, access: 'internal', - options: { - tags: ['access:ml:canGetMlInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetMlInfo'], + }, }, summary: 'Get the number of ML nodes', description: 'Returns the number of ML nodes', @@ -162,8 +166,10 @@ export function systemRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/info`, access: 'internal', - options: { - tags: ['access:ml:canGetMlInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetMlInfo'], + }, }, summary: 'Get ML info', description: 'Returns defaults and limits used by machine learning', @@ -206,8 +212,10 @@ export function systemRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/es_search`, access: 'internal', - options: { - tags: ['access:ml:canGetJobs'], + security: { + authz: { + requiredPrivileges: ['ml:canGetJobs'], + }, }, summary: 'ES Search wrapper', // @ts-expect-error TODO(https://github.com/elastic/kibana/issues/196095): Replace {RouteDeprecationInfo} @@ -238,8 +246,10 @@ export function systemRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/index_exists`, access: 'internal', - options: { - tags: ['access:ml:canGetFieldInfo'], + security: { + authz: { + requiredPrivileges: ['ml:canGetFieldInfo'], + }, }, summary: 'ES Field caps wrapper checks if index exists', }) @@ -281,8 +291,10 @@ export function systemRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/reindex_with_pipeline`, access: 'internal', - options: { - tags: ['access:ml:canCreateTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateTrainedModels'], + }, }, summary: 'ES reindex wrapper to reindex with pipeline', }) diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index 15563f7463265..c0010777ecf18 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -108,8 +108,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId?}`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get info of a trained inference model', description: 'Retrieves configuration information for a trained model.', @@ -278,8 +280,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get stats for all trained models', description: 'Retrieves usage information for all trained models.', @@ -307,8 +311,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}/_stats`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get stats for a trained model', description: 'Retrieves usage information for a trained model.', @@ -342,8 +348,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}/pipelines`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get trained model pipelines', description: 'Retrieves ingest pipelines associated with a trained model.', @@ -376,8 +384,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/ingest_pipelines`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], // TODO: update permissions + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get ingest pipelines', description: 'Retrieves ingest pipelines.', @@ -403,8 +413,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/create_inference_pipeline`, access: 'internal', - options: { - tags: ['access:ml:canCreateTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateTrainedModels'], + }, }, summary: 'Create an inference pipeline', description: 'Creates a pipeline with inference processor', @@ -438,8 +450,10 @@ export function trainedModelsRoutes( .put({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateTrainedModels'], + }, }, summary: 'Put a trained model', description: 'Adds a new trained model', @@ -478,8 +492,10 @@ export function trainedModelsRoutes( .delete({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}`, access: 'internal', - options: { - tags: ['access:ml:canDeleteTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canDeleteTrainedModels'], + }, }, summary: 'Delete a trained model', description: @@ -523,8 +539,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}/deployment/_start`, access: 'internal', - options: { - tags: ['access:ml:canStartStopTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopTrainedModels'], + }, }, summary: 'Start trained model deployment', description: 'Starts trained model deployment.', @@ -569,8 +587,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}/{deploymentId}/deployment/_update`, access: 'internal', - options: { - tags: ['access:ml:canStartStopTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopTrainedModels'], + }, }, summary: 'Update trained model deployment', description: 'Updates trained model deployment.', @@ -604,8 +624,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/{modelId}/{deploymentId}/deployment/_stop`, access: 'internal', - options: { - tags: ['access:ml:canStartStopTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canStartStopTrainedModels'], + }, }, summary: 'Stop trained model deployment', description: 'Stops trained model deployment.', @@ -653,8 +675,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/pipeline_simulate`, access: 'internal', - options: { - tags: ['access:ml:canTestTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canTestTrainedModels'], + }, }, summary: 'Simulates an ingest pipeline', description: 'Simulates an ingest pipeline.', @@ -688,8 +712,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/infer/{modelId}/{deploymentId}`, access: 'internal', - options: { - tags: ['access:ml:canTestTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canTestTrainedModels'], + }, }, summary: 'Evaluates a trained model.', description: 'Evaluates a trained model.', @@ -732,8 +758,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/model_downloads`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get available models for download', description: @@ -761,8 +789,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/elser_config`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get ELSER config for download', description: 'Gets ELSER config for download based on the cluster OS and CPU architecture.', @@ -797,8 +827,10 @@ export function trainedModelsRoutes( .post({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/install_elastic_trained_model/{modelId}`, access: 'internal', - options: { - tags: ['access:ml:canCreateTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateTrainedModels'], + }, }, summary: 'Install Elastic trained model', description: 'Downloads and installs Elastic trained model.', @@ -835,8 +867,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/download_status`, access: 'internal', - options: { - tags: ['access:ml:canCreateTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canCreateTrainedModels'], + }, }, summary: 'Get models download status', description: 'Gets download status for all currently downloading models.', @@ -865,8 +899,10 @@ export function trainedModelsRoutes( .get({ path: `${ML_INTERNAL_BASE_PATH}/trained_models/curated_model_config/{modelName}`, access: 'internal', - options: { - tags: ['access:ml:canGetTrainedModels'], + security: { + authz: { + requiredPrivileges: ['ml:canGetTrainedModels'], + }, }, summary: 'Get curated model config', description: diff --git a/x-pack/test/api_integration/apis/ml/annotations/create_annotations.ts b/x-pack/test/api_integration/apis/ml/annotations/create_annotations.ts index 192177a086d22..68daee02e5d36 100644 --- a/x-pack/test/api_integration/apis/ml/annotations/create_annotations.ts +++ b/x-pack/test/api_integration/apis/ml/annotations/create_annotations.ts @@ -84,7 +84,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/annotations/delete_annotations.ts b/x-pack/test/api_integration/apis/ml/annotations/delete_annotations.ts index 1c23805e264d1..499142a09f7d1 100644 --- a/x-pack/test/api_integration/apis/ml/annotations/delete_annotations.ts +++ b/x-pack/test/api_integration/apis/ml/annotations/delete_annotations.ts @@ -84,7 +84,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); await ml.api.waitForAnnotationToExist(annotationIdToDelete); }); diff --git a/x-pack/test/api_integration/apis/ml/annotations/get_annotations.ts b/x-pack/test/api_integration/apis/ml/annotations/get_annotations.ts index 00cfda209b4fb..38c0c9d22401f 100644 --- a/x-pack/test/api_integration/apis/ml/annotations/get_annotations.ts +++ b/x-pack/test/api_integration/apis/ml/annotations/get_annotations.ts @@ -126,7 +126,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/annotations/update_annotations.ts b/x-pack/test/api_integration/apis/ml/annotations/update_annotations.ts index 6b7c437eb77ca..c4ae62aafef7c 100644 --- a/x-pack/test/api_integration/apis/ml/annotations/update_annotations.ts +++ b/x-pack/test/api_integration/apis/ml/annotations/update_annotations.ts @@ -129,7 +129,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); const updatedAnnotation = await ml.api.getAnnotationById(originalAnnotation._id!); expect(updatedAnnotation).to.eql(originalAnnotation._source); diff --git a/x-pack/test/api_integration/apis/ml/anomaly_detectors/create.ts b/x-pack/test/api_integration/apis/ml/anomaly_detectors/create.ts index 44854dcaeece6..7aa328bc39d2e 100644 --- a/x-pack/test/api_integration/apis/ml/anomaly_detectors/create.ts +++ b/x-pack/test/api_integration/apis/ml/anomaly_detectors/create.ts @@ -92,7 +92,6 @@ export default ({ getService }: FtrProviderContext) => { responseBody: { statusCode: 403, error: 'Forbidden', - message: 'Forbidden', }, }, }, @@ -133,7 +132,6 @@ export default ({ getService }: FtrProviderContext) => { ); } else { expect(body.error).to.eql(testData.expected.responseBody.error); - expect(body.message).to.eql(testData.expected.responseBody.message); } }); } diff --git a/x-pack/test/api_integration/apis/ml/anomaly_detectors/get.ts b/x-pack/test/api_integration/apis/ml/anomaly_detectors/get.ts index 4dcd8f0d8c98c..b3eae761486ad 100644 --- a/x-pack/test/api_integration/apis/ml/anomaly_detectors/get.ts +++ b/x-pack/test/api_integration/apis/ml/anomaly_detectors/get.ts @@ -91,7 +91,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -129,7 +128,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -162,7 +160,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -214,7 +211,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/calendars/create_calendars.ts b/x-pack/test/api_integration/apis/ml/calendars/create_calendars.ts index fdd3d6b2806fc..352264b599118 100644 --- a/x-pack/test/api_integration/apis/ml/calendars/create_calendars.ts +++ b/x-pack/test/api_integration/apis/ml/calendars/create_calendars.ts @@ -64,7 +64,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); await ml.api.waitForCalendarNotToExist(calendarId); }); @@ -77,7 +76,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); await ml.api.waitForCalendarNotToExist(calendarId); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/create_job.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/create_job.ts index bc256d71e4a3e..64745218555a5 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/create_job.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/create_job.ts @@ -168,7 +168,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it('should not allow analytics job creation for the user with only view permission', async () => { @@ -183,7 +182,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts index f7e3d16666342..2a2b062e4f07c 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts @@ -96,7 +96,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); await ml.api.waitForDataFrameAnalyticsJobToExist(analyticsId); }); @@ -109,7 +108,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); await ml.api.waitForDataFrameAnalyticsJobToExist(analyticsId); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/evaluate.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/evaluate.ts index 03cddf6a0668e..7515f8ddc2b87 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/evaluate.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/evaluate.ts @@ -185,7 +185,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/explain.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/explain.ts index f270834e7da71..a50ae4b824dca 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/explain.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/explain.ts @@ -119,7 +119,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it(`should not allow unauthorized user to use explain endpoint for ${testConfig.jobType} job`, async () => { @@ -131,7 +130,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/get.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/get.ts index 2459f81b188b7..370542b585cae 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/get.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/get.ts @@ -115,7 +115,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -153,7 +152,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -186,7 +184,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -238,7 +235,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); @@ -307,7 +303,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/start.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/start.ts index dd1f9fd33aaf0..5a0b1fb0d5451 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/start.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/start.ts @@ -118,7 +118,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it('should not allow to start analytics job for user with view only permission', async () => { @@ -131,7 +130,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/stop.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/stop.ts index 972a78e433932..e5084deb4e13d 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/stop.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/stop.ts @@ -73,7 +73,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it('should not allow to stop analytics job for user with view only permission', async () => { @@ -84,7 +83,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/update.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/update.ts index cb2854723bfba..f15e63af61608 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/update.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/update.ts @@ -226,7 +226,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); const fetchedJob = await getDFAJob(analyticsId); // Description should not have changed @@ -247,7 +246,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); const fetchedJob = await getDFAJob(analyticsId); // Description should not have changed diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/validate.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/validate.ts index b274a1bae4fbc..f16039ef79085 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/validate.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/validate.ts @@ -114,7 +114,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it('should not allow analytics job validation for the user with only view permission', async () => { @@ -128,7 +127,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }); diff --git a/x-pack/test/api_integration/apis/ml/filters/create_filters.ts b/x-pack/test/api_integration/apis/ml/filters/create_filters.ts index 91d468593df82..00f230b883569 100644 --- a/x-pack/test/api_integration/apis/ml/filters/create_filters.ts +++ b/x-pack/test/api_integration/apis/ml/filters/create_filters.ts @@ -42,7 +42,8 @@ export default ({ getService }: FtrProviderContext) => { responseCode: 403, responseBody: { error: 'Forbidden', - message: 'Forbidden', + message: + 'API [PUT /internal/ml/filters] is unauthorized for user, this action is granted by the Kibana privileges [ml:canCreateFilter]', }, }, }, @@ -58,7 +59,8 @@ export default ({ getService }: FtrProviderContext) => { responseCode: 403, responseBody: { error: 'Forbidden', - message: 'Forbidden', + message: + 'API [PUT /internal/ml/filters] is unauthorized for user, this action is granted by the Kibana privileges [ml:canCreateFilter]', }, }, }, diff --git a/x-pack/test/api_integration/apis/ml/filters/get_filters.ts b/x-pack/test/api_integration/apis/ml/filters/get_filters.ts index 791d14ad24089..64d78ac795090 100644 --- a/x-pack/test/api_integration/apis/ml/filters/get_filters.ts +++ b/x-pack/test/api_integration/apis/ml/filters/get_filters.ts @@ -60,7 +60,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it(`should not allow to retrieve filters for unauthorized user`, async () => { @@ -71,7 +70,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it(`should fetch single filter by id`, async () => { diff --git a/x-pack/test/api_integration/apis/ml/filters/get_filters_stats.ts b/x-pack/test/api_integration/apis/ml/filters/get_filters_stats.ts index c0c2e115eb7ba..06f6d17466322 100644 --- a/x-pack/test/api_integration/apis/ml/filters/get_filters_stats.ts +++ b/x-pack/test/api_integration/apis/ml/filters/get_filters_stats.ts @@ -212,7 +212,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); it(`should not allow retrieving filters stats for unauthorized user`, async () => { @@ -223,7 +222,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/job_audit_messages/clear_messages.ts b/x-pack/test/api_integration/apis/ml/job_audit_messages/clear_messages.ts index 4d266055dc54a..a0e214ffe7882 100644 --- a/x-pack/test/api_integration/apis/ml/job_audit_messages/clear_messages.ts +++ b/x-pack/test/api_integration/apis/ml/job_audit_messages/clear_messages.ts @@ -92,7 +92,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); const { body: getBody, status: getStatus } = await supertest .get(`/internal/ml/job_audit_messages/messages/test_get_job_audit_messages_2`) @@ -115,7 +114,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); const { body: getBody, status: getStatus } = await supertest .get(`/internal/ml/job_audit_messages/messages/test_get_job_audit_messages_2`) diff --git a/x-pack/test/api_integration/apis/ml/job_audit_messages/get_job_audit_messages.ts b/x-pack/test/api_integration/apis/ml/job_audit_messages/get_job_audit_messages.ts index 907624586a641..02a4b6e8be3e8 100644 --- a/x-pack/test/api_integration/apis/ml/job_audit_messages/get_job_audit_messages.ts +++ b/x-pack/test/api_integration/apis/ml/job_audit_messages/get_job_audit_messages.ts @@ -118,7 +118,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/job_validation/cardinality.ts b/x-pack/test/api_integration/apis/ml/job_validation/cardinality.ts index 7f625efc9d776..99663f5c57a81 100644 --- a/x-pack/test/api_integration/apis/ml/job_validation/cardinality.ts +++ b/x-pack/test/api_integration/apis/ml/job_validation/cardinality.ts @@ -191,7 +191,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/job_validation/validate.ts b/x-pack/test/api_integration/apis/ml/job_validation/validate.ts index b1481ffe183d5..58dbd5560e661 100644 --- a/x-pack/test/api_integration/apis/ml/job_validation/validate.ts +++ b/x-pack/test/api_integration/apis/ml/job_validation/validate.ts @@ -285,7 +285,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts index def9774a55ddc..c8d5b12f10f55 100644 --- a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts @@ -669,7 +669,8 @@ export default ({ getService }: FtrProviderContext) => { expected: { responseCode: 403, error: 'Forbidden', - message: 'Forbidden', + message: + 'API [POST /internal/ml/modules/setup/sample_data_weblogs] is unauthorized for user, this action is granted by the Kibana privileges [ml:canCreateJob]', }, }, ]; diff --git a/x-pack/test/api_integration/apis/ml/results/get_anomalies_table_data.ts b/x-pack/test/api_integration/apis/ml/results/get_anomalies_table_data.ts index 5cff7bca0981b..d495015b00a51 100644 --- a/x-pack/test/api_integration/apis/ml/results/get_anomalies_table_data.ts +++ b/x-pack/test/api_integration/apis/ml/results/get_anomalies_table_data.ts @@ -130,7 +130,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts b/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts index 87154beb07efe..9b5c945047ac9 100644 --- a/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts +++ b/x-pack/test/api_integration/apis/ml/results/get_categorizer_stats.ts @@ -102,7 +102,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.be('Forbidden'); - expect(body.message).to.be('Forbidden'); }); it('should fetch all the categorizer stats with per-partition value for job id', async () => { @@ -146,7 +145,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.be('Forbidden'); - expect(body.message).to.be('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/results/get_datafeed_results_chart.ts b/x-pack/test/api_integration/apis/ml/results/get_datafeed_results_chart.ts index 07e544fd1ec2c..d8b632dbc8657 100644 --- a/x-pack/test/api_integration/apis/ml/results/get_datafeed_results_chart.ts +++ b/x-pack/test/api_integration/apis/ml/results/get_datafeed_results_chart.ts @@ -120,7 +120,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.eql('Forbidden'); - expect(body.message).to.eql('Forbidden'); }); }); }; diff --git a/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts b/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts index 3dc4686102c3d..6e546df2a58e1 100644 --- a/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts +++ b/x-pack/test/api_integration/apis/ml/results/get_stopped_partitions.ts @@ -152,7 +152,6 @@ export default ({ getService }: FtrProviderContext) => { ml.api.assertResponseStatusCode(403, status, body); expect(body.error).to.be('Forbidden'); - expect(body.message).to.be('Forbidden'); }); it('should fetch stopped partitions for multiple job ids', async () => { diff --git a/x-pack/test/api_integration/apis/ml/system/has_privileges.ts b/x-pack/test/api_integration/apis/ml/system/has_privileges.ts index 2e705240b403e..ac4872ec9c70f 100644 --- a/x-pack/test/api_integration/apis/ml/system/has_privileges.ts +++ b/x-pack/test/api_integration/apis/ml/system/has_privileges.ts @@ -7,7 +7,6 @@ import expect from '@kbn/expect'; -import { MlHasPrivilegesResponse } from '@kbn/ml-plugin/public/application/services/ml_api_service'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getCommonRequestHeader } from '../../../../functional/services/ml/common_api'; import { USER } from '../../../../functional/services/ml/security_common'; @@ -17,11 +16,7 @@ export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertestWithoutAuth'); const ml = getService('ml'); - async function runRequest( - user: USER, - index: any, - expectedStatusCode = 200 - ): Promise { + async function runRequest(user: USER, index: any, expectedStatusCode = 200) { const { body, status } = await supertest .post(`/internal/ml/_has_privileges`) .auth(user, ml.securityCommon.getPasswordForUser(user)) @@ -104,7 +99,10 @@ export default ({ getService }: FtrProviderContext) => { privileges: ['write'], }, ], - expectedResponse: { statusCode: 403, error: 'Forbidden', message: 'Forbidden' }, + expectedResponse: { + statusCode: 403, + error: 'Forbidden', + }, expectedStatusCode: 403, }, ]; @@ -120,9 +118,17 @@ export default ({ getService }: FtrProviderContext) => { it('should return correct privileges for test data', async () => { for (const { user, index, expectedResponse, expectedStatusCode } of testData) { const response = await runRequest(user, index, expectedStatusCode); - expect(response).to.eql( - expectedResponse, - `expected ${JSON.stringify(expectedResponse)}, got ${JSON.stringify(response)}` + expect(response.statusCode).to.eql( + expectedResponse.statusCode, + `expected ${JSON.stringify(expectedResponse.statusCode)}, got ${JSON.stringify( + response.statusCode + )}` + ); + expect(response.error).to.eql( + expectedResponse.error, + `expected ${JSON.stringify(expectedResponse.error)}, got ${JSON.stringify( + response.error + )}` ); } }); From fcc939281d504a368f96c89d302c860ccb7c4dfe Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Wed, 13 Nov 2024 08:13:54 -0500 Subject: [PATCH 13/53] [OAS][Docs] Use correct bump dependency in makefile (#199876) ## Summary Updates the Open API docs make targets to use the correct bump.sh dependency. Unless I'm missing something obvious in my local configuration, `@npx bump ...` uses [node-bump](https://www.npmjs.com/package/bump) not the [bump.sh library](https://www.npmjs.com/package/bump-cli). I discovered this while trying to run the make targets locally. --- oas_docs/makefile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/oas_docs/makefile b/oas_docs/makefile index 2ea80877771c3..1bfd2b8db2323 100644 --- a/oas_docs/makefile +++ b/oas_docs/makefile @@ -40,14 +40,14 @@ api-docs-lint-serverless: ## Run redocly API docs linter on kibana.serverless.ya .PHONY: api-docs-overlay api-docs-overlay: ## Run spectral API docs linter on kibana.serverless.yaml - @npx bump overlay "output/kibana.serverless.yaml" "overlays/kibana.overlays.serverless.yaml" > "output/kibana.serverless.tmp1.yaml" - @npx bump overlay "output/kibana.serverless.tmp1.yaml" "overlays/alerting.overlays.yaml" > "output/kibana.serverless.tmp2.yaml" - @npx bump overlay "output/kibana.serverless.tmp2.yaml" "overlays/connectors.overlays.yaml" > "output/kibana.serverless.tmp3.yaml" - @npx bump overlay "output/kibana.serverless.tmp3.yaml" "overlays/kibana.overlays.shared.yaml" > "output/kibana.serverless.tmp4.yaml" - @npx bump overlay "output/kibana.yaml" "overlays/kibana.overlays.yaml" > "output/kibana.tmp1.yaml" - @npx bump overlay "output/kibana.tmp1.yaml" "overlays/alerting.overlays.yaml" > "output/kibana.tmp2.yaml" - @npx bump overlay "output/kibana.tmp2.yaml" "overlays/connectors.overlays.yaml" > "output/kibana.tmp3.yaml" - @npx bump overlay "output/kibana.tmp3.yaml" "overlays/kibana.overlays.shared.yaml" > "output/kibana.tmp4.yaml" + @npx bump-cli overlay "output/kibana.serverless.yaml" "overlays/kibana.overlays.serverless.yaml" > "output/kibana.serverless.tmp1.yaml" + @npx bump-cli overlay "output/kibana.serverless.tmp1.yaml" "overlays/alerting.overlays.yaml" > "output/kibana.serverless.tmp2.yaml" + @npx bump-cli overlay "output/kibana.serverless.tmp2.yaml" "overlays/connectors.overlays.yaml" > "output/kibana.serverless.tmp3.yaml" + @npx bump-cli overlay "output/kibana.serverless.tmp3.yaml" "overlays/kibana.overlays.shared.yaml" > "output/kibana.serverless.tmp4.yaml" + @npx bump-cli overlay "output/kibana.yaml" "overlays/kibana.overlays.yaml" > "output/kibana.tmp1.yaml" + @npx bump-cli overlay "output/kibana.tmp1.yaml" "overlays/alerting.overlays.yaml" > "output/kibana.tmp2.yaml" + @npx bump-cli overlay "output/kibana.tmp2.yaml" "overlays/connectors.overlays.yaml" > "output/kibana.tmp3.yaml" + @npx bump-cli overlay "output/kibana.tmp3.yaml" "overlays/kibana.overlays.shared.yaml" > "output/kibana.tmp4.yaml" @npx @redocly/cli bundle output/kibana.serverless.tmp4.yaml --ext yaml -o output/kibana.serverless.new.yaml @npx @redocly/cli bundle output/kibana.tmp4.yaml --ext yaml -o output/kibana.new.yaml rm output/kibana.tmp*.yaml @@ -55,13 +55,13 @@ api-docs-overlay: ## Run spectral API docs linter on kibana.serverless.yaml .PHONY: api-docs-preview api-docs-preview: ## Generate a preview for kibana.yaml and kibana.serverless.yaml - @npx bump preview "output/kibana.yaml" - @npx bump preview "output/kibana.serverless.yaml" + @npx bump-cli preview "output/kibana.yaml" + @npx bump-cli preview "output/kibana.serverless.yaml" .PHONY: api-docs-overlay-preview api-docs-overlay-preview: ## Generate a preview for kibana.new.yaml and kibana.serverless.new.yaml - @npx bump preview "output/kibana.new.yaml" - @npx bump preview "output/kibana.serverless.new.yaml" + @npx bump-cli preview "output/kibana.new.yaml" + @npx bump-cli preview "output/kibana.serverless.new.yaml" help: ## Display help @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) From 710c4cc9b147260f027e5103167599f7a0fcfdec Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 13 Nov 2024 14:33:56 +0100 Subject: [PATCH 14/53] Remove functionbeat tutorial and translations (#199301) ## Summary Remove functionbeat tutorial and translations. It's been deprecated and won't be shipped on 9.0 ### Checklist Delete any items that are not applicable to this PR. ### Risk Matrix I'm not sure, its just deleting code/documentation ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels) - [ ] This will appear in the **Release Notes** and follow the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) ## Related issues - Relates https://github.com/elastic/beats/issues/40745 - https://github.com/elastic/kibana/issues/193030 --- api_docs/telemetry.devdocs.json | 4 +- .../server/tutorials/cloudwatch_logs/index.ts | 54 -- .../instructions/functionbeat_instructions.ts | 538 ------------------ src/plugins/home/server/tutorials/register.ts | 2 - .../apis/custom_integration/integrations.ts | 2 +- .../translations/translations/fr-FR.json | 38 -- .../translations/translations/ja-JP.json | 37 -- .../translations/translations/zh-CN.json | 38 -- 8 files changed, 3 insertions(+), 710 deletions(-) delete mode 100644 src/plugins/home/server/tutorials/cloudwatch_logs/index.ts delete mode 100644 src/plugins/home/server/tutorials/instructions/functionbeat_instructions.ts diff --git a/api_docs/telemetry.devdocs.json b/api_docs/telemetry.devdocs.json index 4ea397d13220c..e5f56d87cc9ba 100644 --- a/api_docs/telemetry.devdocs.json +++ b/api_docs/telemetry.devdocs.json @@ -630,7 +630,7 @@ "When the data comes from a matching index-pattern, the name of the pattern" ], "signature": [ - "\"search\" | \"logstash\" | \"alerts\" | \"apm\" | \"metricbeat\" | \"suricata\" | \"zeek\" | \"enterprise-search\" | \"app-search\" | \"magento2\" | \"magento\" | \"shopify\" | \"wordpress\" | \"drupal\" | \"joomla\" | \"sharepoint\" | \"squarespace\" | \"sitecore\" | \"weebly\" | \"acquia\" | \"filebeat\" | \"generic-filebeat\" | \"generic-metricbeat\" | \"functionbeat\" | \"generic-functionbeat\" | \"heartbeat\" | \"generic-heartbeat\" | \"generic-logstash\" | \"fluentd\" | \"telegraf\" | \"prometheusbeat\" | \"fluentbit\" | \"nginx\" | \"apache\" | \"dsns-logs\" | \"generic-logs\" | \"endgame\" | \"logs-endpoint\" | \"metrics-endpoint\" | \"siem-signals\" | \"auditbeat\" | \"winlogbeat\" | \"packetbeat\" | \"tomcat\" | \"artifactory\" | \"aruba\" | \"barracuda\" | \"bluecoat\" | \"arcsight\" | \"checkpoint\" | \"cisco\" | \"citrix\" | \"cyberark\" | \"cylance\" | \"fireeye\" | \"fortinet\" | \"infoblox\" | \"kaspersky\" | \"mcafee\" | \"paloaltonetworks\" | \"rsa\" | \"snort\" | \"sonicwall\" | \"sophos\" | \"squid\" | \"symantec\" | \"tippingpoint\" | \"trendmicro\" | \"tripwire\" | \"zscaler\" | \"sigma_doc\" | \"ecs-corelight\" | \"wazuh\" | \"meow\" | \"host_risk_score\" | \"user_risk_score\" | undefined" + "\"search\" | \"logstash\" | \"alerts\" | \"apm\" | \"metricbeat\" | \"suricata\" | \"zeek\" | \"enterprise-search\" | \"app-search\" | \"magento2\" | \"magento\" | \"shopify\" | \"wordpress\" | \"drupal\" | \"joomla\" | \"sharepoint\" | \"squarespace\" | \"sitecore\" | \"weebly\" | \"acquia\" | \"filebeat\" | \"generic-filebeat\" | \"generic-metricbeat\" | \"heartbeat\" | \"generic-heartbeat\" | \"generic-logstash\" | \"fluentd\" | \"telegraf\" | \"prometheusbeat\" | \"fluentbit\" | \"nginx\" | \"apache\" | \"dsns-logs\" | \"generic-logs\" | \"endgame\" | \"logs-endpoint\" | \"metrics-endpoint\" | \"siem-signals\" | \"auditbeat\" | \"winlogbeat\" | \"packetbeat\" | \"tomcat\" | \"artifactory\" | \"aruba\" | \"barracuda\" | \"bluecoat\" | \"arcsight\" | \"checkpoint\" | \"cisco\" | \"citrix\" | \"cyberark\" | \"cylance\" | \"fireeye\" | \"fortinet\" | \"infoblox\" | \"kaspersky\" | \"mcafee\" | \"paloaltonetworks\" | \"rsa\" | \"snort\" | \"sonicwall\" | \"sophos\" | \"squid\" | \"symantec\" | \"tippingpoint\" | \"trendmicro\" | \"tripwire\" | \"zscaler\" | \"sigma_doc\" | \"ecs-corelight\" | \"wazuh\" | \"meow\" | \"host_risk_score\" | \"user_risk_score\" | undefined" ], "path": "src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts", "deprecated": false, @@ -860,4 +860,4 @@ "misc": [], "objects": [] } -} \ No newline at end of file +} diff --git a/src/plugins/home/server/tutorials/cloudwatch_logs/index.ts b/src/plugins/home/server/tutorials/cloudwatch_logs/index.ts deleted file mode 100644 index a6e0213ddc366..0000000000000 --- a/src/plugins/home/server/tutorials/cloudwatch_logs/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { i18n } from '@kbn/i18n'; -import { TutorialsCategory } from '../../services/tutorials'; -import { - onPremInstructions, - cloudInstructions, - onPremCloudInstructions, -} from '../instructions/functionbeat_instructions'; -import { - TutorialContext, - TutorialSchema, -} from '../../services/tutorials/lib/tutorials_registry_types'; - -export function cloudwatchLogsSpecProvider(context: TutorialContext): TutorialSchema { - const moduleName = 'aws'; - return { - id: 'cloudwatchLogs', - name: i18n.translate('home.tutorials.cloudwatchLogs.nameTitle', { - defaultMessage: 'AWS Cloudwatch Logs', - }), - moduleName, - category: TutorialsCategory.LOGGING, - shortDescription: i18n.translate('home.tutorials.cloudwatchLogs.shortDescription', { - defaultMessage: 'Collect and parse logs from AWS Cloudwatch with Functionbeat.', - }), - longDescription: i18n.translate('home.tutorials.cloudwatchLogs.longDescription', { - defaultMessage: - 'Collect Cloudwatch logs by deploying Functionbeat to run as \ - an AWS Lambda function.', - }), - euiIconType: 'logoAWS', - artifacts: { - dashboards: [ - // TODO - ], - exportedFields: { - documentationUrl: '{config.docs.beats.functionbeat}/exported-fields.html', - }, - }, - completionTimeMinutes: 10, - onPrem: onPremInstructions([], context), - elasticCloud: cloudInstructions(context), - onPremElasticCloud: onPremCloudInstructions(context), - integrationBrowserCategories: ['aws', 'observability', 'monitoring'], - }; -} diff --git a/src/plugins/home/server/tutorials/instructions/functionbeat_instructions.ts b/src/plugins/home/server/tutorials/instructions/functionbeat_instructions.ts deleted file mode 100644 index 74369f044a19a..0000000000000 --- a/src/plugins/home/server/tutorials/instructions/functionbeat_instructions.ts +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { i18n } from '@kbn/i18n'; -import { INSTRUCTION_VARIANT } from '../../../common/instruction_variant'; -import { createTrycloudOption1, createTrycloudOption2 } from './onprem_cloud_instructions'; -import { getSpaceIdForBeatsTutorial } from './get_space_id_for_beats_tutorial'; -import { Platform, TutorialContext } from '../../services/tutorials/lib/tutorials_registry_types'; -import { cloudPasswordAndResetLink } from './cloud_instructions'; - -export const createFunctionbeatInstructions = (context: TutorialContext) => { - const SSL_DOC_URL = `https://www.elastic.co/guide/en/beats/functionbeat/${context.kibanaBranch}/configuration-ssl.html#ca-sha256`; - - return { - INSTALL: { - OSX: { - title: i18n.translate('home.tutorials.common.functionbeatInstructions.install.osxTitle', { - defaultMessage: 'Download and install Functionbeat', - }), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.install.osxTextPre', - { - defaultMessage: 'First time using Functionbeat? See the [Quick Start]({link}).', - values: { - link: '{config.docs.beats.functionbeat}/functionbeat-installation-configuration.html', - }, - } - ), - commands: [ - 'curl -L -O https://artifacts.elastic.co/downloads/beats/functionbeat/functionbeat-{config.kibana.version}-darwin-x86_64.tar.gz', - 'tar xzvf functionbeat-{config.kibana.version}-darwin-x86_64.tar.gz', - 'cd functionbeat-{config.kibana.version}-darwin-x86_64/', - ], - }, - LINUX: { - title: i18n.translate('home.tutorials.common.functionbeatInstructions.install.linuxTitle', { - defaultMessage: 'Download and install Functionbeat', - }), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.install.linuxTextPre', - { - defaultMessage: 'First time using Functionbeat? See the [Quick Start]({link}).', - values: { - link: '{config.docs.beats.functionbeat}/functionbeat-installation-configuration.html', - }, - } - ), - commands: [ - 'curl -L -O https://artifacts.elastic.co/downloads/beats/functionbeat/functionbeat-{config.kibana.version}-linux-x86_64.tar.gz', - 'tar xzvf functionbeat-{config.kibana.version}-linux-x86_64.tar.gz', - 'cd functionbeat-{config.kibana.version}-linux-x86_64/', - ], - }, - WINDOWS: { - title: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.install.windowsTitle', - { - defaultMessage: 'Download and install Functionbeat', - } - ), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.install.windowsTextPre', - { - defaultMessage: - 'First time using Functionbeat? See the [Quick Start]({functionbeatLink}).\n\ - 1. Download the Functionbeat Windows zip file from the [Download]({elasticLink}) page.\n\ - 2. Extract the contents of the zip file into {folderPath}.\n\ - 3. Rename the {directoryName} directory to `Functionbeat`.\n\ - 4. Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select \ -**Run As Administrator**). If you are running Windows XP, you might need to download and install PowerShell.\n\ - 5. From the PowerShell prompt, go to the Functionbeat directory:', - values: { - directoryName: '`functionbeat-{config.kibana.version}-windows`', - folderPath: '`C:\\Program Files`', - functionbeatLink: - '{config.docs.beats.functionbeat}/functionbeat-installation-configuration.html', - elasticLink: 'https://www.elastic.co/downloads/beats/functionbeat', - }, - } - ), - commands: ['cd "C:\\Program Files\\Functionbeat"'], - }, - }, - DEPLOY: { - OSX_LINUX: { - title: i18n.translate('home.tutorials.common.functionbeatInstructions.deploy.osxTitle', { - defaultMessage: 'Deploy Functionbeat to AWS Lambda', - }), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.deploy.osxTextPre', - { - defaultMessage: - 'This installs Functionbeat as a Lambda function.\ -The `setup` command checks the Elasticsearch configuration and loads the \ -Kibana index pattern. It is normally safe to omit this command.', - } - ), - commands: ['./functionbeat setup', './functionbeat deploy fn-cloudwatch-logs'], - }, - WINDOWS: { - title: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.deploy.windowsTitle', - { - defaultMessage: 'Deploy Functionbeat to AWS Lambda', - } - ), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.deploy.windowsTextPre', - { - defaultMessage: - 'This installs Functionbeat as a Lambda function.\ -The `setup` command checks the Elasticsearch configuration and loads the \ -Kibana index pattern. It is normally safe to omit this command.', - } - ), - commands: ['.\\functionbeat.exe setup', '.\\functionbeat.exe deploy fn-cloudwatch-logs'], - }, - }, - CONFIG: { - OSX_LINUX: { - title: i18n.translate('home.tutorials.common.functionbeatInstructions.config.osxTitle', { - defaultMessage: 'Configure the Elastic cluster', - }), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.config.osxTextPre', - { - defaultMessage: 'Modify {path} to set the connection information:', - values: { - path: '`functionbeat.yml`', - }, - } - ), - commands: [ - 'output.elasticsearch:', - ' hosts: [""]', - ' username: "elastic"', - ' password: ""', - " # If using Elasticsearch's default certificate", - ' ssl.ca_trusted_fingerprint: ""', - 'setup.kibana:', - ' host: ""', - getSpaceIdForBeatsTutorial(context), - ], - textPost: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.config.osxTextPostMarkdown', - { - defaultMessage: - 'Where {passwordTemplate} is the password of the `elastic` user, {esUrlTemplate} is the URL of \ - Elasticsearch, and {kibanaUrlTemplate} is the URL of Kibana. To [configure SSL]({configureSslUrl}) with the \ - default certificate generated by Elasticsearch, add its fingerprint in {esCertFingerprintTemplate}.\n\n\ - > **_Important:_** Do not use the built-in `elastic` user to secure clients in a production environment. Instead set up \ - authorized users or API keys, and do not expose passwords in configuration files. [Learn more]({linkUrl}).', - values: { - passwordTemplate: '``', - esUrlTemplate: '``', - kibanaUrlTemplate: '``', - configureSslUrl: SSL_DOC_URL, - esCertFingerprintTemplate: '``', - linkUrl: '{config.docs.beats.functionbeat}/securing-functionbeat.html', - }, - } - ), - }, - WINDOWS: { - title: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.config.windowsTitle', - { - defaultMessage: 'Edit the configuration', - } - ), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.config.windowsTextPre', - { - defaultMessage: 'Modify {path} to set the connection information:', - values: { - path: '`C:\\Program Files\\Functionbeat\\functionbeat.yml`', - }, - } - ), - commands: [ - 'output.elasticsearch:', - ' hosts: [""]', - ' username: "elastic"', - ' password: ""', - " # If using Elasticsearch's default certificate", - ' ssl.ca_trusted_fingerprint: ""', - 'setup.kibana:', - ' host: ""', - getSpaceIdForBeatsTutorial(context), - ], - textPost: i18n.translate( - 'home.tutorials.common.functionbeatInstructions.config.windowsTextPostMarkdown', - { - defaultMessage: - 'Where {passwordTemplate} is the password of the `elastic` user, {esUrlTemplate} is the URL of \ - Elasticsearch, and {kibanaUrlTemplate} is the URL of Kibana. To [configure SSL]({configureSslUrl}) with the \ - default certificate generated by Elasticsearch, add its fingerprint in {esCertFingerprintTemplate}.\n\n\ - > **_Important:_** Do not use the built-in `elastic` user to secure clients in a production environment. Instead set up \ - authorized users or API keys, and do not expose passwords in configuration files. [Learn more]({linkUrl}).', - values: { - passwordTemplate: '``', - esUrlTemplate: '``', - kibanaUrlTemplate: '``', - configureSslUrl: SSL_DOC_URL, - esCertFingerprintTemplate: '``', - linkUrl: '{config.docs.beats.functionbeat}/securing-functionbeat.html', - }, - } - ), - }, - }, - }; -}; - -export const createFunctionbeatCloudInstructions = () => ({ - CONFIG: { - OSX_LINUX: { - title: i18n.translate('home.tutorials.common.functionbeatCloudInstructions.config.osxTitle', { - defaultMessage: 'Edit the configuration', - }), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatCloudInstructions.config.osxTextPre', - { - defaultMessage: 'Modify {path} to set the connection information for Elastic Cloud:', - values: { - path: '`functionbeat.yml`', - }, - } - ), - commands: ['cloud.id: "{config.cloud.id}"', 'cloud.auth: "elastic:"'], - textPost: cloudPasswordAndResetLink, - }, - WINDOWS: { - title: i18n.translate( - 'home.tutorials.common.functionbeatCloudInstructions.config.windowsTitle', - { - defaultMessage: 'Edit the configuration', - } - ), - textPre: i18n.translate( - 'home.tutorials.common.functionbeatCloudInstructions.config.windowsTextPre', - { - defaultMessage: 'Modify {path} to set the connection information for Elastic Cloud:', - values: { - path: '`C:\\Program Files\\Functionbeat\\functionbeat.yml`', - }, - } - ), - commands: ['cloud.id: "{config.cloud.id}"', 'cloud.auth: "elastic:"'], - textPost: cloudPasswordAndResetLink, - }, - }, -}); - -export function functionbeatEnableInstructions() { - const defaultTitle = i18n.translate( - 'home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTitle', - { - defaultMessage: 'Configure the Cloudwatch log group', - } - ); - const defaultCommands = [ - 'functionbeat.provider.aws.functions:', - ' - name: fn-cloudwatch-logs', - ' enabled: true', - ' type: cloudwatch_logs', - ' triggers:', - ' - log_group_name: ', - 'functionbeat.provider.aws.deploy_bucket: ', - ]; - const defaultTextPost = i18n.translate( - 'home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTextPost', - { - defaultMessage: - "Where `''` is the name of the log group you want to ingest, \ -and `''` is a valid S3 bucket name which will be used for staging the \ -Functionbeat deploy.", - } - ); - return { - OSX_LINUX: { - title: defaultTitle, - textPre: i18n.translate( - 'home.tutorials.common.functionbeatEnableOnPremInstructionsOSXLinux.textPre', - { - defaultMessage: 'Modify the settings in the `functionbeat.yml` file.', - } - ), - commands: defaultCommands, - textPost: defaultTextPost, - }, - WINDOWS: { - title: defaultTitle, - textPre: i18n.translate( - 'home.tutorials.common.functionbeatEnableOnPremInstructionsWindows.textPre', - { - defaultMessage: 'Modify the settings in the {path} file.', - values: { - path: '`C:\\Program Files\\Functionbeat\\functionbeat.yml`', - }, - } - ), - commands: defaultCommands, - textPost: defaultTextPost, - }, - }; -} - -export function functionbeatAWSInstructions() { - const defaultTitle = i18n.translate('home.tutorials.common.functionbeatAWSInstructions.title', { - defaultMessage: 'Set AWS credentials', - }); - const defaultPre = i18n.translate('home.tutorials.common.functionbeatAWSInstructions.textPre', { - defaultMessage: 'Set your AWS account credentials in the environment:', - }); - const defaultPost = i18n.translate('home.tutorials.common.functionbeatAWSInstructions.textPost', { - defaultMessage: - 'Where {accessKey} and {secretAccessKey} are your account credentials and `us-east-1` is the desired region.', - values: { - accessKey: '``', - secretAccessKey: '``', - }, - }); - - return { - OSX_LINUX: { - title: defaultTitle, - textPre: defaultPre, - commands: [ - 'export AWS_ACCESS_KEY_ID=', - 'export AWS_SECRET_ACCESS_KEY=', - 'export AWS_DEFAULT_REGION=us-east-1', - ], - textPost: defaultPost, - }, - WINDOWS: { - title: defaultTitle, - textPre: defaultPre, - commands: [ - 'set AWS_ACCESS_KEY_ID=', - 'set AWS_SECRET_ACCESS_KEY=', - 'set AWS_DEFAULT_REGION=us-east-1', - ], - textPost: defaultPost, - }, - }; -} - -export function functionbeatStatusCheck() { - return { - title: i18n.translate('home.tutorials.common.functionbeatStatusCheck.title', { - defaultMessage: 'Functionbeat status', - }), - text: i18n.translate('home.tutorials.common.functionbeatStatusCheck.text', { - defaultMessage: 'Check that data is received from Functionbeat', - }), - btnLabel: i18n.translate('home.tutorials.common.functionbeatStatusCheck.buttonLabel', { - defaultMessage: 'Check data', - }), - success: i18n.translate('home.tutorials.common.functionbeatStatusCheck.successText', { - defaultMessage: 'Data successfully received from Functionbeat', - }), - error: i18n.translate('home.tutorials.common.functionbeatStatusCheck.errorText', { - defaultMessage: 'No data has been received from Functionbeat yet', - }), - esHitsCheck: { - index: 'functionbeat-*', - query: { - match_all: {}, - }, - }, - }; -} - -export function onPremInstructions(platforms: Platform[], context: TutorialContext) { - const FUNCTIONBEAT_INSTRUCTIONS = createFunctionbeatInstructions(context); - - return { - instructionSets: [ - { - title: i18n.translate( - 'home.tutorials.common.functionbeat.premInstructions.gettingStarted.title', - { - defaultMessage: 'Getting Started', - } - ), - instructionVariants: [ - { - id: INSTRUCTION_VARIANT.OSX, - instructions: [ - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.OSX, - functionbeatAWSInstructions().OSX_LINUX, - functionbeatEnableInstructions().OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.CONFIG.OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.OSX_LINUX, - ], - }, - { - id: INSTRUCTION_VARIANT.LINUX, - instructions: [ - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.LINUX, - functionbeatAWSInstructions().OSX_LINUX, - functionbeatEnableInstructions().OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.CONFIG.OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.OSX_LINUX, - ], - }, - { - id: INSTRUCTION_VARIANT.WINDOWS, - instructions: [ - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.WINDOWS, - functionbeatAWSInstructions().WINDOWS, - functionbeatEnableInstructions().WINDOWS, - FUNCTIONBEAT_INSTRUCTIONS.CONFIG.WINDOWS, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.WINDOWS, - ], - }, - ], - statusCheck: functionbeatStatusCheck(), - }, - ], - }; -} - -export function onPremCloudInstructions(context: TutorialContext) { - const TRYCLOUD_OPTION1 = createTrycloudOption1(); - const TRYCLOUD_OPTION2 = createTrycloudOption2(); - const FUNCTIONBEAT_INSTRUCTIONS = createFunctionbeatInstructions(context); - - return { - instructionSets: [ - { - title: i18n.translate( - 'home.tutorials.common.functionbeat.premCloudInstructions.gettingStarted.title', - { - defaultMessage: 'Getting Started', - } - ), - instructionVariants: [ - { - id: INSTRUCTION_VARIANT.OSX, - instructions: [ - TRYCLOUD_OPTION1, - TRYCLOUD_OPTION2, - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.OSX, - functionbeatAWSInstructions().OSX_LINUX, - functionbeatEnableInstructions().OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.CONFIG.OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.OSX_LINUX, - ], - }, - { - id: INSTRUCTION_VARIANT.LINUX, - instructions: [ - TRYCLOUD_OPTION1, - TRYCLOUD_OPTION2, - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.LINUX, - functionbeatAWSInstructions().OSX_LINUX, - functionbeatEnableInstructions().OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.CONFIG.OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.OSX_LINUX, - ], - }, - { - id: INSTRUCTION_VARIANT.WINDOWS, - instructions: [ - TRYCLOUD_OPTION1, - TRYCLOUD_OPTION2, - functionbeatAWSInstructions().WINDOWS, - functionbeatEnableInstructions().WINDOWS, - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.WINDOWS, - FUNCTIONBEAT_INSTRUCTIONS.CONFIG.WINDOWS, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.WINDOWS, - ], - }, - ], - statusCheck: functionbeatStatusCheck(), - }, - ], - }; -} - -export function cloudInstructions(context: TutorialContext) { - const FUNCTIONBEAT_INSTRUCTIONS = createFunctionbeatInstructions(context); - const FUNCTIONBEAT_CLOUD_INSTRUCTIONS = createFunctionbeatCloudInstructions(); - - return { - instructionSets: [ - { - title: i18n.translate( - 'home.tutorials.common.functionbeat.cloudInstructions.gettingStarted.title', - { - defaultMessage: 'Getting Started', - } - ), - instructionVariants: [ - { - id: INSTRUCTION_VARIANT.OSX, - instructions: [ - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.OSX, - functionbeatAWSInstructions().OSX_LINUX, - functionbeatEnableInstructions().OSX_LINUX, - FUNCTIONBEAT_CLOUD_INSTRUCTIONS.CONFIG.OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.OSX_LINUX, - ], - }, - { - id: INSTRUCTION_VARIANT.LINUX, - instructions: [ - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.LINUX, - functionbeatAWSInstructions().OSX_LINUX, - functionbeatEnableInstructions().OSX_LINUX, - FUNCTIONBEAT_CLOUD_INSTRUCTIONS.CONFIG.OSX_LINUX, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.OSX_LINUX, - ], - }, - { - id: INSTRUCTION_VARIANT.WINDOWS, - instructions: [ - FUNCTIONBEAT_INSTRUCTIONS.INSTALL.WINDOWS, - functionbeatAWSInstructions().WINDOWS, - functionbeatEnableInstructions().WINDOWS, - FUNCTIONBEAT_CLOUD_INSTRUCTIONS.CONFIG.WINDOWS, - FUNCTIONBEAT_INSTRUCTIONS.DEPLOY.WINDOWS, - ], - }, - ], - statusCheck: functionbeatStatusCheck(), - }, - ], - }; -} diff --git a/src/plugins/home/server/tutorials/register.ts b/src/plugins/home/server/tutorials/register.ts index 55364b99a10c2..7b687a6fbb15a 100644 --- a/src/plugins/home/server/tutorials/register.ts +++ b/src/plugins/home/server/tutorials/register.ts @@ -24,7 +24,6 @@ import { cefLogsSpecProvider } from './cef_logs'; import { cephMetricsSpecProvider } from './ceph_metrics'; import { checkpointLogsSpecProvider } from './checkpoint_logs'; import { ciscoLogsSpecProvider } from './cisco_logs'; -import { cloudwatchLogsSpecProvider } from './cloudwatch_logs'; import { cockroachdbMetricsSpecProvider } from './cockroachdb_metrics'; import { consulMetricsSpecProvider } from './consul_metrics'; import { corednsLogsSpecProvider } from './coredns_logs'; @@ -162,7 +161,6 @@ export const builtInTutorials = [ prometheusMetricsSpecProvider, zookeeperMetricsSpecProvider, uptimeMonitorsSpecProvider, - cloudwatchLogsSpecProvider, awsMetricsSpecProvider, mssqlMetricsSpecProvider, natsMetricsSpecProvider, diff --git a/test/api_integration/apis/custom_integration/integrations.ts b/test/api_integration/apis/custom_integration/integrations.ts index e04a119aa06ea..d4e0ed4397703 100644 --- a/test/api_integration/apis/custom_integration/integrations.ts +++ b/test/api_integration/apis/custom_integration/integrations.ts @@ -44,7 +44,7 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body).to.be.an('array'); - expect(resp.body.length).to.be(108); // the beats + expect(resp.body.length).to.be(107); // the beats }); }); }); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 27d974d49bcaa..5ccbe70bd8fe1 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -4340,9 +4340,6 @@ "home.tutorials.ciscoLogs.longDescription": "Il s'agit d'un module pour les logs de dispositifs réseau Cisco (ASA, FTD, IOS, Nexus). Il inclut les ensembles de fichiers suivants pour la réception des logs par le biais de Syslog ou d'un ficher. [En savoir plus]({learnMoreLink}).", "home.tutorials.ciscoLogs.nameTitle": "Logs Cisco", "home.tutorials.ciscoLogs.shortDescription": "Collectez et analysez les logs à partir des périphériques réseau Cisco avec Filebeat.", - "home.tutorials.cloudwatchLogs.longDescription": "Collectez les logs Cloudwatch en déployant Functionbeat à des fins d'exécution en tant que fonction AWS Lambda.", - "home.tutorials.cloudwatchLogs.nameTitle": "Logs Cloudwatch AWS", - "home.tutorials.cloudwatchLogs.shortDescription": "Collectez et analysez les logs à partir d'AWS Cloudwatch avec Functionbeat.", "home.tutorials.cockroachdbMetrics.artifacts.dashboards.linkLabel": "Tableau de bord des indicateurs CockroachDB", "home.tutorials.cockroachdbMetrics.longDescription": "Le module Metricbeat `cockroachbd` récupère les indicateurs depuis CockroachDB. [En savoir plus]({learnMoreLink}).", "home.tutorials.cockroachdbMetrics.nameTitle": "Indicateurs CockroachDB", @@ -4451,41 +4448,6 @@ "home.tutorials.common.filebeatStatusCheck.successText": "Des données ont été reçues de ce module.", "home.tutorials.common.filebeatStatusCheck.text": "Vérifier que des données sont reçues du module Filebeat `{moduleName}`", "home.tutorials.common.filebeatStatusCheck.title": "Statut du module", - "home.tutorials.common.functionbeat.cloudInstructions.gettingStarted.title": "Commencer", - "home.tutorials.common.functionbeat.premCloudInstructions.gettingStarted.title": "Commencer", - "home.tutorials.common.functionbeat.premInstructions.gettingStarted.title": "Commencer", - "home.tutorials.common.functionbeatAWSInstructions.textPost": "Où {accessKey} et {secretAccessKey} sont vos informations d'identification et `us-east-1` est la région désirée.", - "home.tutorials.common.functionbeatAWSInstructions.textPre": "Définissez vos informations d'identification AWS dans l'environnement :", - "home.tutorials.common.functionbeatAWSInstructions.title": "Définir des informations d'identification AWS", - "home.tutorials.common.functionbeatCloudInstructions.config.osxTextPre": "Modifiez {path} afin de définir les informations de connexion pour Elastic Cloud :", - "home.tutorials.common.functionbeatCloudInstructions.config.osxTitle": "Modifier la configuration", - "home.tutorials.common.functionbeatCloudInstructions.config.windowsTextPre": "Modifiez {path} afin de définir les informations de connexion pour Elastic Cloud :", - "home.tutorials.common.functionbeatCloudInstructions.config.windowsTitle": "Modifier la configuration", - "home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTextPost": "Où `''` est le nom du groupe de logs à importer et `''` un nom de compartiment S3 valide pour la mise en œuvre du déploiement de Functionbeat.", - "home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTitle": "Configurer le groupe de logs Cloudwatch", - "home.tutorials.common.functionbeatEnableOnPremInstructionsOSXLinux.textPre": "Modifiez les paramètres dans le fichier `functionbeat.yml`.", - "home.tutorials.common.functionbeatEnableOnPremInstructionsWindows.textPre": "Modifiez les paramètres dans le fichier {path}.", - "home.tutorials.common.functionbeatInstructions.config.osxTextPostMarkdown": "Où {passwordTemplate} est le mot de passe de l'utilisateur \"elastic\", {esUrlTemplate} est l'URL d'Elasticsearch et {kibanaUrlTemplate} est l'URL de Kibana. Pour [configurer SSL]({configureSslUrl}) avec le certificat par défaut généré par Elasticsearch, ajoutez son empreinte dans {esCertFingerprintTemplate}. > **_Important :_** N'utilisez pas l'utilisateur intégré `elastic` pour sécuriser les clients dans un environnement de production. À la place, configurez des utilisateurs autorisés ou des clés d'API, et n'exposez pas les mots de passe dans les fichiers de configuration. [Learn more]({linkUrl}).", - "home.tutorials.common.functionbeatInstructions.config.osxTextPre": "Modifiez {path} afin de définir les informations de connexion :", - "home.tutorials.common.functionbeatInstructions.config.osxTitle": "Configurer le cluster Elastic", - "home.tutorials.common.functionbeatInstructions.config.windowsTextPostMarkdown": "Où {passwordTemplate} est le mot de passe de l'utilisateur \"elastic\", {esUrlTemplate} est l'URL d'Elasticsearch et {kibanaUrlTemplate} est l'URL de Kibana. Pour [configurer SSL]({configureSslUrl}) avec le certificat par défaut généré par Elasticsearch, ajoutez son empreinte dans {esCertFingerprintTemplate}. > **_Important :_** N'utilisez pas l'utilisateur intégré `elastic` pour sécuriser les clients dans un environnement de production. À la place, configurez des utilisateurs autorisés ou des clés d'API, et n'exposez pas les mots de passe dans les fichiers de configuration. [Learn more]({linkUrl}).", - "home.tutorials.common.functionbeatInstructions.config.windowsTextPre": "Modifiez {path} afin de définir les informations de connexion :", - "home.tutorials.common.functionbeatInstructions.config.windowsTitle": "Modifier la configuration", - "home.tutorials.common.functionbeatInstructions.deploy.osxTextPre": "Ceci permet d'installer Functionbeat en tant que fonction Lambda. La commande `setup` vérifie la configuration d'Elasticsearch et charge le modèle d'indexation Kibana. L'omission de cette commande est normalement sans risque.", - "home.tutorials.common.functionbeatInstructions.deploy.osxTitle": "Déployer Functionbeat en tant que fonction AWS Lambda", - "home.tutorials.common.functionbeatInstructions.deploy.windowsTextPre": "Ceci permet d'installer Functionbeat en tant que fonction Lambda. La commande `setup` vérifie la configuration d'Elasticsearch et charge le modèle d'indexation Kibana. L'omission de cette commande est normalement sans risque.", - "home.tutorials.common.functionbeatInstructions.deploy.windowsTitle": "Déployer Functionbeat en tant que fonction AWS Lambda", - "home.tutorials.common.functionbeatInstructions.install.linuxTextPre": "Vous utilisez Functionbeat pour la première fois ? Consultez le [guide de démarrage rapide]({link}).", - "home.tutorials.common.functionbeatInstructions.install.linuxTitle": "Télécharger et installer Functionbeat", - "home.tutorials.common.functionbeatInstructions.install.osxTextPre": "Vous utilisez Functionbeat pour la première fois ? Consultez le [guide de démarrage rapide]({link}).", - "home.tutorials.common.functionbeatInstructions.install.osxTitle": "Télécharger et installer Functionbeat", - "home.tutorials.common.functionbeatInstructions.install.windowsTextPre": "Vous utilisez Functionbeat pour la première fois ? Consultez le [guide de démarrage rapide]({functionbeatLink}). 1. Téléchargez le fichier .zip Functionbeat pour Windows via la page [Télécharger]({elasticLink}). 2. Extrayez le contenu du fichier compressé sous {folderPath}. 3. Renommez le répertoire `{directoryName}` en `Functionbeat`. 4. Ouvrez une invite PowerShell en tant qu'administrateur (faites un clic droit sur l'icône PowerShell et sélectionnez **Exécuter en tant qu'administrateur**). Si vous exécutez Windows XP, vous devrez peut-être télécharger et installer PowerShell. 5. Depuis l'invite PowerShell, accédez au répertoire Functionbeat :", - "home.tutorials.common.functionbeatInstructions.install.windowsTitle": "Télécharger et installer Functionbeat", - "home.tutorials.common.functionbeatStatusCheck.buttonLabel": "Vérifier les données", - "home.tutorials.common.functionbeatStatusCheck.errorText": "Aucune donnée n'a encore été reçue de Functionbeat.", - "home.tutorials.common.functionbeatStatusCheck.successText": "Des données ont été reçues de Functionbeat.", - "home.tutorials.common.functionbeatStatusCheck.text": "Vérifier que des données sont reçues de Functionbeat", - "home.tutorials.common.functionbeatStatusCheck.title": "Statut de Functionbeat", "home.tutorials.common.heartbeat.cloudInstructions.gettingStarted.title": "Commencer", "home.tutorials.common.heartbeat.premCloudInstructions.gettingStarted.title": "Commencer", "home.tutorials.common.heartbeat.premInstructions.gettingStarted.title": "Commencer", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 722631f102426..8aadc59451b36 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -4334,9 +4334,6 @@ "home.tutorials.ciscoLogs.longDescription": "これは Cisco ネットワークデバイスのログ用のモジュールです（ASA、FTD、IOS、Nexus）。Syslog のログまたはファイルから読み取られたログを受信するための次のファイルセットが含まれます。[詳細]（{learnMoreLink}）。", "home.tutorials.ciscoLogs.nameTitle": "Ciscoログ", "home.tutorials.ciscoLogs.shortDescription": "Filebeatを使用してCiscoネットワークデバイスからログを収集して解析します。", - "home.tutorials.cloudwatchLogs.longDescription": "FunctionbeatをAWS Lambda関数として実行するようデプロイし、Cloudwatchログを収集します。", - "home.tutorials.cloudwatchLogs.nameTitle": "AWS Cloudwatchログ", - "home.tutorials.cloudwatchLogs.shortDescription": "Functionbeatを使用してAWS Cloudwatchからログを収集して解析します。", "home.tutorials.cockroachdbMetrics.artifacts.dashboards.linkLabel": "CockroachDB メトリックダッシュボード", "home.tutorials.cockroachdbMetrics.longDescription": "Metricbeat モジュール「cockroachdb」は、CockroachDB からメトリックを取得します。[詳細]（{learnMoreLink}）。", "home.tutorials.cockroachdbMetrics.nameTitle": "CockroachDB Metrics", @@ -4446,40 +4443,6 @@ "home.tutorials.common.filebeatStatusCheck.successText": "このモジュールからデータを受け取りました", "home.tutorials.common.filebeatStatusCheck.text": "Filebeat の「{moduleName}」モジュールからデータを受け取ったことを確認してください。", "home.tutorials.common.filebeatStatusCheck.title": "モジュールステータス", - "home.tutorials.common.functionbeat.cloudInstructions.gettingStarted.title": "はじめに", - "home.tutorials.common.functionbeat.premCloudInstructions.gettingStarted.title": "はじめに", - "home.tutorials.common.functionbeat.premInstructions.gettingStarted.title": "はじめに", - "home.tutorials.common.functionbeatAWSInstructions.textPost": "{accessKey}と{secretAccessKey}がアカウント認証情報で、us-east-1が希望の地域です。", - "home.tutorials.common.functionbeatAWSInstructions.textPre": "環境で AWS アカウント認証情報を設定します。", - "home.tutorials.common.functionbeatAWSInstructions.title": "AWS 認証情報の設定", - "home.tutorials.common.functionbeatCloudInstructions.config.osxTextPre": "{path} を変更して Elastic Cloud への接続情報を設定します：", - "home.tutorials.common.functionbeatCloudInstructions.config.osxTitle": "構成を編集する", - "home.tutorials.common.functionbeatCloudInstructions.config.windowsTextPre": "{path} を変更して Elastic Cloud への接続情報を設定します：", - "home.tutorials.common.functionbeatCloudInstructions.config.windowsTitle": "構成を編集する", - "home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTitle": "Cloudwatch ロググループの構成", - "home.tutorials.common.functionbeatEnableOnPremInstructionsOSXLinux.textPre": "「functionbeat.yml」ファイルで設定を変更します。", - "home.tutorials.common.functionbeatEnableOnPremInstructionsWindows.textPre": "{path} ファイルで設定を変更します。", - "home.tutorials.common.functionbeatInstructions.config.osxTextPostMarkdown": "{passwordTemplate} が「Elastic」ユーザーのパスワード、{esUrlTemplate} が Elasticsearch の URL、{kibanaUrlTemplate} が Kibana の URL です。Elasticsearchによって生成されたデフォルトの証明書を使用して[SSLを設定]({configureSslUrl})するには、{esCertFingerprintTemplate}にそのフィンガープリントを追加します。> **_重要：_** 本番環境でクライアントを保護するために、組み込みの「elastic」ユーザーを使用しないでください。許可されたユーザーまたはAPIキーを設定し、パスワードは構成ファイルで公開しないでください。[Learn more]({linkUrl})。", - "home.tutorials.common.functionbeatInstructions.config.osxTextPre": "{path} を変更して Elastic Cloud への接続情報を設定します：", - "home.tutorials.common.functionbeatInstructions.config.osxTitle": "Elastic クラスターの構成", - "home.tutorials.common.functionbeatInstructions.config.windowsTextPostMarkdown": "{passwordTemplate} が「Elastic」ユーザーのパスワード、{esUrlTemplate} が Elasticsearch の URL、{kibanaUrlTemplate} が Kibana の URL です。Elasticsearchによって生成されたデフォルトの証明書を使用して[SSLを設定]({configureSslUrl})するには、{esCertFingerprintTemplate}にそのフィンガープリントを追加します。> **_重要：_** 本番環境でクライアントを保護するために、組み込みの「elastic」ユーザーを使用しないでください。許可されたユーザーまたはAPIキーを設定し、パスワードは構成ファイルで公開しないでください。[Learn more]({linkUrl})。", - "home.tutorials.common.functionbeatInstructions.config.windowsTextPre": "{path} を変更して Elastic Cloud への接続情報を設定します：", - "home.tutorials.common.functionbeatInstructions.config.windowsTitle": "構成を編集する", - "home.tutorials.common.functionbeatInstructions.deploy.osxTextPre": "これにより Functionbeat が Lambda 関数としてインストールされます「setup」コマンドで Elasticsearch の構成を確認し、Kibana インデックスパターンを読み込みます。通常このコマンドを省いても大丈夫です。", - "home.tutorials.common.functionbeatInstructions.deploy.osxTitle": "Functionbeat を AWS Lambda にデプロイ", - "home.tutorials.common.functionbeatInstructions.deploy.windowsTextPre": "これにより Functionbeat が Lambda 関数としてインストールされます「setup」コマンドで Elasticsearch の構成を確認し、Kibana インデックスパターンを読み込みます。通常このコマンドを省いても大丈夫です。", - "home.tutorials.common.functionbeatInstructions.deploy.windowsTitle": "Functionbeat を AWS Lambda にデプロイ", - "home.tutorials.common.functionbeatInstructions.install.linuxTextPre": "Functionbeatは初めてですか？[クイックスタート]（{link}）を参照してください。", - "home.tutorials.common.functionbeatInstructions.install.linuxTitle": "Functionbeat のダウンロードとインストール", - "home.tutorials.common.functionbeatInstructions.install.osxTextPre": "Functionbeatは初めてですか？[クイックスタート]（{link}）を参照してください。", - "home.tutorials.common.functionbeatInstructions.install.osxTitle": "Functionbeat のダウンロードとインストール", - "home.tutorials.common.functionbeatInstructions.install.windowsTextPre": "Functionbeatは初めてですか？[クイックスタート]（{functionbeatLink}）を参照してください。1.[ダウンロード]（{elasticLink}）ページからFunctionbeat Windows zipファイルをダウンロードします。2.zipファイルのコンテンツを{folderPath}に解凍します。3.「{directoryName} ディレクトリの名前を「Functionbeat」に変更します。4.管理者としてPowerShellプロンプトを開きます（PowerShellアイコンを右クリックして「管理者として実行」を選択します）。Windows XPをご使用の場合、PowerShellのダウンロードとインストールが必要な場合があります。5.PowerShell プロンプトから、Functionbeat ディレクトリに移動します：", - "home.tutorials.common.functionbeatInstructions.install.windowsTitle": "Functionbeat のダウンロードとインストール", - "home.tutorials.common.functionbeatStatusCheck.buttonLabel": "データを確認してください", - "home.tutorials.common.functionbeatStatusCheck.errorText": "Functionbeat からまだデータを受け取っていません", - "home.tutorials.common.functionbeatStatusCheck.successText": "Functionbeat からデータを受け取りました", - "home.tutorials.common.functionbeatStatusCheck.text": "Functionbeat からデータを受け取ったことを確認してください。", - "home.tutorials.common.functionbeatStatusCheck.title": "Functionbeat ステータス", "home.tutorials.common.heartbeat.cloudInstructions.gettingStarted.title": "はじめに", "home.tutorials.common.heartbeat.premCloudInstructions.gettingStarted.title": "はじめに", "home.tutorials.common.heartbeat.premInstructions.gettingStarted.title": "はじめに", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index c666d4e60b726..6bb17f1a2bdaa 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -4276,9 +4276,6 @@ "home.tutorials.ciscoLogs.longDescription": "这是用于 Cisco 网络设备日志（ASA、FTD、IOS、Nexus）的模块。其包含以下用于从 Syslog 接收或从文件读取日志的文件集：[了解详情]({learnMoreLink})。", "home.tutorials.ciscoLogs.nameTitle": "Cisco 日志", "home.tutorials.ciscoLogs.shortDescription": "使用 Filebeat 从 Cisco 网络设备收集并解析日志。", - "home.tutorials.cloudwatchLogs.longDescription": "通过部署将运行为 AWS Lambda 函数的 Functionbeat 来收集 Cloudwatch 日志。", - "home.tutorials.cloudwatchLogs.nameTitle": "AWS Cloudwatch 日志", - "home.tutorials.cloudwatchLogs.shortDescription": "使用 Functionbeat 从 AWS Cloudwatch 收集并解析日志。", "home.tutorials.cockroachdbMetrics.artifacts.dashboards.linkLabel": "CockroachDB 指标仪表板", "home.tutorials.cockroachdbMetrics.longDescription": "Metricbeat 模块 `cockroachdb` 从 CockroachDB 提取指标。[了解详情]({learnMoreLink})。", "home.tutorials.cockroachdbMetrics.nameTitle": "CockroachDB 指标", @@ -4388,41 +4385,6 @@ "home.tutorials.common.filebeatStatusCheck.successText": "已从此模块成功接收数据", "home.tutorials.common.filebeatStatusCheck.text": "确认已从 Filebeat `{moduleName}` 模块成功收到数据", "home.tutorials.common.filebeatStatusCheck.title": "模块状态", - "home.tutorials.common.functionbeat.cloudInstructions.gettingStarted.title": "入门", - "home.tutorials.common.functionbeat.premCloudInstructions.gettingStarted.title": "入门", - "home.tutorials.common.functionbeat.premInstructions.gettingStarted.title": "入门", - "home.tutorials.common.functionbeatAWSInstructions.textPost": "其中 {accessKey} 和 {secretAccessKey} 是您的帐户凭据，`us-east-1` 是所需的地区。", - "home.tutorials.common.functionbeatAWSInstructions.textPre": "在环境中设置您的 AWS 帐户凭据：", - "home.tutorials.common.functionbeatAWSInstructions.title": "设置 AWS 凭据", - "home.tutorials.common.functionbeatCloudInstructions.config.osxTextPre": "修改 {path} 以设置 Elastic Cloud 的连接信息：", - "home.tutorials.common.functionbeatCloudInstructions.config.osxTitle": "编辑配置", - "home.tutorials.common.functionbeatCloudInstructions.config.windowsTextPre": "修改 {path} 以设置 Elastic Cloud 的连接信息：", - "home.tutorials.common.functionbeatCloudInstructions.config.windowsTitle": "编辑配置", - "home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTextPost": "其中 `''` 是要采集的日志组名称，`''` 是将用于暂存 Functionbeat 部署的有效 S3 存储桶名称。", - "home.tutorials.common.functionbeatEnableOnPremInstructions.defaultTitle": "配置 Cloudwatch 日志组", - "home.tutorials.common.functionbeatEnableOnPremInstructionsOSXLinux.textPre": "在 `functionbeat.yml` 文件中修改设置。", - "home.tutorials.common.functionbeatEnableOnPremInstructionsWindows.textPre": "在 {path} 文件中修改设置。", - "home.tutorials.common.functionbeatInstructions.config.osxTextPostMarkdown": "其中，{passwordTemplate} 是 `elastic` 用户的密码，{esUrlTemplate} 是 Elasticsearch 的 URL，{kibanaUrlTemplate} 是 Kibana 的 URL。要使用 Elasticsearch 生成的默认证书[配置 SSL]({configureSslUrl})，请在 {esCertFingerprintTemplate} 中添加其指纹。> **_重要说明：_**在生产环境中，请勿使用内置 `elastic` 用户来保护客户端。而要设置授权用户或 API 密钥，并且不要在配置文件中暴露密码。[了解详情]({linkUrl})。", - "home.tutorials.common.functionbeatInstructions.config.osxTextPre": "修改 {path} 以设置连接信息：", - "home.tutorials.common.functionbeatInstructions.config.osxTitle": "配置 Elastic 集群", - "home.tutorials.common.functionbeatInstructions.config.windowsTextPostMarkdown": "其中，{passwordTemplate} 是 `elastic` 用户的密码，{esUrlTemplate} 是 Elasticsearch 的 URL，{kibanaUrlTemplate} 是 Kibana 的 URL。要使用 Elasticsearch 生成的默认证书[配置 SSL]({configureSslUrl})，请在 {esCertFingerprintTemplate} 中添加其指纹。> **_重要说明：_**在生产环境中，请勿使用内置 `elastic` 用户来保护客户端。而要设置授权用户或 API 密钥，并且不要在配置文件中暴露密码。[了解详情]({linkUrl})。", - "home.tutorials.common.functionbeatInstructions.config.windowsTextPre": "修改 {path} 以设置连接信息：", - "home.tutorials.common.functionbeatInstructions.config.windowsTitle": "编辑配置", - "home.tutorials.common.functionbeatInstructions.deploy.osxTextPre": "这会将 Functionbeat 安装为 Lambda 函数。`setup` 命令检查 Elasticsearch 配置并加载 Kibana 索引模式。通常可省略此命令。", - "home.tutorials.common.functionbeatInstructions.deploy.osxTitle": "将 Functionbeat 部署到 AWS Lambda", - "home.tutorials.common.functionbeatInstructions.deploy.windowsTextPre": "这会将 Functionbeat 安装为 Lambda 函数。`setup` 命令检查 Elasticsearch 配置并加载 Kibana 索引模式。通常可省略此命令。", - "home.tutorials.common.functionbeatInstructions.deploy.windowsTitle": "将 Functionbeat 部署到 AWS Lambda", - "home.tutorials.common.functionbeatInstructions.install.linuxTextPre": "首次使用 Functionbeat？查看[快速入门]({link})。", - "home.tutorials.common.functionbeatInstructions.install.linuxTitle": "下载并安装 Functionbeat", - "home.tutorials.common.functionbeatInstructions.install.osxTextPre": "首次使用 Functionbeat？查看[快速入门]({link})。", - "home.tutorials.common.functionbeatInstructions.install.osxTitle": "下载并安装 Functionbeat", - "home.tutorials.common.functionbeatInstructions.install.windowsTextPre": "首次使用 Functionbeat？查看[快速入门]({functionbeatLink})。1.从[下载]({elasticLink})页面下载 Functionbeat Windows zip 文件。2.将该 zip 文件的内容解压缩到 {folderPath}。3.将 {directoryName} 目录重命名为'Functionbeat'。4.以管理员身份打开 PowerShell 提示符（右键单击 PowerShell 图标，然后选择**以管理员身份运行**）。如果运行的是 Windows XP，则可能需要下载并安装 PowerShell。5.从 PowerShell 提示符处，前往 Functionbeat 目录：", - "home.tutorials.common.functionbeatInstructions.install.windowsTitle": "下载并安装 Functionbeat", - "home.tutorials.common.functionbeatStatusCheck.buttonLabel": "检查数据", - "home.tutorials.common.functionbeatStatusCheck.errorText": "尚未从 Functionbeat 收到任何数据", - "home.tutorials.common.functionbeatStatusCheck.successText": "已从 Functionbeat 成功接收数据", - "home.tutorials.common.functionbeatStatusCheck.text": "确认从 Functionbeat 收到数据", - "home.tutorials.common.functionbeatStatusCheck.title": "Functionbeat 状态", "home.tutorials.common.heartbeat.cloudInstructions.gettingStarted.title": "入门", "home.tutorials.common.heartbeat.premCloudInstructions.gettingStarted.title": "入门", "home.tutorials.common.heartbeat.premInstructions.gettingStarted.title": "入门", From e9671937bacfaa911d32de0e8885e7f26425888a Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Wed, 13 Nov 2024 15:42:41 +0200 Subject: [PATCH 15/53] fix: [Search:WebCrawlers:ViewCrawler:Manage Domains page]Incorrect total number of options announced for Policy and Rules combo box (#199745) Closes: #199132 ## Description Visible total number of options should the same as announced for the user as not to confuse them. Especially for the users using assistive technologies. ## What was changed: 1. value of `hasNoInitialSelection` attribute value has been updated for the mentioned cases. This removes the extra selection option from the DOM and fixes the `a11y` issue. ## Screen: image --- .../crawler/components/crawl_rules_table.tsx | 14 +++++++++++--- .../crawler_domain_detail/crawl_rules_table.tsx | 14 +++++++++++--- .../public/connector_types/openai/connector.tsx | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx index 2f66dc455442e..89cf2248201ce 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/crawler/components/crawl_rules_table.tsx @@ -53,7 +53,12 @@ const DEFAULT_DESCRIPTION = ( defaultMessage="Create a crawl rule to include or exclude pages whose URL matches the rule. Rules run in sequential order, and each URL is evaluated according to the first match. {link}" values={{ link: ( - + {i18n.translate( 'xpack.enterpriseSearch.appSearch.crawler.crawlRulesTable.descriptionLinkText', { defaultMessage: 'Learn more about crawl rules' } @@ -78,9 +83,10 @@ export const CrawlRulesTable: React.FC = ({ { editingRender: (crawlRule, onChange, { isInvalid, isLoading }) => ( onChange(e.target.value)} disabled={isLoading} isInvalid={isInvalid} @@ -106,9 +112,10 @@ export const CrawlRulesTable: React.FC = ({ { editingRender: (crawlRule, onChange, { isInvalid, isLoading }) => ( onChange(e.target.value)} disabled={isLoading} isInvalid={isInvalid} @@ -139,6 +146,7 @@ export const CrawlRulesTable: React.FC = ({ onChange(e.target.value)} diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_domain_detail/crawl_rules_table.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_domain_detail/crawl_rules_table.tsx index 5c96617e1fb2d..1912792e463ef 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_domain_detail/crawl_rules_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawler_domain_detail/crawl_rules_table.tsx @@ -104,7 +104,12 @@ const DEFAULT_DESCRIPTION = ( defaultMessage="Create a crawl rule to include or exclude pages whose URL matches the rule. Rules run in sequential order, and each URL is evaluated according to the first match." /> - + {i18n.translate('xpack.enterpriseSearch.crawler.crawlRulesTable.descriptionLinkText', { defaultMessage: 'Learn more about crawl rules', })} @@ -126,10 +131,11 @@ export const CrawlRulesTable: React.FC = ({ { editingRender: (crawlRule, onChange, { isInvalid, isLoading }) => ( onChange(e.target.value)} disabled={isLoading} isInvalid={isInvalid} @@ -152,10 +158,11 @@ export const CrawlRulesTable: React.FC = ({ { editingRender: (crawlRule, onChange, { isInvalid, isLoading }) => ( onChange(e.target.value)} disabled={isLoading} isInvalid={isInvalid} @@ -183,6 +190,7 @@ export const CrawlRulesTable: React.FC = ({ onChange(e.target.value)} diff --git a/x-pack/plugins/stack_connectors/public/connector_types/openai/connector.tsx b/x-pack/plugins/stack_connectors/public/connector_types/openai/connector.tsx index 27cbb9a4dac08..7bcb818893087 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/openai/connector.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/openai/connector.tsx @@ -63,7 +63,7 @@ const ConnectorFields: React.FC = ({ readOnly, isEdi 'data-test-subj': 'config.apiProvider-select', options: providerOptions, fullWidth: true, - hasNoInitialSelection: true, + hasNoInitialSelection: false, disabled: readOnly, readOnly, }, From 4861f183733d811be1aa26e0a8956412068061dd Mon Sep 17 00:00:00 2001 From: Ash <1849116+ashokaditya@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:17:14 +0100 Subject: [PATCH 16/53] [Serverless][SecuritySolution][Endpoint] Re-enable `scan` and `release` e2e tests (#199416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Re-enable `scan` and `release` e2e tests on serverless that failed due to the build artifacts page not being available momentarily. closes https://github.com/elastic/kibana/issues/172326 closes https://github.com/elastic/kibana/issues/187932 ### Flaky runner - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7352 x 150 (tries to run all tests) - https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/7388 x 150 (scan | release) (all 🟢 ) ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed --- .../e2e/response_actions/response_console/release.cy.ts | 3 +-- .../cypress/e2e/response_actions/response_console/scan.cy.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts index 5ad7395efbe21..d11b7210713a8 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/release.cy.ts @@ -27,8 +27,7 @@ import { enableAllPolicyProtections } from '../../../tasks/endpoint_policy'; import { createEndpointHost } from '../../../tasks/create_endpoint_host'; import { deleteAllLoadedEndpointData } from '../../../tasks/delete_all_endpoint_data'; -// FLAKY: https://github.com/elastic/kibana/issues/172326 -describe.skip('Response console', { tags: ['@ess', '@serverless'] }, () => { +describe('Response console', { tags: ['@ess', '@serverless'] }, () => { let indexedPolicy: IndexedFleetEndpointPolicyResponse; let policy: PolicyData; let createdHost: CreateAndEnrollEndpointHostResponse; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/scan.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/scan.cy.ts index 543961ef9900b..04630647ed35f 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/scan.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/response_actions/response_console/scan.cy.ts @@ -41,8 +41,7 @@ describe( login(); }); - // FLAKY: https://github.com/elastic/kibana/issues/187932 - describe.skip('Scan operation:', () => { + describe('Scan operation:', () => { const homeFilePath = Cypress.env('IS_CI') ? '/home/vagrant' : '/home'; const fileContent = 'This is a test file for the scan command.'; From 9670a744606e9ce7fb58bc0121688a1d97966970 Mon Sep 17 00:00:00 2001 From: seanrathier Date: Wed, 13 Nov 2024 09:20:21 -0500 Subject: [PATCH 17/53] [Cloud Security] Add AWS account credentials for the Agentless tests in Serverless Quality Gates (#199547) --- .../agentless_api/create_agent.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless_api/create_agent.ts b/x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless_api/create_agent.ts index b26581fb46dfd..7611061398f18 100644 --- a/x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless_api/create_agent.ts +++ b/x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless_api/create_agent.ts @@ -54,6 +54,22 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await cisIntegration.selectSetupTechnology('agentless'); await cisIntegration.selectAwsCredentials('direct'); + if ( + process.env.TEST_CLOUD && + process.env.CSPM_AWS_ACCOUNT_ID && + process.env.CSPM_AWS_SECRET_KEY + ) { + await cisIntegration.fillInTextField( + cisIntegration.testSubjectIds.DIRECT_ACCESS_KEY_ID_TEST_ID, + process.env.CSPM_AWS_ACCOUNT_ID + ); + + await cisIntegration.fillInTextField( + cisIntegration.testSubjectIds.DIRECT_ACCESS_SECRET_KEY_TEST_ID, + process.env.CSPM_AWS_SECRET_KEY + ); + } + await pageObjects.header.waitUntilLoadingHasFinished(); await cisIntegration.clickSaveButton(); From 7a61d10eef804f3695071969116bbe64afd864b6 Mon Sep 17 00:00:00 2001 From: Sergi Romeu Date: Wed, 13 Nov 2024 15:21:54 +0100 Subject: [PATCH 18/53] [APM] Migrate `/observability_overview` to deployment agnostic test (#199817) ## Summary Closes https://github.com/elastic/kibana/issues/198981 Part of https://github.com/elastic/kibana/issues/193245 This PR contains the changes to migrate `observability_overview` test folder to Deployment-agnostic testing strategy. ### How to test - Serverless ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM" ``` It's recommended to be run against [MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki) - Stateful ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM" ``` ## Checks - [x] (OPTIONAL, only if a test has been unskipped) Run flaky test suite - [x] local run for serverless - [x] local run for stateful - [x] MKI run for serverless --- .../apis/observability/apm/index.ts | 1 + .../observability_overview/has_data.spec.ts | 50 +++++++++------- .../apm/observability_overview/index.ts | 15 +++++ .../observability_overview.spec.ts | 58 +++++++++---------- .../es_archiver/apm_8.0.0/mappings.json | 30 ++-------- .../observability_overview/mappings.json | 3 +- 6 files changed, 77 insertions(+), 80 deletions(-) rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/observability_overview/has_data.spec.ts (57%) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/index.ts rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/observability_overview/observability_overview.spec.ts (80%) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts index fc98e85850bd8..3833aa94c6db4 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts @@ -22,6 +22,7 @@ export default function apmApiIntegrationTests({ loadTestFile(require.resolve('./correlations')); loadTestFile(require.resolve('./entities')); loadTestFile(require.resolve('./cold_start')); + loadTestFile(require.resolve('./observability_overview')); loadTestFile(require.resolve('./latency')); loadTestFile(require.resolve('./infrastructure')); }); diff --git a/x-pack/test/apm_api_integration/tests/observability_overview/has_data.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/has_data.spec.ts similarity index 57% rename from x-pack/test/apm_api_integration/tests/observability_overview/has_data.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/has_data.spec.ts index 3aab948cd4f69..63620d514603e 100644 --- a/x-pack/test/apm_api_integration/tests/observability_overview/has_data.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/has_data.spec.ts @@ -7,16 +7,15 @@ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; +import { ARCHIVER_ROUTES } from '../constants/archiver'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const esArchiver = getService('esArchiver'); - registry.when( - 'Observability overview when data is not loaded', - { config: 'basic', archives: [] }, - () => { + describe('has data', () => { + describe('when no data is loaded', () => { it('returns false when there is no data', async () => { const response = await apmApiClient.readUser({ endpoint: 'GET /internal/apm/observability_overview/has_data', @@ -24,13 +23,17 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(response.status).to.be(200); expect(response.body.hasData).to.eql(false); }); - } - ); + }); + + describe('when only onboarding data is loaded', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES.observability_overview); + }); + + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES.observability_overview); + }); - registry.when( - 'Observability overview when only onboarding data is loaded', - { config: 'basic', archives: ['observability_overview'] }, - () => { it('returns false when there is only onboarding data', async () => { const response = await apmApiClient.readUser({ endpoint: 'GET /internal/apm/observability_overview/has_data', @@ -38,13 +41,16 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(response.status).to.be(200); expect(response.body.hasData).to.eql(false); }); - } - ); + }); + + describe('when data is loaded', () => { + before(async () => { + await esArchiver.load(ARCHIVER_ROUTES['8.0.0']); + }); + after(async () => { + await esArchiver.unload(ARCHIVER_ROUTES['8.0.0']); + }); - registry.when( - 'Observability overview when APM data is loaded', - { config: 'basic', archives: ['apm_8.0.0'] }, - () => { it('returns true when there is at least one document on transaction, error or metrics indices', async () => { const response = await apmApiClient.readUser({ endpoint: 'GET /internal/apm/observability_overview/has_data', @@ -52,6 +58,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(response.status).to.be(200); expect(response.body.hasData).to.eql(true); }); - } - ); + }); + }); } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/index.ts new file mode 100644 index 0000000000000..c43e15d005bb9 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) { + describe('observability_overview', () => { + loadTestFile(require.resolve('./has_data.spec.ts')); + loadTestFile(require.resolve('./observability_overview.spec.ts')); + }); +} diff --git a/x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/observability_overview.spec.ts similarity index 80% rename from x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/observability_overview.spec.ts index 763d8eee929d2..740dd432b670b 100644 --- a/x-pack/test/apm_api_integration/tests/observability_overview/observability_overview.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/observability_overview/observability_overview.spec.ts @@ -9,14 +9,13 @@ import expect from '@kbn/expect'; import { meanBy, sumBy } from 'lodash'; import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; -import { roundNumber } from '../../utils'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { roundNumber } from '../../../../../../apm_api_integration/utils'; +import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); const start = new Date('2021-01-01T00:00:00.000Z').getTime(); const end = new Date('2021-01-01T00:15:00.000Z').getTime() - 1; @@ -62,35 +61,30 @@ export default function ApiTest({ getService }: FtrProviderContext) { }; } - registry.when( - 'Observability overview when data is not loaded', - { config: 'basic', archives: [] }, - () => { - describe('when data is not loaded', () => { - it('handles the empty state', async () => { - const response = await apmApiClient.readUser({ - endpoint: `GET /internal/apm/observability_overview`, - params: { - query: { - start: new Date(start).toISOString(), - end: new Date(end).toISOString(), - bucketSize, - intervalString, - }, + describe('Observability overview', () => { + describe('when data is not loaded', () => { + it('handles the empty state', async () => { + const response = await apmApiClient.readUser({ + endpoint: `GET /internal/apm/observability_overview`, + params: { + query: { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + bucketSize, + intervalString, }, - }); - expect(response.status).to.be(200); - - expect(response.body.serviceCount).to.be(0); - expect(response.body.transactionPerMinute.timeseries.length).to.be(0); + }, }); + expect(response.status).to.be(200); + + expect(response.body.serviceCount).to.be(0); + expect(response.body.transactionPerMinute.timeseries.length).to.be(0); }); - } - ); + }); - // FLAKY: https://github.com/elastic/kibana/issues/177497 - registry.when('data is loaded', { config: 'basic', archives: [] }, () => { describe('Observability overview api ', () => { + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + const GO_PROD_RATE = 50; const GO_DEV_RATE = 5; const JAVA_PROD_RATE = 45; @@ -106,6 +100,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { .service({ name: 'synth-java', environment: 'production', agentName: 'java' }) .instance('instance-c'); + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + await apmSynthtraceEsClient.index([ timerange(start, end) .interval('1m') diff --git a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/apm_8.0.0/mappings.json b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/apm_8.0.0/mappings.json index ee8b97f7ac0ae..c6f64e5026456 100644 --- a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/apm_8.0.0/mappings.json +++ b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/apm_8.0.0/mappings.json @@ -6213,10 +6213,6 @@ "read_only_allow_delete": "false" }, "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-7.14.0-error" - }, "mapping": { "total_fields": { "limit": "2000" @@ -6225,8 +6221,7 @@ "max_docvalue_fields_search": "200", "number_of_replicas": "1", "number_of_shards": "1", - "priority": "100", - "refresh_interval": "5s" + "priority": "100" } } } @@ -11748,10 +11743,6 @@ "read_only_allow_delete": "false" }, "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-7.14.0-metric" - }, "mapping": { "total_fields": { "limit": "2000" @@ -11760,8 +11751,7 @@ "max_docvalue_fields_search": "200", "number_of_replicas": "1", "number_of_shards": "1", - "priority": "100", - "refresh_interval": "5s" + "priority": "100" } } } @@ -16847,10 +16837,6 @@ "read_only_allow_delete": "false" }, "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-7.14.0-span" - }, "mapping": { "total_fields": { "limit": "2000" @@ -16859,8 +16845,7 @@ "max_docvalue_fields_search": "200", "number_of_replicas": "1", "number_of_shards": "1", - "priority": "100", - "refresh_interval": "5s" + "priority": "100" } } } @@ -22037,10 +22022,6 @@ "read_only_allow_delete": "false" }, "codec": "best_compression", - "lifecycle": { - "name": "apm-rollover-30-days", - "rollover_alias": "apm-7.14.0-transaction" - }, "mapping": { "total_fields": { "limit": "2000" @@ -22049,9 +22030,8 @@ "max_docvalue_fields_search": "200", "number_of_replicas": "1", "number_of_shards": "1", - "priority": "100", - "refresh_interval": "5s" + "priority": "100" } } } -} \ No newline at end of file +} diff --git a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/observability_overview/mappings.json b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/observability_overview/mappings.json index 544ad95203c6a..95636d5ee1c1d 100644 --- a/x-pack/test/apm_api_integration/common/fixtures/es_archiver/observability_overview/mappings.json +++ b/x-pack/test/apm_api_integration/common/fixtures/es_archiver/observability_overview/mappings.json @@ -4222,8 +4222,7 @@ "transaction.message.queue.name", "fields.*" ] - }, - "refresh_interval": "1ms" + } } } } From 4ad25cf88df179334db72a593d05ff5bc0e3eb3e Mon Sep 17 00:00:00 2001 From: Robert Jaszczurek <92210485+rbrtj@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:56:56 +0100 Subject: [PATCH 19/53] [ML] AiOps: Action for adding Log Pattern embeddable to a dashboard and case (#199478) ## Summary Part of [#197247](https://github.com/elastic/kibana/issues/197247) - Added the ability to add a Log Pattern Embeddable to a dashboard and case. - Fixed the Change Point Detection embeddable in cases and added a functional test to cover this scenario. https://github.com/user-attachments/assets/d09eccc1-6738-4c8b-9a54-7c78d9ac9017 ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-optimizer/limits.yml | 2 +- .../aiops_log_pattern_analysis/constants.ts | 2 + .../public/cases/log_pattern_attachment.tsx | 61 +++++ ...arts_attachment.tsx => register_cases.tsx} | 32 ++- .../change_point_detection/fields_config.tsx | 2 +- .../log_categorization/attachments_menu.tsx | 243 ++++++++++++++++++ .../loading_categorization.tsx | 5 +- .../log_categorization_app_state.tsx | 23 +- .../log_categorization_for_embeddable.tsx | 13 +- .../log_categorization_page.tsx | 19 +- .../embeddable_chart_component_wrapper.tsx | 12 +- .../embeddable_pattern_analysis_factory.tsx | 2 + .../aiops/public/hooks/use_cases_modal.ts | 15 +- x-pack/plugins/aiops/public/plugin.tsx | 4 +- x-pack/plugins/aiops/server/register_cases.ts | 8 +- .../application/aiops/log_categorization.tsx | 2 + .../registered_persistable_state_trial.ts | 1 + .../apps/aiops/change_point_detection.ts | 24 ++ .../apps/aiops/log_pattern_analysis.ts | 42 +++ .../aiops/change_point_detection_page.ts | 23 ++ .../aiops/log_pattern_analysis_page.ts | 61 +++++ x-pack/test/functional/services/ml/cases.ts | 13 + 22 files changed, 575 insertions(+), 34 deletions(-) create mode 100644 x-pack/plugins/aiops/public/cases/log_pattern_attachment.tsx rename x-pack/plugins/aiops/public/cases/{register_change_point_charts_attachment.tsx => register_cases.tsx} (59%) create mode 100644 x-pack/plugins/aiops/public/components/log_categorization/attachments_menu.tsx diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 79145938fa109..8e1cd3e8fb7a1 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -2,7 +2,7 @@ pageLoadAssetSize: actions: 20000 advancedSettings: 27596 aiAssistantManagementSelection: 19146 - aiops: 16000 + aiops: 16526 alerting: 106936 apm: 64385 banners: 17946 diff --git a/x-pack/packages/ml/aiops_log_pattern_analysis/constants.ts b/x-pack/packages/ml/aiops_log_pattern_analysis/constants.ts index e88944a83b8bb..ff068307425bc 100644 --- a/x-pack/packages/ml/aiops_log_pattern_analysis/constants.ts +++ b/x-pack/packages/ml/aiops_log_pattern_analysis/constants.ts @@ -5,6 +5,8 @@ * 2.0. */ +export const CASES_ATTACHMENT_LOG_PATTERN = 'aiopsPatternAnalysisEmbeddable'; + export const EMBEDDABLE_PATTERN_ANALYSIS_TYPE = 'aiopsPatternAnalysisEmbeddable' as const; export const PATTERN_ANALYSIS_DATA_VIEW_REF_NAME = 'aiopsPatternAnalysisEmbeddableDataViewId'; diff --git a/x-pack/plugins/aiops/public/cases/log_pattern_attachment.tsx b/x-pack/plugins/aiops/public/cases/log_pattern_attachment.tsx new file mode 100644 index 0000000000000..33a64d26d38ff --- /dev/null +++ b/x-pack/plugins/aiops/public/cases/log_pattern_attachment.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; +import { memoize } from 'lodash'; +import React from 'react'; +import type { PersistableStateAttachmentViewProps } from '@kbn/cases-plugin/public/client/attachment_framework/types'; +import { FIELD_FORMAT_IDS } from '@kbn/field-formats-plugin/common'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { EuiDescriptionList } from '@elastic/eui'; +import deepEqual from 'fast-deep-equal'; +import type { + PatternAnalysisProps, + PatternAnalysisSharedComponent, +} from '../shared_components/pattern_analysis'; + +export const initComponent = memoize( + (fieldFormats: FieldFormatsStart, PatternAnalysisComponent: PatternAnalysisSharedComponent) => { + return React.memo( + (props: PersistableStateAttachmentViewProps) => { + const { persistableStateAttachmentState } = props; + + const dataFormatter = fieldFormats.deserialize({ + id: FIELD_FORMAT_IDS.DATE, + }); + + const inputProps = persistableStateAttachmentState as unknown as PatternAnalysisProps; + + const listItems = [ + { + title: ( + + ), + description: `${dataFormatter.convert( + inputProps.timeRange.from + )} - ${dataFormatter.convert(inputProps.timeRange.to)}`, + }, + ]; + + return ( + <> + + + + ); + }, + (prevProps, nextProps) => + deepEqual( + prevProps.persistableStateAttachmentState, + nextProps.persistableStateAttachmentState + ) + ); + } +); diff --git a/x-pack/plugins/aiops/public/cases/register_change_point_charts_attachment.tsx b/x-pack/plugins/aiops/public/cases/register_cases.tsx similarity index 59% rename from x-pack/plugins/aiops/public/cases/register_change_point_charts_attachment.tsx rename to x-pack/plugins/aiops/public/cases/register_cases.tsx index 4dc364836d185..b3b6efaf16d28 100644 --- a/x-pack/plugins/aiops/public/cases/register_change_point_charts_attachment.tsx +++ b/x-pack/plugins/aiops/public/cases/register_cases.tsx @@ -11,10 +11,14 @@ import { FormattedMessage } from '@kbn/i18n-react'; import type { CasesPublicSetup } from '@kbn/cases-plugin/public'; import type { CoreStart } from '@kbn/core/public'; import { CASES_ATTACHMENT_CHANGE_POINT_CHART } from '@kbn/aiops-change-point-detection/constants'; -import { getChangePointDetectionComponent } from '../shared_components'; +import { CASES_ATTACHMENT_LOG_PATTERN } from '@kbn/aiops-log-pattern-analysis/constants'; +import { + getChangePointDetectionComponent, + getPatternAnalysisComponent, +} from '../shared_components'; import type { AiopsPluginStartDeps } from '../types'; -export function registerChangePointChartsAttachment( +export function registerCases( cases: CasesPublicSetup, coreStart: CoreStart, pluginStart: AiopsPluginStartDeps @@ -44,4 +48,28 @@ export function registerChangePointChartsAttachment( }), }), }); + + const LogPatternAttachmentComponent = getPatternAnalysisComponent(coreStart, pluginStart); + + cases.attachmentFramework.registerPersistableState({ + id: CASES_ATTACHMENT_LOG_PATTERN, + icon: 'machineLearningApp', + displayName: i18n.translate('xpack.aiops.logPatternAnalysis.cases.attachmentName', { + defaultMessage: 'Log pattern analysis', + }), + getAttachmentViewObject: () => ({ + event: ( + + ), + timelineAvatar: 'machineLearningApp', + children: React.lazy(async () => { + const { initComponent } = await import('./log_pattern_attachment'); + + return { default: initComponent(pluginStart.fieldFormats, LogPatternAttachmentComponent) }; + }), + }), + }); } diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/fields_config.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/fields_config.tsx index f967fffd45647..90d356809acf5 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/fields_config.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/fields_config.tsx @@ -400,7 +400,7 @@ const FieldPanel: FC = ({ content: ( - + { diff --git a/x-pack/plugins/aiops/public/components/log_categorization/attachments_menu.tsx b/x-pack/plugins/aiops/public/components/log_categorization/attachments_menu.tsx new file mode 100644 index 0000000000000..37d0a828aa607 --- /dev/null +++ b/x-pack/plugins/aiops/public/components/log_categorization/attachments_menu.tsx @@ -0,0 +1,243 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiContextMenuProps } from '@elastic/eui'; +import { + EuiButton, + EuiButtonIcon, + EuiContextMenu, + EuiFlexItem, + EuiForm, + EuiFormRow, + EuiPanel, + EuiPopover, + EuiSpacer, + EuiSwitch, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import type { SaveModalDashboardProps } from '@kbn/presentation-util-plugin/public'; +import { + LazySavedObjectSaveModalDashboard, + withSuspense, +} from '@kbn/presentation-util-plugin/public'; +import React, { useCallback, useState } from 'react'; +import { useMemo } from 'react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import { EMBEDDABLE_PATTERN_ANALYSIS_TYPE } from '@kbn/aiops-log-pattern-analysis/constants'; +import { useTimeRangeUpdates } from '@kbn/ml-date-picker'; +import type { PatternAnalysisEmbeddableState } from '../../embeddables/pattern_analysis/types'; +import type { RandomSamplerOption, RandomSamplerProbability } from './sampling_menu/random_sampler'; +import { useCasesModal } from '../../hooks/use_cases_modal'; +import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; + +const SavedObjectSaveModalDashboard = withSuspense(LazySavedObjectSaveModalDashboard); + +interface AttachmentsMenuProps { + randomSamplerMode: RandomSamplerOption; + randomSamplerProbability: RandomSamplerProbability; + dataView: DataView; + selectedField?: string; +} + +export const AttachmentsMenu = ({ + randomSamplerMode, + randomSamplerProbability, + dataView, + selectedField, +}: AttachmentsMenuProps) => { + const { + application: { capabilities }, + cases, + embeddable, + } = useAiopsAppContext(); + + const [applyTimeRange, setApplyTimeRange] = useState(false); + + const [dashboardAttachmentReady, setDashboardAttachmentReady] = useState(false); + const [isActionMenuOpen, setIsActionMenuOpen] = useState(false); + + const { create: canCreateCase, update: canUpdateCase } = cases?.helpers?.canUseCases() ?? { + create: false, + update: false, + }; + + const openCasesModalCallback = useCasesModal(EMBEDDABLE_PATTERN_ANALYSIS_TYPE); + + const timeRange = useTimeRangeUpdates(); + + const canEditDashboards = capabilities.dashboard.createNew; + + const onSave: SaveModalDashboardProps['onSave'] = useCallback( + ({ dashboardId, newTitle, newDescription }) => { + const stateTransfer = embeddable!.getStateTransfer(); + + const embeddableInput: Partial = { + title: newTitle, + description: newDescription, + dataViewId: dataView.id, + fieldName: selectedField, + randomSamplerMode, + randomSamplerProbability, + minimumTimeRangeOption: 'No minimum', + ...(applyTimeRange && { timeRange }), + }; + + const state = { + input: embeddableInput, + type: EMBEDDABLE_PATTERN_ANALYSIS_TYPE, + }; + + const path = dashboardId === 'new' ? '#/create' : `#/view/${dashboardId}`; + + stateTransfer.navigateToWithEmbeddablePackage('dashboards', { + state, + path, + }); + }, + [ + embeddable, + dataView.id, + selectedField, + randomSamplerMode, + randomSamplerProbability, + applyTimeRange, + timeRange, + ] + ); + + const panels = useMemo(() => { + return [ + { + id: 'attachMainPanel', + size: 's', + items: [ + ...(canEditDashboards + ? [ + { + name: i18n.translate('xpack.aiops.logCategorization.addToDashboardTitle', { + defaultMessage: 'Add to dashboard', + }), + panel: 'attachToDashboardPanel', + 'data-test-subj': 'aiopsLogPatternAnalysisAttachToDashboardButton', + }, + ] + : []), + ...(canUpdateCase || canCreateCase + ? [ + { + name: i18n.translate('xpack.aiops.logCategorization.attachToCaseLabel', { + defaultMessage: 'Add to case', + }), + 'data-test-subj': 'aiopsLogPatternAnalysisAttachToCaseButton', + onClick: () => { + setIsActionMenuOpen(false); + openCasesModalCallback({ + dataViewId: dataView.id, + fieldName: selectedField, + minimumTimeRangeOption: 'No minimum', + randomSamplerMode, + randomSamplerProbability, + timeRange, + }); + }, + }, + ] + : []), + ], + }, + { + id: 'attachToDashboardPanel', + size: 's', + title: i18n.translate('xpack.aiops.logCategorization.addToDashboardTitle', { + defaultMessage: 'Add to dashboard', + }), + content: ( + + + + + setApplyTimeRange(e.target.checked)} + /> + + + { + setIsActionMenuOpen(false); + setDashboardAttachmentReady(true); + }} + > + + + + + ), + }, + ]; + }, [ + canEditDashboards, + canUpdateCase, + canCreateCase, + applyTimeRange, + openCasesModalCallback, + dataView.id, + selectedField, + randomSamplerMode, + randomSamplerProbability, + timeRange, + ]); + + return ( + + setIsActionMenuOpen(!isActionMenuOpen)} + /> + } + isOpen={isActionMenuOpen} + closePopover={() => setIsActionMenuOpen(false)} + panelPaddingSize="none" + anchorPosition="downRight" + > + + + {dashboardAttachmentReady ? ( + setDashboardAttachmentReady(false)} + onSave={onSave} + /> + ) : null} + + ); +}; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/loading_categorization.tsx b/x-pack/plugins/aiops/public/components/log_categorization/loading_categorization.tsx index 1d98325f2d987..17eac431dfba2 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/loading_categorization.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/loading_categorization.tsx @@ -19,7 +19,7 @@ import { import { FormattedMessage } from '@kbn/i18n-react'; interface Props { - onCancel: () => void; + onCancel?: () => void; } export const LoadingCategorization: FC = ({ onCancel }) => ( @@ -46,7 +46,8 @@ export const LoadingCategorization: FC = ({ onCancel }) => ( onCancel()} + onClick={onCancel} + disabled={!onCancel} > Cancel diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx index 85e81ec0f2996..073316455cb53 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_app_state.tsx @@ -60,17 +60,22 @@ export const LogCategorizationAppState: FC = ({ showFrozenDataTierChoice, }; + const CasesContext = appContextValue.cases?.ui.getCasesContext() ?? React.Fragment; + const casesPermissions = appContextValue.cases?.helpers.canUseCases(); + return ( - - - - - - - - - + + + + + + + + + + + ); }; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx index 5ca3cd947f7fe..fde191c42aa3e 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx @@ -18,7 +18,7 @@ import type { Category } from '@kbn/aiops-log-pattern-analysis/types'; import type { CategorizationAdditionalFilter } from '@kbn/aiops-log-pattern-analysis/create_category_request'; import type { EmbeddablePatternAnalysisInput } from '@kbn/aiops-log-pattern-analysis/embeddable'; import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state'; -import { AIOPS_ANALYSIS_RUN_ORIGIN } from '@kbn/aiops-common/constants'; +import { AIOPS_ANALYSIS_RUN_ORIGIN, AIOPS_EMBEDDABLE_ORIGIN } from '@kbn/aiops-common/constants'; import datemath from '@elastic/datemath'; import useMountedState from 'react-use/lib/useMountedState'; import { getEsQueryConfig } from '@kbn/data-service'; @@ -354,12 +354,19 @@ export const LogCategorizationEmbeddable: FC = [lastReloadRequestTime] ); - const actions = [...getActions(false), ...getActions(true)]; + const isCasesEmbeddable = embeddingOrigin === AIOPS_EMBEDDABLE_ORIGIN.CASES; + + // When in cases, we can only show the "Filter for pattern in Discover" actions as Cases does not have full filter management. + const actions = isCasesEmbeddable + ? getActions(true) + : [...getActions(false), ...getActions(true)]; return ( <> - {(loading ?? true) === true ? : null} + {(loading ?? true) === true ? ( + + ) : null} { const actions = getActions(true); + const attachmentsMenuProps = { + dataView, + selectedField, + randomSamplerMode: randomSampler.getMode(), + randomSamplerProbability: randomSampler.getProbability(), + }; + return ( @@ -390,9 +398,14 @@ export const LogCategorizationPage: FC = () => { )} - - loadCategories()} /> - + + + loadCategories()} /> + + + + + {eventRate.length ? ( diff --git a/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx b/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx index 69da4be087a14..15159d7adb60c 100644 --- a/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx +++ b/x-pack/plugins/aiops/public/embeddables/change_point_chart/embeddable_chart_component_wrapper.tsx @@ -6,11 +6,12 @@ */ import type { FC } from 'react'; -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import { css } from '@emotion/react'; import { CHANGE_POINT_DETECTION_VIEW_TYPE } from '@kbn/aiops-change-point-detection/constants'; import { getEsQueryConfig } from '@kbn/data-service'; import { buildEsQuery } from '@kbn/es-query'; +import { EuiLoadingSpinner } from '@elastic/eui'; import type { ChangePointDetectionProps } from '../../shared_components/change_point_detection'; import { ChangePointsTable } from '../../components/change_point_detection/change_points_table'; import { @@ -48,7 +49,6 @@ export const ChartGridEmbeddableWrapper: FC = ({ splitField, partitions, onError, - onLoading, onRenderComplete, onChange, emptyState, @@ -101,10 +101,6 @@ export const ChartGridEmbeddableWrapper: FC = ({ 10000 ); - useEffect(() => { - onLoading(isLoading); - }, [onLoading, isLoading]); - const changePoints = useMemo(() => { let resultChangePoints: ChangePointAnnotation[] = results.sort((a, b) => { if (defaultSort.direction === 'asc') { @@ -125,6 +121,10 @@ export const ChartGridEmbeddableWrapper: FC = ({ return resultChangePoints; }, [results, maxSeriesToPlot, onChange]); + if (isLoading) { + return ; + } + return (
) => { diff --git a/x-pack/plugins/aiops/public/hooks/use_cases_modal.ts b/x-pack/plugins/aiops/public/hooks/use_cases_modal.ts index a59fb8983b794..8ec73a21f9bbd 100644 --- a/x-pack/plugins/aiops/public/hooks/use_cases_modal.ts +++ b/x-pack/plugins/aiops/public/hooks/use_cases_modal.ts @@ -11,11 +11,22 @@ import { AttachmentType } from '@kbn/cases-plugin/common'; import type { ChangePointEmbeddableRuntimeState } from '../embeddables/change_point_chart/types'; import type { EmbeddableChangePointChartType } from '../embeddables/change_point_chart/embeddable_change_point_chart_factory'; import { useAiopsAppContext } from './use_aiops_app_context'; +import type { EmbeddablePatternAnalysisType } from '../embeddables/pattern_analysis/embeddable_pattern_analysis_factory'; +import type { PatternAnalysisEmbeddableRuntimeState } from '../embeddables/pattern_analysis/types'; + +type SupportedEmbeddableTypes = EmbeddableChangePointChartType | EmbeddablePatternAnalysisType; + +type EmbeddableRuntimeState = + T extends EmbeddableChangePointChartType + ? ChangePointEmbeddableRuntimeState + : T extends EmbeddablePatternAnalysisType + ? PatternAnalysisEmbeddableRuntimeState + : never; /** * Returns a callback for opening the cases modal with provided attachment state. */ -export const useCasesModal = ( +export const useCasesModal = ( embeddableType: EmbeddableType ) => { const { cases } = useAiopsAppContext(); @@ -23,7 +34,7 @@ export const useCasesModal = >) => { + (persistableState: Partial, 'id'>>) => { const persistableStateAttachmentState = { ...persistableState, // Creates unique id based on the input diff --git a/x-pack/plugins/aiops/public/plugin.tsx b/x-pack/plugins/aiops/public/plugin.tsx index 5863ea03b3072..d8c3dfd4c3636 100755 --- a/x-pack/plugins/aiops/public/plugin.tsx +++ b/x-pack/plugins/aiops/public/plugin.tsx @@ -19,7 +19,7 @@ import type { } from './types'; import { registerEmbeddables } from './embeddables'; import { registerAiopsUiActions } from './ui_actions'; -import { registerChangePointChartsAttachment } from './cases/register_change_point_charts_attachment'; +import { registerCases } from './cases/register_cases'; export type AiopsCoreSetup = CoreSetup; @@ -44,7 +44,7 @@ export class AiopsPlugin } if (cases) { - registerChangePointChartsAttachment(cases, coreStart, pluginStart); + registerCases(cases, coreStart, pluginStart); } } } diff --git a/x-pack/plugins/aiops/server/register_cases.ts b/x-pack/plugins/aiops/server/register_cases.ts index 8877c2ef9b5ee..5649c88ca6327 100644 --- a/x-pack/plugins/aiops/server/register_cases.ts +++ b/x-pack/plugins/aiops/server/register_cases.ts @@ -8,6 +8,7 @@ import type { Logger } from '@kbn/core/server'; import type { CasesServerSetup } from '@kbn/cases-plugin/server'; import { CASES_ATTACHMENT_CHANGE_POINT_CHART } from '@kbn/aiops-change-point-detection/constants'; +import { CASES_ATTACHMENT_LOG_PATTERN } from '@kbn/aiops-log-pattern-analysis/constants'; export function registerCasesPersistableState(cases: CasesServerSetup | undefined, logger: Logger) { if (cases) { @@ -15,10 +16,11 @@ export function registerCasesPersistableState(cases: CasesServerSetup | undefine cases.attachmentFramework.registerPersistableState({ id: CASES_ATTACHMENT_CHANGE_POINT_CHART, }); + cases.attachmentFramework.registerPersistableState({ + id: CASES_ATTACHMENT_LOG_PATTERN, + }); } catch (error) { - logger.warn( - `AIOPs failed to register cases persistable state for ${CASES_ATTACHMENT_CHANGE_POINT_CHART}` - ); + logger.warn(`AIOPs failed to register cases persistable state`); } } } diff --git a/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx b/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx index fea9b0d7e8810..2dc34ba80a080 100644 --- a/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx +++ b/x-pack/plugins/ml/public/application/aiops/log_categorization.tsx @@ -63,6 +63,8 @@ export const LogCategorizationPage: FC = () => { 'uiActions', 'uiSettings', 'unifiedSearch', + 'embeddable', + 'cases', ]), }} /> diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/attachments_framework/registered_persistable_state_trial.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/attachments_framework/registered_persistable_state_trial.ts index c5e5a23032f66..0969a9261df26 100644 --- a/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/attachments_framework/registered_persistable_state_trial.ts +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/trial/attachments_framework/registered_persistable_state_trial.ts @@ -39,6 +39,7 @@ export default ({ getService }: FtrProviderContext): void => { ml_anomaly_charts: '23e92e824af9db6e8b8bb1d63c222e04f57d2147', ml_anomaly_swimlane: 'a3517f3e53fb041e9cbb150477fb6ef0f731bd5f', ml_single_metric_viewer: '8b9532b0a40dfdfa282e262949b82cc1a643147c', + aiopsPatternAnalysisEmbeddable: '6c2809a0c51e668d11794de0815b293fdb3a9060', }); }); }); diff --git a/x-pack/test/functional/apps/aiops/change_point_detection.ts b/x-pack/test/functional/apps/aiops/change_point_detection.ts index 22177a0a9166d..c0ac744e687b5 100644 --- a/x-pack/test/functional/apps/aiops/change_point_detection.ts +++ b/x-pack/test/functional/apps/aiops/change_point_detection.ts @@ -7,11 +7,13 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; +import { USER } from '../../services/ml/security_common'; export default function ({ getPageObjects, getService }: FtrProviderContext) { const elasticChart = getService('elasticChart'); const esArchiver = getService('esArchiver'); const aiops = getService('aiops'); + const cases = getService('cases'); // aiops lives in the ML UI so we need some related services. const ml = getService('ml'); @@ -26,6 +28,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { after(async () => { await ml.testResources.deleteDataViewByTitle('ft_ecommerce'); + await cases.api.deleteAllCases(); }); it(`loads the change point detection page`, async () => { @@ -108,5 +111,26 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { maxSeries: 1, }); }); + + it('attaches change point charts to a case', async () => { + await ml.navigation.navigateToMl(); + await elasticChart.setNewChartUiDebugFlag(true); + await aiops.changePointDetectionPage.navigateToDataViewSelection(); + await ml.jobSourceSelection.selectSourceForChangePointDetection('ft_ecommerce'); + await aiops.changePointDetectionPage.assertChangePointDetectionPageExists(); + + await aiops.changePointDetectionPage.clickUseFullDataButton(); + await aiops.changePointDetectionPage.selectMetricField(0, 'products.discount_amount'); + + const caseParams = { + title: 'ML Change Point Detection case', + description: 'Case with a change point detection attachment', + tag: 'ml_change_point_detection', + reporter: USER.ML_POWERUSER, + }; + + await aiops.changePointDetectionPage.attachChartsToCases(0, caseParams); + await ml.cases.assertCaseWithChangePointDetectionChartsAttachment(caseParams); + }); }); } diff --git a/x-pack/test/functional/apps/aiops/log_pattern_analysis.ts b/x-pack/test/functional/apps/aiops/log_pattern_analysis.ts index 4cfca6d4d82b5..b056b3d6ec8fb 100644 --- a/x-pack/test/functional/apps/aiops/log_pattern_analysis.ts +++ b/x-pack/test/functional/apps/aiops/log_pattern_analysis.ts @@ -6,6 +6,7 @@ */ import { FtrProviderContext } from '../../ftr_provider_context'; +import { USER } from '../../services/ml/security_common'; export default function ({ getPageObjects, getService }: FtrProviderContext) { const elasticChart = getService('elasticChart'); @@ -16,6 +17,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const ml = getService('ml'); const selectedField = '@message'; const totalDocCount = 14005; + const cases = getService('cases'); async function retrySwitchTab(tabIndex: number, seconds: number) { await retry.tryForTime(seconds * 1000, async () => { @@ -43,6 +45,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { after(async () => { await ml.testResources.deleteDataViewByTitle('logstash-*'); + await cases.api.deleteAllCases(); }); it(`loads the log pattern analysis page and filters in patterns in discover`, async () => { @@ -97,5 +100,44 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { // ensure the discover doc count is greater than 0 await aiops.logPatternAnalysisPage.assertDiscoverDocCountGreaterThan(0); }); + + it('attaches log pattern analysis table to a dashboard', async () => { + // Start navigation from the base of the ML app. + await ml.navigation.navigateToMl(); + await elasticChart.setNewChartUiDebugFlag(true); + await aiops.logPatternAnalysisPage.navigateToDataViewSelection(); + await ml.jobSourceSelection.selectSourceForLogPatternAnalysisDetection('logstash-*'); + await aiops.logPatternAnalysisPage.assertLogPatternAnalysisPageExists(); + + await aiops.logPatternAnalysisPage.clickUseFullDataButton(totalDocCount); + await aiops.logPatternAnalysisPage.selectCategoryField(selectedField); + await aiops.logPatternAnalysisPage.clickRunButton(); + + await aiops.logPatternAnalysisPage.attachToDashboard(); + }); + + it('attaches log pattern analysis table to a case', async () => { + // Start navigation from the base of the ML app. + await ml.navigation.navigateToMl(); + await elasticChart.setNewChartUiDebugFlag(true); + await aiops.logPatternAnalysisPage.navigateToDataViewSelection(); + await ml.jobSourceSelection.selectSourceForLogPatternAnalysisDetection('logstash-*'); + await aiops.logPatternAnalysisPage.assertLogPatternAnalysisPageExists(); + + await aiops.logPatternAnalysisPage.clickUseFullDataButton(totalDocCount); + await aiops.logPatternAnalysisPage.selectCategoryField(selectedField); + await aiops.logPatternAnalysisPage.clickRunButton(); + + const caseParams = { + title: 'ML Log pattern analysis case', + description: 'Case with a log pattern analysis attachment', + tag: 'ml_log_pattern_analysis', + reporter: USER.ML_POWERUSER, + }; + + await aiops.logPatternAnalysisPage.attachToCase(caseParams); + + await ml.cases.assertCaseWithLogPatternAnalysisAttachment(caseParams); + }); }); } diff --git a/x-pack/test/functional/services/aiops/change_point_detection_page.ts b/x-pack/test/functional/services/aiops/change_point_detection_page.ts index 79bc4c378fb1a..e4eceb5539856 100644 --- a/x-pack/test/functional/services/aiops/change_point_detection_page.ts +++ b/x-pack/test/functional/services/aiops/change_point_detection_page.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; import { MlTableService } from '../ml/common_table_service'; +import { CreateCaseParams } from '../cases/create'; export interface DashboardAttachmentOptions { applyTimeRange: boolean; @@ -24,6 +25,7 @@ export function ChangePointDetectionPageProvider( const browser = getService('browser'); const elasticChart = getService('elasticChart'); const dashboardPage = getPageObject('dashboard'); + const cases = getService('cases'); return { async navigateToDataViewSelection() { @@ -160,6 +162,17 @@ export function ChangePointDetectionPageProvider( }); }, + async clickAttachCasesButton() { + await testSubjects.click('aiopsChangePointDetectionAttachToCaseButton'); + await retry.tryForTime(30 * 1000, async () => { + await testSubjects.existOrFail('aiopsChangePointDetectionCaseAttachmentForm'); + }); + }, + + async clickSubmitCaseAttachButton() { + await testSubjects.click('aiopsChangePointDetectionSubmitCaseAttachButton'); + }, + async assertApplyTimeRangeControl(expectedValue: boolean) { const isChecked = await testSubjects.isEuiSwitchChecked( `aiopsChangePointDetectionAttachToDashboardApplyTimeRangeSwitch` @@ -281,5 +294,15 @@ export function ChangePointDetectionPageProvider( `aiopsChangePointPanel_${index}` ); }, + + async attachChartsToCases(panelIndex: number, params: CreateCaseParams) { + await this.assertPanelExist(panelIndex); + await this.openPanelContextMenu(panelIndex); + await this.clickAttachChartsButton(); + await this.clickAttachCasesButton(); + await this.clickSubmitCaseAttachButton(); + + await cases.create.createCaseFromModal(params); + }, }; } diff --git a/x-pack/test/functional/services/aiops/log_pattern_analysis_page.ts b/x-pack/test/functional/services/aiops/log_pattern_analysis_page.ts index 558cfb0af9f0b..f1c62cf63ae5e 100644 --- a/x-pack/test/functional/services/aiops/log_pattern_analysis_page.ts +++ b/x-pack/test/functional/services/aiops/log_pattern_analysis_page.ts @@ -8,11 +8,14 @@ import expect from '@kbn/expect'; import type { FtrProviderContext } from '../../ftr_provider_context'; +import { CreateCaseParams } from '../cases/create'; export function LogPatternAnalysisPageProvider({ getService, getPageObject }: FtrProviderContext) { const retry = getService('retry'); const testSubjects = getService('testSubjects'); const comboBox = getService('comboBox'); + const dashboardPage = getPageObject('dashboard'); + const cases = getService('cases'); type RandomSamplerOption = | 'aiopsRandomSamplerOptionOnAutomatic' @@ -249,5 +252,63 @@ export function LogPatternAnalysisPageProvider({ getService, getPageObject }: Ft await testSubjects.missingOrFail('aiopsRandomSamplerOptionsFormRow', { timeout: 1000 }); }); }, + + async openAttachmentsMenu() { + await testSubjects.click('aiopsLogPatternAnalysisAttachmentsMenuButton'); + }, + + async clickAttachToDashboard() { + await testSubjects.click('aiopsLogPatternAnalysisAttachToDashboardButton'); + }, + + async clickAttachToCase() { + await testSubjects.click('aiopsLogPatternAnalysisAttachToCaseButton'); + }, + + async confirmAttachToDashboard() { + await testSubjects.click('aiopsLogPatternAnalysisAttachToDashboardSubmitButton'); + }, + + async completeSaveToDashboardForm(createNew?: boolean) { + const dashboardSelector = await testSubjects.find('add-to-dashboard-options'); + if (createNew) { + const label = await dashboardSelector.findByCssSelector( + `label[for="new-dashboard-option"]` + ); + await label.click(); + } + + await testSubjects.click('confirmSaveSavedObjectButton'); + await retry.waitForWithTimeout('Save modal to disappear', 1000, () => + testSubjects + .missingOrFail('confirmSaveSavedObjectButton') + .then(() => true) + .catch(() => false) + ); + + // make sure the dashboard page actually loaded + const dashboardItemCount = await dashboardPage.getSharedItemsCount(); + expect(dashboardItemCount).to.not.eql(undefined); + + const embeddable = await testSubjects.find('aiopsEmbeddablePatternAnalysis', 30 * 1000); + expect(await embeddable.isDisplayed()).to.eql( + true, + 'Log pattern analysis chart should be displayed in dashboard' + ); + }, + + async attachToDashboard() { + await this.openAttachmentsMenu(); + await this.clickAttachToDashboard(); + await this.confirmAttachToDashboard(); + await this.completeSaveToDashboardForm(true); + }, + + async attachToCase(params: CreateCaseParams) { + await this.openAttachmentsMenu(); + await this.clickAttachToCase(); + + await cases.create.createCaseFromModal(params); + }, }; } diff --git a/x-pack/test/functional/services/ml/cases.ts b/x-pack/test/functional/services/ml/cases.ts index 6c481df8b99a8..2245410985592 100644 --- a/x-pack/test/functional/services/ml/cases.ts +++ b/x-pack/test/functional/services/ml/cases.ts @@ -81,5 +81,18 @@ export function MachineLearningCasesProvider( expectedChartsCount ); }, + + async assertCaseWithLogPatternAnalysisAttachment(params: CaseParams) { + await this.assertBasicCaseProps(params); + await testSubjects.existOrFail('comment-persistableState-aiopsPatternAnalysisEmbeddable'); + await testSubjects.existOrFail('aiopsEmbeddablePatternAnalysis'); + await testSubjects.existOrFail('aiopsLogPatternsTable'); + }, + + async assertCaseWithChangePointDetectionChartsAttachment(params: CaseParams) { + await this.assertBasicCaseProps(params); + await testSubjects.existOrFail('comment-persistableState-aiopsChangePointChart'); + await testSubjects.existOrFail('aiopsEmbeddableChangePointChart'); + }, }; } From 4ba25f211ab6fa8d256d6385583b1ad6f2868717 Mon Sep 17 00:00:00 2001 From: Thom Heymann <190132+thomheymann@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:28:44 +0000 Subject: [PATCH 20/53] [Observability] Populate service.name for custom logs as part of auto-detect flow (#199827) Resolves https://github.com/elastic/observability-dev/issues/4038 ## Summary Populate service.name for custom logs as part of auto-detect flow ## Screenshot Screenshot 2024-11-12 at 17 10 29 --- .../server/routes/flow/route.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts index 5aa2bc489ab59..f0663aa27ccad 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts +++ b/x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts @@ -300,7 +300,8 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({ * --header "Accept: application/x-tar" \ * --header "Content-Type: text/tab-separated-values" \ * --header "kbn-xsrf: true" \ - * --data $'system\tregistry\nproduct_service\tcustom\t/path/to/access.log\ncheckout_service\tcustom\t/path/to/access.log' \ + * --header "x-elastic-internal-origin: Kibana" \ + * --data $'system\tregistry\twebserver01\nproduct_service\tcustom\t/path/to/access.log\ncheckout_service\tcustom\t/path/to/access.log' \ * --output - | tar -tvf - * ``` */ @@ -456,6 +457,16 @@ async function ensureInstalledIntegrations( id: `filestream-${pkgName}`, data_stream: dataStream, paths: integration.logFilePaths, + processors: [ + { + add_fields: { + target: 'service', + fields: { + name: pkgName, + }, + }, + }, + ], }, ], }, From 20dd7f1bbe61e6263476228d2225d68871849e25 Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:38:05 +0100 Subject: [PATCH 21/53] [Index Mgmt] Improve accessibility of templates table (#199980) ## Summary This improves the accessibility and understandability of the content column in the index templates page by: 1) Improving the tooltip text to be explicit about whether something is present or not 2) Adding an aria-label with the same content, as tooltips are not keyboard accessible ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) --- .../components/template_content_indicator.tsx | 63 +++++++++++++------ .../translations/translations/fr-FR.json | 3 - .../translations/translations/ja-JP.json | 3 - .../translations/translations/zh-CN.json | 3 - 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/index_management/public/application/components/shared/components/template_content_indicator.tsx b/x-pack/plugins/index_management/public/application/components/shared/components/template_content_indicator.tsx index 7ccc4971ef97c..c2df923018a39 100644 --- a/x-pack/plugins/index_management/public/application/components/shared/components/template_content_indicator.tsx +++ b/x-pack/plugins/index_management/public/application/components/shared/components/template_content_indicator.tsx @@ -16,17 +16,7 @@ interface Props { contentWhenEmpty?: JSX.Element | null; } -const texts = { - settings: i18n.translate('xpack.idxMgmt.templateContentIndicator.indexSettingsTooltipLabel', { - defaultMessage: 'Index settings', - }), - mappings: i18n.translate('xpack.idxMgmt.templateContentIndicator.mappingsTooltipLabel', { - defaultMessage: 'Mappings', - }), - aliases: i18n.translate('xpack.idxMgmt.templateContentIndicator.aliasesTooltipLabel', { - defaultMessage: 'Aliases', - }), -}; +const getColor = (flag: boolean) => (flag ? 'primary' : 'hollow'); export const TemplateContentIndicator = ({ mappings, @@ -34,28 +24,63 @@ export const TemplateContentIndicator = ({ aliases, contentWhenEmpty = null, }: Props) => { - const getColor = (flag: boolean) => (flag ? 'primary' : 'hollow'); - if (!mappings && !settings && !aliases) { return contentWhenEmpty; } + const texts = { + settingsTrue: i18n.translate('xpack.idxMgmt.templateContentIndicator.indexSettingsTrueLabel', { + defaultMessage: 'This template contains index settings', + }), + settingsFalse: i18n.translate( + 'xpack.idxMgmt.templateContentIndicator.indexSettingsFalseLabel', + { + defaultMessage: 'This template does not contain index settings', + } + ), + mappingsTrue: i18n.translate('xpack.idxMgmt.templateContentIndicator.indexMappingsTrueLabel', { + defaultMessage: 'This template contains index mappings', + }), + mappingsFalse: i18n.translate( + 'xpack.idxMgmt.templateContentIndicator.indexMappingsFalseLabel', + { + defaultMessage: 'This template does not contain index mappings', + } + ), + aliasesTrue: i18n.translate('xpack.idxMgmt.templateContentIndicator.indexAliasesTrueLabel', { + defaultMessage: 'This template contains index aliases', + }), + aliasesFalse: i18n.translate('xpack.idxMgmt.templateContentIndicator.indexAliasesFalseLabel', { + defaultMessage: 'This template does not contain index aliases', + }), + }; + + const mappingsText = mappings ? texts.mappingsTrue : texts.mappingsFalse; + const settingsText = settings ? texts.settingsTrue : texts.settingsFalse; + const aliasesText = aliases ? texts.aliasesTrue : texts.aliasesFalse; + return ( <> - + <> - M + + M +   - + <> - S + + S +   - - A + + + A + ); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 5ccbe70bd8fe1..4339c2fb0310c 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -23492,9 +23492,6 @@ "xpack.idxMgmt.templateBadgeType.cloudManaged": "Géré dans le cloud", "xpack.idxMgmt.templateBadgeType.managed": "Géré", "xpack.idxMgmt.templateBadgeType.system": "Système", - "xpack.idxMgmt.templateContentIndicator.aliasesTooltipLabel": "Alias", - "xpack.idxMgmt.templateContentIndicator.indexSettingsTooltipLabel": "Paramètres des index", - "xpack.idxMgmt.templateContentIndicator.mappingsTooltipLabel": "Mappings", "xpack.idxMgmt.templateCreate.loadingTemplateToCloneDescription": "Chargement du modèle à cloner en cours…", "xpack.idxMgmt.templateCreate.loadingTemplateToCloneErrorMessage": "Erreur lors du chargement du modèle à cloner", "xpack.idxMgmt.templateDetails.aliasesTabTitle": "Alias", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 8aadc59451b36..b4c96ee85f3d1 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -23464,9 +23464,6 @@ "xpack.idxMgmt.templateBadgeType.cloudManaged": "クラウド管理", "xpack.idxMgmt.templateBadgeType.managed": "管理中", "xpack.idxMgmt.templateBadgeType.system": "システム", - "xpack.idxMgmt.templateContentIndicator.aliasesTooltipLabel": "エイリアス", - "xpack.idxMgmt.templateContentIndicator.indexSettingsTooltipLabel": "インデックス設定", - "xpack.idxMgmt.templateContentIndicator.mappingsTooltipLabel": "マッピング", "xpack.idxMgmt.templateCreate.loadingTemplateToCloneDescription": "クローンを作成するテンプレートを読み込み中…", "xpack.idxMgmt.templateCreate.loadingTemplateToCloneErrorMessage": "クローンを作成するテンプレートを読み込み中にエラーが発生", "xpack.idxMgmt.templateDetails.aliasesTabTitle": "エイリアス", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 6bb17f1a2bdaa..661a37969e6a5 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -23063,9 +23063,6 @@ "xpack.idxMgmt.templateBadgeType.cloudManaged": "云托管", "xpack.idxMgmt.templateBadgeType.managed": "托管", "xpack.idxMgmt.templateBadgeType.system": "系统", - "xpack.idxMgmt.templateContentIndicator.aliasesTooltipLabel": "别名", - "xpack.idxMgmt.templateContentIndicator.indexSettingsTooltipLabel": "索引设置", - "xpack.idxMgmt.templateContentIndicator.mappingsTooltipLabel": "映射", "xpack.idxMgmt.templateCreate.loadingTemplateToCloneDescription": "正在加载要克隆的模板……", "xpack.idxMgmt.templateCreate.loadingTemplateToCloneErrorMessage": "加载要克隆的模板时出错", "xpack.idxMgmt.templateDetails.aliasesTabTitle": "别名", From 9a33211acab2f6c7110498f53c4de33bb89cd59c Mon Sep 17 00:00:00 2001 From: Chenhui Wang <54903978+wangch079@users.noreply.github.com> Date: Wed, 13 Nov 2024 23:57:15 +0800 Subject: [PATCH 22/53] Update the elastic-connectors docker namespace to integrations (#194537) ## Summary This PR updates the `elastic-connectors` docker namespace to `integrations` ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) Co-authored-by: Elastic Machine --- .../components/search_index/connector/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/constants.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/constants.ts index 65120d78cec84..d6f8b72f2e4f4 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/constants.ts @@ -39,6 +39,6 @@ export const getRunFromDockerSnippet = ({ version }: { version: string }) => `do -v "$HOME/elastic-connectors:/config" \\ --tty \\ --rm \\ -docker.elastic.co/enterprise-search/elastic-connectors:${version} \\ +docker.elastic.co/integrations/elastic-connectors:${version} \\ /app/bin/elastic-ingest \\ -c /config/config.yml`; From 9af4bb5b7f70c9f281f728b24292e89171dcc846 Mon Sep 17 00:00:00 2001 From: Catherine Liu Date: Wed, 13 Nov 2024 08:46:30 -0800 Subject: [PATCH 23/53] [Dashboard Usability] Remove borders from presentation panel and CSS clean up for hover actions (#198454) ## Summary Closes https://github.com/elastic/kibana/issues/198370. Follow up to #182535. This removes border style overrides from the apps that support embeddables and cleans up outline styles for presentation panel borders/hover actions. I removed all of the dashboard specific border styles and made them more generic for any embeddable panel. Now, the borders respect the `showBorder` prop on the embeddable input. However even when `showBorder` is `false`, a border is shown on focus/hover to align with the border styles of the hover actions. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#_add_your_labels) - [ ] This will appear in the **Release Notes** and follow the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../component/grid/_dashboard_grid.scss | 2 + .../component/grid/_dashboard_panel.scss | 39 +--- .../__stories__/embeddable_panel.stories.tsx | 2 +- .../group_editor_flyout/lens_attributes.ts | 1 + .../panel_component/_presentation_panel.scss | 98 +------- .../presentation_panel_hover_actions.tsx | 219 ++++++++++++------ .../panel_component/presentation_panel.tsx | 8 +- .../presentation_panel_internal.tsx | 3 +- .../renderers/embeddable/embeddable.scss | 15 -- .../embeddable/embeddable.tsx | 3 - .../visualization_actions/lens_embeddable.tsx | 13 -- .../components/embeddables/embedded_map.tsx | 5 - 12 files changed, 165 insertions(+), 243 deletions(-) diff --git a/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_grid.scss b/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_grid.scss index 49a6b01049da7..ce010aa4cf9a5 100644 --- a/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_grid.scss +++ b/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_grid.scss @@ -113,10 +113,12 @@ z-index: $euiZLevel9; top: -$euiSizeXL; + // Show hover actions with drag handle .embPanel__hoverActions:has(.embPanel--dragHandle) { opacity: 1; } + // Hide hover actions without drag handle .embPanel__hoverActions:not(:has(.embPanel--dragHandle)) { opacity: 0; } diff --git a/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_panel.scss b/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_panel.scss index 93a95e1ef37e5..2b7ec068f827d 100644 --- a/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_panel.scss +++ b/src/plugins/dashboard/public/dashboard_container/component/grid/_dashboard_panel.scss @@ -1,44 +1,11 @@ -/** - * EDITING MODE - * Use .dshLayout--editing to target editing state because - * .embPanel--editing doesn't get updating without a hard refresh - */ - -.dshLayout--editing { - // change the style of the hover actions border to a dashed line in edit mode - .embPanel__hoverActionsAnchor { - .embPanel__hoverActionsWrapper { - .embPanel__hoverActions { - border-color: $euiColorMediumShade; - border-style: dashed; - } - } - } -} - // LAYOUT MODES // Adjust borders/etc... for non-spaced out and expanded panels .dshLayout-withoutMargins { - .embPanel, - .embPanel__hoverActionsAnchor { - box-shadow: none; - outline: none; - border-radius: 0; - } - - &.dshLayout--editing { - .embPanel__hoverActionsAnchor:hover { - outline: 1px dashed $euiColorMediumShade; - } - } - - .embPanel__hoverActionsAnchor:hover { - outline: $euiBorderThin; - z-index: $euiZLevel2; - } + margin-top: $euiSizeS; .embPanel__content, - .dshDashboardGrid__item--highlighted, + .embPanel, + .embPanel__hoverActionsAnchor, .lnsExpressionRenderer { border-radius: 0; } diff --git a/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx b/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx index 6e1f58d9689a6..bcb0af1cf9064 100644 --- a/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx +++ b/src/plugins/embeddable/public/__stories__/embeddable_panel.stories.tsx @@ -128,7 +128,7 @@ Default.args = { hideHeader: false, loading: false, showShadow: false, - showBorder: true, + showBorder: false, title: 'Hello World', viewMode: true, }; diff --git a/src/plugins/event_annotation_listing/public/components/group_editor_flyout/lens_attributes.ts b/src/plugins/event_annotation_listing/public/components/group_editor_flyout/lens_attributes.ts index 9de624c7bcf6a..416c18c7f457f 100644 --- a/src/plugins/event_annotation_listing/public/components/group_editor_flyout/lens_attributes.ts +++ b/src/plugins/event_annotation_listing/public/components/group_editor_flyout/lens_attributes.ts @@ -72,6 +72,7 @@ export const getLensAttributes = (group: EventAnnotationGroupConfig, timeField: language: 'kuery', }, filters: [], + showBorder: false, datasourceStates: { formBased: { layers: { diff --git a/src/plugins/presentation_panel/public/panel_component/_presentation_panel.scss b/src/plugins/presentation_panel/public/panel_component/_presentation_panel.scss index 5094cf6b02ba3..86f9e8d378e5a 100644 --- a/src/plugins/presentation_panel/public/panel_component/_presentation_panel.scss +++ b/src/plugins/presentation_panel/public/panel_component/_presentation_panel.scss @@ -6,8 +6,6 @@ height: 100%; min-height: $euiSizeL + 2px; // + 2px to account for border position: relative; - border: none; - outline: $euiBorderThin; &-isLoading { // completely center the loading indicator @@ -105,47 +103,9 @@ } } -// OPTIONS MENU - -/** - * 1. Use opacity to make this element accessible to screen readers and keyboard. - * 2. Show on focus to enable keyboard accessibility. - * 3. Always show in editing mode - */ - -.embPanel__optionsMenuButton { - background-color: transparentize($euiColorLightShade, .5); - border-bottom-right-radius: 0; - border-top-left-radius: 0; - - &:focus { - background-color: transparentize($euiColorLightestShade, .5); - } -} - -.embPanel__optionsMenuPopover-loading { - width: $euiSizeS * 32; -} - -.embPanel__optionsMenuPopover-notification::after { - position: absolute; - top: 0; - right: 0; - content: '•'; - transform: translate(50%, -50%); - color: $euiColorAccent; - font-size: $euiSizeL; -} - // EDITING MODE - .embPanel--editing { - transition: all $euiAnimSpeedFast $euiAnimSlightResistance; - outline: 1px dashed $euiColorMediumShade; - .embPanel--dragHandle { - transition: background-color $euiAnimSpeedFast $euiAnimSlightResistance; - .embPanel--dragHandle:hover { background-color: transparentize($euiColorWarning, lightOrDarkTheme(.9, .7)); cursor: move; @@ -170,56 +130,14 @@ z-index: $euiZLevel1; } -.embPanel__hoverActionsAnchor { - position: relative; - height: 100%; - - .embPanel__hoverActionsWrapper { - height: $euiSizeXL; - position: absolute; - top: 0; - display: flex; - justify-content: space-between; - padding: 0 $euiSize; - flex-wrap: nowrap; - min-width: 100%; - z-index: -1; - pointer-events: none; // Prevent hover actions wrapper from blocking interactions with other panels - } - - .embPanel__hoverActions { - opacity: 0; - padding: calc($euiSizeXS - 1px); - display: flex; - flex-wrap: nowrap; - border: $euiBorderThin; - - background-color: $euiColorEmptyShade; - height: $euiSizeXL; - - pointer-events: all; // Re-enable pointer-events for hover actions - } - - .embPanel--dragHandle { - cursor: move; +.embPanel--dragHandle { + cursor: move; - img { - pointer-events: all !important; - } + img { + pointer-events: all !important; } +} - .embPanel__descriptionTooltipAnchor { - padding: $euiSizeXS; - } - - &:hover .embPanel__hoverActionsWrapper, - &:focus-within .embPanel__hoverActionsWrapper, - .embPanel__hoverActionsWrapper--lockHoverActions { - z-index: $euiZLevel9; - top: -$euiSizeXL; - - .embPanel__hoverActions { - opacity: 1; - } - } -} \ No newline at end of file +.embPanel__descriptionTooltipAnchor { + padding: $euiSizeXS; +} diff --git a/src/plugins/presentation_panel/public/panel_component/panel_header/presentation_panel_hover_actions.tsx b/src/plugins/presentation_panel/public/panel_component/panel_header/presentation_panel_hover_actions.tsx index 469a1f8c4f6e3..b29563713d365 100644 --- a/src/plugins/presentation_panel/public/panel_component/panel_header/presentation_panel_hover_actions.tsx +++ b/src/plugins/presentation_panel/public/panel_component/panel_header/presentation_panel_hover_actions.tsx @@ -54,6 +54,9 @@ import { getContextMenuAriaLabel } from '../presentation_panel_strings'; import { DefaultPresentationPanelApi, PresentationPanelInternalProps } from '../types'; import { AnyApiAction } from '../../panel_actions/types'; +const DASHED_OUTLINE = `1px dashed ${euiThemeVars.euiColorMediumShade}`; +const SOLID_OUTLINE = `1px solid ${euiThemeVars.euiBorderColor}`; + const QUICK_ACTION_IDS = { edit: [ 'editPanel', @@ -67,12 +70,14 @@ const QUICK_ACTION_IDS = { const ALLOWED_NOTIFICATIONS = ['ACTION_FILTERS_NOTIFICATION'] as const; -const ALL_ROUNDED_CORNERS = `border-radius: ${euiThemeVars.euiBorderRadius}; +const ALL_ROUNDED_CORNERS = ` + border-radius: ${euiThemeVars.euiBorderRadius}; +`; +const TOP_ROUNDED_CORNERS = ` + border-top-left-radius: ${euiThemeVars.euiBorderRadius}; + border-top-right-radius: ${euiThemeVars.euiBorderRadius}; + border-bottom: 0px; `; -const TOP_ROUNDED_CORNERS = `border-top-left-radius: ${euiThemeVars.euiBorderRadius}; - border-top-right-radius: ${euiThemeVars.euiBorderRadius}; - border-bottom: 0 !important; - `; const createClickHandler = (action: AnyApiAction, context: ActionExecutionContext) => @@ -101,6 +106,7 @@ export const PresentationPanelHoverActions = ({ className, viewMode, showNotifications = true, + showBorder, }: { index?: number; api: DefaultPresentationPanelApi | null; @@ -110,6 +116,7 @@ export const PresentationPanelHoverActions = ({ className?: string; viewMode?: ViewMode; showNotifications?: boolean; + showBorder?: boolean; }) => { const [quickActions, setQuickActions] = useState([]); const [contextMenuPanels, setContextMenuPanels] = useState([]); @@ -195,7 +202,6 @@ export const PresentationPanelHoverActions = ({ ); const hideTitle = hidePanelTitle || parentHideTitle; - const showDescription = description && (!title || hideTitle); const quickActionIds = useMemo( @@ -429,7 +435,7 @@ export const PresentationPanelHoverActions = ({ onClick={() => { setIsContextMenuOpen(!isContextMenuOpen); if (apiCanLockHoverActions(api)) { - api?.lockHoverActions(!hasLockedHoverActions); + api.lockHoverActions(!hasLockedHoverActions); } }} iconType="boxesVertical" @@ -451,26 +457,81 @@ export const PresentationPanelHoverActions = ({ /> ); + const hasHoverActions = quickActionElements.length || contextMenuPanels.lastIndexOf.length; + return (
{children} {api ? (
{viewMode === 'edit' && !combineHoverActions ? (
@@ -490,72 +552,75 @@ export const PresentationPanelHoverActions = ({ ) : (
// necessary for the right hover actions to align correctly when left hover actions are not present )} -
- {viewMode === 'edit' && combineHoverActions && dragHandle} - {showNotifications && notificationElements} - {showDescription && ( - - )} - {quickActionElements.map( - ({ iconType, 'data-test-subj': dataTestSubj, onClick, name }, i) => ( - - - - ) - )} - {contextMenuPanels.length ? ( - - + {viewMode === 'edit' && combineHoverActions && dragHandle} + {showNotifications && notificationElements} + {showDescription && ( + - - ) : null} -
+ )} + {quickActionElements.map( + ({ iconType, 'data-test-subj': dataTestSubj, onClick, name }, i) => ( + + + + ) + )} + {contextMenuPanels.length ? ( + + + + ) : null} +
+ ) : null}
) : null}
diff --git a/src/plugins/presentation_panel/public/panel_component/presentation_panel.tsx b/src/plugins/presentation_panel/public/panel_component/presentation_panel.tsx index 7a403261b5995..3f509ebdc0a6c 100644 --- a/src/plugins/presentation_panel/public/panel_component/presentation_panel.tsx +++ b/src/plugins/presentation_panel/public/panel_component/presentation_panel.tsx @@ -14,6 +14,8 @@ import { PanelLoader } from '@kbn/panel-loader'; import { isPromise } from '@kbn/std'; import React from 'react'; import useAsync from 'react-use/lib/useAsync'; +import { css } from '@emotion/react'; +import { euiThemeVars } from '@kbn/ui-theme'; import { untilPluginStartServicesReady } from '../kibana_services'; import { PresentationPanelError } from './presentation_panel_error'; import { DefaultPresentationPanelApi, PresentationPanelProps } from './types'; @@ -55,7 +57,11 @@ export const PresentationPanel = < ); diff --git a/src/plugins/presentation_panel/public/panel_component/presentation_panel_internal.tsx b/src/plugins/presentation_panel/public/panel_component/presentation_panel_internal.tsx index ccf2e694d1b7a..d8c85350f3801 100644 --- a/src/plugins/presentation_panel/public/panel_component/presentation_panel_internal.tsx +++ b/src/plugins/presentation_panel/public/panel_component/presentation_panel_internal.tsx @@ -92,7 +92,7 @@ export const PresentationPanelInternal = < return ( .embPanel--dragHandle { diff --git a/x-pack/plugins/observability_solution/exploratory_view/public/components/shared/exploratory_view/embeddable/embeddable.tsx b/x-pack/plugins/observability_solution/exploratory_view/public/components/shared/exploratory_view/embeddable/embeddable.tsx index a7760014dec8c..a0079568803b6 100644 --- a/x-pack/plugins/observability_solution/exploratory_view/public/components/shared/exploratory_view/embeddable/embeddable.tsx +++ b/x-pack/plugins/observability_solution/exploratory_view/public/components/shared/exploratory_view/embeddable/embeddable.tsx @@ -288,8 +288,5 @@ const Wrapper = styled.div<{ right: 50%; transform: translate(50%, -50%); } - .embPanel { - outline: none; - } } `; diff --git a/x-pack/plugins/security_solution/public/common/components/visualization_actions/lens_embeddable.tsx b/x-pack/plugins/security_solution/public/common/components/visualization_actions/lens_embeddable.tsx index 0461cb8888be5..6b264a4dc759f 100644 --- a/x-pack/plugins/security_solution/public/common/components/visualization_actions/lens_embeddable.tsx +++ b/x-pack/plugins/security_solution/public/common/components/visualization_actions/lens_embeddable.tsx @@ -43,19 +43,6 @@ const LensComponentWrapper = styled.div<{ height: ${({ $height }) => ($height ? `${$height}px` : 'auto')}; width: ${({ width }) => width ?? 'auto'}; - .embPanel { - outline: none; - } - - .embPanel__hoverActions.embPanel__hoverActionsRight { - border-radius: 6px !important; - border-bottom: 1px solid #d3dae6 !important; - } - - .embPanel__hoverActionsAnchor .embPanel__hoverActionsWrapper { - top: -20px; - } - .expExpressionRenderer__expression { padding: 2px 0 0 0 !important; } diff --git a/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx b/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx index 3f3a5431c2c1d..bdd86a09ba231 100644 --- a/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx +++ b/x-pack/plugins/security_solution/public/explore/network/components/embeddables/embedded_map.tsx @@ -40,11 +40,6 @@ interface EmbeddableMapProps { const EmbeddableMapRatioHolder = styled.div.attrs(() => ({ className: 'siemEmbeddable__map', }))` - .embPanel { - border: none; - box-shadow: none; - } - .mapToolbarOverlay__button { display: none; } From 55da11e9188ad1e68a8cc9b45013cafa6e2c3b33 Mon Sep 17 00:00:00 2001 From: Irene Blanco Date: Wed, 13 Nov 2024 17:50:15 +0100 Subject: [PATCH 24/53] [APM] Migrate historical data API tests to be deployment-agnostic (#199765) ### How to test Closes https://github.com/elastic/kibana/issues/198975 Part of https://github.com/elastic/kibana/issues/193245 This PR contains the changes to migrate `historical_data` test folder to deployment-agnostic testing strategy. ### How to test - Serverless ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep="APM" ``` It's recommended to be run against [MKI](https://github.com/crespocarlos/kibana/blob/main/x-pack/test_serverless/README.md#run-tests-on-mki) - Stateful ``` node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep="APM" ``` ## Checks - [x] (OPTIONAL, only if a test has been unskipped) Run flaky test suite - [x] local run for serverless - [x] local run for stateful - [x] MKI run for serverless --- .../apm}/historical_data/has_data.spec.ts | 17 ++++++++++------- .../observability/apm/historical_data/index.ts | 14 ++++++++++++++ .../apis/observability/apm/index.ts | 1 + 3 files changed, 25 insertions(+), 7 deletions(-) rename x-pack/test/{apm_api_integration/tests => api_integration/deployment_agnostic/apis/observability/apm}/historical_data/has_data.spec.ts (77%) create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/index.ts diff --git a/x-pack/test/apm_api_integration/tests/historical_data/has_data.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/has_data.spec.ts similarity index 77% rename from x-pack/test/apm_api_integration/tests/historical_data/has_data.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/has_data.spec.ts index e0b5a0e076ffd..6ac96b8e38154 100644 --- a/x-pack/test/apm_api_integration/tests/historical_data/has_data.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/has_data.spec.ts @@ -8,15 +8,14 @@ import expect from '@kbn/expect'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; import moment from 'moment'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; -export default function ApiTest({ getService }: FtrProviderContext) { - const registry = getService('registry'); - const apmApiClient = getService('apmApiClient'); - const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); +export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { + const apmApiClient = getService('apmApi'); + const synthtrace = getService('synthtrace'); - // FLAKY: https://github.com/elastic/kibana/issues/177385 - registry.when('Historical data ', { config: 'basic', archives: [] }, () => { + describe('Historical data ', () => { describe('when there is not data', () => { it('returns hasData=false', async () => { const response = await apmApiClient.readUser({ endpoint: `GET /internal/apm/has_data` }); @@ -26,7 +25,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when there is data', () => { + let apmSynthtraceEsClient: ApmSynthtraceEsClient; + before(async () => { + apmSynthtraceEsClient = await synthtrace.createApmSynthtraceEsClient(); + const start = moment().subtract(30, 'minutes').valueOf(); const end = moment().valueOf(); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/index.ts new file mode 100644 index 0000000000000..49f0068ee313b --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/historical_data/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context'; + +export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) { + describe('historical_data', () => { + loadTestFile(require.resolve('./has_data.spec.ts')); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts index 3833aa94c6db4..aaf095cfb9425 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/apm/index.ts @@ -22,6 +22,7 @@ export default function apmApiIntegrationTests({ loadTestFile(require.resolve('./correlations')); loadTestFile(require.resolve('./entities')); loadTestFile(require.resolve('./cold_start')); + loadTestFile(require.resolve('./historical_data')); loadTestFile(require.resolve('./observability_overview')); loadTestFile(require.resolve('./latency')); loadTestFile(require.resolve('./infrastructure')); From 53d4580a8959a9e4b166df4e4a4cc83de61f7928 Mon Sep 17 00:00:00 2001 From: Andrew Macri Date: Wed, 13 Nov 2024 12:37:54 -0500 Subject: [PATCH 25/53] [Security Solution] [Attack discovery] Additional Attack discovery tests (#199659) ### [Security Solution] [Attack discovery] Additional Attack discovery tests This PR adds additional unit test coverage to Attack discovery. --- .../impl/knowledge_base/alerts_range.test.tsx | 86 +++++ .../__mocks__/raw_attack_discoveries.ts | 106 ++++++ .../__mocks__/mock_anonymization_fields.ts | 65 ++++ .../get_alerts_context_prompt/index.test.ts | 2 +- .../nodes/generate/index.test.ts | 247 ++++++++++++- .../nodes/generate/index.ts | 6 +- .../index.test.ts | 46 +++ .../nodes/helpers/extract_json/index.test.ts | 18 + .../nodes/helpers/extract_json/index.ts | 6 +- .../nodes/helpers/get_combined/index.test.ts | 37 ++ .../helpers/get_continue_prompt/index.test.ts | 22 ++ .../index.test.ts | 16 + .../parse_combined_or_throw/index.test.ts | 118 ++++++ .../index.test.ts | 82 +++++ .../get_combined_refine_prompt/index.test.ts | 77 ++++ .../get_default_refine_prompt/index.test.ts | 19 + .../get_use_unrefined_results/index.test.ts | 46 +++ .../nodes/refine/index.test.ts | 342 ++++++++++++++++++ .../nodes/refine/index.ts | 2 +- .../anonymized_alerts_retriever/index.test.ts | 104 ++++++ .../nodes/retriever/index.test.ts | 111 ++++++ .../nodes/retriever/index.ts | 8 - .../state/index.test.ts | 115 ++++++ .../helpers/show_empty_states/index.test.ts | 87 +++++ .../pages/generate/index.test.tsx | 46 +++ .../alerts_settings/index.test.tsx | 39 ++ .../settings_modal/alerts_settings/index.tsx | 4 +- .../settings_modal/footer/index.test.tsx | 42 +++ .../header/settings_modal/footer/index.tsx | 6 +- .../header/settings_modal/index.test.tsx | 72 ++++ .../is_tour_enabled/index.test.ts | 76 ++++ .../index.test.ts | 83 +++++ .../pages/results/index.test.tsx | 88 +++++ 33 files changed, 2195 insertions(+), 29 deletions(-) create mode 100644 x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/alerts_range.test.tsx create mode 100644 x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/__mocks__/mock_anonymization_fields.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/add_trailing_backticks_if_necessary/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_combined/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_continue_prompt/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_default_attack_discovery_prompt/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/parse_combined_or_throw/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/discard_previous_refinements/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_combined_refine_prompt/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_default_refine_prompt/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_use_unrefined_results/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/anonymized_alerts_retriever/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.test.ts create mode 100644 x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/state/index.test.ts create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/empty_states/helpers/show_empty_states/index.test.ts create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/generate/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/footer/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/index.test.tsx create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/is_tour_enabled/index.test.ts create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/loading_callout/loading_messages/get_loading_callout_alerts_count/index.test.ts create mode 100644 x-pack/plugins/security_solution/public/attack_discovery/pages/results/index.test.tsx diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/alerts_range.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/alerts_range.test.tsx new file mode 100644 index 0000000000000..1aaf7879b1c0b --- /dev/null +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/alerts_range.test.tsx @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fireEvent, render, screen } from '@testing-library/react'; +import React from 'react'; + +import { AlertsRange } from './alerts_range'; +import { + MAX_LATEST_ALERTS, + MIN_LATEST_ALERTS, +} from '../assistant/settings/alerts_settings/alerts_settings'; +import { KnowledgeBaseConfig } from '../assistant/types'; + +const nonDefaultMin = MIN_LATEST_ALERTS + 5000; +const nonDefaultMax = nonDefaultMin + 5000; + +describe('AlertsRange', () => { + beforeEach(() => jest.clearAllMocks()); + + it('renders the expected default min alerts', () => { + render(); + + expect(screen.getByText(`${MIN_LATEST_ALERTS}`)).toBeInTheDocument(); + }); + + it('renders the expected NON-default min alerts', () => { + render( + + ); + + expect(screen.getByText(`${nonDefaultMin}`)).toBeInTheDocument(); + }); + + it('renders the expected default max alerts', () => { + render(); + + expect(screen.getByText(`${MAX_LATEST_ALERTS}`)).toBeInTheDocument(); + }); + + it('renders the expected NON-default max alerts', () => { + render( + + ); + + expect(screen.getByText(`${nonDefaultMax}`)).toBeInTheDocument(); + }); + + it('calls onChange when the range value changes', () => { + const mockOnChange = jest.fn(); + render(); + + fireEvent.click(screen.getByText(`${MAX_LATEST_ALERTS}`)); + + expect(mockOnChange).toHaveBeenCalled(); + }); + + it('calls setUpdatedKnowledgeBaseSettings with the expected arguments', () => { + const mockSetUpdatedKnowledgeBaseSettings = jest.fn(); + const knowledgeBase: KnowledgeBaseConfig = { latestAlerts: 150 }; + + render( + + ); + + fireEvent.click(screen.getByText(`${MAX_LATEST_ALERTS}`)); + + expect(mockSetUpdatedKnowledgeBaseSettings).toHaveBeenCalledWith({ + ...knowledgeBase, + latestAlerts: MAX_LATEST_ALERTS, + }); + }); + + it('renders with the correct initial value', () => { + render(); + + expect(screen.getByTestId('alertsRange')).toHaveValue('250'); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts new file mode 100644 index 0000000000000..3c2df8aa1ab45 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/raw_attack_discoveries.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AttackDiscovery } from '@kbn/elastic-assistant-common'; + +/** + * A mock response from invoking the `attack-discovery` tool. + * This is a JSON string that represents the response from the tool + */ +export const getRawAttackDiscoveriesMock = () => + '{\n "insights": [\n {\n "alertIds": [\n "cced5cec88026ccb68fc0c01c096d6330873ee80838fa367a24c5cd04b679df1",\n "40a4242b163d2552ad24c208dc7ab754f3b2c9cd76fb961ea72391cb5f654580",\n "42ac2ecf60173edff8ef10b32c3b706b866845e75e5107870d7f43f681c819dc",\n "bd8204c37db970bf86c2713325652710d8e5ac2cd43a0f0f2234a65e8e5a0157",\n "b7a073c94cccde9fc4164a1f5aba5169b3ef5e349797326f8b166314c8cdb60d"\n ],\n "detailsMarkdown": "- {{ user.name 1ee7566b-9b26-4f3e-8d2f-0eaafc40cd5d }} executed a suspicious process {{ process.name unix1 }} on {{ host.name 3abc855f-65b6-49b0-ac2f-123e34355b83 }}. The process was located at {{ file.path /Users/james/unix1 }} and had a hash of {{ file.hash.sha256 0b18d6880dc9670ab2b955914598c96fc3d0097dc40ea61157b8c79e75edf231 }}.\\n- The process {{ process.name unix1 }} attempted to access sensitive files such as {{ process.args /Users/james/library/Keychains/login.keychain-db }}.\\n- The process {{ process.name unix1 }} was executed with the command line {{ process.command_line /Users/james/unix1 /Users/james/library/Keychains/login.keychain-db TempTemp1234!! }}.\\n- The process {{ process.name unix1 }} was not trusted as indicated by the code signature status {{ process.code_signature.status code failed to satisfy specified code requirement(s) }}.\\n- Another process {{ process.name My Go Application.app }} was also detected on the same host, indicating potential lateral movement or additional malicious activity.",\n "entitySummaryMarkdown": "{{ host.name 3abc855f-65b6-49b0-ac2f-123e34355b83 }} and {{ user.name 1ee7566b-9b26-4f3e-8d2f-0eaafc40cd5d }} were involved in the suspicious activity.",\n "mitreAttackTactics": [\n "Execution",\n "Credential Access",\n "Persistence"\n ],\n "summaryMarkdown": "A critical malware detection alert was triggered on {{ host.name 3abc855f-65b6-49b0-ac2f-123e34355b83 }} involving the user {{ user.name 1ee7566b-9b26-4f3e-8d2f-0eaafc40cd5d }}. The process {{ process.name unix1 }} was executed with suspicious arguments and attempted to access sensitive files.",\n "title": "Critical Malware Detection on macOS Host"\n },\n {\n "alertIds": [\n "1b9c52673b184e6b9bd29b3378f90ec5e7b917c17018ce2d40188a065f145087",\n "881c8cd24296c3efc066f894b2f60e28c86b6398e8d81fcdb0a21e2d4e6f37fb",\n "6ae56534e1246b42afbb0658586bfe03717ee9853cc80d462b9f0aceb44194d3",\n "94dda5ac846d122cf2e582ade68123f036b1b78c63752a30bcf8acdbbbba83ce",\n "250f7967181328c67d1de251c606fd4a791fd81964f431e3d7d76149f531be00"\n ],\n "detailsMarkdown": "- {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} executed a suspicious PowerShell script via Microsoft Office on {{ host.name 5e15d911-50a1-486a-a520-baa449451358 }}. The script was located at {{ file.path C:\\\\ProgramData\\\\WindowsAppPool\\\\AppPool.ps1 }}.\\n- The process {{ process.name powershell.exe }} was executed with the command line {{ process.command_line \\"C:\\\\Windows\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe\\" -exec bypass -file C:\\\\ProgramData\\\\WindowsAppPool\\\\AppPool.ps1 }}.\\n- The parent process {{ process.parent.name wscript.exe }} was executed by {{ process.parent.command_line wscript C:\\\\ProgramData\\\\WindowsAppPool\\\\AppPool.vbs }}.\\n- The process {{ process.name powershell.exe }} was not trusted as indicated by the code signature status {{ process.code_signature.status trusted }}.\\n- The process {{ process.name powershell.exe }} was detected with a high integrity level, indicating potential privilege escalation.",\n "entitySummaryMarkdown": "{{ host.name 5e15d911-50a1-486a-a520-baa449451358 }} and {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} were involved in the suspicious activity.",\n "mitreAttackTactics": [\n "Initial Access",\n "Execution",\n "Persistence"\n ],\n "summaryMarkdown": "A critical malicious behavior detection alert was triggered on {{ host.name 5e15d911-50a1-486a-a520-baa449451358 }} involving the user {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }}. The process {{ process.name powershell.exe }} was executed with suspicious arguments and attempted to execute a PowerShell script.",\n "title": "Suspicious PowerShell Execution via Microsoft Office"\n },\n {\n "alertIds": [\n "6cbbf7fb63ffed6e091ae21866043df699c839603ec573d3173b36e2d0e66ea3",\n "e7b6f978336961522b0753ffe79cc4a2aa6e2c08c491657ade3eccdb58033852",\n "d3ef244bda90960c091f516874a87b9cf01d206844c2e6ba324e3034472787f5"\n ],\n "detailsMarkdown": "- {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} executed a suspicious PowerShell script via MsiExec on {{ host.name 2068fbbd-341a-477a-b06c-7097ddecd024 }}. The script was located at {{ file.path C:\\\\Users\\\\ADMINI~1\\\\AppData\\\\Local\\\\Temp\\\\2\\\\Package Installation Dir\\\\chch.ps1 }}.\\n- The process {{ process.name powershell.exe }} was executed with the command line {{ process.command_line \\"C:\\\\Windows\\\\System32\\\\WindowsPowerShell\\\\v1.0\\\\powershell.exe\\" -ep bypass -file \\"C:\\\\Users\\\\ADMINI~1\\\\AppData\\\\Local\\\\Temp\\\\2\\\\Package Installation Dir\\\\chch.ps1\\" }}.\\n- The parent process {{ process.parent.name msiexec.exe }} was executed by {{ process.parent.command_line C:\\\\Windows\\\\system32\\\\msiexec.exe /V }}.\\n- The process {{ process.name powershell.exe }} was not trusted as indicated by the code signature status {{ process.code_signature.status trusted }}.\\n- The process {{ process.name powershell.exe }} was detected with a high integrity level, indicating potential privilege escalation.",\n "entitySummaryMarkdown": "{{ host.name 2068fbbd-341a-477a-b06c-7097ddecd024 }} and {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} were involved in the suspicious activity.",\n "mitreAttackTactics": [\n "Defense Evasion",\n "Execution"\n ],\n "summaryMarkdown": "A critical malicious behavior detection alert was triggered on {{ host.name 2068fbbd-341a-477a-b06c-7097ddecd024 }} involving the user {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }}. The process {{ process.name powershell.exe }} was executed with suspicious arguments and attempted to execute a PowerShell script.",\n "title": "Suspicious PowerShell Execution via MsiExec"\n },\n {\n "alertIds": [\n "8b1ccd0bfb927caeb5f9818098eebde9a091b99334c84bfffd36aa83db8b36ee",\n "0ae1370d0c08d651a05421009ed8358d9037f3d6af0cf5f3417979489ca80f12",\n "bed4a026232fb8e67f248771a99af722116556ace7ef9aaddefc082da4209c61",\n "d28f2c32ae8e6bc33edfe51ace4621c0e7b826c087386c46ce9138be92baf3f9"\n ],\n "detailsMarkdown": "- {{ user.name 00468e82-e37f-4224-80c1-c62e594c74b1 }} executed a suspicious process {{ process.name unzip }} on {{ host.name b557bb12-8206-44b6-b2a5-dbcce5b1e65e }}. The process was located at {{ file.path /home/ubuntu/74ef6cc38f5a1a80148752b63c117e6846984debd2af806c65887195a8eccc56 }} and had a hash of {{ file.hash.sha256 74ef6cc38f5a1a80148752b63c117e6846984debd2af806c65887195a8eccc56 }}.\\n- The process {{ process.name unzip }} was executed with the command line {{ process.command_line unzip 9415656314.zip }}.\\n- The process {{ process.name unzip }} was detected with a high integrity level, indicating potential privilege escalation.\\n- Another process {{ process.name kdmtmpflush }} was also detected on the same host, indicating potential lateral movement or additional malicious activity.",\n "entitySummaryMarkdown": "{{ host.name b557bb12-8206-44b6-b2a5-dbcce5b1e65e }} and {{ user.name 00468e82-e37f-4224-80c1-c62e594c74b1 }} were involved in the suspicious activity.",\n "mitreAttackTactics": [\n "Execution",\n "Persistence"\n ],\n "summaryMarkdown": "A critical malware detection alert was triggered on {{ host.name b557bb12-8206-44b6-b2a5-dbcce5b1e65e }} involving the user {{ user.name 00468e82-e37f-4224-80c1-c62e594c74b1 }}. The process {{ process.name unzip }} was executed with suspicious arguments and attempted to extract a potentially malicious file.",\n "title": "Suspicious File Extraction on Linux Host"\n },\n {\n "alertIds": [\n "15c3053659b3bccbcc2c75eb90963596bbba707496e6b8c4927b5dc3995e0e11",\n "461fedbfddd0d8d42c11630d5cdb9a103fac05327dff5bcdbf51505f01ec39da",\n "03ef2d6a825993d08f545cfa25e8dab765dd1f4688124e7d12d8d81a2f324464",\n "bfd4f9a71c9ca6a8dc68a41ea96b5ca14380da9669fb62ccae06769ad931eef2"\n ],\n "detailsMarkdown": "- {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} executed a suspicious process {{ process.name MsMpEng.exe }} on {{ host.name b808feb3-7ab3-4006-9c67-3cf7aeffe572 }}. The process was located at {{ file.path C:\\\\Windows\\\\MsMpEng.exe }} and had a hash of {{ file.hash.sha256 33bc14d231a4afaa18f06513766d5f69d8b88f1e697cd127d24fb4b72ad44c7a }}.\\n- The process {{ process.name MsMpEng.exe }} was executed with the command line {{ process.command_line \\"C:\\\\Windows\\\\MsMpEng.exe\\" }}.\\n- The parent process {{ process.parent.name d55f983c994caa160ec63a59f6b4250fe67fb3e8c43a388aec60a4a6978e9f1e.exe }} was executed by {{ process.parent.command_line \\"C:\\\\Users\\\\Administrator\\\\Desktop\\\\8813719803\\\\d55f983c994caa160ec63a59f6b4250fe67fb3e8c43a388aec60a4a6978e9f1e.exe\\" }}.\\n- The process {{ process.name MsMpEng.exe }} was not trusted as indicated by the code signature status {{ process.code_signature.status trusted }}.\\n- The process {{ process.name MsMpEng.exe }} was detected with a high integrity level, indicating potential privilege escalation.",\n "entitySummaryMarkdown": "{{ host.name b808feb3-7ab3-4006-9c67-3cf7aeffe572 }} and {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} were involved in the suspicious activity.",\n "mitreAttackTactics": [\n "Execution",\n "Persistence"\n ],\n "summaryMarkdown": "A critical malware detection alert was triggered on {{ host.name b808feb3-7ab3-4006-9c67-3cf7aeffe572 }} involving the user {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }}. The process {{ process.name MsMpEng.exe }} was executed with suspicious arguments and attempted to execute a potentially malicious file.",\n "title": "Suspicious Process Execution on Windows Host"\n }\n ]\n}'; + +export const getParsedAttackDiscoveriesMock = ( + attackDiscoveryTimestamp: string +): AttackDiscovery[] => [ + { + alertIds: [ + 'cced5cec88026ccb68fc0c01c096d6330873ee80838fa367a24c5cd04b679df1', + '40a4242b163d2552ad24c208dc7ab754f3b2c9cd76fb961ea72391cb5f654580', + '42ac2ecf60173edff8ef10b32c3b706b866845e75e5107870d7f43f681c819dc', + 'bd8204c37db970bf86c2713325652710d8e5ac2cd43a0f0f2234a65e8e5a0157', + 'b7a073c94cccde9fc4164a1f5aba5169b3ef5e349797326f8b166314c8cdb60d', + ], + detailsMarkdown: + '- {{ user.name 1ee7566b-9b26-4f3e-8d2f-0eaafc40cd5d }} executed a suspicious process {{ process.name unix1 }} on {{ host.name 3abc855f-65b6-49b0-ac2f-123e34355b83 }}. The process was located at {{ file.path /Users/james/unix1 }} and had a hash of {{ file.hash.sha256 0b18d6880dc9670ab2b955914598c96fc3d0097dc40ea61157b8c79e75edf231 }}.\n- The process {{ process.name unix1 }} attempted to access sensitive files such as {{ process.args /Users/james/library/Keychains/login.keychain-db }}.\n- The process {{ process.name unix1 }} was executed with the command line {{ process.command_line /Users/james/unix1 /Users/james/library/Keychains/login.keychain-db TempTemp1234!! }}.\n- The process {{ process.name unix1 }} was not trusted as indicated by the code signature status {{ process.code_signature.status code failed to satisfy specified code requirement(s) }}.\n- Another process {{ process.name My Go Application.app }} was also detected on the same host, indicating potential lateral movement or additional malicious activity.', + entitySummaryMarkdown: + '{{ host.name 3abc855f-65b6-49b0-ac2f-123e34355b83 }} and {{ user.name 1ee7566b-9b26-4f3e-8d2f-0eaafc40cd5d }} were involved in the suspicious activity.', + mitreAttackTactics: ['Execution', 'Credential Access', 'Persistence'], + summaryMarkdown: + 'A critical malware detection alert was triggered on {{ host.name 3abc855f-65b6-49b0-ac2f-123e34355b83 }} involving the user {{ user.name 1ee7566b-9b26-4f3e-8d2f-0eaafc40cd5d }}. The process {{ process.name unix1 }} was executed with suspicious arguments and attempted to access sensitive files.', + title: 'Critical Malware Detection on macOS Host', + timestamp: attackDiscoveryTimestamp, + }, + { + alertIds: [ + '1b9c52673b184e6b9bd29b3378f90ec5e7b917c17018ce2d40188a065f145087', + '881c8cd24296c3efc066f894b2f60e28c86b6398e8d81fcdb0a21e2d4e6f37fb', + '6ae56534e1246b42afbb0658586bfe03717ee9853cc80d462b9f0aceb44194d3', + '94dda5ac846d122cf2e582ade68123f036b1b78c63752a30bcf8acdbbbba83ce', + '250f7967181328c67d1de251c606fd4a791fd81964f431e3d7d76149f531be00', + ], + detailsMarkdown: + '- {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} executed a suspicious PowerShell script via Microsoft Office on {{ host.name 5e15d911-50a1-486a-a520-baa449451358 }}. The script was located at {{ file.path C:\\ProgramData\\WindowsAppPool\\AppPool.ps1 }}.\n- The process {{ process.name powershell.exe }} was executed with the command line {{ process.command_line "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" -exec bypass -file C:\\ProgramData\\WindowsAppPool\\AppPool.ps1 }}.\n- The parent process {{ process.parent.name wscript.exe }} was executed by {{ process.parent.command_line wscript C:\\ProgramData\\WindowsAppPool\\AppPool.vbs }}.\n- The process {{ process.name powershell.exe }} was not trusted as indicated by the code signature status {{ process.code_signature.status trusted }}.\n- The process {{ process.name powershell.exe }} was detected with a high integrity level, indicating potential privilege escalation.', + entitySummaryMarkdown: + '{{ host.name 5e15d911-50a1-486a-a520-baa449451358 }} and {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} were involved in the suspicious activity.', + mitreAttackTactics: ['Initial Access', 'Execution', 'Persistence'], + summaryMarkdown: + 'A critical malicious behavior detection alert was triggered on {{ host.name 5e15d911-50a1-486a-a520-baa449451358 }} involving the user {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }}. The process {{ process.name powershell.exe }} was executed with suspicious arguments and attempted to execute a PowerShell script.', + title: 'Suspicious PowerShell Execution via Microsoft Office', + timestamp: attackDiscoveryTimestamp, + }, + { + alertIds: [ + '6cbbf7fb63ffed6e091ae21866043df699c839603ec573d3173b36e2d0e66ea3', + 'e7b6f978336961522b0753ffe79cc4a2aa6e2c08c491657ade3eccdb58033852', + 'd3ef244bda90960c091f516874a87b9cf01d206844c2e6ba324e3034472787f5', + ], + detailsMarkdown: + '- {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} executed a suspicious PowerShell script via MsiExec on {{ host.name 2068fbbd-341a-477a-b06c-7097ddecd024 }}. The script was located at {{ file.path C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\2\\Package Installation Dir\\chch.ps1 }}.\n- The process {{ process.name powershell.exe }} was executed with the command line {{ process.command_line "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" -ep bypass -file "C:\\Users\\ADMINI~1\\AppData\\Local\\Temp\\2\\Package Installation Dir\\chch.ps1" }}.\n- The parent process {{ process.parent.name msiexec.exe }} was executed by {{ process.parent.command_line C:\\Windows\\system32\\msiexec.exe /V }}.\n- The process {{ process.name powershell.exe }} was not trusted as indicated by the code signature status {{ process.code_signature.status trusted }}.\n- The process {{ process.name powershell.exe }} was detected with a high integrity level, indicating potential privilege escalation.', + entitySummaryMarkdown: + '{{ host.name 2068fbbd-341a-477a-b06c-7097ddecd024 }} and {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} were involved in the suspicious activity.', + mitreAttackTactics: ['Defense Evasion', 'Execution'], + summaryMarkdown: + 'A critical malicious behavior detection alert was triggered on {{ host.name 2068fbbd-341a-477a-b06c-7097ddecd024 }} involving the user {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }}. The process {{ process.name powershell.exe }} was executed with suspicious arguments and attempted to execute a PowerShell script.', + title: 'Suspicious PowerShell Execution via MsiExec', + timestamp: attackDiscoveryTimestamp, + }, + { + alertIds: [ + '8b1ccd0bfb927caeb5f9818098eebde9a091b99334c84bfffd36aa83db8b36ee', + '0ae1370d0c08d651a05421009ed8358d9037f3d6af0cf5f3417979489ca80f12', + 'bed4a026232fb8e67f248771a99af722116556ace7ef9aaddefc082da4209c61', + 'd28f2c32ae8e6bc33edfe51ace4621c0e7b826c087386c46ce9138be92baf3f9', + ], + detailsMarkdown: + '- {{ user.name 00468e82-e37f-4224-80c1-c62e594c74b1 }} executed a suspicious process {{ process.name unzip }} on {{ host.name b557bb12-8206-44b6-b2a5-dbcce5b1e65e }}. The process was located at {{ file.path /home/ubuntu/74ef6cc38f5a1a80148752b63c117e6846984debd2af806c65887195a8eccc56 }} and had a hash of {{ file.hash.sha256 74ef6cc38f5a1a80148752b63c117e6846984debd2af806c65887195a8eccc56 }}.\n- The process {{ process.name unzip }} was executed with the command line {{ process.command_line unzip 9415656314.zip }}.\n- The process {{ process.name unzip }} was detected with a high integrity level, indicating potential privilege escalation.\n- Another process {{ process.name kdmtmpflush }} was also detected on the same host, indicating potential lateral movement or additional malicious activity.', + entitySummaryMarkdown: + '{{ host.name b557bb12-8206-44b6-b2a5-dbcce5b1e65e }} and {{ user.name 00468e82-e37f-4224-80c1-c62e594c74b1 }} were involved in the suspicious activity.', + mitreAttackTactics: ['Execution', 'Persistence'], + summaryMarkdown: + 'A critical malware detection alert was triggered on {{ host.name b557bb12-8206-44b6-b2a5-dbcce5b1e65e }} involving the user {{ user.name 00468e82-e37f-4224-80c1-c62e594c74b1 }}. The process {{ process.name unzip }} was executed with suspicious arguments and attempted to extract a potentially malicious file.', + title: 'Suspicious File Extraction on Linux Host', + timestamp: attackDiscoveryTimestamp, + }, + { + alertIds: [ + '15c3053659b3bccbcc2c75eb90963596bbba707496e6b8c4927b5dc3995e0e11', + '461fedbfddd0d8d42c11630d5cdb9a103fac05327dff5bcdbf51505f01ec39da', + '03ef2d6a825993d08f545cfa25e8dab765dd1f4688124e7d12d8d81a2f324464', + 'bfd4f9a71c9ca6a8dc68a41ea96b5ca14380da9669fb62ccae06769ad931eef2', + ], + detailsMarkdown: + '- {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} executed a suspicious process {{ process.name MsMpEng.exe }} on {{ host.name b808feb3-7ab3-4006-9c67-3cf7aeffe572 }}. The process was located at {{ file.path C:\\Windows\\MsMpEng.exe }} and had a hash of {{ file.hash.sha256 33bc14d231a4afaa18f06513766d5f69d8b88f1e697cd127d24fb4b72ad44c7a }}.\n- The process {{ process.name MsMpEng.exe }} was executed with the command line {{ process.command_line "C:\\Windows\\MsMpEng.exe" }}.\n- The parent process {{ process.parent.name d55f983c994caa160ec63a59f6b4250fe67fb3e8c43a388aec60a4a6978e9f1e.exe }} was executed by {{ process.parent.command_line "C:\\Users\\Administrator\\Desktop\\8813719803\\d55f983c994caa160ec63a59f6b4250fe67fb3e8c43a388aec60a4a6978e9f1e.exe" }}.\n- The process {{ process.name MsMpEng.exe }} was not trusted as indicated by the code signature status {{ process.code_signature.status trusted }}.\n- The process {{ process.name MsMpEng.exe }} was detected with a high integrity level, indicating potential privilege escalation.', + entitySummaryMarkdown: + '{{ host.name b808feb3-7ab3-4006-9c67-3cf7aeffe572 }} and {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }} were involved in the suspicious activity.', + mitreAttackTactics: ['Execution', 'Persistence'], + summaryMarkdown: + 'A critical malware detection alert was triggered on {{ host.name b808feb3-7ab3-4006-9c67-3cf7aeffe572 }} involving the user {{ user.name 37764a98-eeb5-459f-ab04-f8b70e8239cb }}. The process {{ process.name MsMpEng.exe }} was executed with suspicious arguments and attempted to execute a potentially malicious file.', + title: 'Suspicious Process Execution on Windows Host', + timestamp: attackDiscoveryTimestamp, + }, +]; diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/__mocks__/mock_anonymization_fields.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/__mocks__/mock_anonymization_fields.ts new file mode 100644 index 0000000000000..ed487e4705c27 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/__mocks__/mock_anonymization_fields.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AnonymizationFieldResponse } from '@kbn/elastic-assistant-common/impl/schemas/anonymization_fields/bulk_crud_anonymization_fields_route.gen'; + +export const getMockAnonymizationFieldResponse = (): AnonymizationFieldResponse[] => [ + { + id: '6UDO45IBoEQSo_rIK1EW', + timestamp: '2024-10-31T18:19:52.468Z', + field: '_id', + allowed: true, + anonymized: false, + createdAt: '2024-10-31T18:19:52.468Z', + namespace: 'default', + }, + { + id: '6kDO45IBoEQSo_rIK1EW', + timestamp: '2024-10-31T18:19:52.468Z', + field: '@timestamp', + allowed: true, + anonymized: false, + createdAt: '2024-10-31T18:19:52.468Z', + namespace: 'default', + }, + { + id: '60DO45IBoEQSo_rIK1EW', + timestamp: '2024-10-31T18:19:52.468Z', + field: 'cloud.availability_zone', + allowed: true, + anonymized: false, + createdAt: '2024-10-31T18:19:52.468Z', + namespace: 'default', + }, + { + id: '_EDO45IBoEQSo_rIK1EW', + timestamp: '2024-10-31T18:19:52.468Z', + field: 'host.name', + allowed: true, + anonymized: true, + createdAt: '2024-10-31T18:19:52.468Z', + namespace: 'default', + }, + { + id: 'SkDO45IBoEQSo_rIK1IW', + timestamp: '2024-10-31T18:19:52.468Z', + field: 'user.name', + allowed: true, + anonymized: true, + createdAt: '2024-10-31T18:19:52.468Z', + namespace: 'default', + }, + { + id: 'TUDO45IBoEQSo_rIK1IW', + timestamp: '2024-10-31T18:19:52.468Z', + field: 'user.target.name', + allowed: true, + anonymized: true, + createdAt: '2024-10-31T18:19:52.468Z', + namespace: 'default', + }, +]; diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/helpers/get_alerts_context_prompt/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/helpers/get_alerts_context_prompt/index.test.ts index 287f5e6b2130a..098d2b81f4914 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/helpers/get_alerts_context_prompt/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/helpers/get_alerts_context_prompt/index.test.ts @@ -12,7 +12,7 @@ describe('getAlertsContextPrompt', () => { it('generates the correct prompt', () => { const anonymizedAlerts = ['Alert 1', 'Alert 2', 'Alert 3']; - const expected = `You are a cyber security analyst tasked with analyzing security events from Elastic Security to identify and report on potential cyber attacks or progressions. Your report should focus on high-risk incidents that could severely impact the organization, rather than isolated alerts. Present your findings in a way that can be easily understood by anyone, regardless of their technical expertise, as if you were briefing the CISO. Break down your response into sections based on timing, hosts, and users involved. When correlating alerts, use kibana.alert.original_time when it's available, otherwise use @timestamp. Include appropriate context about the affected hosts and users. Describe how the attack progression might have occurred and, if feasible, attribute it to known threat groups. Prioritize high and critical alerts, but include lower-severity alerts if desired. In the description field, provide as much detail as possible, in a bulleted list explaining any attack progressions. Accuracy is of utmost importance. You MUST escape all JSON special characters (i.e. backslashes, double quotes, newlines, tabs, carriage returns, backspaces, and form feeds). + const expected = `${getDefaultAttackDiscoveryPrompt()} Use context from the following alerts to provide insights: diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.test.ts index da815aad9795a..07c3e0007f851 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.test.ts @@ -5,8 +5,8 @@ * 2.0. */ +import type { Logger } from '@kbn/core/server'; import type { ActionsClientLlm } from '@kbn/langchain/server'; -import { loggerMock } from '@kbn/logging-mocks'; import { FakeLLM } from '@langchain/core/utils/testing'; import { getGenerateNode } from '.'; @@ -16,7 +16,15 @@ import { } from '../../../../evaluation/__mocks__/mock_anonymized_alerts'; import { getAnonymizedAlertsFromState } from './helpers/get_anonymized_alerts_from_state'; import { getChainWithFormatInstructions } from '../helpers/get_chain_with_format_instructions'; +import { getDefaultAttackDiscoveryPrompt } from '../helpers/get_default_attack_discovery_prompt'; +import { getDefaultRefinePrompt } from '../refine/helpers/get_default_refine_prompt'; import { GraphState } from '../../types'; +import { + getParsedAttackDiscoveriesMock, + getRawAttackDiscoveriesMock, +} from '../../../../../../__mocks__/raw_attack_discoveries'; + +const attackDiscoveryTimestamp = '2024-10-11T17:55:59.702Z'; jest.mock('../helpers/get_chain_with_format_instructions', () => { const mockInvoke = jest.fn().mockResolvedValue(''); @@ -27,19 +35,21 @@ jest.mock('../helpers/get_chain_with_format_instructions', () => { invoke: mockInvoke, }, formatInstructions: ['mock format instructions'], - llmType: 'fake', + llmType: 'openai', mockInvoke, // <-- added for testing }), }; }); -const mockLogger = loggerMock.create(); +const mockLogger = { + debug: (x: Function) => x(), +} as unknown as Logger; + let mockLlm: ActionsClientLlm; const initialGraphState: GraphState = { attackDiscoveries: null, - attackDiscoveryPrompt: - "You are a cyber security analyst tasked with analyzing security events from Elastic Security to identify and report on potential cyber attacks or progressions. Your report should focus on high-risk incidents that could severely impact the organization, rather than isolated alerts. Present your findings in a way that can be easily understood by anyone, regardless of their technical expertise, as if you were briefing the CISO. Break down your response into sections based on timing, hosts, and users involved. When correlating alerts, use kibana.alert.original_time when it's available, otherwise use @timestamp. Include appropriate context about the affected hosts and users. Describe how the attack progression might have occurred and, if feasible, attribute it to known threat groups. Prioritize high and critical alerts, but include lower-severity alerts if desired. In the description field, provide as much detail as possible, in a bulleted list explaining any attack progressions. Accuracy is of utmost importance. You MUST escape all JSON special characters (i.e. backslashes, double quotes, newlines, tabs, carriage returns, backspaces, and form feeds).", + attackDiscoveryPrompt: getDefaultAttackDiscoveryPrompt(), anonymizedAlerts: [...mockAnonymizedAlerts], combinedGenerations: '', combinedRefinements: '', @@ -51,8 +61,7 @@ const initialGraphState: GraphState = { maxHallucinationFailures: 5, maxRepeatedGenerations: 3, refinements: [], - refinePrompt: - 'You previously generated the following insights, but sometimes they represent the same attack.\n\nCombine the insights below, when they represent the same attack; leave any insights that are not combined unchanged:', + refinePrompt: getDefaultRefinePrompt(), replacements: { ...mockAnonymizedAlertsReplacements, }, @@ -63,11 +72,18 @@ describe('getGenerateNode', () => { beforeEach(() => { jest.clearAllMocks(); + jest.useFakeTimers(); + jest.setSystemTime(new Date(attackDiscoveryTimestamp)); + mockLlm = new FakeLLM({ - response: JSON.stringify({}, null, 2), + response: '', }) as unknown as ActionsClientLlm; }); + afterEach(() => { + jest.useRealTimers(); + }); + it('returns a function', () => { const generateNode = getGenerateNode({ llm: mockLlm, @@ -77,9 +93,8 @@ describe('getGenerateNode', () => { expect(typeof generateNode).toBe('function'); }); - it('invokes the chain with the alerts from state and format instructions', async () => { - // @ts-expect-error - const { mockInvoke } = getChainWithFormatInstructions(mockLlm); + it('invokes the chain with the expected alerts from state and formatting instructions', async () => { + const mockInvoke = getChainWithFormatInstructions(mockLlm).chain.invoke as jest.Mock; const generateNode = getGenerateNode({ llm: mockLlm, @@ -100,4 +115,214 @@ ${getAnonymizedAlertsFromState(initialGraphState).join('\n\n')} `, }); }); + + it('removes the surrounding json from the response', async () => { + const response = + 'You asked for some JSON, here it is:\n```json\n{"key": "value"}\n```\nI hope that works for you.'; + + const mockLlmWithResponse = new FakeLLM({ response }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(response); + + const generateNode = getGenerateNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const state = await generateNode(initialGraphState); + + expect(state).toEqual({ + ...initialGraphState, + combinedGenerations: '{"key": "value"}', + errors: [ + 'generate node is unable to parse (fake) response from attempt 0; (this may be an incomplete response from the model): [\n {\n "code": "invalid_type",\n "expected": "array",\n "received": "undefined",\n "path": [\n "insights"\n ],\n "message": "Required"\n }\n]', + ], + generationAttempts: 1, + generations: ['{"key": "value"}'], + }); + }); + + it('handles hallucinations', async () => { + const hallucinatedResponse = + 'tactics like **Credential Access**, **Command and Control**, and **Persistence**.",\n "entitySummaryMarkdown": "Malware detected on host **{{ host.name hostNameValue }}**'; + + const mockLlmWithHallucination = new FakeLLM({ + response: hallucinatedResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithHallucination).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(hallucinatedResponse); + + const generateNode = getGenerateNode({ + llm: mockLlmWithHallucination, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedGenerations: '{"key": "value"}', + generationAttempts: 1, + generations: ['{"key": "value"}'], + }; + + const state = await generateNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + combinedGenerations: '', // <-- reset + generationAttempts: 2, // <-- incremented + generations: [], // <-- reset + hallucinationFailures: 1, // <-- incremented + }); + }); + + it('discards previous generations and starts over when the maxRepeatedGenerations limit is reached', async () => { + const repeatedResponse = 'gen1'; + + const mockLlmWithRepeatedGenerations = new FakeLLM({ + response: repeatedResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithRepeatedGenerations).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(repeatedResponse); + + const generateNode = getGenerateNode({ + llm: mockLlmWithRepeatedGenerations, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedGenerations: 'gen1gen1', + generationAttempts: 2, + generations: ['gen1', 'gen1'], + }; + + const state = await generateNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + combinedGenerations: '', + generationAttempts: 3, // <-- incremented + generations: [], + }); + }); + + it('combines the response with the previous generations', async () => { + const response = 'gen1'; + + const mockLlmWithResponse = new FakeLLM({ + response, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(response); + + const generateNode = getGenerateNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedGenerations: 'gen0', + generationAttempts: 1, + generations: ['gen0'], + }; + + const state = await generateNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + combinedGenerations: 'gen0gen1', + errors: [ + 'generate node is unable to parse (fake) response from attempt 1; (this may be an incomplete response from the model): SyntaxError: Unexpected token \'g\', "gen0gen1" is not valid JSON', + ], + generationAttempts: 2, + generations: ['gen0', 'gen1'], + }); + }); + + it('returns unrefined results when combined responses pass validation', async () => { + // split the response into two parts to simulate a valid response + const splitIndex = 100; // arbitrary index + const firstResponse = getRawAttackDiscoveriesMock().slice(0, splitIndex); + const secondResponse = getRawAttackDiscoveriesMock().slice(splitIndex); + + const mockLlmWithResponse = new FakeLLM({ + response: secondResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(secondResponse); + + const generateNode = getGenerateNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedGenerations: firstResponse, + generationAttempts: 1, + generations: [firstResponse], + }; + + const state = await generateNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + attackDiscoveries: null, + combinedGenerations: firstResponse.concat(secondResponse), + errors: [], + generationAttempts: 2, + generations: [firstResponse, secondResponse], + unrefinedResults: getParsedAttackDiscoveriesMock(attackDiscoveryTimestamp), // <-- generated from the combined response + }); + }); + + it('skips the refinements step if the max number of retries has already been reached', async () => { + // split the response into two parts to simulate a valid response + const splitIndex = 100; // arbitrary index + const firstResponse = getRawAttackDiscoveriesMock().slice(0, splitIndex); + const secondResponse = getRawAttackDiscoveriesMock().slice(splitIndex); + + const mockLlmWithResponse = new FakeLLM({ + response: secondResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(secondResponse); + + const generateNode = getGenerateNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedGenerations: firstResponse, + generationAttempts: 9, + generations: [firstResponse], + }; + + const state = await generateNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + attackDiscoveries: getParsedAttackDiscoveriesMock(attackDiscoveryTimestamp), // <-- skip the refinement step + combinedGenerations: firstResponse.concat(secondResponse), + errors: [], + generationAttempts: 10, + generations: [firstResponse, secondResponse], + unrefinedResults: getParsedAttackDiscoveriesMock(attackDiscoveryTimestamp), // <-- generated from the combined response + }); + }); }); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.ts index 1fcd81622f0fe..0dfe1b0629f58 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/generate/index.ts @@ -58,10 +58,10 @@ export const getGenerateNode = ({ () => `generate node is invoking the chain (${llmType}), attempt ${generationAttempts}` ); - const rawResponse = (await chain.invoke({ + const rawResponse = await chain.invoke({ format_instructions: formatInstructions, query, - })) as unknown as string; + }); // LOCAL MUTATION: partialResponse = extractJson(rawResponse); // remove the surrounding ```json``` @@ -86,7 +86,7 @@ export const getGenerateNode = ({ generationsAreRepeating({ currentGeneration: partialResponse, previousGenerations: generations, - sampleLastNGenerations: maxRepeatedGenerations, + sampleLastNGenerations: maxRepeatedGenerations - 1, }) ) { logger?.debug( diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/add_trailing_backticks_if_necessary/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/add_trailing_backticks_if_necessary/index.test.ts new file mode 100644 index 0000000000000..4c95cb05faae0 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/add_trailing_backticks_if_necessary/index.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { addTrailingBackticksIfNecessary } from '.'; + +describe('addTrailingBackticksIfNecessary', () => { + it('adds trailing backticks when necessary', () => { + const input = '```json\n{\n "key": "value"\n}'; + const expected = '```json\n{\n "key": "value"\n}\n```'; + const result = addTrailingBackticksIfNecessary(input); + + expect(result).toEqual(expected); + }); + + it('does NOT add trailing backticks when they are already present', () => { + const input = '```json\n{\n "key": "value"\n}\n```'; + const result = addTrailingBackticksIfNecessary(input); + + expect(result).toEqual(input); + }); + + it("does NOT add trailing backticks when there's no leading JSON wrapper", () => { + const input = '{\n "key": "value"\n}'; + const result = addTrailingBackticksIfNecessary(input); + + expect(result).toEqual(input); + }); + + it('handles empty string input', () => { + const input = ''; + const result = addTrailingBackticksIfNecessary(input); + + expect(result).toEqual(input); + }); + + it('handles input without a JSON wrapper, but with trailing backticks', () => { + const input = '{\n "key": "value"\n}\n```'; + const result = addTrailingBackticksIfNecessary(input); + + expect(result).toEqual(input); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.test.ts index 5e13ec9f0dafe..7a2ced64163c5 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.test.ts @@ -8,6 +8,24 @@ import { extractJson } from '.'; describe('extractJson', () => { + it('returns an empty string if input is undefined', () => { + const input = undefined; + + expect(extractJson(input)).toBe(''); + }); + + it('returns an empty string if input an array', () => { + const input = ['some', 'array']; + + expect(extractJson(input)).toBe(''); + }); + + it('returns an empty string if input is an object', () => { + const input = {}; + + expect(extractJson(input)).toBe(''); + }); + it('returns the JSON text surrounded by ```json and ``` with no whitespace or additional text', () => { const input = '```json{"key": "value"}```'; diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.ts index 79d3f9c0d0599..089756840e568 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/extract_json/index.ts @@ -5,7 +5,11 @@ * 2.0. */ -export const extractJson = (input: string): string => { +export const extractJson = (input: unknown): string => { + if (typeof input !== 'string') { + return ''; + } + const regex = /```json\s*([\s\S]*?)(?:\s*```|$)/; const match = input.match(regex); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_combined/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_combined/index.test.ts new file mode 100644 index 0000000000000..75d7d83db3e92 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_combined/index.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getCombined } from '.'; + +describe('getCombined', () => { + it('combines two strings correctly', () => { + const combinedGenerations = 'generation1'; + const partialResponse = 'response1'; + const expected = 'generation1response1'; + const result = getCombined({ combinedGenerations, partialResponse }); + + expect(result).toEqual(expected); + }); + + it('handles empty combinedGenerations', () => { + const combinedGenerations = ''; + const partialResponse = 'response1'; + const expected = 'response1'; + const result = getCombined({ combinedGenerations, partialResponse }); + + expect(result).toEqual(expected); + }); + + it('handles an empty partialResponse', () => { + const combinedGenerations = 'generation1'; + const partialResponse = ''; + const expected = 'generation1'; + const result = getCombined({ combinedGenerations, partialResponse }); + + expect(result).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_continue_prompt/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_continue_prompt/index.test.ts new file mode 100644 index 0000000000000..35dae31a3ae6a --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_continue_prompt/index.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getContinuePrompt } from '.'; + +describe('getContinuePrompt', () => { + it('returns the expected prompt string', () => { + const expectedPrompt = `Continue exactly where you left off in the JSON output below, generating only the additional JSON output when it's required to complete your work. The additional JSON output MUST ALWAYS follow these rules: +1) it MUST conform to the schema above, because it will be checked against the JSON schema +2) it MUST escape all JSON special characters (i.e. backslashes, double quotes, newlines, tabs, carriage returns, backspaces, and form feeds), because it will be parsed as JSON +3) it MUST NOT repeat any the previous output, because that would prevent partial results from being combined +4) it MUST NOT restart from the beginning, because that would prevent partial results from being combined +5) it MUST NOT be prefixed or suffixed with additional text outside of the JSON, because that would prevent it from being combined and parsed as JSON: +`; + + expect(getContinuePrompt()).toBe(expectedPrompt); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_default_attack_discovery_prompt/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_default_attack_discovery_prompt/index.test.ts new file mode 100644 index 0000000000000..6fce86bfceb6f --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/get_default_attack_discovery_prompt/index.test.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDefaultAttackDiscoveryPrompt } from '.'; + +describe('getDefaultAttackDiscoveryPrompt', () => { + it('returns the default attack discovery prompt', () => { + expect(getDefaultAttackDiscoveryPrompt()).toEqual( + "You are a cyber security analyst tasked with analyzing security events from Elastic Security to identify and report on potential cyber attacks or progressions. Your report should focus on high-risk incidents that could severely impact the organization, rather than isolated alerts. Present your findings in a way that can be easily understood by anyone, regardless of their technical expertise, as if you were briefing the CISO. Break down your response into sections based on timing, hosts, and users involved. When correlating alerts, use kibana.alert.original_time when it's available, otherwise use @timestamp. Include appropriate context about the affected hosts and users. Describe how the attack progression might have occurred and, if feasible, attribute it to known threat groups. Prioritize high and critical alerts, but include lower-severity alerts if desired. In the description field, provide as much detail as possible, in a bulleted list explaining any attack progressions. Accuracy is of utmost importance. You MUST escape all JSON special characters (i.e. backslashes, double quotes, newlines, tabs, carriage returns, backspaces, and form feeds)." + ); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/parse_combined_or_throw/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/parse_combined_or_throw/index.test.ts new file mode 100644 index 0000000000000..cfbc837a83f66 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/helpers/parse_combined_or_throw/index.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; + +import { parseCombinedOrThrow } from '.'; +import { getRawAttackDiscoveriesMock } from '../../../../../../../__mocks__/raw_attack_discoveries'; + +describe('parseCombinedOrThrow', () => { + const mockLogger: Logger = { + debug: jest.fn(), + } as unknown as Logger; + + const nodeName = 'testNodeName'; + const llmType = 'testLlmType'; + + const validCombinedResponse = getRawAttackDiscoveriesMock(); + + const invalidCombinedResponse = 'invalid json'; + + const defaultArgs = { + combinedResponse: validCombinedResponse, + generationAttempts: 0, + nodeName, + llmType, + logger: mockLogger, + }; + + it('returns an Attack discovery for each insight in a valid combined response', () => { + const discoveries = parseCombinedOrThrow({ + ...defaultArgs, + }); + + expect(discoveries).toHaveLength(5); + }); + + it('adds a timestamp to all discoveries in a valid response', () => { + const discoveries = parseCombinedOrThrow({ + ...defaultArgs, + }); + + expect(discoveries.every((discovery) => discovery.timestamp != null)).toBe(true); + }); + + it('adds trailing backticks to the combined response if necessary', () => { + const withLeadingJson = '```json\n'.concat(validCombinedResponse); + + const discoveries = parseCombinedOrThrow({ + ...defaultArgs, + combinedResponse: withLeadingJson, + }); + + expect(discoveries).toHaveLength(5); + }); + + it('logs the parsing step', () => { + const generationAttempts = 0; + + parseCombinedOrThrow({ + ...defaultArgs, + generationAttempts, + }); + + expect((mockLogger.debug as jest.Mock).mock.calls[0][0]()).toBe( + `${nodeName} node is parsing extractedJson (${llmType}) from attempt ${generationAttempts}` + ); + }); + + it('logs the validation step', () => { + const generationAttempts = 0; + + parseCombinedOrThrow({ + ...defaultArgs, + generationAttempts, + }); + + expect((mockLogger.debug as jest.Mock).mock.calls[1][0]()).toBe( + `${nodeName} node is validating combined response (${llmType}) from attempt ${generationAttempts}` + ); + }); + + it('logs the successful validation step', () => { + const generationAttempts = 0; + + parseCombinedOrThrow({ + ...defaultArgs, + generationAttempts, + }); + + expect((mockLogger.debug as jest.Mock).mock.calls[2][0]()).toBe( + `${nodeName} node successfully validated Attack discoveries response (${llmType}) from attempt ${generationAttempts}` + ); + }); + + it('throws the expected error when JSON parsing fails', () => { + expect(() => + parseCombinedOrThrow({ + ...defaultArgs, + combinedResponse: invalidCombinedResponse, + }) + ).toThrowError('Unexpected token \'i\', "invalid json" is not valid JSON'); + }); + + it('throws the expected error when JSON validation fails', () => { + const invalidJson = '{ "insights": "not an array" }'; + + expect(() => + parseCombinedOrThrow({ + ...defaultArgs, + combinedResponse: invalidJson, + }) + ).toThrowError('Expected array, received string'); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/discard_previous_refinements/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/discard_previous_refinements/index.test.ts new file mode 100644 index 0000000000000..1409b3d47473c --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/discard_previous_refinements/index.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { discardPreviousRefinements } from '.'; +import { mockAttackDiscoveries } from '../../../../../../evaluation/__mocks__/mock_attack_discoveries'; +import { GraphState } from '../../../../types'; + +const initialState: GraphState = { + anonymizedAlerts: [], + attackDiscoveries: null, + attackDiscoveryPrompt: 'attackDiscoveryPrompt', + combinedGenerations: 'generation1generation2', + combinedRefinements: 'refinement1', // <-- existing refinements + errors: [], + generationAttempts: 3, + generations: ['generation1', 'generation2'], + hallucinationFailures: 0, + maxGenerationAttempts: 10, + maxHallucinationFailures: 5, + maxRepeatedGenerations: 3, + refinements: ['refinement1'], + refinePrompt: 'refinePrompt', + replacements: {}, + unrefinedResults: [...mockAttackDiscoveries], +}; + +describe('discardPreviousRefinements', () => { + describe('common state updates', () => { + let result: GraphState; + + beforeEach(() => { + result = discardPreviousRefinements({ + generationAttempts: initialState.generationAttempts, + hallucinationFailures: initialState.hallucinationFailures, + isHallucinationDetected: true, + state: initialState, + }); + }); + + it('resets the combined refinements', () => { + expect(result.combinedRefinements).toBe(''); + }); + + it('increments the generation attempts', () => { + expect(result.generationAttempts).toBe(initialState.generationAttempts + 1); + }); + + it('resets the refinements', () => { + expect(result.refinements).toEqual([]); + }); + + it('increments the hallucination failures when hallucinations are detected', () => { + expect(result.hallucinationFailures).toBe(initialState.hallucinationFailures + 1); + }); + }); + + it('increments the hallucination failures when hallucinations are detected', () => { + const result = discardPreviousRefinements({ + generationAttempts: initialState.generationAttempts, + hallucinationFailures: initialState.hallucinationFailures, + isHallucinationDetected: true, // <-- hallucinations detected + state: initialState, + }); + + expect(result.hallucinationFailures).toBe(initialState.hallucinationFailures + 1); + }); + + it('does NOT increment the hallucination failures when hallucinations are NOT detected', () => { + const result = discardPreviousRefinements({ + generationAttempts: initialState.generationAttempts, + hallucinationFailures: initialState.hallucinationFailures, + isHallucinationDetected: false, // <-- no hallucinations detected + state: initialState, + }); + + expect(result.hallucinationFailures).toBe(initialState.hallucinationFailures); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_combined_refine_prompt/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_combined_refine_prompt/index.test.ts new file mode 100644 index 0000000000000..f955f1b175b5b --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_combined_refine_prompt/index.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getCombinedRefinePrompt } from '.'; +import { mockAttackDiscoveries } from '../../../../../../evaluation/__mocks__/mock_attack_discoveries'; +import { getContinuePrompt } from '../../../helpers/get_continue_prompt'; + +describe('getCombinedRefinePrompt', () => { + it('returns the base query when combinedRefinements is empty', () => { + const result = getCombinedRefinePrompt({ + attackDiscoveryPrompt: 'Initial query', + combinedRefinements: '', + refinePrompt: 'Refine prompt', + unrefinedResults: [...mockAttackDiscoveries], + }); + + expect(result).toEqual(`Initial query + +Refine prompt + +""" +${JSON.stringify(mockAttackDiscoveries, null, 2)} +""" + +`); + }); + + it('returns the combined prompt when combinedRefinements is not empty', () => { + const result = getCombinedRefinePrompt({ + attackDiscoveryPrompt: 'Initial query', + combinedRefinements: 'Combined refinements', + refinePrompt: 'Refine prompt', + unrefinedResults: [...mockAttackDiscoveries], + }); + + expect(result).toEqual(`Initial query + +Refine prompt + +""" +${JSON.stringify(mockAttackDiscoveries, null, 2)} +""" + + + +${getContinuePrompt()} + +""" +Combined refinements +""" + +`); + }); + + it('handles null unrefinedResults', () => { + const result = getCombinedRefinePrompt({ + attackDiscoveryPrompt: 'Initial query', + combinedRefinements: '', + refinePrompt: 'Refine prompt', + unrefinedResults: null, + }); + + expect(result).toEqual(`Initial query + +Refine prompt + +""" +null +""" + +`); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_default_refine_prompt/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_default_refine_prompt/index.test.ts new file mode 100644 index 0000000000000..95a68524ca31e --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_default_refine_prompt/index.test.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDefaultRefinePrompt } from '.'; + +describe('getDefaultRefinePrompt', () => { + it('returns the default refine prompt string', () => { + const result = getDefaultRefinePrompt(); + + expect(result) + .toEqual(`You previously generated the following insights, but sometimes they represent the same attack. + +Combine the insights below, when they represent the same attack; leave any insights that are not combined unchanged:`); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_use_unrefined_results/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_use_unrefined_results/index.test.ts new file mode 100644 index 0000000000000..3b9aa160b4918 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/helpers/get_use_unrefined_results/index.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getUseUnrefinedResults } from '.'; + +describe('getUseUnrefinedResults', () => { + it('returns true if both maxHallucinationFailuresReached and maxRetriesReached are true', () => { + const result = getUseUnrefinedResults({ + maxHallucinationFailuresReached: true, + maxRetriesReached: true, + }); + + expect(result).toBe(true); + }); + + it('returns true if maxHallucinationFailuresReached is true and maxRetriesReached is false', () => { + const result = getUseUnrefinedResults({ + maxHallucinationFailuresReached: true, + maxRetriesReached: false, + }); + + expect(result).toBe(true); + }); + + it('returns true if maxHallucinationFailuresReached is false and maxRetriesReached is true', () => { + const result = getUseUnrefinedResults({ + maxHallucinationFailuresReached: false, + maxRetriesReached: true, + }); + + expect(result).toBe(true); + }); + + it('returns false if both maxHallucinationFailuresReached and maxRetriesReached are false', () => { + const result = getUseUnrefinedResults({ + maxHallucinationFailuresReached: false, + maxRetriesReached: false, + }); + + expect(result).toBe(false); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.test.ts new file mode 100644 index 0000000000000..d5b5a333f48f2 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.test.ts @@ -0,0 +1,342 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AttackDiscovery } from '@kbn/elastic-assistant-common'; +import type { ActionsClientLlm } from '@kbn/langchain/server'; +import { loggerMock } from '@kbn/logging-mocks'; +import { FakeLLM } from '@langchain/core/utils/testing'; + +import { getRefineNode } from '.'; +import { + mockAnonymizedAlerts, + mockAnonymizedAlertsReplacements, +} from '../../../../evaluation/__mocks__/mock_anonymized_alerts'; +import { getChainWithFormatInstructions } from '../helpers/get_chain_with_format_instructions'; +import { getDefaultAttackDiscoveryPrompt } from '../helpers/get_default_attack_discovery_prompt'; +import { getDefaultRefinePrompt } from './helpers/get_default_refine_prompt'; +import { GraphState } from '../../types'; +import { + getParsedAttackDiscoveriesMock, + getRawAttackDiscoveriesMock, +} from '../../../../../../__mocks__/raw_attack_discoveries'; + +const attackDiscoveryTimestamp = '2024-10-11T17:55:59.702Z'; + +export const mockUnrefinedAttackDiscoveries: AttackDiscovery[] = [ + { + title: 'unrefinedTitle1', + alertIds: ['unrefinedAlertId1', 'unrefinedAlertId2', 'unrefinedAlertId3'], + timestamp: '2024-10-10T22:59:52.749Z', + detailsMarkdown: 'unrefinedDetailsMarkdown1', + summaryMarkdown: 'unrefinedSummaryMarkdown1 - entity A', + mitreAttackTactics: ['Input Capture'], + entitySummaryMarkdown: 'entitySummaryMarkdown1', + }, + { + title: 'unrefinedTitle2', + alertIds: ['unrefinedAlertId3', 'unrefinedAlertId4', 'unrefinedAlertId5'], + timestamp: '2024-10-10T22:59:52.749Z', + detailsMarkdown: 'unrefinedDetailsMarkdown2', + summaryMarkdown: 'unrefinedSummaryMarkdown2 - also entity A', + mitreAttackTactics: ['Credential Access'], + entitySummaryMarkdown: 'entitySummaryMarkdown2', + }, +]; + +jest.mock('../helpers/get_chain_with_format_instructions', () => { + const mockInvoke = jest.fn().mockResolvedValue(''); + + return { + getChainWithFormatInstructions: jest.fn().mockReturnValue({ + chain: { + invoke: mockInvoke, + }, + formatInstructions: ['mock format instructions'], + llmType: 'openai', + mockInvoke, // <-- added for testing + }), + }; +}); + +const mockLogger = loggerMock.create(); +let mockLlm: ActionsClientLlm; + +const initialGraphState: GraphState = { + attackDiscoveries: null, + attackDiscoveryPrompt: getDefaultAttackDiscoveryPrompt(), + anonymizedAlerts: [...mockAnonymizedAlerts], + combinedGenerations: 'gen1', + combinedRefinements: '', + errors: [], + generationAttempts: 1, + generations: ['gen1'], + hallucinationFailures: 0, + maxGenerationAttempts: 10, + maxHallucinationFailures: 5, + maxRepeatedGenerations: 3, + refinements: [], + refinePrompt: getDefaultRefinePrompt(), + replacements: { + ...mockAnonymizedAlertsReplacements, + }, + unrefinedResults: [...mockUnrefinedAttackDiscoveries], +}; + +describe('getRefineNode', () => { + beforeEach(() => { + jest.clearAllMocks(); + + jest.useFakeTimers(); + jest.setSystemTime(new Date(attackDiscoveryTimestamp)); + + mockLlm = new FakeLLM({ + response: '', + }) as unknown as ActionsClientLlm; + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('returns a function', () => { + const refineNode = getRefineNode({ + llm: mockLlm, + logger: mockLogger, + }); + + expect(typeof refineNode).toBe('function'); + }); + + it('invokes the chain with the unrefinedResults from state and format instructions', async () => { + const mockInvoke = getChainWithFormatInstructions(mockLlm).chain.invoke as jest.Mock; + + const refineNode = getRefineNode({ + llm: mockLlm, + logger: mockLogger, + }); + + await refineNode(initialGraphState); + + expect(mockInvoke).toHaveBeenCalledWith({ + format_instructions: ['mock format instructions'], + query: `${initialGraphState.attackDiscoveryPrompt} + +${getDefaultRefinePrompt()} + +\"\"\" +${JSON.stringify(initialGraphState.unrefinedResults, null, 2)} +\"\"\" + +`, + }); + }); + + it('removes the surrounding json from the response', async () => { + const response = + 'You asked for some JSON, here it is:\n```json\n{"key": "value"}\n```\nI hope that works for you.'; + + const mockLlmWithResponse = new FakeLLM({ response }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(response); + + const refineNode = getRefineNode({ + llm: mockLlm, + logger: mockLogger, + }); + + const state = await refineNode(initialGraphState); + + expect(state).toEqual({ + ...initialGraphState, + combinedRefinements: '{"key": "value"}', + errors: [ + 'refine node is unable to parse (fake) response from attempt 1; (this may be an incomplete response from the model): [\n {\n "code": "invalid_type",\n "expected": "array",\n "received": "undefined",\n "path": [\n "insights"\n ],\n "message": "Required"\n }\n]', + ], + generationAttempts: 2, + refinements: ['{"key": "value"}'], + }); + }); + + it('handles hallucinations', async () => { + const hallucinatedResponse = + 'tactics like **Credential Access**, **Command and Control**, and **Persistence**.",\n "entitySummaryMarkdown": "Malware detected on host **{{ host.name hostNameValue }}**'; + + const mockLlmWithHallucination = new FakeLLM({ + response: hallucinatedResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithHallucination).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(hallucinatedResponse); + + const refineNode = getRefineNode({ + llm: mockLlmWithHallucination, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedRefinements: '{"key": "value"}', + refinements: ['{"key": "value"}'], + }; + + const state = await refineNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + combinedRefinements: '', // <-- reset + generationAttempts: 2, // <-- incremented + refinements: [], // <-- reset + hallucinationFailures: 1, // <-- incremented + }); + }); + + it('discards previous refinements and starts over when the maxRepeatedGenerations limit is reached', async () => { + const repeatedResponse = '{"key": "value"}'; + + const mockLlmWithRepeatedGenerations = new FakeLLM({ + response: repeatedResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithRepeatedGenerations).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(repeatedResponse); + + const refineNode = getRefineNode({ + llm: mockLlmWithRepeatedGenerations, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedRefinements: '{"key": "value"}{"key": "value"}', + generationAttempts: 3, + refinements: ['{"key": "value"}', '{"key": "value"}'], + }; + + const state = await refineNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + combinedRefinements: '', + generationAttempts: 4, // <-- incremented + refinements: [], + }); + }); + + it('combines the response with the previous refinements', async () => { + const response = 'refine1'; + + const mockLlmWithResponse = new FakeLLM({ + response, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(response); + + const refineNode = getRefineNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedRefinements: 'refine0', + generationAttempts: 2, + refinements: ['refine0'], + }; + + const state = await refineNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + combinedRefinements: 'refine0refine1', + errors: [ + 'refine node is unable to parse (fake) response from attempt 2; (this may be an incomplete response from the model): SyntaxError: Unexpected token \'r\', "refine0refine1" is not valid JSON', + ], + generationAttempts: 3, + refinements: ['refine0', 'refine1'], + }); + }); + + it('returns refined results when combined responses pass validation', async () => { + // split the response into two parts to simulate a valid response + const splitIndex = 100; // arbitrary index + const firstResponse = getRawAttackDiscoveriesMock().slice(0, splitIndex); + const secondResponse = getRawAttackDiscoveriesMock().slice(splitIndex); + + const mockLlmWithResponse = new FakeLLM({ + response: secondResponse, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(secondResponse); + + const refineNode = getRefineNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedRefinements: firstResponse, + generationAttempts: 2, + refinements: [firstResponse], + }; + + const state = await refineNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + attackDiscoveries: getParsedAttackDiscoveriesMock(attackDiscoveryTimestamp), + combinedRefinements: firstResponse.concat(secondResponse), + generationAttempts: 3, + refinements: [firstResponse, secondResponse], + }); + }); + + it('uses the unrefined results when the max number of retries has already been reached', async () => { + const response = 'this will not pass JSON parsing'; + + const mockLlmWithResponse = new FakeLLM({ + response, + }) as unknown as ActionsClientLlm; + const mockInvoke = getChainWithFormatInstructions(mockLlmWithResponse).chain + .invoke as jest.Mock; + + mockInvoke.mockResolvedValue(response); + + const refineNode = getRefineNode({ + llm: mockLlmWithResponse, + logger: mockLogger, + }); + + const withPreviousGenerations = { + ...initialGraphState, + combinedRefinements: 'refine1', + generationAttempts: 9, + refinements: ['refine1'], + }; + + const state = await refineNode(withPreviousGenerations); + + expect(state).toEqual({ + ...withPreviousGenerations, + attackDiscoveries: state.unrefinedResults, // <-- the unrefined results are returned + combinedRefinements: 'refine1this will not pass JSON parsing', + errors: [ + 'refine node is unable to parse (fake) response from attempt 9; (this may be an incomplete response from the model): SyntaxError: Unexpected token \'r\', "refine1thi"... is not valid JSON', + ], + generationAttempts: 10, + refinements: ['refine1', response], + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.ts index 0c7987eef92bc..d1bed136f6a1c 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/refine/index.ts @@ -89,7 +89,7 @@ export const getRefineNode = ({ generationsAreRepeating({ currentGeneration: partialResponse, previousGenerations: refinements, - sampleLastNGenerations: maxRepeatedGenerations, + sampleLastNGenerations: maxRepeatedGenerations - 1, }) ) { logger?.debug( diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/anonymized_alerts_retriever/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/anonymized_alerts_retriever/index.test.ts new file mode 100644 index 0000000000000..dab2d57b20edc --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/anonymized_alerts_retriever/index.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; + +import { AnonymizedAlertsRetriever } from '.'; +import { getMockAnonymizationFieldResponse } from '../../../../../evaluation/__mocks__/mock_anonymization_fields'; +import { getAnonymizedAlerts } from '../helpers/get_anonymized_alerts'; + +const anonymizationFields = getMockAnonymizationFieldResponse(); + +const rawAlerts = [ + '@timestamp,2024-11-05T15:42:48.034Z\n_id,07d86d116ff754f4aa57c00e23a5273c2efbc9450416823ebd1d7b343b42d11a\nevent.category,malware,intrusion_detection,process\nevent.dataset,endpoint.alerts\nevent.module,endpoint\nevent.outcome,success\nfile.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nfile.name,My Go Application.app\nfile.path,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/6D63F08A-011C-4511-8556-EAEF9AFD6340/d/Setup.app/Contents/MacOS/My Go Application.app\nhost.name,d26e9abd-6cbb-4620-a802-a22b97845d5c\nhost.os.name,macOS\nhost.os.version,13.4\nkibana.alert.original_time,2023-06-19T00:28:06.888Z\nkibana.alert.risk_score,99\nkibana.alert.rule.description,Generates a detection alert each time an Elastic Endpoint Security alert is received. Enabling this rule allows you to immediately begin investigating your Endpoint alerts.\nkibana.alert.rule.name,Malware Detection Alert\nkibana.alert.severity,critical\nkibana.alert.workflow_status,open\nmessage,Malware Detection Alert\nprocess.args,xpcproxy,application.Appify by Machine Box.My Go Application.20.23\nprocess.code_signature.exists,true\nprocess.code_signature.signing_id,a.out\nprocess.code_signature.status,code failed to satisfy specified code requirement(s)\nprocess.code_signature.subject_name,\nprocess.code_signature.trusted,false\nprocess.command_line,xpcproxy application.Appify by Machine Box.My Go Application.20.23\nprocess.executable,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/6D63F08A-011C-4511-8556-EAEF9AFD6340/d/Setup.app/Contents/MacOS/My Go Application.app\nprocess.hash.md5,e62bdd3eaf2be436fca2e67b7eede603\nprocess.hash.sha1,58a3bddbc7c45193ecbefa22ad0496b60a29dff2\nprocess.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nprocess.name,My Go Application.app\nprocess.parent.args,/sbin/launchd\nprocess.parent.args_count,1\nprocess.parent.code_signature.exists,true\nprocess.parent.code_signature.status,No error.\nprocess.parent.code_signature.subject_name,Software Signing\nprocess.parent.code_signature.trusted,true\nprocess.parent.command_line,/sbin/launchd\nprocess.parent.executable,/sbin/launchd\nprocess.parent.name,launchd\nprocess.pid,1200\nuser.name,81c3db40-f3da-4c6a-b3c8-48c776148102', + '@timestamp,2024-11-05T15:42:48.033Z\n_id,f2d2d8bd15402e8efff81d48b70ef8cb890d5502576fb92365ee2328f5fcb123\nevent.category,malware,intrusion_detection,process\nevent.dataset,endpoint.alerts\nevent.module,endpoint\nevent.outcome,success\nfile.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nfile.name,My Go Application.app\nfile.path,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/3C4D44B9-4838-4613-BACC-BD00A9CE4025/d/Setup.app/Contents/MacOS/My Go Application.app\nhost.name,d26e9abd-6cbb-4620-a802-a22b97845d5c\nhost.os.name,macOS\nhost.os.version,13.4\nkibana.alert.original_time,2023-06-19T00:27:47.362Z\nkibana.alert.risk_score,99\nkibana.alert.rule.description,Generates a detection alert each time an Elastic Endpoint Security alert is received. Enabling this rule allows you to immediately begin investigating your Endpoint alerts.\nkibana.alert.rule.name,Malware Detection Alert\nkibana.alert.severity,critical\nkibana.alert.workflow_status,open\nmessage,Malware Detection Alert\nprocess.args,xpcproxy,application.Appify by Machine Box.My Go Application.20.23\nprocess.code_signature.exists,true\nprocess.code_signature.signing_id,a.out\nprocess.code_signature.status,code failed to satisfy specified code requirement(s)\nprocess.code_signature.subject_name,\nprocess.code_signature.trusted,false\nprocess.command_line,xpcproxy application.Appify by Machine Box.My Go Application.20.23\nprocess.executable,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/3C4D44B9-4838-4613-BACC-BD00A9CE4025/d/Setup.app/Contents/MacOS/My Go Application.app\nprocess.hash.md5,e62bdd3eaf2be436fca2e67b7eede603\nprocess.hash.sha1,58a3bddbc7c45193ecbefa22ad0496b60a29dff2\nprocess.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nprocess.name,My Go Application.app\nprocess.parent.args,/sbin/launchd\nprocess.parent.args_count,1\nprocess.parent.code_signature.exists,true\nprocess.parent.code_signature.status,No error.\nprocess.parent.code_signature.subject_name,Software Signing\nprocess.parent.code_signature.trusted,true\nprocess.parent.command_line,/sbin/launchd\nprocess.parent.executable,/sbin/launchd\nprocess.parent.name,launchd\nprocess.pid,1169\nuser.name,81c3db40-f3da-4c6a-b3c8-48c776148102', +]; + +jest.mock('../helpers/get_anonymized_alerts', () => ({ + getAnonymizedAlerts: jest.fn(), +})); + +describe('AnonymizedAlertsRetriever', () => { + let esClient: ElasticsearchClient; + + beforeEach(() => { + jest.clearAllMocks(); + + esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + + (getAnonymizedAlerts as jest.Mock).mockResolvedValue([...rawAlerts]); + }); + + it('returns the expected pageContent and metadata', async () => { + const retriever = new AnonymizedAlertsRetriever({ + alertsIndexPattern: 'test-pattern', + anonymizationFields, + esClient, + size: 10, + }); + + const documents = await retriever._getRelevantDocuments('test-query'); + + expect(documents).toEqual([ + { + pageContent: + '@timestamp,2024-11-05T15:42:48.034Z\n_id,07d86d116ff754f4aa57c00e23a5273c2efbc9450416823ebd1d7b343b42d11a\nevent.category,malware,intrusion_detection,process\nevent.dataset,endpoint.alerts\nevent.module,endpoint\nevent.outcome,success\nfile.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nfile.name,My Go Application.app\nfile.path,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/6D63F08A-011C-4511-8556-EAEF9AFD6340/d/Setup.app/Contents/MacOS/My Go Application.app\nhost.name,d26e9abd-6cbb-4620-a802-a22b97845d5c\nhost.os.name,macOS\nhost.os.version,13.4\nkibana.alert.original_time,2023-06-19T00:28:06.888Z\nkibana.alert.risk_score,99\nkibana.alert.rule.description,Generates a detection alert each time an Elastic Endpoint Security alert is received. Enabling this rule allows you to immediately begin investigating your Endpoint alerts.\nkibana.alert.rule.name,Malware Detection Alert\nkibana.alert.severity,critical\nkibana.alert.workflow_status,open\nmessage,Malware Detection Alert\nprocess.args,xpcproxy,application.Appify by Machine Box.My Go Application.20.23\nprocess.code_signature.exists,true\nprocess.code_signature.signing_id,a.out\nprocess.code_signature.status,code failed to satisfy specified code requirement(s)\nprocess.code_signature.subject_name,\nprocess.code_signature.trusted,false\nprocess.command_line,xpcproxy application.Appify by Machine Box.My Go Application.20.23\nprocess.executable,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/6D63F08A-011C-4511-8556-EAEF9AFD6340/d/Setup.app/Contents/MacOS/My Go Application.app\nprocess.hash.md5,e62bdd3eaf2be436fca2e67b7eede603\nprocess.hash.sha1,58a3bddbc7c45193ecbefa22ad0496b60a29dff2\nprocess.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nprocess.name,My Go Application.app\nprocess.parent.args,/sbin/launchd\nprocess.parent.args_count,1\nprocess.parent.code_signature.exists,true\nprocess.parent.code_signature.status,No error.\nprocess.parent.code_signature.subject_name,Software Signing\nprocess.parent.code_signature.trusted,true\nprocess.parent.command_line,/sbin/launchd\nprocess.parent.executable,/sbin/launchd\nprocess.parent.name,launchd\nprocess.pid,1200\nuser.name,81c3db40-f3da-4c6a-b3c8-48c776148102', + metadata: {}, + }, + { + pageContent: + '@timestamp,2024-11-05T15:42:48.033Z\n_id,f2d2d8bd15402e8efff81d48b70ef8cb890d5502576fb92365ee2328f5fcb123\nevent.category,malware,intrusion_detection,process\nevent.dataset,endpoint.alerts\nevent.module,endpoint\nevent.outcome,success\nfile.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nfile.name,My Go Application.app\nfile.path,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/3C4D44B9-4838-4613-BACC-BD00A9CE4025/d/Setup.app/Contents/MacOS/My Go Application.app\nhost.name,d26e9abd-6cbb-4620-a802-a22b97845d5c\nhost.os.name,macOS\nhost.os.version,13.4\nkibana.alert.original_time,2023-06-19T00:27:47.362Z\nkibana.alert.risk_score,99\nkibana.alert.rule.description,Generates a detection alert each time an Elastic Endpoint Security alert is received. Enabling this rule allows you to immediately begin investigating your Endpoint alerts.\nkibana.alert.rule.name,Malware Detection Alert\nkibana.alert.severity,critical\nkibana.alert.workflow_status,open\nmessage,Malware Detection Alert\nprocess.args,xpcproxy,application.Appify by Machine Box.My Go Application.20.23\nprocess.code_signature.exists,true\nprocess.code_signature.signing_id,a.out\nprocess.code_signature.status,code failed to satisfy specified code requirement(s)\nprocess.code_signature.subject_name,\nprocess.code_signature.trusted,false\nprocess.command_line,xpcproxy application.Appify by Machine Box.My Go Application.20.23\nprocess.executable,/private/var/folders/_b/rmcpc65j6nv11ygrs50ctcjr0000gn/T/AppTranslocation/3C4D44B9-4838-4613-BACC-BD00A9CE4025/d/Setup.app/Contents/MacOS/My Go Application.app\nprocess.hash.md5,e62bdd3eaf2be436fca2e67b7eede603\nprocess.hash.sha1,58a3bddbc7c45193ecbefa22ad0496b60a29dff2\nprocess.hash.sha256,2c63ba2b1a5131b80e567b7a1a93997a2de07ea20d0a8f5149701c67b832c097\nprocess.name,My Go Application.app\nprocess.parent.args,/sbin/launchd\nprocess.parent.args_count,1\nprocess.parent.code_signature.exists,true\nprocess.parent.code_signature.status,No error.\nprocess.parent.code_signature.subject_name,Software Signing\nprocess.parent.code_signature.trusted,true\nprocess.parent.command_line,/sbin/launchd\nprocess.parent.executable,/sbin/launchd\nprocess.parent.name,launchd\nprocess.pid,1169\nuser.name,81c3db40-f3da-4c6a-b3c8-48c776148102', + metadata: {}, + }, + ]); + }); + + it('calls getAnonymizedAlerts with the expected parameters', async () => { + const onNewReplacements = jest.fn(); + const mockReplacements = { + replacement1: 'SRVMAC08', + replacement2: 'SRVWIN01', + replacement3: 'SRVWIN02', + }; + + const retriever = new AnonymizedAlertsRetriever({ + alertsIndexPattern: 'test-pattern', + anonymizationFields, + esClient, + onNewReplacements, + replacements: mockReplacements, + size: 10, + }); + + await retriever._getRelevantDocuments('test-query'); + + expect(getAnonymizedAlerts as jest.Mock).toHaveBeenCalledWith({ + alertsIndexPattern: 'test-pattern', + anonymizationFields, + esClient, + onNewReplacements, + replacements: mockReplacements, + size: 10, + }); + }); + + it('handles empty anonymized alerts', async () => { + (getAnonymizedAlerts as jest.Mock).mockResolvedValue([]); + + const retriever = new AnonymizedAlertsRetriever({ + esClient, + alertsIndexPattern: 'test-pattern', + anonymizationFields, + size: 10, + }); + + const documents = await retriever._getRelevantDocuments('test-query'); + + expect(documents).toHaveLength(0); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.test.ts new file mode 100644 index 0000000000000..bfd8bf2ce6953 --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import { Replacements } from '@kbn/elastic-assistant-common'; + +import { getRetrieveAnonymizedAlertsNode } from '.'; +import { mockAnonymizedAlerts } from '../../../../evaluation/__mocks__/mock_anonymized_alerts'; +import { getDefaultAttackDiscoveryPrompt } from '../helpers/get_default_attack_discovery_prompt'; +import { getDefaultRefinePrompt } from '../refine/helpers/get_default_refine_prompt'; +import type { GraphState } from '../../types'; + +const initialGraphState: GraphState = { + attackDiscoveries: null, + attackDiscoveryPrompt: getDefaultAttackDiscoveryPrompt(), + anonymizedAlerts: [], + combinedGenerations: '', + combinedRefinements: '', + errors: [], + generationAttempts: 0, + generations: [], + hallucinationFailures: 0, + maxGenerationAttempts: 10, + maxHallucinationFailures: 5, + maxRepeatedGenerations: 3, + refinements: [], + refinePrompt: getDefaultRefinePrompt(), + replacements: {}, + unrefinedResults: null, +}; + +jest.mock('./anonymized_alerts_retriever', () => ({ + AnonymizedAlertsRetriever: jest + .fn() + .mockImplementation( + ({ + onNewReplacements, + replacements, + }: { + onNewReplacements?: (replacements: Replacements) => void; + replacements?: Replacements; + }) => ({ + withConfig: jest.fn().mockReturnValue({ + invoke: jest.fn(async () => { + if (onNewReplacements != null && replacements != null) { + onNewReplacements(replacements); + } + + return mockAnonymizedAlerts; + }), + }), + }) + ), +})); + +describe('getRetrieveAnonymizedAlertsNode', () => { + const logger = { + debug: jest.fn(), + } as unknown as Logger; + + let esClient: ElasticsearchClient; + + beforeEach(() => { + esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + }); + + it('returns a function', () => { + const result = getRetrieveAnonymizedAlertsNode({ + esClient, + logger, + }); + expect(typeof result).toBe('function'); + }); + + it('updates state with anonymized alerts', async () => { + const state: GraphState = { ...initialGraphState }; + + const retrieveAnonymizedAlerts = getRetrieveAnonymizedAlertsNode({ + esClient, + logger, + }); + + const result = await retrieveAnonymizedAlerts(state); + + expect(result).toHaveProperty('anonymizedAlerts', mockAnonymizedAlerts); + }); + + it('calls onNewReplacements with updated replacements', async () => { + const state: GraphState = { ...initialGraphState }; + const onNewReplacements = jest.fn(); + const replacements = { key: 'value' }; + + const retrieveAnonymizedAlerts = getRetrieveAnonymizedAlertsNode({ + esClient, + logger, + onNewReplacements, + replacements, + }); + + await retrieveAnonymizedAlerts(state); + + expect(onNewReplacements).toHaveBeenCalledWith({ + ...replacements, + }); + }); +}); diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.ts index 951ae3bca8854..a5d31fa14770a 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/nodes/retriever/index.ts @@ -60,11 +60,3 @@ export const getRetrieveAnonymizedAlertsNode = ({ return retrieveAnonymizedAlerts; }; - -/** - * Retrieve documents - * - * @param {GraphState} state The current state of the graph. - * @param {RunnableConfig | undefined} config The configuration object for tracing. - * @returns {Promise} The new state object. - */ diff --git a/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/state/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/state/index.test.ts new file mode 100644 index 0000000000000..dcc8ee1e4292d --- /dev/null +++ b/x-pack/plugins/elastic_assistant/server/lib/attack_discovery/graphs/default_attack_discovery_graph/state/index.test.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getDefaultGraphState } from '.'; +import { + DEFAULT_MAX_GENERATION_ATTEMPTS, + DEFAULT_MAX_HALLUCINATION_FAILURES, + DEFAULT_MAX_REPEATED_GENERATIONS, +} from '../constants'; +import { getDefaultAttackDiscoveryPrompt } from '../nodes/helpers/get_default_attack_discovery_prompt'; +import { getDefaultRefinePrompt } from '../nodes/refine/helpers/get_default_refine_prompt'; + +const defaultAttackDiscoveryPrompt = getDefaultAttackDiscoveryPrompt(); +const defaultRefinePrompt = getDefaultRefinePrompt(); + +describe('getDefaultGraphState', () => { + it('returns the expected default attackDiscoveries', () => { + const state = getDefaultGraphState(); + + expect(state.attackDiscoveries?.default?.()).toBeNull(); + }); + + it('returns the expected default attackDiscoveryPrompt', () => { + const state = getDefaultGraphState(); + + expect(state.attackDiscoveryPrompt?.default?.()).toEqual(defaultAttackDiscoveryPrompt); + }); + + it('returns the expected default empty collection of anonymizedAlerts', () => { + const state = getDefaultGraphState(); + + expect(state.anonymizedAlerts?.default?.()).toHaveLength(0); + }); + + it('returns the expected default combinedGenerations state', () => { + const state = getDefaultGraphState(); + + expect(state.combinedGenerations?.default?.()).toBe(''); + }); + + it('returns the expected default combinedRefinements state', () => { + const state = getDefaultGraphState(); + + expect(state.combinedRefinements?.default?.()).toBe(''); + }); + + it('returns the expected default errors state', () => { + const state = getDefaultGraphState(); + + expect(state.errors?.default?.()).toHaveLength(0); + }); + + it('return the expected default generationAttempts state', () => { + const state = getDefaultGraphState(); + + expect(state.generationAttempts?.default?.()).toBe(0); + }); + + it('returns the expected default generations state', () => { + const state = getDefaultGraphState(); + + expect(state.generations?.default?.()).toHaveLength(0); + }); + + it('returns the expected default hallucinationFailures state', () => { + const state = getDefaultGraphState(); + + expect(state.hallucinationFailures?.default?.()).toBe(0); + }); + + it('returns the expected default refinePrompt state', () => { + const state = getDefaultGraphState(); + + expect(state.refinePrompt?.default?.()).toEqual(defaultRefinePrompt); + }); + + it('returns the expected default maxGenerationAttempts state', () => { + const state = getDefaultGraphState(); + + expect(state.maxGenerationAttempts?.default?.()).toBe(DEFAULT_MAX_GENERATION_ATTEMPTS); + }); + + it('returns the expected default maxHallucinationFailures state', () => { + const state = getDefaultGraphState(); + expect(state.maxHallucinationFailures?.default?.()).toBe(DEFAULT_MAX_HALLUCINATION_FAILURES); + }); + + it('returns the expected default maxRepeatedGenerations state', () => { + const state = getDefaultGraphState(); + + expect(state.maxRepeatedGenerations?.default?.()).toBe(DEFAULT_MAX_REPEATED_GENERATIONS); + }); + + it('returns the expected default refinements state', () => { + const state = getDefaultGraphState(); + + expect(state.refinements?.default?.()).toHaveLength(0); + }); + + it('returns the expected default replacements state', () => { + const state = getDefaultGraphState(); + + expect(state.replacements?.default?.()).toEqual({}); + }); + + it('returns the expected default unrefinedResults state', () => { + const state = getDefaultGraphState(); + + expect(state.unrefinedResults?.default?.()).toBeNull(); + }); +}); diff --git a/x-pack/plugins/security_solution/public/attack_discovery/pages/empty_states/helpers/show_empty_states/index.test.ts b/x-pack/plugins/security_solution/public/attack_discovery/pages/empty_states/helpers/show_empty_states/index.test.ts new file mode 100644 index 0000000000000..0211dc8d51eba --- /dev/null +++ b/x-pack/plugins/security_solution/public/attack_discovery/pages/empty_states/helpers/show_empty_states/index.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { showEmptyStates } from '.'; +import { + showEmptyPrompt, + showFailurePrompt, + showNoAlertsPrompt, + showWelcomePrompt, +} from '../../../helpers'; + +jest.mock('../../../helpers', () => ({ + showEmptyPrompt: jest.fn().mockReturnValue(false), + showFailurePrompt: jest.fn().mockReturnValue(false), + showNoAlertsPrompt: jest.fn().mockReturnValue(false), + showWelcomePrompt: jest.fn().mockReturnValue(false), +})); + +const defaultArgs = { + aiConnectorsCount: 0, + alertsContextCount: 0, + attackDiscoveriesCount: 0, + connectorId: undefined, + failureReason: null, + isLoading: false, +}; + +describe('showEmptyStates', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns true if showWelcomePrompt returns true', () => { + (showWelcomePrompt as jest.Mock).mockReturnValue(true); + + const result = showEmptyStates({ + ...defaultArgs, + }); + expect(result).toBe(true); + }); + + it('returns true if showFailurePrompt returns true', () => { + (showFailurePrompt as jest.Mock).mockReturnValue(true); + + const result = showEmptyStates({ + ...defaultArgs, + connectorId: 'test', + failureReason: 'error', + }); + expect(result).toBe(true); + }); + + it('returns true if showNoAlertsPrompt returns true', () => { + (showNoAlertsPrompt as jest.Mock).mockReturnValue(true); + + const result = showEmptyStates({ + ...defaultArgs, + connectorId: 'test', + }); + expect(result).toBe(true); + }); + + it('returns true if showEmptyPrompt returns true', () => { + (showEmptyPrompt as jest.Mock).mockReturnValue(true); + + const result = showEmptyStates({ + ...defaultArgs, + }); + expect(result).toBe(true); + }); + + it('returns false if all prompts return false', () => { + (showWelcomePrompt as jest.Mock).mockReturnValue(false); + (showFailurePrompt as jest.Mock).mockReturnValue(false); + (showNoAlertsPrompt as jest.Mock).mockReturnValue(false); + (showEmptyPrompt as jest.Mock).mockReturnValue(false); + + const result = showEmptyStates({ + ...defaultArgs, + }); + expect(result).toBe(false); + }); +}); diff --git a/x-pack/plugins/security_solution/public/attack_discovery/pages/generate/index.test.tsx b/x-pack/plugins/security_solution/public/attack_discovery/pages/generate/index.test.tsx new file mode 100644 index 0000000000000..e818d1c4140f6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/attack_discovery/pages/generate/index.test.tsx @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import React from 'react'; + +import { Generate } from '.'; +import * as i18n from '../empty_prompt/translations'; + +describe('Generate Component', () => { + it('calls onGenerate when the button is clicked', () => { + const onGenerate = jest.fn(); + + render(); + + fireEvent.click(screen.getByTestId('generate')); + + expect(onGenerate).toHaveBeenCalled(); + }); + + it('disables the generate button when isLoading is true', () => { + render(); + + expect(screen.getByTestId('generate')).toBeDisabled(); + }); + + it('disables the generate button when isDisabled is true', () => { + render(); + + expect(screen.getByTestId('generate')).toBeDisabled(); + }); + + it('shows tooltip content when the button is disabled', async () => { + render(); + + fireEvent.mouseOver(screen.getByTestId('generate')); + + await waitFor(() => { + expect(screen.getByText(i18n.SELECT_A_CONNECTOR)).toBeInTheDocument(); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.test.tsx b/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.test.tsx new file mode 100644 index 0000000000000..958c9094fabf3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.test.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { render, screen, fireEvent } from '@testing-library/react'; +import React from 'react'; + +import { AlertsSettings, MAX_ALERTS } from '.'; + +const maxAlerts = '150'; + +const setMaxAlerts = jest.fn(); + +describe('AlertsSettings', () => { + it('calls setMaxAlerts when the alerts range changes', () => { + render(); + + fireEvent.click(screen.getByText(`${MAX_ALERTS}`)); + + expect(setMaxAlerts).toHaveBeenCalledWith(`${MAX_ALERTS}`); + }); + + it('displays the correct maxAlerts value', () => { + render(); + + expect(screen.getByTestId('alertsRange')).toHaveValue(maxAlerts); + }); + + it('displays the expected text for anonymization settings', () => { + render(); + + expect(screen.getByTestId('latestAndRiskiest')).toHaveTextContent( + 'Send Attack discovery information about your 150 newest and riskiest open or acknowledged alerts.' + ); + }); +}); diff --git a/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.tsx b/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.tsx index b51a1fc3f85c8..336da549f55ea 100644 --- a/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.tsx +++ b/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/alerts_settings/index.tsx @@ -51,7 +51,9 @@ const AlertsSettingsComponent: React.FC = ({ maxAlerts, setMaxAlerts }) = - {i18n.LATEST_AND_RISKIEST_OPEN_ALERTS(Number(maxAlerts))} + + {i18n.LATEST_AND_RISKIEST_OPEN_ALERTS(Number(maxAlerts))} + diff --git a/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/footer/index.test.tsx b/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/footer/index.test.tsx new file mode 100644 index 0000000000000..e487304c41350 --- /dev/null +++ b/x-pack/plugins/security_solution/public/attack_discovery/pages/header/settings_modal/footer/index.test.tsx @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; + +import { Footer } from '.'; + +describe('Footer', () => { + const closeModal = jest.fn(); + const onReset = jest.fn(); + const onSave = jest.fn(); + + beforeEach(() => jest.clearAllMocks()); + + it('calls onReset when the reset button is clicked', () => { + render(