From 9036b15be0c75e3541738d6d121bc2583affd447 Mon Sep 17 00:00:00 2001 From: Ievgen Sorokopud Date: Mon, 31 Jul 2023 11:43:54 +0200 Subject: [PATCH 01/33] Flaky test: Detection rule telemetry #162407 (#162681) ## Summary Original ticket: https://github.com/elastic/kibana/issues/156088 It looks like flakiness happens due to another rule from previous test case being scheduled by alerting framework and running even after we deleted it (since we called delete rule after it was scheduled). That rule scheduled earlier produces some unexpected artifacts in metrics, specifically `stats.detection_rules.detection_rule_status` where occasionally see Screenshot 2023-07-25 at 15 13 29 Since, in this test case the rule is disabled, we should not see this data. As a fix, we decided to test against specific metrics attributes `stats.detection_rules.detection_rule_usage` which we expect being updated. --- .../usage_collector/detection_rules.ts | 1173 +++++++---------- 1 file changed, 447 insertions(+), 726 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts index 575da7ec63f5c..c913b8d6e8a68 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group4/telemetry/usage_collector/detection_rules.ts @@ -6,14 +6,13 @@ */ import expect from '@kbn/expect'; -import type { DetectionMetrics } from '@kbn/security-solution-plugin/server/usage/detections/types'; import type { ThreatMatchRuleCreateProps, ThresholdRuleCreateProps, } from '@kbn/security-solution-plugin/common/api/detection_engine'; import { getInitialDetectionMetrics } from '@kbn/security-solution-plugin/server/usage/detections/get_initial_usage'; -import { getInitialEventLogUsage } from '@kbn/security-solution-plugin/server/usage/detections/rules/get_initial_usage'; import { ELASTIC_SECURITY_RULE_ID } from '@kbn/security-solution-plugin/common'; +import { RulesTypeUsage } from '@kbn/security-solution-plugin/server/usage/detections/rules/types'; import type { FtrProviderContext } from '../../../../common/ftr_provider_context'; import { createLegacyRuleAction, @@ -73,32 +72,26 @@ export default ({ getService }: FtrProviderContext) => { await createRule(supertest, log, rule); await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - query: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + query: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -109,38 +102,28 @@ export default ({ getService }: FtrProviderContext) => { await waitForSignalsToBePresent(supertest, log, 4, [id]); await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - query: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + query: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -152,30 +135,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - query: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, - notifications_disabled: 1, - disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - notifications_disabled: 1, - disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + query: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, + notifications_disabled: 1, + disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + notifications_disabled: 1, + disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -189,32 +162,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - query: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + query: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, + enabled: 1, + alerts: 4, + notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -226,26 +189,21 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - query: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, - disabled: 1, - legacy_notifications_disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - legacy_notifications_disabled: 1, - }, - }, + + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + query: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, + disabled: 1, + legacy_notifications_disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + legacy_notifications_disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -259,69 +217,52 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - query: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + query: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.query, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); }); - // FLAKY: https://github.com/elastic/kibana/issues/156088 - describe.skip('"eql" rule type', () => { + describe('"eql" rule type', () => { it('should show "notifications_enabled", "notifications_disabled" "legacy_notifications_enabled", "legacy_notifications_disabled", all to be "0" for "disabled"/"in-active" rule that does not have any actions', async () => { const rule = getEqlRuleForSignalTesting(['telemetry'], 'rule-1', false); await createRule(supertest, log, rule); await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - eql: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + eql: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -332,38 +273,28 @@ export default ({ getService }: FtrProviderContext) => { await waitForSignalsToBePresent(supertest, log, 4, [id]); await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - eql: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + eql: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -375,30 +306,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - eql: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, - notifications_disabled: 1, - disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - notifications_disabled: 1, - disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + eql: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, + notifications_disabled: 1, + disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + notifications_disabled: 1, + disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -412,32 +333,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - eql: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + eql: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, + enabled: 1, + alerts: 4, + notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -449,26 +360,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - eql: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, - disabled: 1, - legacy_notifications_disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - legacy_notifications_disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + eql: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, + disabled: 1, + legacy_notifications_disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + legacy_notifications_disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -482,32 +387,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial (see detection_rule_status.ts for more in-depth testing of this structure) - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - eql: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + eql: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.eql, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); }); @@ -524,32 +419,26 @@ export default ({ getService }: FtrProviderContext) => { await createRule(supertest, log, rule); await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threshold: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threshold: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -566,38 +455,28 @@ export default ({ getService }: FtrProviderContext) => { await waitForSignalsToBePresent(supertest, log, 4, [id]); await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threshold: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threshold: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -615,30 +494,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threshold: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, - notifications_disabled: 1, - disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - notifications_disabled: 1, - disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threshold: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, + notifications_disabled: 1, + disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + notifications_disabled: 1, + disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -658,32 +527,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threshold: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threshold: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, + enabled: 1, + alerts: 4, + notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -701,30 +560,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threshold: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, - disabled: 1, - legacy_notifications_disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - legacy_notifications_disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threshold: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, + disabled: 1, + legacy_notifications_disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + legacy_notifications_disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -744,32 +593,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threshold: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threshold: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threshold, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); }); @@ -781,33 +620,26 @@ export default ({ getService }: FtrProviderContext) => { await createRule(supertest, log, rule); await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - machine_learning: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage - .machine_learning, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + machine_learning: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.machine_learning, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -816,37 +648,26 @@ export default ({ getService }: FtrProviderContext) => { await createRule(supertest, log, rule); await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - machine_learning: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage - .machine_learning, - enabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + machine_learning: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.machine_learning, + enabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -858,31 +679,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - machine_learning: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage - .machine_learning, - notifications_disabled: 1, - disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - notifications_disabled: 1, - disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + machine_learning: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.machine_learning, + notifications_disabled: 1, + disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + notifications_disabled: 1, + disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -894,31 +704,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - machine_learning: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage - .machine_learning, - enabled: 1, - notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + machine_learning: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.machine_learning, + enabled: 1, + notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -930,31 +729,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - machine_learning: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage - .machine_learning, - disabled: 1, - legacy_notifications_disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - legacy_notifications_disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + machine_learning: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.machine_learning, + disabled: 1, + legacy_notifications_disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + legacy_notifications_disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -966,31 +754,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - machine_learning: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage - .machine_learning, - enabled: 1, - legacy_notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - legacy_notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + machine_learning: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.machine_learning, + enabled: 1, + legacy_notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + legacy_notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); }); @@ -1001,32 +778,26 @@ export default ({ getService }: FtrProviderContext) => { await createRule(supertest, log, rule); await retry.try(async () => { const stats = await getStats(supertest, log); - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threat_match: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threat_match: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -1052,38 +823,28 @@ export default ({ getService }: FtrProviderContext) => { await waitForSignalsToBePresent(supertest, log, 4, [id]); await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threat_match: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 0, - notifications_disabled: 0, - legacy_notifications_disabled: 0, - legacy_notifications_enabled: 0, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threat_match: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 0, + notifications_disabled: 0, + legacy_notifications_disabled: 0, + legacy_notifications_enabled: 0, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -1095,30 +856,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threat_match: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, - notifications_disabled: 1, - disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - notifications_disabled: 1, - disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threat_match: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, + notifications_disabled: 1, + disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + notifications_disabled: 1, + disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -1147,32 +898,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threat_match: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - enabled: 1, - alerts: 4, - notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threat_match: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, + enabled: 1, + alerts: 4, + notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + enabled: 1, + alerts: 4, + notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -1184,30 +925,20 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threat_match: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, - disabled: 1, - legacy_notifications_disabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - disabled: 1, - legacy_notifications_disabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threat_match: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, + disabled: 1, + legacy_notifications_disabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + disabled: 1, + legacy_notifications_disabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); @@ -1236,32 +967,22 @@ export default ({ getService }: FtrProviderContext) => { await retry.try(async () => { const stats = await getStats(supertest, log); - - // remove "detection_rule_status" from the test by resetting it to initial - stats.detection_rules.detection_rule_status = getInitialEventLogUsage(); - - const expected: DetectionMetrics = { - ...getInitialDetectionMetrics(), - detection_rules: { - ...getInitialDetectionMetrics().detection_rules, - detection_rule_usage: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, - threat_match: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - custom_total: { - ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, - alerts: 4, - enabled: 1, - legacy_notifications_enabled: 1, - }, - }, + const expected: RulesTypeUsage = { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage, + threat_match: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.threat_match, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, + }, + custom_total: { + ...getInitialDetectionMetrics().detection_rules.detection_rule_usage.custom_total, + alerts: 4, + enabled: 1, + legacy_notifications_enabled: 1, }, }; - expect(stats).to.eql(expected); + expect(stats.detection_rules.detection_rule_usage).to.eql(expected); }); }); }); From 8ca90fbfc3f5f201e12053d9675c41b0906b8f9e Mon Sep 17 00:00:00 2001 From: Ievgen Sorokopud Date: Mon, 31 Jul 2023 11:51:38 +0200 Subject: [PATCH 02/33] [Security Solution] Legacy actions are deleted when user tries to save a rule and the run action interval is slower than rule run interval (#160798) ## Summary Original ticket: https://github.com/elastic/kibana/issues/157462 With these changes we fix the legacy actions data loss (on migration) issue. One of the first steps of the migration we retrieve legacy actions and immediately delete them. Then we do validation which might throw an exception and all legacy actions will be lost in this case. As a solution we will do legacy actions validation before deleting them and throwing exception in case those are broken. This means that in case legacy action is broken user will need to export the rule, fix it manually and import it again. Or just re-create it from scratch. https://github.com/elastic/kibana/assets/2700761/a23f5d43-3758-4ab7-8e63-bd93016e338d --- .../migrate_legacy_actions.test.ts | 63 +++++-------------- .../migrate_legacy_actions.ts | 43 ++++++++----- .../retrieve_migrated_legacy_actions.test.ts | 60 ++++++++++++++++-- .../retrieve_migrated_legacy_actions.ts | 59 ++++++++++------- 4 files changed, 132 insertions(+), 93 deletions(-) diff --git a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.test.ts b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.test.ts index 50848be43a73f..3e6b010380751 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.test.ts @@ -124,18 +124,17 @@ describe('migrateLegacyActions', () => { jest.clearAllMocks(); }); - it('should return empty migratedActions when error is thrown within method', async () => { + it('should throw an exception when error is thrown within method', async () => { (retrieveMigratedLegacyActions as jest.Mock).mockRejectedValueOnce(new Error('test failure')); - const migratedActions = await migrateLegacyActions(context, { - ruleId, - attributes, - }); + await expect( + migrateLegacyActions(context, { + ruleId, + attributes, + }) + ).rejects.toThrowError( + `Failed to migrate legacy actions for SIEM rule ${ruleId}: test failure` + ); - expect(migratedActions).toEqual({ - resultedActions: [], - hasLegacyActions: false, - resultedReferences: [], - }); expect(context.logger.error).toHaveBeenCalledWith( `migrateLegacyActions(): Failed to migrate legacy actions for SIEM rule ${ruleId}: test failure` ); @@ -168,7 +167,11 @@ describe('migrateLegacyActions', () => { }); await migrateLegacyActions(context, { ruleId, attributes }); - expect(retrieveMigratedLegacyActions).toHaveBeenCalledWith(context, { ruleId }); + expect(retrieveMigratedLegacyActions).toHaveBeenCalledWith( + context, + { ruleId }, + expect.any(Function) + ); }); it('should not call validateActions and injectReferencesIntoActions if skipActionsValidation=true', async () => { @@ -178,44 +181,6 @@ describe('migrateLegacyActions', () => { expect(injectReferencesIntoActions).not.toHaveBeenCalled(); }); - it('should call validateActions and injectReferencesIntoActions if attributes provided', async () => { - (retrieveMigratedLegacyActions as jest.Mock).mockResolvedValueOnce({ - legacyActions: legacyActionsMock, - legacyActionsReferences: legacyReferencesMock, - }); - - (injectReferencesIntoActions as jest.Mock).mockReturnValue('actions-with-references'); - await migrateLegacyActions(context, { ruleId, attributes }); - - expect(validateActions).toHaveBeenCalledWith(context, ruleType, { - ...attributes, - actions: 'actions-with-references', - }); - - expect(injectReferencesIntoActions).toHaveBeenCalledWith( - 'rule_id_1', - [ - { - actionRef: 'action_0', - actionTypeId: '.email', - group: 'default', - params: { - message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', - subject: 'Test Actions', - to: ['test@test.com'], - }, - uuid: '11403909-ca9b-49ba-9d7a-7e5320e68d05', - frequency: { - notifyWhen: 'onThrottleInterval', - summary: true, - throttle: '1d', - }, - }, - ], - [{ id: 'cc85da20-d480-11ed-8e69-1df522116c28', name: 'action_0', type: 'action' }] - ); - }); - it('should set frequency props from rule level to existing actions', async () => { const result = await migrateLegacyActions(context, { ruleId, diff --git a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.ts b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.ts index c092e3fed982b..75bcb39e522b9 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/migrate_legacy_actions.ts @@ -5,6 +5,9 @@ * 2.0. */ +import Boom from '@hapi/boom'; +import { i18n } from '@kbn/i18n'; + import { AlertConsumers } from '@kbn/rule-data-utils'; import type { SavedObjectReference } from '@kbn/core/server'; @@ -49,15 +52,14 @@ export const migrateLegacyActions: MigrateLegacyActions = async ( }; } - const { legacyActions, legacyActionsReferences } = await retrieveMigratedLegacyActions( - context, - { - ruleId, + const validateLegacyActions = async ( + legacyActions: RawRuleAction[], + legacyActionsReferences: SavedObjectReference[] + ) => { + // sometimes we don't need to validate legacy actions. For example, when delete rules or update rule from payload + if (skipActionsValidation === true) { + return; } - ); - - // sometimes we don't need to validate legacy actions. For example, when delete rules or update rule from payload - if (skipActionsValidation !== true) { const ruleType = context.ruleTypeRegistry.get(attributes.alertTypeId); await validateActions(context, ruleType, { ...attributes, @@ -66,7 +68,15 @@ export const migrateLegacyActions: MigrateLegacyActions = async ( notifyWhen: undefined, actions: injectReferencesIntoActions(ruleId, legacyActions, legacyActionsReferences), }); - } + }; + + const { legacyActions, legacyActionsReferences } = await retrieveMigratedLegacyActions( + context, + { + ruleId, + }, + validateLegacyActions + ); // fix references for a case when actions present in a rule if (actions.length) { @@ -103,11 +113,14 @@ export const migrateLegacyActions: MigrateLegacyActions = async ( context.logger.error( `migrateLegacyActions(): Failed to migrate legacy actions for SIEM rule ${ruleId}: ${e.message}` ); - - return { - resultedActions: [], - hasLegacyActions: false, - resultedReferences: [], - }; + throw Boom.badRequest( + i18n.translate('xpack.alerting.rulesClient.validateLegacyActions.errorSummary', { + defaultMessage: 'Failed to migrate legacy actions for SIEM rule {ruleId}: {errorMessage}', + values: { + ruleId, + errorMessage: e.message, + }, + }) + ); } }; diff --git a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.test.ts b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.test.ts index b49497d086efe..043754dff226e 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.test.ts @@ -69,7 +69,8 @@ describe('Legacy rule action migration logic', () => { unsecuredSavedObjectsClient: savedObjectsClient, logger, } as unknown as RulesClientContext, - { ruleId } + { ruleId }, + () => Promise.resolve() ); expect(deleteRuleMock).not.toHaveBeenCalled(); @@ -92,7 +93,8 @@ describe('Legacy rule action migration logic', () => { unsecuredSavedObjectsClient: savedObjectsClient, logger, } as unknown as RulesClientContext, - { ruleId } + { ruleId }, + () => Promise.resolve() ); expect(deleteRuleMock).not.toHaveBeenCalled(); @@ -116,7 +118,8 @@ describe('Legacy rule action migration logic', () => { unsecuredSavedObjectsClient: savedObjectsClient, logger, } as unknown as RulesClientContext, - { ruleId } + { ruleId }, + () => Promise.resolve() ); expect(deleteRuleMock).not.toHaveBeenCalled(); @@ -146,7 +149,8 @@ describe('Legacy rule action migration logic', () => { unsecuredSavedObjectsClient: savedObjectsClient, logger, } as unknown as RulesClientContext, - { ruleId } + { ruleId }, + () => Promise.resolve() ); expect(deleteRuleMock).toHaveBeenCalledWith(expect.any(Object), { id: '456' }); @@ -192,7 +196,8 @@ describe('Legacy rule action migration logic', () => { unsecuredSavedObjectsClient: savedObjectsClient, logger, } as unknown as RulesClientContext, - { ruleId } + { ruleId }, + () => Promise.resolve() ); expect(deleteRuleMock).toHaveBeenCalledWith(expect.any(Object), { id: '456' }); @@ -237,7 +242,8 @@ describe('Legacy rule action migration logic', () => { unsecuredSavedObjectsClient: savedObjectsClient, logger, } as unknown as RulesClientContext, - { ruleId } + { ruleId }, + () => Promise.resolve() ); expect(deleteRuleMock).toHaveBeenCalledWith(expect.any(Object), { id: '456' }); @@ -263,5 +269,47 @@ describe('Legacy rule action migration logic', () => { legacyActionsReferences: [{ id: '456', name: 'action_0', type: 'action' }], }); }); + + test('it calls validateLegacyActions on migration a rule with legacy actions', async () => { + // siem.notifications is not created for a rule with no actions + findMock.mockResolvedValueOnce({ + page: 1, + perPage: 1, + total: 1, + data: [legacyGetDailyNotificationResult(connectorId, ruleId)], + }); + // siem-detection-engine-rule-actions SO is still created + savedObjectsClient.find.mockResolvedValueOnce( + legacyGetSiemNotificationRuleActionsSOResultWithSingleHit(['daily'], ruleId, connectorId) + ); + + const validateLegacyActions = jest.fn(); + await retrieveMigratedLegacyActions( + { + unsecuredSavedObjectsClient: savedObjectsClient, + logger, + } as unknown as RulesClientContext, + { ruleId }, + validateLegacyActions + ); + + expect(validateLegacyActions).toHaveBeenCalledWith( + [ + { + actionRef: 'action_0', + actionTypeId: '.email', + frequency: { notifyWhen: 'onThrottleInterval', summary: true, throttle: '1d' }, + group: 'default', + params: { + message: 'Rule {{context.rule.name}} generated {{state.signals_count}} alerts', + subject: 'Test Actions', + to: ['test@test.com'], + }, + uuid: expect.any(String), + }, + ], + [{ id: '456', name: 'action_0', type: 'action' }] + ); + }); }); }); diff --git a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.ts b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.ts index 83cd315aaa5a5..9678dff6898c8 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/siem_legacy_actions/retrieve_migrated_legacy_actions.ts @@ -15,7 +15,11 @@ import { transformFromLegacyActions } from './transform_legacy_actions'; type RetrieveMigratedLegacyActions = ( context: RulesClientContext, - { ruleId }: { ruleId: string } + { ruleId }: { ruleId: string }, + validateLegacyActions: ( + legacyActions: RawRuleAction[], + legacyActionsReferences: SavedObjectReference[] + ) => Promise ) => Promise<{ legacyActions: RawRuleAction[]; legacyActionsReferences: SavedObjectReference[] }>; /** @@ -27,7 +31,8 @@ type RetrieveMigratedLegacyActions = ( */ export const retrieveMigratedLegacyActions: RetrieveMigratedLegacyActions = async ( context, - { ruleId } + { ruleId }, + validateLegacyActions ) => { const { unsecuredSavedObjectsClient } = context; try { @@ -71,17 +76,19 @@ export const retrieveMigratedLegacyActions: RetrieveMigratedLegacyActions = asyn return { legacyActions: [], legacyActionsReferences: [] }; } - await Promise.all([ - // If the legacy notification rule type ("siem.notification") exist, - // migration and cleanup are needed - siemNotificationsExist && deleteRule(context, { id: siemNotification.data[0].id }), - // Delete the legacy sidecar SO if it exists - legacyRuleNotificationSOsExist && - unsecuredSavedObjectsClient.delete( - legacyRuleActionsSavedObjectType, - legacyRuleActionsSO.saved_objects[0].id - ), - ]); + const deleteLegacyActions = async () => { + await Promise.all([ + // If the legacy notification rule type ("siem.notification") exist, + // migration and cleanup are needed + siemNotificationsExist && deleteRule(context, { id: siemNotification.data[0].id }), + // Delete the legacy sidecar SO if it exists + legacyRuleNotificationSOsExist && + unsecuredSavedObjectsClient.delete( + legacyRuleActionsSavedObjectType, + legacyRuleActionsSO.saved_objects[0].id + ), + ]); + }; // If legacy notification sidecar ("siem-detection-engine-rule-actions") // exist, migration is needed @@ -95,22 +102,28 @@ export const retrieveMigratedLegacyActions: RetrieveMigratedLegacyActions = asyn legacyRuleActionsSO.saved_objects[0].attributes.ruleThrottle === 'no_actions' || legacyRuleActionsSO.saved_objects[0].attributes.ruleThrottle === 'rule' ) { + await deleteLegacyActions(); return { legacyActions: [], legacyActionsReferences: [] }; } - return { - legacyActions: transformFromLegacyActions( - legacyRuleActionsSO.saved_objects[0].attributes, - legacyRuleActionsSO.saved_objects[0].references - ), - legacyActionsReferences: - // only action references need to be saved - legacyRuleActionsSO.saved_objects[0].references.filter(({ type }) => type === 'action') ?? - [], - }; + const legacyActions = transformFromLegacyActions( + legacyRuleActionsSO.saved_objects[0].attributes, + legacyRuleActionsSO.saved_objects[0].references + ); + // only action references need to be saved + const legacyActionsReferences = + legacyRuleActionsSO.saved_objects[0].references.filter(({ type }) => type === 'action') ?? + []; + await validateLegacyActions(legacyActions, legacyActionsReferences); + + // Delete legacy actions only after the validation + await deleteLegacyActions(); + return { legacyActions, legacyActionsReferences }; } + await deleteLegacyActions(); } catch (e) { context.logger.debug(`Migration has failed for rule ${ruleId}: ${e.message}`); + throw e; } return { legacyActions: [], legacyActionsReferences: [] }; From 7ef303eb362df466e53fe1ae5f38d719a782098f Mon Sep 17 00:00:00 2001 From: amyjtechwriter <61687663+amyjtechwriter@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:26:23 +0100 Subject: [PATCH 03/33] =?UTF-8?q?[DOCS]Adding=20ESS=20icon=20to=20xpack.se?= =?UTF-8?q?curity.session.=D1=81oncurrentSessions.maxSessi=E2=80=A6=20(#16?= =?UTF-8?q?2493)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `xpack.security.session.сoncurrentSessions.maxSessons` in Kibana docs. ## Summary Adding the Elastic Cloud icon to `xpack.security.session.сoncurrentSessions.maxSessons` security setting on [docs page](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html). Need to raise another PR from cloud repo to add security setting to [cloud docs page](https://www.elastic.co/guide/en/cloud/current/ec-manage-kibana-settings.html#ec-kibana-config). Relates to: #160958 --- docs/settings/security-settings.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings/security-settings.asciidoc b/docs/settings/security-settings.asciidoc index 98d7b13a1122b..06c8835d25c3c 100644 --- a/docs/settings/security-settings.asciidoc +++ b/docs/settings/security-settings.asciidoc @@ -210,7 +210,7 @@ Sets the interval at which {kib} tries to remove expired and invalid sessions fr + TIP: Use a string of `[ms\|s\|m\|h\|d\|w\|M\|Y]` (e.g. '20m', '24h', '7d', '1w'). -xpack.security.session.сoncurrentSessions.maxSessions:: +xpack.security.session.concurrentSessions.maxSessions {ess-icon}:: Set the maximum number of sessions each user is allowed to have active at any given time. By default, no limit is applied. If set, the value of this option should be an integer between `1` and `1000`. From 41af3d6c4dd120db9f7d021148f2ef5f22799117 Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Mon, 31 Jul 2023 13:38:05 +0200 Subject: [PATCH 04/33] Skip Serverless Observability Tests / serverless observability UI navigation active sidenav section is auto opened on load (#162782) ## Summary skip https://github.com/elastic/kibana/issues/162781 --- .../functional/test_suites/observability/navigation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/navigation.ts b/x-pack/test_serverless/functional/test_suites/observability/navigation.ts index 2e8745680c528..6c90a2a410b61 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/navigation.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/navigation.ts @@ -69,7 +69,8 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await expectNoPageReload(); }); - it('active sidenav section is auto opened on load', async () => { + // FLAKY/BUG?: https://github.com/elastic/kibana/issues/162781 + it.skip('active sidenav section is auto opened on load', async () => { await svlCommonNavigation.sidenav.openSection('project_settings_project_nav'); await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'management' }); await browser.refresh(); From d74d33f757b2c665c0d0b133f6836f87c93a3663 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 31 Jul 2023 14:01:44 +0200 Subject: [PATCH 05/33] [APM] Improving overflow buckets documentation (#162774) --- .../plugins/apm/dev_docs/overflow_bucket_setup.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md b/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md index 61e8bfb53d581..60bd87714609f 100644 --- a/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md +++ b/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md @@ -154,7 +154,11 @@ For more detailed instructions you can check [apm-server dev documentation](http - Copy paste the below script in a file called `tx_max_group.go` This file is responsible for generating 3 transactions per service. - ``` + +
+ tx_max_group.go + + ```go package main import ( @@ -185,9 +189,14 @@ For more detailed instructions you can check [apm-server dev documentation](http span.End() } ``` - +
+ - Now create a Bash Script file, name it anything - e.g., `service_max_group.sh` This file will generate services and then transactions for each service using the go script above. + +
+ tx_max_group.go + ```sh #!/usr/bin/env bash @@ -203,6 +212,7 @@ For more detailed instructions you can check [apm-server dev documentation](http echo "Ending script" ``` +
- Run `sh service_max_group` to generate the data From e66d949198f60eb6e6cc30eea93d99228b4cce4d Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:08:21 -0400 Subject: [PATCH 06/33] [ResponseOps][Alerting] Flaky test x-pack/plugins/event_log/server/es (#162558) https://github.com/elastic/kibana/issues/156061 ## Summary Fixes flaky test by adding ceil(). When it fails the time elapsed is close to the `MOCK_RETRY_DELAY` of 20 (like 19.7), so rounding up helps with the flakiness --- x-pack/plugins/event_log/server/es/init.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts index 154a98fcb194c..30e220313b26b 100644 --- a/x-pack/plugins/event_log/server/es/init.test.ts +++ b/x-pack/plugins/event_log/server/es/init.test.ts @@ -455,8 +455,7 @@ describe('parseIndexAliases', () => { }); }); -// FLAKY: https://github.com/elastic/kibana/issues/156061 -describe.skip('retries', () => { +describe('retries', () => { let esContext = contextMock.create(); // set up context APIs to return defaults indicating already created beforeEach(() => { @@ -474,7 +473,7 @@ describe.skip('retries', () => { const timeStart = performance.now(); await initializeEs(esContext); - const timeElapsed = performance.now() - timeStart; + const timeElapsed = Math.ceil(performance.now() - timeStart); expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY); From d5761dc0ea5c6d347eff55028a53f23b64050069 Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:09:49 -0400 Subject: [PATCH 07/33] =?UTF-8?q?[ResponseOps][Alerting]=20Flaky=20test=20?= =?UTF-8?q?x-pack/test/alerting=5Fapi=5Fintegration/security=5Fand=5Fspace?= =?UTF-8?q?s/group2/tests/telemetry/index=C2=B7ts=20(#161096)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves https://github.com/elastic/kibana/issues/140973 Resolves https://github.com/elastic/kibana/issues/136153 ## Summary Fixes a telemetry bug and a flaky telemetry test ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2717 x 250 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2718 x 250 --- .../actions/server/usage/actions_telemetry.ts | 2 +- .../actions/server/usage/task_state.ts | 2 +- .../alerting_and_actions_telemetry.ts | 341 ++++++------------ .../group2/tests/telemetry/index.ts | 6 +- 4 files changed, 116 insertions(+), 235 deletions(-) diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.ts index 7959d14775350..49ac9ff1978f8 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.ts @@ -476,7 +476,7 @@ export async function getExecutionsPerDayCount( countFailedByType: Record; avgExecutionTime: number; avgExecutionTimeByType: Record; - countRunOutcomeByConnectorType: Record; + countRunOutcomeByConnectorType: Record>; }> { const scriptedMetric = { scripted_metric: { diff --git a/x-pack/plugins/actions/server/usage/task_state.ts b/x-pack/plugins/actions/server/usage/task_state.ts index c41c76df6133c..3d4d96b525d5e 100644 --- a/x-pack/plugins/actions/server/usage/task_state.ts +++ b/x-pack/plugins/actions/server/usage/task_state.ts @@ -66,7 +66,7 @@ export const stateSchemaByVersion = { avg_execution_time_by_type_per_day: schema.recordOf(schema.string(), schema.number()), count_connector_types_by_action_run_outcome_per_day: schema.recordOf( schema.string(), - schema.number() + schema.recordOf(schema.string(), schema.number()) ), }), }, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts index 7925bfe6f2eb9..02a41b6de7afa 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts @@ -8,7 +8,13 @@ import expect from '@kbn/expect'; import { ESTestIndexTool } from '@kbn/alerting-api-integration-helpers'; import { Spaces, Superuser } from '../../../scenarios'; -import { getUrlPrefix, getEventLog, getTestRuleData, TaskManagerDoc } from '../../../../common/lib'; +import { + getUrlPrefix, + getEventLog, + getTestRuleData, + TaskManagerDoc, + ObjectRemover, +} from '../../../../common/lib'; import { FtrProviderContext } from '../../../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export @@ -20,8 +26,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const esTestIndexTool = new ESTestIndexTool(es, retry); const supertestWithoutAuth = getService('supertestWithoutAuth'); - // FLAKY: https://github.com/elastic/kibana/issues/140973 - describe.skip('telemetry', () => { + describe('telemetry', () => { + const objectRemover = new ObjectRemover(supertest); const alwaysFiringRuleId: { [key: string]: string } = {}; beforeEach(async () => { @@ -29,6 +35,12 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F await esTestIndexTool.setup(); }); + afterEach(() => objectRemover.removeAll()); + + after(async () => { + await esTestIndexTool.destroy(); + }); + async function createConnector(opts: { name: string; space: string; connectorTypeId: string }) { const { name, space, connectorTypeId } = opts; const { body: createdConnector } = await supertestWithoutAuth @@ -42,6 +54,7 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F secrets: {}, }) .expect(200); + objectRemover.add(space, createdConnector.id, 'connector', 'actions'); return createdConnector.id; } @@ -52,7 +65,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F .set('kbn-xsrf', 'foo') .auth(Superuser.username, Superuser.password) .send(getTestRuleData(ruleOverwrites)); - expect(ruleResponse.status).to.eql(200); + expect(ruleResponse.status).to.equal(200); + objectRemover.add(space, ruleResponse.body.id, 'rule', 'alerting'); return ruleResponse.body.id; } @@ -69,127 +83,81 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F space: space.id, connectorTypeId: 'test.throw', }); + // excluded connector await createConnector({ name: 'unused connector', space: space.id, connectorTypeId: 'test.excluded', }); - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.noop', - schedule: { interval: '30s' }, - throttle: '1s', - params: {}, - actions: [ - { - id: noopConnectorId, - group: 'default', - params: {}, - }, - ], - }, - }); - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.onlyContextVariables', - schedule: { interval: '10s' }, - throttle: '10m', - params: {}, - actions: [ - { - id: failingConnectorId, - group: 'default', - params: {}, - }, - ], - }, - }); - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.throw', - schedule: { interval: '1m' }, - throttle: '30s', - params: {}, - actions: [ - { - id: noopConnectorId, - group: 'default', - params: {}, - }, - ], - }, - }); alwaysFiringRuleId[space.id] = await createRule({ space: space.id, ruleOverwrites: { - rule_type_id: 'example.always-firing', - schedule: { interval: '3s' }, + rule_type_id: 'test.patternFiring', + schedule: { interval: '1h' }, throttle: null, - notify_when: 'onActiveAlert', - params: {}, + params: { + pattern: { instance: [true] }, + }, actions: [ { id: noopConnectorId, - group: 'small', + group: 'default', params: {}, }, { id: 'my-slack1', - group: 'medium', + group: 'default', params: {}, }, { id: failingConnectorId, - group: 'large', + group: 'default', params: {}, }, ], }, }); - + // disabled rule await createRule({ space: space.id, ruleOverwrites: { rule_type_id: 'test.noop', - schedule: { interval: '5m' }, + schedule: { interval: '1h' }, throttle: null, enabled: false, params: {}, actions: [], }, }); - + // throwing rule await createRule({ space: space.id, ruleOverwrites: { - rule_type_id: 'test.multipleSearches', - schedule: { interval: '40s' }, - throttle: '1m', - params: { numSearches: 2, delay: `2s` }, - actions: [], - }, - }); - - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.cumulative-firing', - schedule: { interval: '61s' }, - throttle: '2s', - notify_when: 'onActiveAlert', + rule_type_id: 'test.throw', + schedule: { interval: '1h' }, + throttle: null, + params: {}, actions: [ { - id: failingConnectorId, + id: noopConnectorId, group: 'default', params: {}, }, ], }, }); + // ES search rule + await createRule({ + space: space.id, + ruleOverwrites: { + rule_type_id: 'test.multipleSearches', + schedule: { interval: '1h' }, + throttle: '1s', + params: { numSearches: 2, delay: `2s` }, + actions: [], + }, + }); } } @@ -245,49 +213,44 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F ); expect( - telemetry.count_connector_types_by_action_run_outcome_per_day['test.throw'].failure - ).to.greaterThan(0); + telemetry.count_connector_types_by_action_run_outcome_per_day['test.throw'].failure > 0 + ).to.be(true); } function verifyAlertingTelemetry(telemetry: any) { logger.info(`alerting telemetry - ${JSON.stringify(telemetry)}`); // total number of enabled rules - expect(telemetry.count_active_total).to.equal(18); + expect(telemetry.count_active_total).to.equal(9); // total number of disabled rules expect(telemetry.count_disabled_total).to.equal(3); // total number of rules broken down by rule type - expect(telemetry.count_by_type.test__noop).to.equal(6); - expect(telemetry.count_by_type['example__always-firing']).to.equal(3); - expect(telemetry.count_by_type['test__cumulative-firing']).to.equal(3); + expect(telemetry.count_by_type.test__noop).to.equal(3); + expect(telemetry.count_by_type.test__patternFiring).to.equal(3); expect(telemetry.count_by_type.test__multipleSearches).to.equal(3); - expect(telemetry.count_by_type.test__onlyContextVariables).to.equal(3); expect(telemetry.count_by_type.test__throw).to.equal(3); // total number of enabled rules broken down by rule type - expect(telemetry.count_active_by_type['example__always-firing']).to.equal(3); - expect(telemetry.count_active_by_type['test__cumulative-firing']).to.equal(3); + expect(telemetry.count_active_by_type.test__patternFiring).to.equal(3); expect(telemetry.count_active_by_type.test__multipleSearches).to.equal(3); - expect(telemetry.count_active_by_type.test__noop).to.equal(3); - expect(telemetry.count_active_by_type.test__onlyContextVariables).to.equal(3); expect(telemetry.count_active_by_type.test__throw).to.equal(3); // throttle time stats expect(telemetry.throttle_time.min).to.equal('0s'); - expect(telemetry.throttle_time.avg).to.equal('115.5s'); - expect(telemetry.throttle_time.max).to.equal('600s'); + expect(telemetry.throttle_time.avg).to.equal('0.4s'); + expect(telemetry.throttle_time.max).to.equal('1s'); expect(telemetry.throttle_time_number_s.min).to.equal(0); - expect(telemetry.throttle_time_number_s.avg).to.equal(115.5); - expect(telemetry.throttle_time_number_s.max).to.equal(600); + expect(telemetry.throttle_time_number_s.avg).to.equal(0.4); + expect(telemetry.throttle_time_number_s.max).to.equal(1); // schedule interval stats - expect(telemetry.schedule_time.min).to.equal('3s'); - expect(telemetry.schedule_time.avg).to.equal('72s'); - expect(telemetry.schedule_time.max).to.equal('300s'); - expect(telemetry.schedule_time_number_s.min).to.equal(3); - expect(telemetry.schedule_time_number_s.avg).to.equal(72); - expect(telemetry.schedule_time_number_s.max).to.equal(300); + expect(telemetry.schedule_time.min).to.equal('3600s'); + expect(telemetry.schedule_time.avg).to.equal('3600s'); + expect(telemetry.schedule_time.max).to.equal('3600s'); + expect(telemetry.schedule_time_number_s.min).to.equal(3600); + expect(telemetry.schedule_time_number_s.avg).to.equal(3600); + expect(telemetry.schedule_time_number_s.max).to.equal(3600); // attached connectors stats expect(telemetry.connectors_per_alert.min).to.equal(0); @@ -299,12 +262,10 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F // number of rule executions - just checking for non-zero as we can't set an exact number // each rule should have had a chance to execute once - expect(telemetry.count_rules_executions_per_day >= 21).to.be(true); + expect(telemetry.count_rules_executions_per_day > 0).to.be(true); // number of rule executions broken down by rule type - expect(telemetry.count_by_type['example__always-firing'] >= 3).to.be(true); - expect(telemetry.count_by_type.test__onlyContextVariables >= 3).to.be(true); - expect(telemetry.count_by_type['test__cumulative-firing'] >= 3).to.be(true); + expect(telemetry.count_by_type.test__patternFiring >= 3).to.be(true); expect(telemetry.count_by_type.test__noop >= 3).to.be(true); expect(telemetry.count_by_type.test__multipleSearches >= 3).to.be(true); expect(telemetry.count_by_type.test__throw >= 3).to.be(true); @@ -313,16 +274,7 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.avg_execution_time_per_day > 0).to.be(true); // average execution time broken down by rule type - expect(telemetry.avg_execution_time_by_type_per_day['example__always-firing'] > 0).to.be( - true - ); - expect(telemetry.avg_execution_time_by_type_per_day.test__onlyContextVariables > 0).to.be( - true - ); - expect(telemetry.avg_execution_time_by_type_per_day['test__cumulative-firing'] > 0).to.be( - true - ); - expect(telemetry.avg_execution_time_by_type_per_day.test__noop > 0).to.be(true); + expect(telemetry.avg_execution_time_by_type_per_day.test__patternFiring > 0).to.be(true); expect(telemetry.avg_execution_time_by_type_per_day.test__multipleSearches > 0).to.be(true); expect(telemetry.avg_execution_time_by_type_per_day.test__throw > 0).to.be(true); @@ -330,17 +282,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.avg_es_search_duration_per_day > 0).to.be(true); // average es search time broken down by rule type, most of these rule types don't perform ES queries - expect( - telemetry.avg_es_search_duration_by_type_per_day['example__always-firing'] === 0 - ).to.be(true); - expect( - telemetry.avg_es_search_duration_by_type_per_day.test__onlyContextVariables === 0 - ).to.be(true); - expect( - telemetry.avg_es_search_duration_by_type_per_day['test__cumulative-firing'] === 0 - ).to.be(true); - expect(telemetry.avg_es_search_duration_by_type_per_day.test__noop === 0).to.be(true); - expect(telemetry.avg_es_search_duration_by_type_per_day.test__throw === 0).to.be(true); + expect(telemetry.avg_es_search_duration_by_type_per_day.test__patternFiring).to.equal(0); + expect(telemetry.avg_es_search_duration_by_type_per_day.test__throw).to.equal(0); // rule type that performs ES search expect(telemetry.avg_es_search_duration_by_type_per_day.test__multipleSearches > 0).to.be( @@ -351,17 +294,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.avg_total_search_duration_per_day > 0).to.be(true); // average total search time broken down by rule type, most of these rule types don't perform ES queries - expect( - telemetry.avg_total_search_duration_by_type_per_day['example__always-firing'] === 0 - ).to.be(true); - expect( - telemetry.avg_total_search_duration_by_type_per_day.test__onlyContextVariables === 0 - ).to.be(true); - expect( - telemetry.avg_total_search_duration_by_type_per_day['test__cumulative-firing'] === 0 - ).to.be(true); - expect(telemetry.avg_total_search_duration_by_type_per_day.test__noop === 0).to.be(true); - expect(telemetry.avg_total_search_duration_by_type_per_day.test__throw === 0).to.be(true); + expect(telemetry.avg_total_search_duration_by_type_per_day.test__patternFiring).to.equal(0); + expect(telemetry.avg_total_search_duration_by_type_per_day.test__throw).to.equal(0); // rule type that performs ES search expect(telemetry.avg_total_search_duration_by_type_per_day.test__multipleSearches > 0).to.be( @@ -391,32 +325,20 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F // percentile calculations for number of scheduled actions expect(telemetry.percentile_num_generated_actions_per_day.p50 >= 0).to.be(true); expect(telemetry.percentile_num_generated_actions_per_day.p90 >= 0).to.be(true); - expect(telemetry.percentile_num_generated_actions_per_day.p99).to.be.greaterThan(0); + expect(telemetry.percentile_num_generated_actions_per_day.p99 > 0).to.be(true); // percentile calculations by rule type. most of these rule types don't schedule actions so they should all be 0 - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p50['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p90['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p99['example__always-firing'] - ).to.equal(0); + // but this rule type does schedule actions so should be least 1 action scheduled expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__onlyContextVariables - ).to.equal(0); + telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__patternFiring > 0 + ).to.be(true); expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p90.test__onlyContextVariables - ).to.equal(0); + telemetry.percentile_num_generated_actions_by_type_per_day.p90.test__patternFiring > 0 + ).to.be(true); expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__onlyContextVariables - ).to.equal(0); - - expect(telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__noop).to.equal(0); - expect(telemetry.percentile_num_generated_actions_by_type_per_day.p90.test__noop).to.equal(0); - expect(telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__noop).to.equal(0); + telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__patternFiring > 0 + ).to.be(true); expect(telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__throw).to.equal( 0 @@ -438,46 +360,23 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__multipleSearches ).to.equal(0); - // this rule type does schedule actions so should be least 1 action scheduled - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p50['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p90['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p99['test__cumulative-firing'] - ).to.be.greaterThan(0); - // percentile calculations for number of alerts expect(telemetry.percentile_num_alerts_per_day.p50 >= 0).to.be(true); expect(telemetry.percentile_num_alerts_per_day.p90 >= 0).to.be(true); - expect(telemetry.percentile_num_alerts_per_day.p99).to.be.greaterThan(0); + expect(telemetry.percentile_num_alerts_per_day.p99 > 0).to.be(true); // percentile calculations by rule type. most of these rule types don't generate alerts so they should all be 0 - expect( - telemetry.percentile_num_alerts_by_type_per_day.p50['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p90['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p99['example__always-firing'] - ).to.equal(0); - - expect( - telemetry.percentile_num_alerts_by_type_per_day.p50.test__onlyContextVariables - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p90.test__onlyContextVariables - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p99.test__onlyContextVariables - ).to.equal(0); + // but this rule type does generate alerts so should be least 1 alert - expect(telemetry.percentile_num_alerts_by_type_per_day.p50.test__noop).to.equal(0); - expect(telemetry.percentile_num_alerts_by_type_per_day.p90.test__noop).to.equal(0); - expect(telemetry.percentile_num_alerts_by_type_per_day.p99.test__noop).to.equal(0); + expect(telemetry.percentile_num_alerts_by_type_per_day.p50.test__patternFiring > 0).to.be( + true + ); + expect(telemetry.percentile_num_alerts_by_type_per_day.p90.test__patternFiring > 0).to.be( + true + ); + expect(telemetry.percentile_num_alerts_by_type_per_day.p99.test__patternFiring > 0).to.be( + true + ); expect(telemetry.percentile_num_alerts_by_type_per_day.p50.test__throw).to.equal(0); expect(telemetry.percentile_num_alerts_by_type_per_day.p90.test__throw).to.equal(0); @@ -493,45 +392,30 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F 0 ); - // this rule type does generate alerts so should be least 1 alert - expect( - telemetry.percentile_num_alerts_by_type_per_day.p50['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p90['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p99['test__cumulative-firing'] - ).to.be.greaterThan(0); - // rules grouped by execution status - expect(telemetry.count_rules_by_execution_status).to.eql({ - success: 15, - error: 3, - warning: 0, - }); + expect(telemetry.count_rules_by_execution_status.success > 0).to.be(true); + expect(telemetry.count_rules_by_execution_status.error > 0).to.be(true); + expect(telemetry.count_rules_by_execution_status.warning).to.equal(0); + // number of rules that has tags - expect(telemetry.count_rules_with_tags).to.be(21); + expect(telemetry.count_rules_with_tags).to.equal(12); // rules grouped by notify when - expect(telemetry.count_rules_by_notify_when).to.eql({ - on_action_group_change: 0, - on_active_alert: 6, - on_throttle_interval: 15, - }); + expect(telemetry.count_rules_by_notify_when.on_action_group_change).to.equal(0); + expect(telemetry.count_rules_by_notify_when.on_active_alert).to.equal(0); + expect(telemetry.count_rules_by_notify_when.on_throttle_interval).to.equal(12); // rules snoozed - expect(telemetry.count_rules_snoozed).to.be(0); + expect(telemetry.count_rules_snoozed).to.equal(0); // rules muted - expect(telemetry.count_rules_muted).to.be(0); + expect(telemetry.count_rules_muted).to.equal(0); // rules with muted alerts - expect(telemetry.count_rules_with_muted_alerts).to.be(0); + expect(telemetry.count_rules_with_muted_alerts).to.equal(0); // Connector types grouped by consumers - expect(telemetry.count_connector_types_by_consumers).to.eql({ - // eslint-disable-next-line @typescript-eslint/naming-convention - alertsFixture: { test__noop: 9, test__throw: 9, __slack: 3 }, - }); + expect(telemetry.count_connector_types_by_consumers.alertsFixture.test__noop).to.equal(6); + expect(telemetry.count_connector_types_by_consumers.alertsFixture.test__throw).to.equal(3); + expect(telemetry.count_connector_types_by_consumers.alertsFixture.__slack).to.equal(3); - expect(telemetry.count_rules_by_execution_status_per_day.failure).to.greaterThan(0); - expect(telemetry.count_rules_by_execution_status_per_day.success).to.greaterThan(0); + expect(telemetry.count_rules_by_execution_status_per_day.failure > 0).to.be(true); + expect(telemetry.count_rules_by_execution_status_per_day.success > 0).to.be(true); } it('should retrieve telemetry data in the expected format', async () => { @@ -541,11 +425,11 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F await retry.try(async () => { return await getEventLog({ getService, - spaceId: Spaces[0].id, + spaceId: Spaces[2].id, type: 'alert', - id: alwaysFiringRuleId[Spaces[0].id], + id: alwaysFiringRuleId[Spaces[2].id], provider: 'alerting', - actions: new Map([['execute', { gte: 10 }]]), + actions: new Map([['execute', { gte: 1 }]]), }); }); @@ -566,10 +450,12 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const taskState = telemetryTask!._source!.task?.state; expect(taskState).not.to.be(undefined); actionsTelemetry = JSON.parse(taskState!); - expect(actionsTelemetry.runs).to.equal(2); - expect(actionsTelemetry.count_total).to.equal(20); + expect(actionsTelemetry.runs > 0).to.be(true); + expect(actionsTelemetry.count_total).to.equal(21); }); + verifyActionsTelemetry(actionsTelemetry); + // request alerting telemetry task to run await supertest .post('/api/alerting_actions_telemetry/run_soon') @@ -587,11 +473,10 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const taskState = telemetryTask!._source!.task?.state; expect(taskState).not.to.be(undefined); alertingTelemetry = JSON.parse(taskState!); - expect(alertingTelemetry.runs).to.equal(2); - expect(alertingTelemetry.count_total).to.equal(21); + expect(alertingTelemetry.runs > 0).to.be(true); + expect(alertingTelemetry.count_total).to.equal(12); }); - verifyActionsTelemetry(actionsTelemetry); verifyAlertingTelemetry(alertingTelemetry); }); }); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts index 829de0e1c6e71..9d393e73a4640 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts @@ -10,17 +10,13 @@ import { setupSpacesAndUsers, tearDown } from '../../../setup'; // eslint-disable-next-line import/no-default-export export default function telemetryTests({ loadTestFile, getService }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); - // FLAKY: https://github.com/elastic/kibana/issues/136153 - describe.skip('Alerting and Actions Telemetry', () => { + describe('Alerting and Actions Telemetry', () => { before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/event_log_telemetry'); // reset the state in the telemetry task await setupSpacesAndUsers(getService); }); after(async () => { await tearDown(getService); - await esArchiver.unload('x-pack/test/functional/es_archives/event_log_telemetry'); }); loadTestFile(require.resolve('./alerting_and_actions_telemetry')); From 6b44fff5acd9cd10e53965446bc36a77769a6e50 Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Mon, 31 Jul 2023 14:27:12 +0200 Subject: [PATCH 08/33] [APM] Implementing Journey for APM (#162721) ## Summary Closes - https://github.com/elastic/kibana/issues/153844 As part of this PR, as its just the stepping stone, we will only cover a basic navigation flow and analyze the result obtained from Steps Dashboard and data collected by the APM Agents for this journey ## Scope - Generating a data set using Synthtrace instead of Archives - Capturing the flow from Service Inventory to Trace Waterfall loading on Transaction page - Capturing Event loop utilisation metrics enabled for APM Journey ## How to run it ``` node scripts/run_performance.js --journey-path x-pack/performance/journeys/apm_service_inventory.ts --skip-warmup ``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/ftr_configs.yml | 1 + x-pack/performance/configs/apm_config.ts | 37 +++++++++ .../journeys/apm_service_inventory.ts | 53 +++++++++++++ .../performance/services/synthtrace/index.ts | 66 ++++++++++++++++ .../performance/synthtrace_data/apm_data.ts | 77 +++++++++++++++++++ x-pack/performance/tsconfig.json | 3 + 6 files changed, 237 insertions(+) create mode 100644 x-pack/performance/configs/apm_config.ts create mode 100644 x-pack/performance/journeys/apm_service_inventory.ts create mode 100644 x-pack/performance/services/synthtrace/index.ts create mode 100644 x-pack/performance/synthtrace_data/apm_data.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 8a5d4c2ea3f9f..70be71f08184a 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -413,5 +413,6 @@ enabled: - x-pack/performance/journeys/ecommerce_dashboard_tsvb_gauge_only.ts - x-pack/performance/journeys/dashboard_listing_page.ts - x-pack/performance/journeys/cloud_security_dashboard.ts + - x-pack/performance/journeys/apm_service_inventory.ts - x-pack/test/custom_branding/config.ts - x-pack/test/profiling_api_integration/cloud/config.ts diff --git a/x-pack/performance/configs/apm_config.ts b/x-pack/performance/configs/apm_config.ts new file mode 100644 index 0000000000000..ebcfeb1aa3448 --- /dev/null +++ b/x-pack/performance/configs/apm_config.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrConfigProviderContext } from '@kbn/test'; +import { CA_CERT_PATH } from '@kbn/dev-utils'; + +// eslint-disable-next-line import/no-default-export +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const xpackFunctionalConfig = await readConfigFile( + // eslint-disable-next-line @kbn/imports/no_boundary_crossing + require.resolve('../../test/functional/config.base.js') + ); + + return { + ...xpackFunctionalConfig.getAll(), + + kbnTestServer: { + ...xpackFunctionalConfig.get('kbnTestServer'), + serverArgs: [ + ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), + '--home.disableWelcomeScreen=true', + '--csp.strict=false', + '--csp.warnLegacyBrowsers=false', + // define custom kibana server args here + `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, + '--server.eluMonitor.enabled=true', + '--server.eluMonitor.logging.enabled=true', + '--server.eluMonitor.logging.threshold.ela=250', + '--server.eluMonitor.logging.threshold.elu=0.15', + ], + }, + }; +} diff --git a/x-pack/performance/journeys/apm_service_inventory.ts b/x-pack/performance/journeys/apm_service_inventory.ts new file mode 100644 index 0000000000000..03a5fce636dcd --- /dev/null +++ b/x-pack/performance/journeys/apm_service_inventory.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { SynthtraceClient } from '../services/synthtrace'; +import { generateData } from '../synthtrace_data/apm_data'; + +export const journey = new Journey({ + beforeSteps: async ({ kbnUrl, log, auth, es }) => { + // Install APM Package + const synthClient = new SynthtraceClient({ + kbnBaseUrl: kbnUrl.get(), + logger: log, + username: auth.getUsername(), + password: auth.getPassword(), + esClient: es, + }); + + await synthClient.installApmPackage(); + // Setup Synthtrace Client + await synthClient.initialiseEsClient(); + // Generate data using Synthtrace + const start = Date.now() - 1000; + const end = Date.now(); + await synthClient.index( + generateData({ + from: new Date(start).getTime(), + to: new Date(end).getTime(), + }) + ); + }, + ftrConfigPath: 'x-pack/performance/configs/apm_config.ts', +}) + .step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`app/apm/services`)); + await page.waitForSelector(`[data-test-subj="serviceLink_nodejs"]`); + }) + .step('Navigate to Service Overview Page', async ({ page, kbnUrl }) => { + await page.click(`[data-test-subj="serviceLink_nodejs"]`); + await page.waitForSelector(`[data-test-subj="apmMainTemplateHeaderServiceName"]`); + }) + .step('Navigate to Transactions tabs', async ({ page, kbnUrl }) => { + await page.click(`[data-test-subj="transactionsTab"]`); + await page.waitForSelector(`[data-test-subj="apmTransactionDetailLinkLink"]`); + }) + .step('Wait for Trace Waterfall on the page to load', async ({ page, kbnUrl }) => { + await page.click(`[data-test-subj="apmTransactionDetailLinkLink"]`); + await page.waitForSelector(`[data-test-subj="apmWaterfallButton"]`); + }); diff --git a/x-pack/performance/services/synthtrace/index.ts b/x-pack/performance/services/synthtrace/index.ts new file mode 100644 index 0000000000000..fc7ac5c306b62 --- /dev/null +++ b/x-pack/performance/services/synthtrace/index.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApmSynthtraceEsClient, ApmSynthtraceKibanaClient } from '@kbn/apm-synthtrace'; +import Url from 'url'; +import { Readable } from 'stream'; +import { ApmFields, SynthtraceGenerator } from '@kbn/apm-synthtrace-client'; + +export interface SynthtraceClientParams { + kbnBaseUrl: string; + logger: any; + username: string; + password: string; + esClient: any; +} + +export class SynthtraceClient { + private synthtraceEsClient: ApmSynthtraceEsClient | undefined; + private packageVersion: string = ''; + private readonly kibanaUrlWithAuth: string; + + constructor(private readonly baseParams: SynthtraceClientParams) { + const kibanaUrl = new URL(this.baseParams.kbnBaseUrl); + this.kibanaUrlWithAuth = Url.format({ + protocol: kibanaUrl.protocol, + hostname: kibanaUrl.hostname, + port: kibanaUrl.port, + auth: `${this.baseParams.username}:${this.baseParams.password}`, + }); + } + + async installApmPackage() { + const kibanaClient = new ApmSynthtraceKibanaClient({ + logger: this.baseParams.logger, + target: this.kibanaUrlWithAuth, + }); + this.packageVersion = await kibanaClient.fetchLatestApmPackageVersion(); + + await kibanaClient.installApmPackage(this.packageVersion); + } + + async initialiseEsClient() { + this.synthtraceEsClient = new ApmSynthtraceEsClient({ + client: this.baseParams.esClient, + logger: this.baseParams.logger, + refreshAfterIndex: true, + version: this.packageVersion, + }); + + this.synthtraceEsClient.pipeline(this.synthtraceEsClient.getDefaultPipeline(false)); + } + + async index(events: SynthtraceGenerator) { + if (this.synthtraceEsClient) { + await this.synthtraceEsClient.index( + Readable.from(Array.from(events).flatMap((event) => event.serialize())) + ); + } else { + throw new Error('ES Client not initialised'); + } + } +} diff --git a/x-pack/performance/synthtrace_data/apm_data.ts b/x-pack/performance/synthtrace_data/apm_data.ts new file mode 100644 index 0000000000000..4a7ab835d37cd --- /dev/null +++ b/x-pack/performance/synthtrace_data/apm_data.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, httpExitSpan, timerange } from '@kbn/apm-synthtrace-client'; + +export function generateData({ from, to }: { from: number; to: number }) { + const range = timerange(from, to); + const transactionName = '240rpm/75% 1000ms'; + + const synthRum = apm + .service({ name: 'synth-rum', environment: 'production', agentName: 'rum-js' }) + .instance('my-instance'); + const synthNode = apm + .service({ name: 'synth-node', environment: 'production', agentName: 'nodejs' }) + .instance('my-instance'); + const synthGo = apm + .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) + .instance('my-instance'); + + return range.interval('1m').generator((timestamp) => { + return synthRum + .transaction({ transactionName }) + .duration(400) + .timestamp(timestamp) + .children( + // synth-rum -> synth-node + synthRum + .span( + httpExitSpan({ + spanName: 'GET /api/products/top', + destinationUrl: 'http://synth-node:3000', + }) + ) + .duration(300) + .timestamp(timestamp) + + .children( + // synth-node + synthNode + .transaction({ transactionName: 'Initial transaction in synth-node' }) + .duration(300) + .timestamp(timestamp) + .children( + synthNode + // synth-node -> synth-go + .span( + httpExitSpan({ + spanName: 'GET synth-go:3000', + destinationUrl: 'http://synth-go:3000', + }) + ) + .timestamp(timestamp) + .duration(400) + + .children( + // synth-go + synthGo + + .transaction({ transactionName: 'Initial transaction in synth-go' }) + .timestamp(timestamp) + .duration(200) + .children( + synthGo + .span({ spanName: 'custom_operation', spanType: 'custom' }) + .timestamp(timestamp) + .duration(100) + .success() + ) + ) + ) + ) + ); + }); +} diff --git a/x-pack/performance/tsconfig.json b/x-pack/performance/tsconfig.json index fb60be2e310ce..8dcd1caca873f 100644 --- a/x-pack/performance/tsconfig.json +++ b/x-pack/performance/tsconfig.json @@ -14,5 +14,8 @@ "@kbn/tooling-log", "@kbn/test", "@kbn/expect", + "@kbn/dev-utils", + "@kbn/apm-synthtrace", + "@kbn/apm-synthtrace-client", ] } From 1a30dfa5260ee771365d7adb61390a1c7899af64 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Mon, 31 Jul 2023 15:32:01 +0200 Subject: [PATCH 09/33] [files] make download URL `access: public` (#162707) This endpoint is used in `href`s and should continue to work without require any special headers. --- src/plugins/files/server/routes/public_facing/download.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/files/server/routes/public_facing/download.ts b/src/plugins/files/server/routes/public_facing/download.ts index 3155980095229..6eadde2aaa383 100644 --- a/src/plugins/files/server/routes/public_facing/download.ts +++ b/src/plugins/files/server/routes/public_facing/download.ts @@ -73,6 +73,7 @@ export function register(router: FilesRouter) { validate: { ...rt }, options: { authRequired: false, + access: 'public', }, }, handler From 02a43a1dd027863dc7f8458624e8513c3b5628ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= Date: Mon, 31 Jul 2023 15:42:45 +0200 Subject: [PATCH 10/33] Adjust cypress jobs config (#162729) ## Summary Adjust cypress buildkite job configs --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/pipelines/pull_request/defend_workflows.yml | 4 ---- .buildkite/pipelines/pull_request/osquery_cypress.yml | 2 -- .buildkite/pipelines/pull_request/response_ops.yml | 4 ++-- .buildkite/pipelines/pull_request/response_ops_cases.yml | 4 ++-- .buildkite/pipelines/pull_request/security_solution.yml | 8 ++++---- .../pipelines/pull_request/security_solution_explore.yml | 6 +++--- .../pull_request/security_solution_investigations.yml | 6 +++--- .buildkite/pipelines/pull_request/threat_intelligence.yml | 4 +--- .buildkite/scripts/steps/functional/security_solution.sh | 2 +- .../scripts/steps/functional/threat_intelligence.sh | 2 +- .../security_solution/scripts/run_cypress/parallel.ts | 2 +- 11 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.buildkite/pipelines/pull_request/defend_workflows.yml b/.buildkite/pipelines/pull_request/defend_workflows.yml index 22be5122727bf..02f9239a8e6a2 100644 --- a/.buildkite/pipelines/pull_request/defend_workflows.yml +++ b/.buildkite/pipelines/pull_request/defend_workflows.yml @@ -8,8 +8,6 @@ steps: parallelism: 2 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 artifact_paths: @@ -24,8 +22,6 @@ steps: parallelism: 6 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 artifact_paths: diff --git a/.buildkite/pipelines/pull_request/osquery_cypress.yml b/.buildkite/pipelines/pull_request/osquery_cypress.yml index 15d3330f75c56..4eefffdf7ee06 100644 --- a/.buildkite/pipelines/pull_request/osquery_cypress.yml +++ b/.buildkite/pipelines/pull_request/osquery_cypress.yml @@ -8,8 +8,6 @@ steps: parallelism: 6 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 artifact_paths: diff --git a/.buildkite/pipelines/pull_request/response_ops.yml b/.buildkite/pipelines/pull_request/response_ops.yml index 2f71568f38cb5..b2ef5199fab91 100644 --- a/.buildkite/pipelines/pull_request/response_ops.yml +++ b/.buildkite/pipelines/pull_request/response_ops.yml @@ -8,7 +8,7 @@ steps: parallelism: 4 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 + artifact_paths: + - "target/kibana-security-solution/**/*" diff --git a/.buildkite/pipelines/pull_request/response_ops_cases.yml b/.buildkite/pipelines/pull_request/response_ops_cases.yml index 84497c953afc7..af2e58b65ab34 100644 --- a/.buildkite/pipelines/pull_request/response_ops_cases.yml +++ b/.buildkite/pipelines/pull_request/response_ops_cases.yml @@ -7,7 +7,7 @@ steps: timeout_in_minutes: 120 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 + artifact_paths: + - "target/kibana-security-solution/**/*" diff --git a/.buildkite/pipelines/pull_request/security_solution.yml b/.buildkite/pipelines/pull_request/security_solution.yml index 6002756c8889d..a30609ac5ca21 100644 --- a/.buildkite/pipelines/pull_request/security_solution.yml +++ b/.buildkite/pipelines/pull_request/security_solution.yml @@ -1,14 +1,14 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution.sh - label: 'Security Solution Tests' + label: 'Security Solution Cypress Tests' agents: queue: n2-4-spot depends_on: build timeout_in_minutes: 60 - parallelism: 7 + parallelism: 10 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 + artifact_paths: + - "target/kibana-security-solution/**/*" diff --git a/.buildkite/pipelines/pull_request/security_solution_explore.yml b/.buildkite/pipelines/pull_request/security_solution_explore.yml index 1ab77db5b4cfc..3ff38c0e012f5 100644 --- a/.buildkite/pipelines/pull_request/security_solution_explore.yml +++ b/.buildkite/pipelines/pull_request/security_solution_explore.yml @@ -1,6 +1,6 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_explore.sh - label: 'Explore - Security Solution Tests' + label: 'Explore - Security Solution Cypress Tests' agents: queue: n2-4-spot depends_on: build @@ -8,7 +8,7 @@ steps: parallelism: 2 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 + artifact_paths: + - "target/kibana-security-solution/**/*" diff --git a/.buildkite/pipelines/pull_request/security_solution_investigations.yml b/.buildkite/pipelines/pull_request/security_solution_investigations.yml index 7ec4efb5f2a9e..395055db922a8 100644 --- a/.buildkite/pipelines/pull_request/security_solution_investigations.yml +++ b/.buildkite/pipelines/pull_request/security_solution_investigations.yml @@ -1,6 +1,6 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh - label: 'Investigations - Security Solution Tests' + label: 'Investigations - Security Solution Cypress Tests' agents: queue: n2-4-spot depends_on: build @@ -8,7 +8,7 @@ steps: parallelism: 4 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 + artifact_paths: + - "target/kibana-security-solution/**/*" diff --git a/.buildkite/pipelines/pull_request/threat_intelligence.yml b/.buildkite/pipelines/pull_request/threat_intelligence.yml index 9a16eb9527882..f9b9050d28d95 100644 --- a/.buildkite/pipelines/pull_request/threat_intelligence.yml +++ b/.buildkite/pipelines/pull_request/threat_intelligence.yml @@ -1,6 +1,6 @@ steps: - command: .buildkite/scripts/steps/functional/threat_intelligence.sh - label: 'Threat Intelligence Tests' + label: 'Threat Intelligence Cypress Tests' agents: queue: n2-4-spot depends_on: build @@ -8,8 +8,6 @@ steps: parallelism: 2 retry: automatic: - - exit_status: '-1' - limit: 3 - exit_status: '*' limit: 1 artifact_paths: diff --git a/.buildkite/scripts/steps/functional/security_solution.sh b/.buildkite/scripts/steps/functional/security_solution.sh index f021cdc7c8fc3..5890b463f7735 100755 --- a/.buildkite/scripts/steps/functional/security_solution.sh +++ b/.buildkite/scripts/steps/functional/security_solution.sh @@ -8,6 +8,6 @@ source .buildkite/scripts/steps/functional/common_cypress.sh export JOB=kibana-security-solution-chrome export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} -echo "--- Security Solution tests (Chrome)" +echo "--- Security Solution Cypress tests (Chrome)" yarn --cwd x-pack/plugins/security_solution cypress:run diff --git a/.buildkite/scripts/steps/functional/threat_intelligence.sh b/.buildkite/scripts/steps/functional/threat_intelligence.sh index c07ea8605c2d0..0c2c80942e7c6 100755 --- a/.buildkite/scripts/steps/functional/threat_intelligence.sh +++ b/.buildkite/scripts/steps/functional/threat_intelligence.sh @@ -8,6 +8,6 @@ source .buildkite/scripts/steps/functional/common_cypress.sh export JOB=kibana-threat-intelligence-chrome export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} -echo "--- Threat Intelligence tests (Chrome)" +echo "--- Threat Intelligence Cypress tests (Chrome)" yarn --cwd x-pack/plugins/threat_intelligence cypress:run diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts index 5b4da6dcf3daf..bbabfc8bbd0e1 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel.ts @@ -327,7 +327,7 @@ export const cli = () => { concurrency: (argv.concurrency as number | undefined) ? (argv.concurrency as number) : !isOpen - ? 3 + ? 2 : 1, } ).then((results) => { From 2279dcec874e8d44509f96ec169875ab530250b4 Mon Sep 17 00:00:00 2001 From: Aleh Zasypkin Date: Mon, 31 Jul 2023 15:48:36 +0200 Subject: [PATCH 11/33] Re-enable security related tests related to the concurrent access token refresh. (#162693) ## Summary Since https://github.com/elastic/elasticsearch/pull/98011 is merged, we can re-enable our tests as soon as ES snapshot is promoted. I run a few of these tests locally against Elasticsearch built from source and they were passing. __Fixes: https://github.com/elastic/kibana/issues/162581__ __Fixes: https://github.com/elastic/kibana/issues/162583__ __Fixes: https://github.com/elastic/kibana/issues/162584__ __Fixes: https://github.com/elastic/kibana/issues/162586__ --- .../security_api_integration/tests/kerberos/kerberos_login.ts | 3 +-- .../tests/oidc/authorization_code_flow/oidc_auth.ts | 3 +-- x-pack/test/security_api_integration/tests/saml/saml_login.ts | 3 +-- x-pack/test/security_api_integration/tests/token/session.ts | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/test/security_api_integration/tests/kerberos/kerberos_login.ts b/x-pack/test/security_api_integration/tests/kerberos/kerberos_login.ts index b51b36cf4608c..d5f65754739f5 100644 --- a/x-pack/test/security_api_integration/tests/kerberos/kerberos_login.ts +++ b/x-pack/test/security_api_integration/tests/kerberos/kerberos_login.ts @@ -381,8 +381,7 @@ export default function ({ getService }: FtrProviderContext) { expect(nonAjaxResponse.headers['www-authenticate']).to.be(undefined); }); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/162581 - describe.skip('post-authentication stage', () => { + describe('post-authentication stage', () => { for (const client of ['start-contract', 'request-context', 'custom']) { it(`expired access token should be automatically refreshed by the ${client} client`, async function () { this.timeout(60000); diff --git a/x-pack/test/security_api_integration/tests/oidc/authorization_code_flow/oidc_auth.ts b/x-pack/test/security_api_integration/tests/oidc/authorization_code_flow/oidc_auth.ts index 6c86d9ec4c397..f1aef15c081f2 100644 --- a/x-pack/test/security_api_integration/tests/oidc/authorization_code_flow/oidc_auth.ts +++ b/x-pack/test/security_api_integration/tests/oidc/authorization_code_flow/oidc_auth.ts @@ -542,8 +542,7 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); }); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/162583 - describe.skip('post-authentication stage', () => { + describe('post-authentication stage', () => { for (const client of ['start-contract', 'request-context', 'custom']) { it(`expired access token should be automatically refreshed by the ${client} client`, async function () { this.timeout(60000); diff --git a/x-pack/test/security_api_integration/tests/saml/saml_login.ts b/x-pack/test/security_api_integration/tests/saml/saml_login.ts index b055fb56576aa..a10e2e2067eb6 100644 --- a/x-pack/test/security_api_integration/tests/saml/saml_login.ts +++ b/x-pack/test/security_api_integration/tests/saml/saml_login.ts @@ -443,8 +443,7 @@ export default function ({ getService }: FtrProviderContext) { }); }); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/162584 - describe.skip('API access with expired access token.', () => { + describe('API access with expired access token.', () => { let sessionCookie: Cookie; beforeEach(async function () { diff --git a/x-pack/test/security_api_integration/tests/token/session.ts b/x-pack/test/security_api_integration/tests/token/session.ts index e811372755fc8..f42d9cc93585f 100644 --- a/x-pack/test/security_api_integration/tests/token/session.ts +++ b/x-pack/test/security_api_integration/tests/token/session.ts @@ -132,8 +132,7 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); }); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/162586 - describe.skip('post-authentication stage', () => { + describe('post-authentication stage', () => { for (const client of ['start-contract', 'request-context', 'custom']) { it(`expired access token should be automatically refreshed by the ${client} client`, async function () { this.timeout(60000); From 4eac24281823ccd8eac9656d6c01e1a6dec89a64 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 31 Jul 2023 15:53:40 +0200 Subject: [PATCH 12/33] [Synthetics] Implement private location run once (#162582) ## Summary Implement private location run once mode , user will be able to do run once or test now for private locations as well. This implemented a task manager which will clean up temporarily created package policies created for the purpose of run once and test now mode. image --- .../monitor_add_edit/form/run_test_btn.tsx | 42 ++---- .../monitor_details/run_test_manually.tsx | 24 +-- .../overview/overview/actions_popover.tsx | 14 +- .../hooks/use_run_once_errors.ts | 14 +- .../manual_test_run_mode.tsx | 3 - .../test_now_mode_flyout_container.tsx | 1 - x-pack/plugins/synthetics/server/plugin.ts | 1 + .../routes/monitor_cruds/add_monitor.ts | 3 +- .../bulk_cruds/add_monitor_bulk.ts | 3 +- .../bulk_cruds/delete_monitor_bulk.ts | 1 - .../routes/monitor_cruds/delete_monitor.ts | 3 +- .../routes/settings/sync_global_params.ts | 1 - .../synthetics_service/run_once_monitor.ts | 37 +++-- .../synthetics_service/test_now_monitor.ts | 49 +++---- .../saved_objects/synthetics_monitor.ts | 20 ++- .../format_synthetics_policy.ts | 3 + .../private_location/clean_up_task.ts | 138 ++++++++++++++++++ .../synthetics_private_location.test.ts | 10 +- .../synthetics_private_location.ts | 99 +++++++++---- .../synthetics_monitor_client.test.ts | 2 - .../synthetics_monitor_client.ts | 86 ++++++++--- .../synthetics_service/synthetics_service.ts | 9 +- x-pack/plugins/synthetics/server/types.ts | 2 + .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../apis/synthetics/delete_monitor.ts | 2 +- .../check_registered_task_types.ts | 1 + 28 files changed, 373 insertions(+), 198 deletions(-) create mode 100644 x-pack/plugins/synthetics/server/synthetics_service/private_location/clean_up_task.ts diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/form/run_test_btn.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/form/run_test_btn.tsx index 448ec2f5ed3a4..4b73d71b90b09 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/form/run_test_btn.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_add_edit/form/run_test_btn.tsx @@ -13,14 +13,11 @@ import { v4 as uuidv4 } from 'uuid'; import { useFetcher } from '@kbn/observability-shared-plugin/public'; import { TestNowModeFlyout, TestRun } from '../../test_now_mode/test_now_mode_flyout'; import { format } from './formatter'; -import { - Locations, - MonitorFields as MonitorFieldsType, -} from '../../../../../../common/runtime_types'; +import { MonitorFields as MonitorFieldsType } from '../../../../../../common/runtime_types'; import { runOnceMonitor } from '../../../state/manual_test_runs/api'; export const RunTestButton = () => { - const { watch, formState, getValues, handleSubmit } = useFormContext(); + const { formState, getValues, handleSubmit } = useFormContext(); const [inProgress, setInProgress] = useState(false); const [testRun, setTestRun] = useState(); @@ -51,13 +48,7 @@ export const RunTestButton = () => { } }, [testRun?.id]); - const locations = watch('locations') as Locations; - - const { tooltipContent, isDisabled } = useTooltipContent( - locations, - formState.isValid, - inProgress - ); + const { tooltipContent, isDisabled } = useTooltipContent(formState.isValid, inProgress); return ( <> @@ -94,22 +85,12 @@ export const RunTestButton = () => { ); }; -const useTooltipContent = ( - locations: Locations, - isValid: boolean, - isTestRunInProgress?: boolean -) => { - const isAnyPublicLocationSelected = locations?.some((loc) => loc.isServiceManaged); - const isOnlyPrivateLocations = (locations?.length ?? 0) > 0 && !isAnyPublicLocationSelected; - - let tooltipContent = - isOnlyPrivateLocations || (isValid && !isAnyPublicLocationSelected) - ? PRIVATE_AVAILABLE_LABEL - : TEST_NOW_DESCRIPTION; +const useTooltipContent = (isValid: boolean, isTestRunInProgress?: boolean) => { + let tooltipContent = !isValid ? INVALID_DESCRIPTION : TEST_NOW_DESCRIPTION; tooltipContent = isTestRunInProgress ? TEST_SCHEDULED_LABEL : tooltipContent; - const isDisabled = isTestRunInProgress || !isAnyPublicLocationSelected; + const isDisabled = isTestRunInProgress || !isValid; return { tooltipContent, isDisabled }; }; @@ -118,6 +99,10 @@ const TEST_NOW_DESCRIPTION = i18n.translate('xpack.synthetics.testRun.descriptio defaultMessage: 'Test your monitor and verify the results before saving', }); +const INVALID_DESCRIPTION = i18n.translate('xpack.synthetics.testRun.invalid', { + defaultMessage: 'Monitor has to be valid to run test, please fix above required fields.', +}); + export const TEST_SCHEDULED_LABEL = i18n.translate( 'xpack.synthetics.monitorList.testNow.scheduled', { @@ -125,13 +110,6 @@ export const TEST_SCHEDULED_LABEL = i18n.translate( } ); -export const PRIVATE_AVAILABLE_LABEL = i18n.translate( - 'xpack.synthetics.app.testNow.available.private', - { - defaultMessage: `You can't manually start tests on a private location.`, - } -); - export const TEST_NOW_ARIA_LABEL = i18n.translate( 'xpack.synthetics.monitorList.testNow.AriaLabel', { diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/run_test_manually.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/run_test_manually.tsx index 741d342da2875..55fb54dd0d0dc 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/run_test_manually.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitor_details/run_test_manually.tsx @@ -9,39 +9,20 @@ import { EuiButton, EuiToolTip } from '@elastic/eui'; import React from 'react'; import { i18n } from '@kbn/i18n'; import { useDispatch, useSelector } from 'react-redux'; -import { - TEST_NOW_ARIA_LABEL, - TEST_SCHEDULED_LABEL, - PRIVATE_AVAILABLE_LABEL, -} from '../monitor_add_edit/form/run_test_btn'; +import { TEST_NOW_ARIA_LABEL, TEST_SCHEDULED_LABEL } from '../monitor_add_edit/form/run_test_btn'; import { useSelectedMonitor } from './hooks/use_selected_monitor'; import { manualTestMonitorAction, manualTestRunInProgressSelector, } from '../../state/manual_test_runs'; -import { useGetUrlParams } from '../../hooks/use_url_params'; export const RunTestManually = () => { const dispatch = useDispatch(); const { monitor } = useSelectedMonitor(); - - const hasPublicLocation = monitor?.locations.some((loc) => loc.isServiceManaged); - - const { locationId } = useGetUrlParams(); - - const isSelectedLocationPrivate = monitor?.locations.some( - (loc) => loc.isServiceManaged === false && loc.id === locationId - ); - const testInProgress = useSelector(manualTestRunInProgressSelector(monitor?.config_id)); - const content = - !hasPublicLocation || isSelectedLocationPrivate - ? PRIVATE_AVAILABLE_LABEL - : testInProgress - ? TEST_SCHEDULED_LABEL - : TEST_NOW_ARIA_LABEL; + const content = testInProgress ? TEST_SCHEDULED_LABEL : TEST_NOW_ARIA_LABEL; return ( @@ -49,7 +30,6 @@ export const RunTestManually = () => { data-test-subj="syntheticsRunTestManuallyButton" color="success" iconType="beaker" - isDisabled={!hasPublicLocation || isSelectedLocationPrivate} isLoading={!Boolean(monitor) || testInProgress} onClick={() => { if (monitor) { diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/overview/overview/actions_popover.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/overview/overview/actions_popover.tsx index 9348bae13b253..2a6f71aeca9f2 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/overview/overview/actions_popover.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/monitors_page/overview/overview/actions_popover.tsx @@ -14,13 +14,11 @@ import { EuiPanel, EuiLoadingSpinner, EuiContextMenuPanelItemDescriptor, - EuiToolTip, } from '@elastic/eui'; import { FETCH_STATUS } from '@kbn/observability-shared-plugin/public'; import { useDispatch, useSelector } from 'react-redux'; import styled from 'styled-components'; import { toggleStatusAlert } from '../../../../../../../common/runtime_types/monitor_management/alert_config'; -import { PRIVATE_AVAILABLE_LABEL } from '../../../monitor_add_edit/form/run_test_btn'; import { manualTestMonitorAction, manualTestRunInProgressSelector, @@ -106,8 +104,6 @@ export function ActionsPopover({ const location = useLocationName({ locationId }); const locationName = location?.label || monitor.location.id; - const isPrivateLocation = !Boolean(location?.isServiceManaged); - const detailUrl = useMonitorDetailLocator({ configId: monitor.configId, locationId: locationId ?? monitor.location.id, @@ -176,15 +172,9 @@ export function ActionsPopover({ }, quickInspectPopoverItem, { - name: isPrivateLocation ? ( - - {runTestManually} - - ) : ( - runTestManually - ), + name: runTestManually, icon: 'beaker', - disabled: testInProgress || isPrivateLocation, + disabled: testInProgress, onClick: () => { dispatch(manualTestMonitorAction.get({ configId: monitor.configId, name: monitor.name })); dispatch(setFlyoutConfig(null)); diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/hooks/use_run_once_errors.ts b/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/hooks/use_run_once_errors.ts index 36611914264db..f4f3f4d98ccf8 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/hooks/use_run_once_errors.ts +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/hooks/use_run_once_errors.ts @@ -14,9 +14,7 @@ export function useRunOnceErrors({ serviceError, errors, locations, - showErrors = true, }: { - showErrors?: boolean; testRunId: string; serviceError?: Error; errors: ServiceLocationErrors; @@ -24,10 +22,6 @@ export function useRunOnceErrors({ }) { const [locationErrors, setLocationErrors] = useState([]); const [runOnceServiceError, setRunOnceServiceError] = useState(null); - const publicLocations = useMemo( - () => (locations ?? []).filter((loc) => loc.isServiceManaged), - [locations] - ); useEffect(() => { setLocationErrors([]); @@ -49,12 +43,12 @@ export function useRunOnceErrors({ }, [serviceError]); const locationsById: Record = useMemo( - () => (publicLocations as Locations).reduce((acc, cur) => ({ ...acc, [cur.id]: cur }), {}), - [publicLocations] + () => (locations as Locations).reduce((acc, cur) => ({ ...acc, [cur.id]: cur }), {}), + [locations] ); const expectPings = - publicLocations.length - (locationErrors ?? []).filter(({ locationId }) => !!locationId).length; + locations.length - (locationErrors ?? []).filter(({ locationId }) => !!locationId).length; const locationErrorReasons = useMemo(() => { return (locationErrors ?? []) @@ -64,7 +58,7 @@ export function useRunOnceErrors({ }, [locationErrors]); const hasBlockingError = !!runOnceServiceError || - (locationErrors?.length && locationErrors?.length === publicLocations.length); + (locationErrors?.length && locationErrors?.length === locations.length); const errorMessages = useMemo(() => { if (hasBlockingError) { diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/manual_test_run_mode/manual_test_run_mode.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/manual_test_run_mode/manual_test_run_mode.tsx index 986b975a03175..3bf9e8fd4bb07 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/manual_test_run_mode/manual_test_run_mode.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/manual_test_run_mode/manual_test_run_mode.tsx @@ -15,14 +15,11 @@ import { Locations } from '../../../../../../common/runtime_types'; export function ManualTestRunMode({ manualTestRun, onDone, - showErrors, }: { - showErrors: boolean; manualTestRun: ManualTestRun; onDone: (testRunId: string) => void; }) { const { expectPings } = useRunOnceErrors({ - showErrors, testRunId: manualTestRun.testRunId!, locations: (manualTestRun.monitor!.locations ?? []) as Locations, errors: manualTestRun.errors ?? [], diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/test_now_mode_flyout_container.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/test_now_mode_flyout_container.tsx index ddf21c869c1a2..ff3114c98f845 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/test_now_mode_flyout_container.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/test_now_mode/test_now_mode_flyout_container.tsx @@ -83,7 +83,6 @@ export function TestNowModeFlyoutContainer() { key={manualTestRun.testRunId} manualTestRun={manualTestRun} onDone={onDone} - showErrors={flyoutOpenTestRun?.testRunId !== manualTestRun.testRunId} /> ))} {flyout} diff --git a/x-pack/plugins/synthetics/server/plugin.ts b/x-pack/plugins/synthetics/server/plugin.ts index d308dccab2093..e35ea0f9549bc 100644 --- a/x-pack/plugins/synthetics/server/plugin.ts +++ b/x-pack/plugins/synthetics/server/plugin.ts @@ -104,6 +104,7 @@ export class Plugin implements PluginType { if (this.server) { this.server.coreStart = coreStart; + this.server.pluginsStart = pluginsStart; this.server.security = pluginsStart.security; this.server.fleet = pluginsStart.fleet; this.server.encryptedSavedObjects = pluginsStart.encryptedSavedObjects; diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts index d78f2233ad46c..3db44c1a6c0c4 100644 --- a/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts @@ -174,7 +174,7 @@ export const syncNewMonitor = async ({ routeContext: RouteContext; privateLocations: PrivateLocationAttributes[]; }) => { - const { savedObjectsClient, server, syntheticsMonitorClient, request, spaceId } = routeContext; + const { savedObjectsClient, server, syntheticsMonitorClient, spaceId } = routeContext; const newMonitorId = id ?? uuidV4(); let monitorSavedObject: SavedObject | null = null; @@ -193,7 +193,6 @@ export const syncNewMonitor = async ({ const syncErrorsPromise = syntheticsMonitorClient.addMonitors( [{ monitor: monitorWithNamespace as MonitorFields, id: newMonitorId }], - request, savedObjectsClient, privateLocations, spaceId diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts index bc83c6356a67d..5f7a0a10ab66a 100644 --- a/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts @@ -64,7 +64,7 @@ export const syncNewMonitorBulk = async ({ privateLocations: PrivateLocationAttributes[]; spaceId: string; }) => { - const { server, savedObjectsClient, syntheticsMonitorClient, request } = routeContext; + const { server, savedObjectsClient, syntheticsMonitorClient } = routeContext; let newMonitors: CreatedMonitors | null = null; const monitorsToCreate = normalizedMonitors.map((monitor) => { @@ -88,7 +88,6 @@ export const syncNewMonitorBulk = async ({ }), syntheticsMonitorClient.addMonitors( monitorsToCreate, - request, savedObjectsClient, privateLocations, spaceId diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts index de35d48f62e20..7df12b17b6092 100644 --- a/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts @@ -47,7 +47,6 @@ export const deleteMonitorBulk = async ({ ...normalizedMonitor.attributes, id: normalizedMonitor.attributes[ConfigKey.MONITOR_QUERY_ID], })) as SyntheticsMonitorWithId[], - request, savedObjectsClient, spaceId ); diff --git a/x-pack/plugins/synthetics/server/routes/monitor_cruds/delete_monitor.ts b/x-pack/plugins/synthetics/server/routes/monitor_cruds/delete_monitor.ts index ed223bca944b3..4841ca581c077 100644 --- a/x-pack/plugins/synthetics/server/routes/monitor_cruds/delete_monitor.ts +++ b/x-pack/plugins/synthetics/server/routes/monitor_cruds/delete_monitor.ts @@ -68,7 +68,7 @@ export const deleteMonitor = async ({ routeContext: RouteContext; monitorId: string; }) => { - const { spaceId, savedObjectsClient, server, syntheticsMonitorClient, request } = routeContext; + const { spaceId, savedObjectsClient, server, syntheticsMonitorClient } = routeContext; const { logger, telemetry, stackVersion } = server; const { monitor, monitorWithSecret } = await getMonitorToDelete( @@ -92,7 +92,6 @@ export const deleteMonitor = async ({ /* Type cast encrypted saved objects to decrypted saved objects for delete flow only. * Deletion does not require all monitor fields */ ] as SyntheticsMonitorWithId[], - request, savedObjectsClient, spaceId ); diff --git a/x-pack/plugins/synthetics/server/routes/settings/sync_global_params.ts b/x-pack/plugins/synthetics/server/routes/settings/sync_global_params.ts index f56f2ae14d694..4064a706c6a39 100644 --- a/x-pack/plugins/synthetics/server/routes/settings/sync_global_params.ts +++ b/x-pack/plugins/synthetics/server/routes/settings/sync_global_params.ts @@ -26,7 +26,6 @@ export const syncParamsSyntheticsParamsRoute: SyntheticsRestApiRouteFactory = () const allPrivateLocations = await getPrivateLocations(savedObjectsClient); await syntheticsMonitorClient.syncGlobalParams({ - request, spaceId, allPrivateLocations, encryptedSavedObjects: server.encryptedSavedObjects, diff --git a/x-pack/plugins/synthetics/server/routes/synthetics_service/run_once_monitor.ts b/x-pack/plugins/synthetics/server/routes/synthetics_service/run_once_monitor.ts index b87848a2b71ba..be57e38b7978b 100644 --- a/x-pack/plugins/synthetics/server/routes/synthetics_service/run_once_monitor.ts +++ b/x-pack/plugins/synthetics/server/routes/synthetics_service/run_once_monitor.ts @@ -6,6 +6,8 @@ */ import { schema } from '@kbn/config-schema'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; +import { PrivateLocationAttributes } from '../../runtime_types/private_locations'; +import { getPrivateLocationsForMonitor } from '../monitor_cruds/add_monitor'; import { SyntheticsRestApiRouteFactory } from '../types'; import { MonitorFields } from '../../../common/runtime_types'; import { SYNTHETICS_API_URLS } from '../../../common/constants'; @@ -20,7 +22,13 @@ export const runOnceSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () = monitorId: schema.string({ minLength: 1, maxLength: 1024 }), }), }, - handler: async ({ request, response, server, syntheticsMonitorClient }): Promise => { + handler: async ({ + request, + response, + server, + syntheticsMonitorClient, + savedObjectsClient, + }): Promise => { const monitor = request.body as MonitorFields; const { monitorId } = request.params; @@ -33,19 +41,22 @@ export const runOnceSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () = return response.badRequest({ body: { message, attributes: { details, ...payload } } }); } - const { syntheticsService } = syntheticsMonitorClient; + const privateLocations: PrivateLocationAttributes[] = await getPrivateLocationsForMonitor( + savedObjectsClient, + validationResult.decodedMonitor + ); - const paramsBySpace = await syntheticsService.getSyntheticsParams({ spaceId }); - - const errors = await syntheticsService.runOnceConfigs({ - // making it enabled, even if it's disabled in the UI - monitor: { ...validationResult.decodedMonitor, enabled: true }, - configId: monitorId, - heartbeatId: monitorId, - runOnce: true, - testRunId: monitorId, - params: paramsBySpace[spaceId], - }); + const [, errors] = await syntheticsMonitorClient.testNowConfigs( + { + monitor: { ...validationResult.decodedMonitor, config_id: monitorId } as MonitorFields, + id: monitorId, + testRunId: monitorId, + }, + savedObjectsClient, + privateLocations, + spaceId, + true + ); if (errors) { return { errors }; diff --git a/x-pack/plugins/synthetics/server/routes/synthetics_service/test_now_monitor.ts b/x-pack/plugins/synthetics/server/routes/synthetics_service/test_now_monitor.ts index 6515562a36173..08c4f8dbebb3c 100644 --- a/x-pack/plugins/synthetics/server/routes/synthetics_service/test_now_monitor.ts +++ b/x-pack/plugins/synthetics/server/routes/synthetics_service/test_now_monitor.ts @@ -6,14 +6,12 @@ */ import { schema } from '@kbn/config-schema'; import { v4 as uuidv4 } from 'uuid'; +import { getDecryptedMonitor } from '../../saved_objects/synthetics_monitor'; +import { PrivateLocationAttributes } from '../../runtime_types/private_locations'; +import { getPrivateLocationsForMonitor } from '../monitor_cruds/add_monitor'; import { RouteContext, SyntheticsRestApiRouteFactory } from '../types'; -import { syntheticsMonitorType } from '../../../common/types/saved_objects'; import { TestNowResponse } from '../../../common/types'; -import { - ConfigKey, - MonitorFields, - SyntheticsMonitorWithSecretsAttributes, -} from '../../../common/runtime_types'; +import { ConfigKey, MonitorFields } from '../../../common/runtime_types'; import { SYNTHETICS_API_URLS } from '../../../common/constants'; import { normalizeSecrets } from '../../synthetics_service/utils/secrets'; @@ -33,37 +31,32 @@ export const testNowMonitorRoute: SyntheticsRestApiRouteFactory export const triggerTestNow = async ( monitorId: string, - { server, spaceId, syntheticsMonitorClient }: RouteContext + routeContext: RouteContext ): Promise => { - const encryptedClient = server.encryptedSavedObjects.getClient(); + const { server, spaceId, syntheticsMonitorClient, savedObjectsClient } = routeContext; - const monitorWithSecrets = - await encryptedClient.getDecryptedAsInternalUser( - syntheticsMonitorType, - monitorId, - { - namespace: spaceId, - } - ); + const monitorWithSecrets = await getDecryptedMonitor(server, monitorId, spaceId); const normalizedMonitor = normalizeSecrets(monitorWithSecrets); const { [ConfigKey.SCHEDULE]: schedule, [ConfigKey.LOCATIONS]: locations } = monitorWithSecrets.attributes; - const { syntheticsService } = syntheticsMonitorClient; - + const privateLocations: PrivateLocationAttributes[] = await getPrivateLocationsForMonitor( + savedObjectsClient, + normalizedMonitor.attributes + ); const testRunId = uuidv4(); - const paramsBySpace = await syntheticsService.getSyntheticsParams({ spaceId }); - - const errors = await syntheticsService.runOnceConfigs({ - // making it enabled, even if it's disabled in the UI - monitor: { ...normalizedMonitor.attributes, enabled: true }, - configId: monitorId, - heartbeatId: (normalizedMonitor.attributes as MonitorFields)[ConfigKey.MONITOR_QUERY_ID], - testRunId, - params: paramsBySpace[spaceId], - }); + const [, errors] = await syntheticsMonitorClient.testNowConfigs( + { + monitor: normalizedMonitor.attributes as MonitorFields, + id: monitorId, + testRunId, + }, + savedObjectsClient, + privateLocations, + spaceId + ); if (errors && errors?.length > 0) { return { diff --git a/x-pack/plugins/synthetics/server/saved_objects/synthetics_monitor.ts b/x-pack/plugins/synthetics/server/saved_objects/synthetics_monitor.ts index c5bd20b4c968b..de2296947e182 100644 --- a/x-pack/plugins/synthetics/server/saved_objects/synthetics_monitor.ts +++ b/x-pack/plugins/synthetics/server/saved_objects/synthetics_monitor.ts @@ -7,8 +7,10 @@ import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; import { SavedObjectsType } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; +import { SyntheticsMonitorWithSecretsAttributes } from '../../common/runtime_types'; +import { SyntheticsServerSetup } from '../types'; import { syntheticsMonitorType } from '../../common/types/saved_objects'; -import { secretKeys, ConfigKey, LegacyConfigKey } from '../../common/constants/monitor_management'; +import { ConfigKey, LegacyConfigKey, secretKeys } from '../../common/constants/monitor_management'; import { monitorMigrations } from './migrations/monitors'; const legacyConfigKeys = Object.values(LegacyConfigKey); @@ -194,3 +196,19 @@ export const getSyntheticsMonitorSavedObjectType = ( }, }; }; + +export const getDecryptedMonitor = async ( + server: SyntheticsServerSetup, + monitorId: string, + spaceId: string +) => { + const encryptedClient = server.encryptedSavedObjects.getClient(); + + return await encryptedClient.getDecryptedAsInternalUser( + syntheticsMonitorType, + monitorId, + { + namespace: spaceId, + } + ); +}; diff --git a/x-pack/plugins/synthetics/server/synthetics_service/formatters/private_formatters/format_synthetics_policy.ts b/x-pack/plugins/synthetics/server/synthetics_service/formatters/private_formatters/format_synthetics_policy.ts index 2c42be64e5a1a..19df7c70a86c0 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/formatters/private_formatters/format_synthetics_policy.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/formatters/private_formatters/format_synthetics_policy.ts @@ -23,6 +23,9 @@ export const formatSyntheticsPolicy = ( location_id: string; 'monitor.project.name': string; 'monitor.project.id': string; + 'monitor.id': string; + test_run_id: string; + run_once: boolean; } >, params: Record, diff --git a/x-pack/plugins/synthetics/server/synthetics_service/private_location/clean_up_task.ts b/x-pack/plugins/synthetics/server/synthetics_service/private_location/clean_up_task.ts new file mode 100644 index 0000000000000..7fdc4b7fa1605 --- /dev/null +++ b/x-pack/plugins/synthetics/server/synthetics_service/private_location/clean_up_task.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConcreteTaskInstance, TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import moment from 'moment'; +import { + BROWSER_TEST_NOW_RUN, + LIGHTWEIGHT_TEST_NOW_RUN, +} from '../synthetics_monitor/synthetics_monitor_client'; +import { SyntheticsServerSetup } from '../../types'; + +const SYNTHETICS_SERVICE_CLEAN_UP_TASK_TYPE = 'Synthetics:Clean-Up-Package-Policies'; +const SYNTHETICS_SERVICE_CLEAN_UP_TASK_ID = 'SyntheticsService:clean-up-package-policies-task-id'; +const SYNTHETICS_SERVICE_CLEAN_UP_INTERVAL_DEFAULT = '60m'; +const DELETE_BROWSER_MINUTES = 15; +const DELETE_LIGHTWEIGHT_MINUTES = 2; + +export const registerCleanUpTask = ( + taskManager: TaskManagerSetupContract, + serverSetup: SyntheticsServerSetup +) => { + const { logger } = serverSetup; + const interval = SYNTHETICS_SERVICE_CLEAN_UP_INTERVAL_DEFAULT; + + taskManager.registerTaskDefinitions({ + [SYNTHETICS_SERVICE_CLEAN_UP_TASK_TYPE]: { + title: 'Synthetics Plugin Clean Up Task', + description: 'This task which runs periodically to clean up run once monitors.', + timeout: '1m', + maxAttempts: 3, + + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + return { + // Perform the work of the task. The return value should fit the TaskResult interface. + async run() { + logger.info( + `Executing synthetics clean up task: ${SYNTHETICS_SERVICE_CLEAN_UP_TASK_ID}` + ); + const { state } = taskInstance; + try { + const esClient = serverSetup.coreStart?.elasticsearch?.client.asInternalUser; + if (esClient) { + const { fleet } = serverSetup.pluginsStart; + const { savedObjects } = serverSetup.coreStart; + const soClient = savedObjects.createInternalRepository(); + + const { items } = await fleet.packagePolicyService.list(soClient, { + kuery: getFilterForTestNowRun(), + }); + + const allItems = items.map((item) => { + const minutesAgo = moment().diff(moment(item.created_at), 'minutes'); + const isBrowser = item.name === BROWSER_TEST_NOW_RUN; + if (isBrowser) { + return { + isBrowser: true, + id: item.id, + shouldDelete: minutesAgo > DELETE_BROWSER_MINUTES, + }; + } else { + return { + isBrowser: false, + id: item.id, + shouldDelete: minutesAgo > DELETE_LIGHTWEIGHT_MINUTES, + }; + } + }); + const toDelete = allItems.filter((item) => item.shouldDelete); + if (toDelete.length > 0) { + await fleet.packagePolicyService.delete( + soClient, + esClient, + toDelete.map((item) => item.id), + { + force: true, + } + ); + } + const remaining = allItems.filter((item) => !item.shouldDelete); + if (remaining.length === 0) { + return { state, schedule: { interval: '24h' } }; + } else { + return { state, schedule: { interval: '15m' } }; + } + } + } catch (e) { + logger.error(e); + } + + return { state, schedule: { interval } }; + }, + }; + }, + }, + }); +}; + +export const scheduleCleanUpTask = async ({ logger, pluginsStart }: SyntheticsServerSetup) => { + const interval = SYNTHETICS_SERVICE_CLEAN_UP_INTERVAL_DEFAULT; + + try { + const taskInstance = await pluginsStart.taskManager.ensureScheduled({ + id: SYNTHETICS_SERVICE_CLEAN_UP_TASK_ID, + taskType: SYNTHETICS_SERVICE_CLEAN_UP_TASK_TYPE, + schedule: { + interval, + }, + params: {}, + state: {}, + scope: ['uptime'], + }); + + logger?.info( + `Task ${SYNTHETICS_SERVICE_CLEAN_UP_TASK_ID} scheduled with interval ${taskInstance.schedule?.interval}.` + ); + + await pluginsStart.taskManager.runSoon(SYNTHETICS_SERVICE_CLEAN_UP_TASK_ID); + } catch (e) { + logger?.error(e); + logger?.error( + `Error running synthetics clean up task: ${SYNTHETICS_SERVICE_CLEAN_UP_TASK_ID}, ${e?.message}` + ); + } +}; + +const getFilterForTestNowRun = () => { + const pkg = 'ingest-package-policies'; + + let filter = `${pkg}.package.name:synthetics and ${pkg}.is_managed:true`; + const lightweight = `${pkg}.name: ${LIGHTWEIGHT_TEST_NOW_RUN}`; + const browser = `${pkg}.name: ${BROWSER_TEST_NOW_RUN}`; + filter = `${filter} and (${lightweight} or ${browser})`; + return filter; +}; diff --git a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts index 3c7f33a5723eb..87398936e1543 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server'; +import { SavedObjectsClientContract } from '@kbn/core/server'; import { loggerMock } from '@kbn/logging-mocks'; import { DataStream, @@ -93,7 +93,6 @@ describe('SyntheticsPrivateLocation', () => { try { await syntheticsPrivateLocation.createPackagePolicies( [{ config: testConfig, globalParams: {} }], - {} as unknown as KibanaRequest, [mockPrivateLocation], 'test-space' ); @@ -116,7 +115,6 @@ describe('SyntheticsPrivateLocation', () => { try { await syntheticsPrivateLocation.editMonitors( [{ config: testConfig, globalParams: {} }], - {} as unknown as KibanaRequest, [mockPrivateLocation], 'test-space' ); @@ -152,11 +150,7 @@ describe('SyntheticsPrivateLocation', () => { }, }); try { - await syntheticsPrivateLocation.deleteMonitors( - [testConfig], - {} as unknown as KibanaRequest, - 'test-space' - ); + await syntheticsPrivateLocation.deleteMonitors([testConfig], 'test-space'); } catch (e) { expect(e).toEqual(new Error(error)); } diff --git a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts index e38c3c1aba7e4..8ed63f0539eed 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts @@ -4,11 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { KibanaRequest } from '@kbn/core/server'; import { NewPackagePolicy } from '@kbn/fleet-plugin/common'; import { NewPackagePolicyWithId } from '@kbn/fleet-plugin/server/services/package_policy'; import { cloneDeep } from 'lodash'; import { SavedObjectError } from '@kbn/core-saved-objects-common'; +import { + BROWSER_TEST_NOW_RUN, + LIGHTWEIGHT_TEST_NOW_RUN, +} from '../synthetics_monitor/synthetics_monitor_client'; +import { scheduleCleanUpTask } from './clean_up_task'; import { getAgentPoliciesAsInternalUser } from '../../routes/settings/private_locations/get_agent_policies'; import { SyntheticsServerSetup } from '../../types'; import { formatSyntheticsPolicy } from '../formatters/private_formatters/format_synthetics_policy'; @@ -67,7 +71,9 @@ export class SyntheticsPrivateLocation { privateLocation: PrivateLocationAttributes, newPolicyTemplate: NewPackagePolicy, spaceId: string, - globalParams: Record + globalParams: Record, + testRunId?: string, + runOnce?: boolean ): (NewPackagePolicy & { policy_id: string }) | null { const { label: locName } = privateLocation; @@ -76,10 +82,15 @@ export class SyntheticsPrivateLocation { try { newPolicy.is_managed = true; newPolicy.policy_id = privateLocation.agentPolicyId; - if (config[ConfigKey.MONITOR_SOURCE_TYPE] === SourceType.PROJECT) { - newPolicy.name = `${config.id}-${locName}`; + if (testRunId) { + newPolicy.name = + config.type === 'browser' ? BROWSER_TEST_NOW_RUN : LIGHTWEIGHT_TEST_NOW_RUN; } else { - newPolicy.name = `${config[ConfigKey.NAME]}-${locName}-${spaceId}`; + if (config[ConfigKey.MONITOR_SOURCE_TYPE] === SourceType.PROJECT) { + newPolicy.name = `${config.id}-${locName}`; + } else { + newPolicy.name = `${config[ConfigKey.NAME]}-${locName}-${spaceId}`; + } } newPolicy.namespace = config[ConfigKey.NAMESPACE]; @@ -91,8 +102,20 @@ export class SyntheticsPrivateLocation { config_id: config.fields?.config_id, location_name: stringifyString(privateLocation.label), location_id: privateLocation.id, - 'monitor.project.id': stringifyString(config.fields?.['monitor.project.name']), - 'monitor.project.name': stringifyString(config.fields?.['monitor.project.name']), + 'monitor.project.id': stringifyString( + config.fields?.['monitor.project.id'] ?? config[ConfigKey.PROJECT_ID] + ), + 'monitor.project.name': stringifyString( + config.fields?.['monitor.project.name'] ?? config[ConfigKey.PROJECT_ID] + ), + ...(testRunId + ? { + test_run_id: testRunId, + 'monitor.id': config[ConfigKey.MONITOR_QUERY_ID], + id: testRunId, + } + : {}), + ...(runOnce ? { run_once: runOnce } : {}), }, globalParams ); @@ -106,27 +129,25 @@ export class SyntheticsPrivateLocation { async createPackagePolicies( configs: PrivateConfig[], - request: KibanaRequest, privateLocations: PrivateLocationAttributes[], - spaceId: string + spaceId: string, + testRunId?: string, + runOnce?: boolean ) { if (configs.length === 0) { return { created: [], failed: [] }; } const newPolicies: NewPackagePolicyWithId[] = []; - const newPolicyTemplate = await this.buildNewPolicy(); for (const { config, globalParams } of configs) { try { const { locations } = config; - const fleetManagedLocations = locations.filter((loc) => !loc.isServiceManaged); for (const privateLocation of fleetManagedLocations) { const location = privateLocations?.find((loc) => loc.id === privateLocation.id)!; - if (!location) { throw new Error( `Unable to find Synthetics private location for agentId ${privateLocation.id}` @@ -138,7 +159,9 @@ export class SyntheticsPrivateLocation { location, newPolicyTemplate, spaceId, - globalParams + globalParams, + testRunId, + runOnce ); if (!newPolicy) { @@ -149,7 +172,14 @@ export class SyntheticsPrivateLocation { ); } if (newPolicy) { - newPolicies.push({ ...newPolicy, id: this.getPolicyId(config, location.id, spaceId) }); + if (testRunId) { + newPolicies.push(newPolicy as NewPackagePolicyWithId); + } else { + newPolicies.push({ + ...newPolicy, + id: this.getPolicyId(config, location.id, spaceId), + }); + } } } } catch (e) { @@ -163,7 +193,12 @@ export class SyntheticsPrivateLocation { } try { - return await this.createPolicyBulk(newPolicies); + const result = await this.createPolicyBulk(newPolicies); + if (result?.created && result?.created?.length > 0 && testRunId) { + // ignore await here, we don't want to wait for this to finish + scheduleCleanUpTask(this.server); + } + return result; } catch (e) { this.server.logger.error(e); throw e; @@ -215,7 +250,6 @@ export class SyntheticsPrivateLocation { async editMonitors( configs: Array<{ config: HeartbeatConfig; globalParams: Record }>, - request: KibanaRequest, allPrivateLocations: PrivateLocationAttributes[], spaceId: string ) { @@ -354,18 +388,22 @@ export class SyntheticsPrivateLocation { const soClient = this.server.coreStart.savedObjects.createInternalRepository(); const esClient = this.server.uptimeEsClient.baseESClient; if (soClient && esClient && policyIdsToDelete.length > 0) { - return await this.server.fleet.packagePolicyService.delete( - soClient, - esClient, - policyIdsToDelete, - { - force: true, - } - ); + try { + return await this.server.fleet.packagePolicyService.delete( + soClient, + esClient, + policyIdsToDelete, + { + force: true, + } + ); + } catch (e) { + this.server.logger.error(e); + } } } - async deleteMonitors(configs: HeartbeatConfig[], request: KibanaRequest, spaceId: string) { + async deleteMonitors(configs: HeartbeatConfig[], spaceId: string) { const soClient = this.server.coreStart.savedObjects.createInternalRepository(); const esClient = this.server.uptimeEsClient.baseESClient; @@ -376,12 +414,7 @@ export class SyntheticsPrivateLocation { const monitorPrivateLocations = locations.filter((loc) => !loc.isServiceManaged); for (const privateLocation of monitorPrivateLocations) { - try { - policyIdsToDelete.push(this.getPolicyId(config, privateLocation.id, spaceId)); - } catch (e) { - this.server.logger.error(e); - throw new Error(deletePolicyError(config[ConfigKey.NAME], privateLocation.label)); - } + policyIdsToDelete.push(this.getPolicyId(config, privateLocation.id, spaceId)); } } if (policyIdsToDelete.length > 0) { @@ -393,7 +426,9 @@ export class SyntheticsPrivateLocation { force: true, } ); - const failedPolicies = result?.filter((policy) => !policy.success); + const failedPolicies = result?.filter((policy) => { + return !policy.success && policy?.statusCode !== 404; + }); if (failedPolicies?.length === policyIdsToDelete.length) { throw new Error(deletePolicyError(configs[0][ConfigKey.NAME])); } diff --git a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.test.ts b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.test.ts index bcda186237d09..50a3284c76e3c 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.test.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.test.ts @@ -133,7 +133,6 @@ describe('SyntheticsMonitorClient', () => { await client.addMonitors( [{ monitor, id }], - mockRequest, savedObjectsClientMock, privateLocations, 'test-space' @@ -223,7 +222,6 @@ describe('SyntheticsMonitorClient', () => { await client.deleteMonitors( [monitor as unknown as SyntheticsMonitorWithId], - mockRequest, savedObjectsClientMock, 'test-space' ); diff --git a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts index 2af1c2ff2ef60..c5c158c750beb 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts @@ -4,12 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { - KibanaRequest, - SavedObject, - SavedObjectsClientContract, - SavedObjectsFindResult, -} from '@kbn/core/server'; +import { SavedObject, SavedObjectsClientContract, SavedObjectsFindResult } from '@kbn/core/server'; import { EncryptedSavedObjectsPluginStart } from '@kbn/encrypted-saved-objects-plugin/server'; import { RouteContext } from '../../routes/types'; import { SyntheticsServerSetup } from '../../types'; @@ -21,10 +16,12 @@ import { } from '../private_location/synthetics_private_location'; import { SyntheticsService } from '../synthetics_service'; import { + ConfigKey, EncryptedSyntheticsMonitorAttributes, HeartbeatConfig, MonitorFields, MonitorServiceLocation, + ScheduleUnit, SyntheticsMonitorWithId, SyntheticsMonitorWithSecretsAttributes, } from '../../../common/runtime_types'; @@ -34,19 +31,23 @@ import { mixParamsWithGlobalParams, } from '../formatters/public_formatters/format_configs'; import type { PrivateLocationAttributes } from '../../runtime_types/private_locations'; +export const LIGHTWEIGHT_TEST_NOW_RUN = 'LIGHTWEIGHT_SYNTHETICS_TEST_NOW_RUN'; +export const BROWSER_TEST_NOW_RUN = 'BROWSER_SYNTHETICS_TEST_NOW_RUN'; +const LONG_TIME_MONTH = '43800'; export class SyntheticsMonitorClient { + public server: SyntheticsServerSetup; public syntheticsService: SyntheticsService; public privateLocationAPI: SyntheticsPrivateLocation; constructor(syntheticsService: SyntheticsService, server: SyntheticsServerSetup) { + this.server = server; this.syntheticsService = syntheticsService; this.privateLocationAPI = new SyntheticsPrivateLocation(server); } async addMonitors( monitors: Array<{ monitor: MonitorFields; id: string }>, - request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract, allPrivateLocations: PrivateLocationAttributes[], spaceId: string @@ -75,7 +76,6 @@ export class SyntheticsMonitorClient { const newPolicies = this.privateLocationAPI.createPackagePolicies( privateConfigs, - request, allPrivateLocations, spaceId ); @@ -96,7 +96,6 @@ export class SyntheticsMonitorClient { allPrivateLocations: PrivateLocationAttributes[], spaceId: string ) { - const { request } = routeContext; const privateConfigs: Array<{ config: HeartbeatConfig; globalParams: Record }> = []; @@ -143,7 +142,6 @@ export class SyntheticsMonitorClient { const privateEditPromise = this.privateLocationAPI.editMonitors( privateConfigs, - request, allPrivateLocations, spaceId ); @@ -161,11 +159,10 @@ export class SyntheticsMonitorClient { } async deleteMonitors( monitors: SyntheticsMonitorWithId[], - request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract, spaceId: string ) { - const privateDeletePromise = this.privateLocationAPI.deleteMonitors(monitors, request, spaceId); + const privateDeletePromise = this.privateLocationAPI.deleteMonitors(monitors, spaceId); const publicDeletePromise = this.syntheticsService.deleteConfigs( monitors.map((monitor) => ({ monitor, configId: monitor.config_id, params: {} })) @@ -175,6 +172,62 @@ export class SyntheticsMonitorClient { return pubicResponse; } + async testNowConfigs( + monitor: { monitor: MonitorFields; id: string; testRunId: string }, + savedObjectsClient: SavedObjectsClientContract, + allPrivateLocations: PrivateLocationAttributes[], + spaceId: string, + runOnce?: true + ) { + let privateConfig: PrivateConfig | undefined; + let publicConfig: ConfigData | undefined; + + const paramsBySpace = await this.syntheticsService.getSyntheticsParams({ spaceId }); + + const { formattedConfig, params, config } = await this.formatConfigWithParams( + monitor, + spaceId, + paramsBySpace + ); + + const { privateLocations, publicLocations } = this.parseLocations(formattedConfig); + if (privateLocations.length > 0) { + privateConfig = { + config: { + ...formattedConfig, + [ConfigKey.SCHEDULE]: { + number: LONG_TIME_MONTH, + unit: ScheduleUnit.MINUTES, + }, + [ConfigKey.ENABLED]: true, + }, + globalParams: params, + }; + } + + if (publicLocations.length > 0) { + publicConfig = config; + // making it enabled, even if it's disabled in the UI + publicConfig.monitor.enabled = true; + publicConfig.testRunId = monitor.testRunId; + if (runOnce) { + publicConfig.runOnce = true; + } + } + + const newPolicies = this.privateLocationAPI.createPackagePolicies( + privateConfig ? [privateConfig] : [], + allPrivateLocations, + spaceId, + monitor.testRunId, + runOnce + ); + + const syncErrors = this.syntheticsService.runOnceConfigs(publicConfig); + + return await Promise.all([newPolicies, syncErrors]); + } + hasPrivateLocations(previousMonitor: SavedObject) { const { locations } = previousMonitor.attributes; @@ -213,13 +266,11 @@ export class SyntheticsMonitorClient { } async syncGlobalParams({ - request, spaceId, allPrivateLocations, encryptedSavedObjects, }: { spaceId: string; - request: KibanaRequest; allPrivateLocations: PrivateLocationAttributes[]; encryptedSavedObjects: EncryptedSavedObjectsPluginStart; }) { @@ -243,12 +294,7 @@ export class SyntheticsMonitorClient { } } if (privateConfigs.length > 0) { - await this.privateLocationAPI.editMonitors( - privateConfigs, - request, - allPrivateLocations, - spaceId - ); + await this.privateLocationAPI.editMonitors(privateConfigs, allPrivateLocations, spaceId); } if (publicConfigs.length > 0) { diff --git a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts index 2242d2576ddc5..eae3fb0082030 100644 --- a/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts +++ b/x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts @@ -19,6 +19,7 @@ import { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin import pMap from 'p-map'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; import { ALL_SPACES_ID } from '@kbn/spaces-plugin/common/constants'; +import { registerCleanUpTask } from './private_location/clean_up_task'; import { SyntheticsServerSetup } from '../types'; import { syntheticsMonitorType, syntheticsParamType } from '../../common/types/saved_objects'; import { sendErrorTelemetryEvents } from '../routes/telemetry/monitor_upgrade_sender'; @@ -92,6 +93,7 @@ export class SyntheticsService { public async setup(taskManager: TaskManagerSetupContract) { this.registerSyncTask(taskManager); + registerCleanUpTask(taskManager, this.server); await this.registerServiceLocations(); @@ -418,12 +420,15 @@ export class SyntheticsService { await this.getMonitorConfigs(subject); } - async runOnceConfigs(configs: ConfigData) { - const license = await this.getLicense(); + async runOnceConfigs(configs?: ConfigData) { + if (!configs) { + return; + } const monitors = this.formatConfigs(configs); if (monitors.length === 0) { return; } + const license = await this.getLicense(); const output = await this.getOutput(); if (!output) { diff --git a/x-pack/plugins/synthetics/server/types.ts b/x-pack/plugins/synthetics/server/types.ts index 847d91579ed2d..a14cf6dcd72c1 100644 --- a/x-pack/plugins/synthetics/server/types.ts +++ b/x-pack/plugins/synthetics/server/types.ts @@ -58,6 +58,7 @@ export interface SyntheticsServerSetup { basePath: IBasePath; isDev?: boolean; coreStart: CoreStart; + pluginsStart: SyntheticsPluginsStartDependencies; } export interface SyntheticsPluginsSetupDependencies { @@ -77,6 +78,7 @@ export interface SyntheticsPluginsSetupDependencies { export interface SyntheticsPluginsStartDependencies { security: SecurityPluginStart; + elasticsearch: SecurityPluginStart; fleet: FleetStartContract; encryptedSavedObjects: EncryptedSavedObjectsPluginStart; taskManager: TaskManagerStartContract; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index fdb057d2eb843..64ffa9ac501f5 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -37043,7 +37043,6 @@ "xpack.synthetics.analyzeDataButtonLabel.message": "La fonctionnalité Explorer les données vous permet de sélectionner et de filtrer les données de résultat dans toute dimension et de rechercher la cause ou l'impact des problèmes de performances.", "xpack.synthetics.app.navigateToAlertingButton.content": "Gérer les règles", "xpack.synthetics.app.navigateToAlertingUi": "Quitter Synthetics et accéder à la page de gestion Alerting", - "xpack.synthetics.app.testNow.available.private": "Vous ne pouvez pas démarrer les tests manuellement dans un emplacement privé.", "xpack.synthetics.badge.readOnly.text": "Lecture seule", "xpack.synthetics.badge.readOnly.tooltip": "Enregistrement impossible", "xpack.synthetics.blocked": "Bloqué", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 31d7c6c52b743..8eabc63eb8ea9 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -37042,7 +37042,6 @@ "xpack.synthetics.analyzeDataButtonLabel.message": "データの探索では、任意のディメンションの結果データを選択してフィルタリングし、パフォーマンスの問題の原因または影響を調査することができます。", "xpack.synthetics.app.navigateToAlertingButton.content": "ルールの管理", "xpack.synthetics.app.navigateToAlertingUi": "Syntheticsを離れてアラート管理ページに移動します", - "xpack.synthetics.app.testNow.available.private": "非公開の場所では手動でテストを開始できません。", "xpack.synthetics.badge.readOnly.text": "読み取り専用", "xpack.synthetics.badge.readOnly.tooltip": "を保存できませんでした", "xpack.synthetics.blocked": "ブロック", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index c18773cc676be..61295a1cc366b 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -37036,7 +37036,6 @@ "xpack.synthetics.analyzeDataButtonLabel.message": "“浏览数据”允许您选择和筛选任意维度中的结果数据以及查找性能问题的原因或影响。", "xpack.synthetics.app.navigateToAlertingButton.content": "管理规则", "xpack.synthetics.app.navigateToAlertingUi": "离开 Synthetics 并前往“Alerting 管理”页面", - "xpack.synthetics.app.testNow.available.private": "不能在专用位置上手动启动测试。", "xpack.synthetics.badge.readOnly.text": "只读", "xpack.synthetics.badge.readOnly.tooltip": "无法保存", "xpack.synthetics.blocked": "已阻止", diff --git a/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts b/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts index 9d96ceb28ef33..e19bbd7ffeb15 100644 --- a/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts +++ b/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts @@ -105,7 +105,7 @@ export default function ({ getService }: FtrProviderContext) { .expect(400); }); - it('handles private location errors and does not delete the monitor if integration policy is unable to be deleted', async () => { + it.skip('handles private location errors and does not delete the monitor if integration policy is unable to be deleted', async () => { const name = `Monitor with a private location ${uuidv4()}`; const newMonitor = { name, diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index a5503568605c1..708eb4a5c3abc 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -47,6 +47,7 @@ export default function ({ getService }: FtrProviderContext) { 'Fleet-Usage-Logger', 'Fleet-Usage-Sender', 'ML:saved-objects-sync', + 'Synthetics:Clean-Up-Package-Policies', 'UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects', 'actions:.cases-webhook', 'actions:.d3security', From 770c4a2b53b28d893546554141c44f9198f8c7f8 Mon Sep 17 00:00:00 2001 From: Julia Rechkunova Date: Mon, 31 Jul 2023 15:54:47 +0200 Subject: [PATCH 13/33] [Discover] Fix grid styles to enable better wrapping (#162325) Closes https://github.com/elastic/kibana/issues/162304 ## Summary This PR fixes the issue with cell content wrapping. Before it was just cut off. Now it falls to the next line. Please check that there is no regression for https://github.com/elastic/kibana/pull/139449 Before: Screenshot 2023-07-20 at 15 03 16 After: Screenshot 2023-07-20 at 15 02 34 ### Checklist - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --- .../public/components/discover_grid/discover_grid.scss | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/discover/public/components/discover_grid/discover_grid.scss b/src/plugins/discover/public/components/discover_grid/discover_grid.scss index 18fc15f024fa4..c21651cec7dd9 100644 --- a/src/plugins/discover/public/components/discover_grid/discover_grid.scss +++ b/src/plugins/discover/public/components/discover_grid/discover_grid.scss @@ -119,8 +119,8 @@ } .dscDiscoverGrid__descriptionListDescription { - word-break: normal !important; - white-space: nowrap; + word-break: break-all; + white-space: normal; // Special handling for images coming from the image field formatter img { From 0cc81d73f726b1e0e7a6e56e1478f9d1b1070250 Mon Sep 17 00:00:00 2001 From: Sandra G Date: Mon, 31 Jul 2023 10:09:30 -0400 Subject: [PATCH 14/33] [Logs Shared] remove observability dependency (#162733) After the obs ai assistant [moved to its own plugin](https://github.com/elastic/kibana/pull/162243) outside the observability plugin, logs_shared shouldn't need to depend on observability Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/logs_shared/kibana.jsonc | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/logs_shared/kibana.jsonc b/x-pack/plugins/logs_shared/kibana.jsonc index e14b6ad8dc837..406b399d6ed17 100644 --- a/x-pack/plugins/logs_shared/kibana.jsonc +++ b/x-pack/plugins/logs_shared/kibana.jsonc @@ -9,7 +9,6 @@ "browser": true, "configPath": ["xpack", "logs_shared"], "requiredPlugins": ["data", "dataViews", "usageCollection", "observabilityShared", "observabilityAIAssistant"], - "optionalPlugins": ["observability"], "requiredBundles": [ "kibanaUtils", "kibanaReact", From 036751463dba5b8fecf4ddddd843bc92d4422429 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Mon, 31 Jul 2023 16:19:11 +0200 Subject: [PATCH 15/33] [Infra UI] Chart load optimization (#162328) Closes https://github.com/elastic/kibana/issues/161445 ## Summary This PR enhances the usage of the intersection observer to control the loading behavior of charts, improving the performance, especially noticeable during scrolling. Before https://github.com/elastic/kibana/assets/2767137/dfb1d7db-4ddb-41c3-b8ae-3b5bdf3fe36e After https://github.com/elastic/kibana/assets/2767137/f348b0b9-4e6f-4163-9eb4-99daea91bbef Besides, the intersection observer threshold has been set to 0, allowing charts to start loading as soon as they begin to enter the viewport. ### How to test - Start a local Kibana instance - Navigate to `Infrastructure > Hosts` - Scroll down and confirm that it's not laggy (same for the flyout) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../visualizations/lens/chart_loader.tsx | 63 ------- .../visualizations/lens/lens_wrapper.tsx | 144 ---------------- .../layers/xy_data_layer.ts | 4 +- .../tabs/overview/kpis/kpi_grid.tsx | 7 +- .../asset_details/tabs/overview/kpis/tile.tsx | 6 +- .../tabs/overview/metrics/metric_chart.tsx | 3 +- .../tabs/overview/metrics/metrics_grid.tsx | 2 +- .../components/lens/chart_placeholder.tsx | 39 +++++ .../infra/public/components/lens/index.tsx | 12 ++ .../public/components/lens/lens_wrapper.tsx | 154 ++++++++++++++++++ .../host_metrics_docs_link.tsx | 2 +- .../metric_explanation/tooltip_content.tsx | 2 +- .../public/hooks/use_intersection_once.ts | 25 --- .../components/chart/metric_chart_wrapper.tsx | 8 +- .../hosts/components/kpis/hosts_tile.tsx | 2 +- .../hosts/components/kpis/kpi_grid.tsx | 9 +- .../metrics/hosts/components/kpis/tile.tsx | 8 +- .../hosts/components/table/column_header.tsx | 2 +- .../components/tabs/metrics/metric_chart.tsx | 5 +- .../components/tabs/metrics/metrics_grid.tsx | 2 +- 20 files changed, 231 insertions(+), 268 deletions(-) delete mode 100644 x-pack/plugins/infra/public/common/visualizations/lens/chart_loader.tsx delete mode 100644 x-pack/plugins/infra/public/common/visualizations/lens/lens_wrapper.tsx create mode 100644 x-pack/plugins/infra/public/components/lens/chart_placeholder.tsx create mode 100644 x-pack/plugins/infra/public/components/lens/index.tsx create mode 100644 x-pack/plugins/infra/public/components/lens/lens_wrapper.tsx rename x-pack/plugins/infra/public/{common/visualizations => components/lens}/metric_explanation/host_metrics_docs_link.tsx (90%) rename x-pack/plugins/infra/public/{common/visualizations => components/lens}/metric_explanation/tooltip_content.tsx (96%) delete mode 100644 x-pack/plugins/infra/public/hooks/use_intersection_once.ts diff --git a/x-pack/plugins/infra/public/common/visualizations/lens/chart_loader.tsx b/x-pack/plugins/infra/public/common/visualizations/lens/chart_loader.tsx deleted file mode 100644 index 34ca3b87bdfdf..0000000000000 --- a/x-pack/plugins/infra/public/common/visualizations/lens/chart_loader.tsx +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { EuiFlexGroup, EuiProgress, EuiFlexItem, EuiLoadingChart, useEuiTheme } from '@elastic/eui'; -import { css } from '@emotion/react'; -import { euiStyled } from '@kbn/kibana-react-plugin/common'; - -export const ChartLoader = ({ - children, - loading, - style, - loadedOnce = false, - hasTitle = false, -}: { - style?: React.CSSProperties; - children: React.ReactNode; - loadedOnce: boolean; - loading: boolean; - hasTitle?: boolean; -}) => { - const { euiTheme } = useEuiTheme(); - return ( - - {loading && ( - - )} - {loading && !loadedOnce ? ( - - - - - - ) : ( - children - )} - - ); -}; - -const LoaderContainer = euiStyled.div` - position: relative; - border-radius: ${({ theme }) => theme.eui.euiSizeS}; - overflow: hidden; - height: 100%; -`; diff --git a/x-pack/plugins/infra/public/common/visualizations/lens/lens_wrapper.tsx b/x-pack/plugins/infra/public/common/visualizations/lens/lens_wrapper.tsx deleted file mode 100644 index 13579621dedc4..0000000000000 --- a/x-pack/plugins/infra/public/common/visualizations/lens/lens_wrapper.tsx +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React, { useEffect, useState, useRef, useCallback } from 'react'; - -import { Action } from '@kbn/ui-actions-plugin/public'; -import { ViewMode } from '@kbn/embeddable-plugin/public'; -import type { TimeRange } from '@kbn/es-query'; -import { TypedLensByValueInput } from '@kbn/lens-plugin/public'; -import { css } from '@emotion/react'; -import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; -import { useIntersectedOnce } from '../../../hooks/use_intersection_once'; -import { ChartLoader } from './chart_loader'; -import type { LensAttributes } from '../types'; - -export type LensWrapperProps = Pick< - TypedLensByValueInput, - | 'id' - | 'filters' - | 'query' - | 'style' - | 'onBrushEnd' - | 'hidePanelTitles' - | 'overrides' - | 'hidePanelTitles' - | 'disabledActions' - | 'disableTriggers' -> & { - attributes: LensAttributes | null; - dateRange: TimeRange; - extraActions: Action[]; - lastReloadRequestTime?: number; - loading?: boolean; - hasTitle?: boolean; -}; - -export const LensWrapper = React.memo( - ({ - attributes, - dateRange, - filters, - id, - query, - extraActions, - style, - onBrushEnd, - lastReloadRequestTime, - overrides, - loading = false, - hasTitle = false, - disableTriggers = false, - }: LensWrapperProps) => { - const intersectionRef = useRef(null); - const [loadedOnce, setLoadedOnce] = useState(false); - - const [state, setState] = useState({ - attributes, - lastReloadRequestTime, - query, - filters, - dateRange, - }); - - const { - services: { lens }, - } = useKibanaContextForPlugin(); - const { intersectedOnce, intersection } = useIntersectedOnce(intersectionRef, { - threshold: 1, - }); - - const EmbeddableComponent = lens.EmbeddableComponent; - - useEffect(() => { - if ((intersection?.intersectionRatio ?? 0) === 1) { - setState({ - attributes, - lastReloadRequestTime, - query, - filters, - dateRange, - }); - } - }, [ - attributes, - dateRange, - filters, - intersection?.intersectionRatio, - lastReloadRequestTime, - query, - ]); - - const isReady = state.attributes && intersectedOnce; - - const onLoad = useCallback(() => { - if (!loadedOnce) { - setLoadedOnce(true); - } - }, [loadedOnce]); - - return ( -
- - {state.attributes && ( - - )} - -
- ); - } -); diff --git a/x-pack/plugins/infra/public/common/visualizations/lens/visualization_types/layers/xy_data_layer.ts b/x-pack/plugins/infra/public/common/visualizations/lens/visualization_types/layers/xy_data_layer.ts index 294cf29701d2b..f0babf519f19a 100644 --- a/x-pack/plugins/infra/public/common/visualizations/lens/visualization_types/layers/xy_data_layer.ts +++ b/x-pack/plugins/infra/public/common/visualizations/lens/visualization_types/layers/xy_data_layer.ts @@ -45,7 +45,7 @@ export class XYDataLayer implements ChartLayer { return this.column[0].getFormulaConfig().label; } - getBaseColumnColumn(dataView: DataView, options?: XYLayerOptions) { + getBaseLayer(dataView: DataView, options?: XYLayerOptions) { return { ...getHistogramColumn({ columnName: HISTOGRAM_COLUMN_NAME, @@ -75,7 +75,7 @@ export class XYDataLayer implements ChartLayer { const baseLayer: PersistedIndexPatternLayer = { columnOrder: [BREAKDOWN_COLUMN_NAME, HISTOGRAM_COLUMN_NAME], columns: { - ...this.getBaseColumnColumn(dataView, this.layerConfig.options), + ...this.getBaseLayer(dataView, this.layerConfig.options), }, }; diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/kpi_grid.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/kpi_grid.tsx index ed0c2fc97d675..b86201a29098c 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/kpi_grid.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/kpi_grid.tsx @@ -12,12 +12,7 @@ import { KPI_CHARTS } from '../../../../../common/visualizations/lens/dashboards export const KPIGrid = React.memo(({ nodeName, dataView, timeRange: dateRange }: TileProps) => { return ( <> - + {KPI_CHARTS.map((chartProp, index) => ( diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/tile.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/tile.tsx index b670867f774c8..9907b81d64fc0 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/tile.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/kpis/tile.tsx @@ -12,11 +12,10 @@ import styled from 'styled-components'; import type { Action } from '@kbn/ui-actions-plugin/public'; import { TimeRange } from '@kbn/es-query'; import { FormattedMessage } from '@kbn/i18n-react'; +import { LensWrapper, TooltipContent } from '../../../../lens'; import type { KPIChartProps } from '../../../../../common/visualizations/lens/dashboards/host/kpi_grid_config'; import { useLensAttributes } from '../../../../../hooks/use_lens_attributes'; -import { LensWrapper } from '../../../../../common/visualizations/lens/lens_wrapper'; import { buildCombinedHostsFilter } from '../../../../../utils/filters/build'; -import { TooltipContent } from '../../../../../common/visualizations/metric_explanation/tooltip_content'; const MIN_HEIGHT = 150; @@ -72,7 +71,6 @@ export const Tile = ({ {error ? ( @@ -109,6 +107,7 @@ export const Tile = ({ dateRange={timeRange} filters={filters} loading={loading} + hidePanelTitles /> )} @@ -117,6 +116,7 @@ export const Tile = ({ }; const EuiPanelStyled = styled(EuiPanel)` + min-height: ${MIN_HEIGHT}px; .echMetric { border-radius: ${({ theme }) => theme.eui.euiBorderRadius}; pointer-events: none; diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metric_chart.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metric_chart.tsx index fae393a3f0ff1..ad75734013a29 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metric_chart.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metric_chart.tsx @@ -12,8 +12,8 @@ import { TypedLensByValueInput } from '@kbn/lens-plugin/public'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { TimeRange } from '@kbn/es-query'; import { FormattedMessage } from '@kbn/i18n-react'; +import { LensWrapper } from '../../../../lens/lens_wrapper'; import { buildCombinedHostsFilter } from '../../../../../utils/filters/build'; -import { LensWrapper } from '../../../../../common/visualizations/lens/lens_wrapper'; import { useLensAttributes, type Layer } from '../../../../../hooks/use_lens_attributes'; import type { FormulaConfig, XYLayerOptions } from '../../../../../common/visualizations'; @@ -109,7 +109,6 @@ export const MetricChart = ({ overrides={overrides} loading={loading} disableTriggers - hasTitle /> )} diff --git a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metrics_grid.tsx b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metrics_grid.tsx index aad7889ba7597..9a62fe1d2fe72 100644 --- a/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metrics_grid.tsx +++ b/x-pack/plugins/infra/public/components/asset_details/tabs/overview/metrics/metrics_grid.tsx @@ -11,7 +11,7 @@ import { i18n } from '@kbn/i18n'; import type { DataView } from '@kbn/data-views-plugin/public'; import { TimeRange } from '@kbn/es-query'; import { FormattedMessage } from '@kbn/i18n-react'; -import { HostMetricsDocsLink } from '../../../../../common/visualizations/metric_explanation/host_metrics_docs_link'; +import { HostMetricsDocsLink } from '../../../../lens'; import { MetricChart, type MetricChartProps } from './metric_chart'; import { hostLensFormulas } from '../../../../../common/visualizations'; diff --git a/x-pack/plugins/infra/public/components/lens/chart_placeholder.tsx b/x-pack/plugins/infra/public/components/lens/chart_placeholder.tsx new file mode 100644 index 0000000000000..7bf35f0e0392a --- /dev/null +++ b/x-pack/plugins/infra/public/components/lens/chart_placeholder.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFlexGroup, EuiProgress, EuiFlexItem, EuiLoadingChart } from '@elastic/eui'; +import { useEuiTheme } from '@elastic/eui'; +import { css } from '@emotion/react'; + +export const ChartLoadingProgress = ({ hasTopMargin = false }: { hasTopMargin?: boolean }) => { + const { euiTheme } = useEuiTheme(); + return ( + + ); +}; + +export const ChartPlaceholder = ({ style }: { style?: React.CSSProperties }) => { + return ( + <> + + + + + + + + ); +}; diff --git a/x-pack/plugins/infra/public/components/lens/index.tsx b/x-pack/plugins/infra/public/components/lens/index.tsx new file mode 100644 index 0000000000000..17a2f5b480442 --- /dev/null +++ b/x-pack/plugins/infra/public/components/lens/index.tsx @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ChartPlaceholder } from './chart_placeholder'; +export { LensWrapper } from './lens_wrapper'; + +export { TooltipContent } from './metric_explanation/tooltip_content'; +export { HostMetricsDocsLink } from './metric_explanation/host_metrics_docs_link'; diff --git a/x-pack/plugins/infra/public/components/lens/lens_wrapper.tsx b/x-pack/plugins/infra/public/components/lens/lens_wrapper.tsx new file mode 100644 index 0000000000000..dc3c11dccacc0 --- /dev/null +++ b/x-pack/plugins/infra/public/components/lens/lens_wrapper.tsx @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useEffect, useState, useRef, useCallback, useMemo } from 'react'; +import type { Action } from '@kbn/ui-actions-plugin/public'; +import { ViewMode } from '@kbn/embeddable-plugin/public'; +import type { TimeRange } from '@kbn/es-query'; +import { TypedLensByValueInput } from '@kbn/lens-plugin/public'; +import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import { useKibanaContextForPlugin } from '../../hooks/use_kibana'; +import { ChartLoadingProgress, ChartPlaceholder } from './chart_placeholder'; +import { parseDateRange } from '../../utils/datemath'; +import { LensAttributes } from '../../common/visualizations'; + +export type LensWrapperProps = Omit< + TypedLensByValueInput, + 'timeRange' | 'attributes' | 'viewMode' +> & { + attributes: LensAttributes | null; + dateRange: TimeRange; + extraActions: Action[]; + loading?: boolean; +}; + +export const LensWrapper = ({ + attributes, + dateRange, + filters, + lastReloadRequestTime, + loading, + query, + ...props +}: LensWrapperProps) => { + const [intersectionObserverEntry, setIntersectionObserverEntry] = + useState(); + const [embeddableLoaded, setEmbeddableLoaded] = useState(false); + const [state, setState] = useState({ + attributes, + dateRange, + filters, + lastReloadRequestTime, + query, + }); + + const ref = useRef(null); + const observerRef = useRef( + new IntersectionObserver(([value]) => setIntersectionObserverEntry(value), { + root: ref.current, + }) + ); + + useEffect(() => { + const { current: currentObserver } = observerRef; + currentObserver.disconnect(); + const { current } = ref; + + if (current) { + currentObserver.observe(current); + } + + return () => currentObserver.disconnect(); + }, [ref]); + + useEffect(() => { + if (intersectionObserverEntry?.isIntersecting) { + setState({ + attributes, + dateRange, + filters, + lastReloadRequestTime, + query, + }); + } + }, [ + attributes, + dateRange, + filters, + intersectionObserverEntry?.isIntersecting, + lastReloadRequestTime, + query, + ]); + + const onLoad = useCallback(() => { + if (!embeddableLoaded) { + setEmbeddableLoaded(true); + } + }, [embeddableLoaded]); + + const parsedDateRange: TimeRange = useMemo(() => { + const { from = state.dateRange.from, to = state.dateRange.to } = parseDateRange( + state.dateRange + ); + + return { from, to }; + }, [state.dateRange]); + + const isLoading = loading || !state.attributes; + + return ( + + <> + {isLoading && !embeddableLoaded ? ( + + ) : ( + <> + {isLoading && } + + + )} + + + ); +}; + +const EmbeddableComponentMemo = React.memo( + ({ + attributes, + ...props + }: Omit & { attributes: LensAttributes | null }) => { + const { + services: { lens }, + } = useKibanaContextForPlugin(); + + const EmbeddableComponent = lens.EmbeddableComponent; + + if (!attributes) { + return ; + } + + return ; + } +); + +const Container = euiStyled.div` + position: relative; + border-radius: ${({ theme }) => theme.eui.euiSizeS}; + overflow: hidden; + height: 100%; + .echLegend .echLegendList { + display: flex; + } +`; diff --git a/x-pack/plugins/infra/public/common/visualizations/metric_explanation/host_metrics_docs_link.tsx b/x-pack/plugins/infra/public/components/lens/metric_explanation/host_metrics_docs_link.tsx similarity index 90% rename from x-pack/plugins/infra/public/common/visualizations/metric_explanation/host_metrics_docs_link.tsx rename to x-pack/plugins/infra/public/components/lens/metric_explanation/host_metrics_docs_link.tsx index eefb88f061440..347c2174b9077 100644 --- a/x-pack/plugins/infra/public/common/visualizations/metric_explanation/host_metrics_docs_link.tsx +++ b/x-pack/plugins/infra/public/components/lens/metric_explanation/host_metrics_docs_link.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { EuiLink, EuiText } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import { HOST_METRICS_DOC_HREF } from '../constants'; +import { HOST_METRICS_DOC_HREF } from '../../../common/visualizations/constants'; export const HostMetricsDocsLink = () => { return ( diff --git a/x-pack/plugins/infra/public/common/visualizations/metric_explanation/tooltip_content.tsx b/x-pack/plugins/infra/public/components/lens/metric_explanation/tooltip_content.tsx similarity index 96% rename from x-pack/plugins/infra/public/common/visualizations/metric_explanation/tooltip_content.tsx rename to x-pack/plugins/infra/public/components/lens/metric_explanation/tooltip_content.tsx index 1b7dc70c813aa..fd46700130ee4 100644 --- a/x-pack/plugins/infra/public/common/visualizations/metric_explanation/tooltip_content.tsx +++ b/x-pack/plugins/infra/public/components/lens/metric_explanation/tooltip_content.tsx @@ -9,7 +9,7 @@ import React, { HTMLAttributes } from 'react'; import { EuiText, EuiLink } from '@elastic/eui'; import { css } from '@emotion/react'; import { FormattedMessage } from '@kbn/i18n-react'; -import { HOST_METRICS_DOC_HREF } from '../constants'; +import { HOST_METRICS_DOC_HREF } from '../../../common/visualizations/constants'; interface Props extends Pick, 'style'> { description: string; diff --git a/x-pack/plugins/infra/public/hooks/use_intersection_once.ts b/x-pack/plugins/infra/public/hooks/use_intersection_once.ts deleted file mode 100644 index 8894e9fec3176..0000000000000 --- a/x-pack/plugins/infra/public/hooks/use_intersection_once.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { RefObject, useEffect, useState } from 'react'; -import useIntersection from 'react-use/lib/useIntersection'; - -export const useIntersectedOnce = ( - ref: RefObject, - options: IntersectionObserverInit -) => { - const [intersectedOnce, setIntersectedOnce] = useState(false); - const intersection = useIntersection(ref, options); - - useEffect(() => { - if (!intersectedOnce && (intersection?.intersectionRatio ?? 0) > 0) { - setIntersectedOnce(true); - } - }, [intersectedOnce, intersection?.intersectionRatio]); - - return { intersectedOnce, intersection }; -}; diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/chart/metric_chart_wrapper.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/chart/metric_chart_wrapper.tsx index f25a26044fe25..97be988142aa0 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/chart/metric_chart_wrapper.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/chart/metric_chart_wrapper.tsx @@ -8,7 +8,7 @@ import React, { useEffect, useRef, CSSProperties } from 'react'; import { Chart, Metric, type MetricWNumber, type MetricWTrend } from '@elastic/charts'; import { EuiPanel, EuiToolTip } from '@elastic/eui'; import styled from 'styled-components'; -import { ChartLoader } from '../../../../../common/visualizations/lens/chart_loader'; +import { ChartPlaceholder } from '../../../../../components/lens'; export interface Props extends Pick { id: string; @@ -43,7 +43,9 @@ export const MetricChartWrapper = React.memo( return ( - + {loading && !loadedOnce.current ? ( + + ) : ( - + )} ); } diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/hosts_tile.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/hosts_tile.tsx index 806570ebac349..0571733b80034 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/hosts_tile.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/hosts_tile.tsx @@ -13,7 +13,7 @@ import { useUnifiedSearchContext } from '../../hooks/use_unified_search'; import { TOOLTIP } from '../../../../../common/visualizations/lens/dashboards/host/translations'; import { type Props, MetricChartWrapper } from '../chart/metric_chart_wrapper'; -import { TooltipContent } from '../../../../../common/visualizations/metric_explanation/tooltip_content'; +import { TooltipContent } from '../../../../../components/lens'; const HOSTS_CHART: Omit = { id: `metric-hostCount`, diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/kpi_grid.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/kpi_grid.tsx index 01fd3ccef909f..66eb84b4e4618 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/kpi_grid.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/kpi_grid.tsx @@ -8,7 +8,7 @@ import React, { CSSProperties } from 'react'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { EuiSpacer } from '@elastic/eui'; -import { HostMetricsDocsLink } from '../../../../../common/visualizations/metric_explanation/host_metrics_docs_link'; +import { HostMetricsDocsLink } from '../../../../../components/lens'; import { Tile } from './tile'; import { HostCountProvider } from '../../hooks/use_host_count'; import { HostsTile } from './hosts_tile'; @@ -24,12 +24,7 @@ export const KPIGrid = () => { - + diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/tile.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/tile.tsx index 8f5ee086cbd5a..950e74478dd4b 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/tile.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/kpis/tile.tsx @@ -12,19 +12,18 @@ import { EuiIcon, EuiPanel, EuiFlexGroup, EuiFlexItem, EuiText, EuiToolTip } fro import styled from 'styled-components'; import { Action } from '@kbn/ui-actions-plugin/public'; import { FormattedMessage } from '@kbn/i18n-react'; +import { LensWrapper, TooltipContent } from '../../../../../components/lens'; import { KPIChartProps } from '../../../../../common/visualizations/lens/dashboards/host/kpi_grid_config'; import { buildCombinedHostsFilter } from '../../../../../utils/filters/build'; import { useLensAttributes } from '../../../../../hooks/use_lens_attributes'; import { useMetricsDataViewContext } from '../../hooks/use_data_view'; import { useUnifiedSearchContext } from '../../hooks/use_unified_search'; import { useHostsViewContext } from '../../hooks/use_hosts_view'; -import { LensWrapper } from '../../../../../common/visualizations/lens/lens_wrapper'; import { useHostCountContext } from '../../hooks/use_host_count'; import { useAfterLoadedState } from '../../hooks/use_after_loaded_state'; -import { TooltipContent } from '../../../../../common/visualizations/metric_explanation/tooltip_content'; import { KPI_CHART_MIN_HEIGHT } from '../../constants'; -export const Tile = ({ id, title, layers, style, toolTip, ...props }: KPIChartProps) => { +export const Tile = ({ id, title, layers, style, toolTip }: KPIChartProps) => { const { searchCriteria, onSubmit } = useUnifiedSearchContext(); const { dataView } = useMetricsDataViewContext(); const { requestTs, hostNodes, loading: hostsLoading } = useHostsViewContext(); @@ -149,6 +148,7 @@ export const Tile = ({ id, title, layers, style, toolTip, ...props }: KPIChartPr query={shouldUseSearchCriteria ? afterLoadedState.query : undefined} onBrushEnd={handleBrushEnd} loading={loading} + hidePanelTitles /> @@ -158,7 +158,7 @@ export const Tile = ({ id, title, layers, style, toolTip, ...props }: KPIChartPr }; const EuiPanelStyled = styled(EuiPanel)` - min-height: ${KPI_CHART_MIN_HEIGHT}; + min-height: ${KPI_CHART_MIN_HEIGHT}px; .echMetric { border-radius: ${({ theme }) => theme.eui.euiBorderRadius}; pointer-events: none; diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/table/column_header.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/table/column_header.tsx index e23275598f990..e865fad082731 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/table/column_header.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/table/column_header.tsx @@ -8,7 +8,7 @@ import React, { useState, useRef, useCallback, useLayoutEffect } from 'react'; import { EuiPopover, EuiIcon, EuiFlexGroup, useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; import { APP_WRAPPER_CLASS } from '@kbn/core/public'; -import { TooltipContent } from '../../../../../common/visualizations/metric_explanation/tooltip_content'; +import { TooltipContent } from '../../../../../components/lens/metric_explanation/tooltip_content'; import { useBoolean } from '../../../../../hooks/use_boolean'; interface Props { diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metric_chart.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metric_chart.tsx index c985df8512e8a..b31e5c1cb08d2 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metric_chart.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metric_chart.tsx @@ -11,7 +11,7 @@ import { EuiIcon, EuiPanel, EuiFlexGroup, EuiFlexItem, EuiText, useEuiTheme } fr import { css } from '@emotion/react'; import { TypedLensByValueInput } from '@kbn/lens-plugin/public'; import { FormattedMessage } from '@kbn/i18n-react'; -import { LensWrapper } from '../../../../../../common/visualizations/lens/lens_wrapper'; +import { LensWrapper } from '../../../../../../components/lens'; import { useLensAttributes, Layer } from '../../../../../../hooks/use_lens_attributes'; import { useMetricsDataViewContext } from '../../../hooks/use_data_view'; import { useUnifiedSearchContext } from '../../../hooks/use_unified_search'; @@ -130,7 +130,7 @@ export const MetricChart = ({ id, title, layers, overrides }: MetricChartProps) ) : ( )} diff --git a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metrics_grid.tsx b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metrics_grid.tsx index 33bad2803a438..7dfa9b63b87b2 100644 --- a/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metrics_grid.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/hosts/components/tabs/metrics/metrics_grid.tsx @@ -10,7 +10,7 @@ import { EuiFlexGrid, EuiFlexItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { EuiSpacer } from '@elastic/eui'; import { hostLensFormulas, type XYLayerOptions } from '../../../../../../common/visualizations'; -import { HostMetricsDocsLink } from '../../../../../../common/visualizations/metric_explanation/host_metrics_docs_link'; +import { HostMetricsDocsLink } from '../../../../../../components/lens'; import { MetricChart, MetricChartProps } from './metric_chart'; const DEFAULT_BREAKDOWN_SIZE = 20; From fa01462487dac36febac123cb4db6f717c8790ee Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 31 Jul 2023 16:22:49 +0200 Subject: [PATCH 16/33] [Synthetics] Clean up common dir (#162692) --- .../common/constants/chart_format_limits.ts | 27 --- .../synthetics/common/constants/index.ts | 3 - .../synthetics/common/constants/query.ts | 28 --- .../synthetics/common/constants/rest_api.ts | 37 --- .../common/constants/synthetics/rest_api.ts | 7 + .../plugins/synthetics/common/constants/ui.ts | 15 -- .../common/constants/uptime_alerts.ts | 49 ---- .../assert_close_to.test.ts.snap | 3 - .../common/lib/assert_close_to.test.ts | 22 -- .../synthetics/common/lib/assert_close_to.ts | 17 -- .../common/lib/get_histogram_interval.test.ts | 21 -- .../common/lib/get_histogram_interval.ts | 46 ---- x-pack/plugins/synthetics/common/lib/index.ts | 2 - .../plugins/synthetics/common/lib/ml.test.ts | 35 --- x-pack/plugins/synthetics/common/lib/ml.ts | 28 --- .../common/rules/alert_actions.test.ts | 228 ------------------ .../synthetics/common/rules/alert_actions.ts | 66 +---- .../rules/legacy_uptime/translations.ts | 160 ------------ .../plugins/synthetics/common/translations.ts | 22 -- .../plugins/synthetics/common/types/index.ts | 2 - .../common/types/monitor_duration.ts | 26 -- .../common/utils/get_monitor_url.ts | 40 --- .../utils/get_synthetics_monitor_url.ts | 6 +- .../apps/synthetics/state/settings/api.ts | 8 +- x-pack/plugins/synthetics/server/feature.ts | 7 +- x-pack/plugins/synthetics/server/lib.ts | 4 +- .../translations/translations/fr-FR.json | 21 +- .../translations/translations/ja-JP.json | 21 +- .../translations/translations/zh-CN.json | 21 +- .../common/index.ts} | 4 +- .../observability/synthetics_rule.ts | 6 +- .../apis/uptime/feature_controls.ts | 2 +- .../apis/uptime/get_all_pings.ts | 2 +- .../apis/uptime/rest/dynamic_settings.ts | 4 +- .../apis/uptime/rest/index_status.ts | 2 +- .../apis/uptime/rest/monitor_duration.ts | 2 +- .../apis/uptime/rest/monitor_latest_status.ts | 2 +- .../uptime/rest/monitor_states_generated.ts | 2 +- .../uptime/rest/monitor_states_real_data.ts | 2 +- .../apis/uptime/rest/ping_histogram.ts | 2 +- .../apis/uptime/rest/ping_list.ts | 4 +- .../apis/uptime/rest/snapshot.ts | 2 +- .../uptime/simple_down_alert.ts | 2 +- 43 files changed, 57 insertions(+), 953 deletions(-) delete mode 100644 x-pack/plugins/synthetics/common/constants/chart_format_limits.ts delete mode 100644 x-pack/plugins/synthetics/common/constants/query.ts delete mode 100644 x-pack/plugins/synthetics/common/constants/rest_api.ts delete mode 100644 x-pack/plugins/synthetics/common/constants/uptime_alerts.ts delete mode 100644 x-pack/plugins/synthetics/common/lib/__snapshots__/assert_close_to.test.ts.snap delete mode 100644 x-pack/plugins/synthetics/common/lib/assert_close_to.test.ts delete mode 100644 x-pack/plugins/synthetics/common/lib/assert_close_to.ts delete mode 100644 x-pack/plugins/synthetics/common/lib/get_histogram_interval.test.ts delete mode 100644 x-pack/plugins/synthetics/common/lib/get_histogram_interval.ts delete mode 100644 x-pack/plugins/synthetics/common/lib/ml.test.ts delete mode 100644 x-pack/plugins/synthetics/common/lib/ml.ts delete mode 100644 x-pack/plugins/synthetics/common/rules/legacy_uptime/translations.ts delete mode 100644 x-pack/plugins/synthetics/common/translations.ts delete mode 100644 x-pack/plugins/synthetics/common/types/monitor_duration.ts delete mode 100644 x-pack/plugins/synthetics/common/utils/get_monitor_url.ts rename x-pack/plugins/{synthetics/common/types/integration_deprecation.ts => uptime/common/index.ts} (72%) diff --git a/x-pack/plugins/synthetics/common/constants/chart_format_limits.ts b/x-pack/plugins/synthetics/common/constants/chart_format_limits.ts deleted file mode 100644 index 44f1c5cd0c7d3..0000000000000 --- a/x-pack/plugins/synthetics/common/constants/chart_format_limits.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -const MINUTE = 1000 * 60; -const HOUR = MINUTE * 60; -const DAY = 24 * 60 * 60 * 1000; -const WEEK = DAY * 7; -const MONTH = WEEK * 4; - -/** - * These constants are used by the charting code to determine - * what label should be applied to chart axes so as to help users - * understand the timeseries data they're being shown. - */ -export const CHART_FORMAT_LIMITS = { - DAY, - EIGHT_MINUTES: MINUTE * 8, - FOUR_YEARS: 4 * 12 * 4 * WEEK, - THIRTY_SIX_HOURS: HOUR * 36, - THREE_WEEKS: WEEK * 3, - SIX_MONTHS: MONTH * 7, - NINE_DAYS: DAY * 9, -}; diff --git a/x-pack/plugins/synthetics/common/constants/index.ts b/x-pack/plugins/synthetics/common/constants/index.ts index d94fa976fb5e0..46a415f8a41c6 100644 --- a/x-pack/plugins/synthetics/common/constants/index.ts +++ b/x-pack/plugins/synthetics/common/constants/index.ts @@ -5,12 +5,9 @@ * 2.0. */ -export { CHART_FORMAT_LIMITS } from './chart_format_limits'; export { CLIENT_DEFAULTS } from './client_defaults'; export { CONTEXT_DEFAULTS } from './context_defaults'; export * from './capabilities'; export * from './settings_defaults'; -export { QUERY } from './query'; export * from './ui'; -export * from './rest_api'; export * from './synthetics'; diff --git a/x-pack/plugins/synthetics/common/constants/query.ts b/x-pack/plugins/synthetics/common/constants/query.ts deleted file mode 100644 index 5bd376ae02393..0000000000000 --- a/x-pack/plugins/synthetics/common/constants/query.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** - * These are the fields that will be used when users enter simple_query_string - * searches into the FilterBar component. - */ -export const QUERY = { - DEFAULT_BUCKET_COUNT: 25, - // the maximum buckets allowed by most aggregations - DEFAULT_AGGS_CAP: 10000, - SIMPLE_QUERY_STRING_FIELDS: [ - 'monitor.id', - 'monitor.url', - 'monitor.type', - 'monitor.status', - 'monitor.name', - 'url.full', - 'url.path', - 'url.scheme', - 'url.domain', - 'error.type', - ], -}; diff --git a/x-pack/plugins/synthetics/common/constants/rest_api.ts b/x-pack/plugins/synthetics/common/constants/rest_api.ts deleted file mode 100644 index 2499c178da1db..0000000000000 --- a/x-pack/plugins/synthetics/common/constants/rest_api.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export enum API_URLS { - DYNAMIC_SETTINGS = `/internal/uptime/dynamic_settings`, - INDEX_STATUS = '/internal/uptime/index_status', - MONITOR_LIST = `/internal/uptime/monitor/list`, - MONITOR_LOCATIONS = `/internal/uptime/monitor/locations`, - MONITOR_DURATION = `/internal/uptime/monitor/duration`, - MONITOR_DETAILS = `/internal/uptime/monitor/details`, - MONITOR_STATUS = `/internal/uptime/monitor/status`, - NETWORK_EVENTS = `/internal/uptime/network_events`, - PINGS = '/internal/uptime/pings', - PING_HISTOGRAM = `/internal/uptime/ping/histogram`, - SNAPSHOT_COUNT = `/internal/uptime/snapshot/count`, - SYNTHETICS_SUCCESSFUL_CHECK = `/internal/uptime/synthetics/check/success`, - JOURNEY = `/internal/uptime/journey/{checkGroup}`, - JOURNEY_FAILED_STEPS = `/internal/uptime/journeys/failed_steps`, - JOURNEY_SCREENSHOT = `/internal/uptime/journey/screenshot/{checkGroup}/{stepIndex}`, - JOURNEY_SCREENSHOT_BLOCKS = `/internal/uptime/journey/screenshot/block`, - - ML_MODULE_JOBS = `/internal/ml/modules/jobs_exist/`, - ML_SETUP_MODULE = '/internal/ml/modules/setup/', - ML_DELETE_JOB = `/internal/ml/jobs/delete_jobs`, - ML_CAPABILITIES = '/internal/ml/ml_capabilities', - ML_ANOMALIES_RESULT = `/internal/ml/results/anomalies_table_data`, - - RULE_CONNECTORS = '/api/actions/connectors', - CREATE_RULE = '/api/alerting/rule', - DELETE_RULE = '/api/alerting/rule/', - RULES_FIND = '/api/alerting/rules/_find', - CONNECTOR_TYPES = '/api/actions/connector_types', -} diff --git a/x-pack/plugins/synthetics/common/constants/synthetics/rest_api.ts b/x-pack/plugins/synthetics/common/constants/synthetics/rest_api.ts index d43eb00d96b84..97f14b77f7665 100644 --- a/x-pack/plugins/synthetics/common/constants/synthetics/rest_api.ts +++ b/x-pack/plugins/synthetics/common/constants/synthetics/rest_api.ts @@ -45,4 +45,11 @@ export enum SYNTHETICS_API_URLS { SYNTHETICS_MONITORS_PROJECT = '/api/synthetics/project/{projectName}/monitors', SYNTHETICS_MONITORS_PROJECT_UPDATE = '/api/synthetics/project/{projectName}/monitors/_bulk_update', SYNTHETICS_MONITORS_PROJECT_DELETE = '/api/synthetics/project/{projectName}/monitors/_bulk_delete', + + DYNAMIC_SETTINGS = `/internal/uptime/dynamic_settings`, + RULE_CONNECTORS = '/api/actions/connectors', + CREATE_RULE = '/api/alerting/rule', + DELETE_RULE = '/api/alerting/rule/', + RULES_FIND = '/api/alerting/rules/_find', + CONNECTOR_TYPES = '/api/actions/connector_types', } diff --git a/x-pack/plugins/synthetics/common/constants/ui.ts b/x-pack/plugins/synthetics/common/constants/ui.ts index 189dd40660ae7..64259a676189c 100644 --- a/x-pack/plugins/synthetics/common/constants/ui.ts +++ b/x-pack/plugins/synthetics/common/constants/ui.ts @@ -42,8 +42,6 @@ export const SYNTHETIC_CHECK_STEPS_ROUTE = '/journey/:checkGroupId/steps'; export const TEST_RUN_DETAILS_ROUTE = '/monitor/:monitorId/test-run/:checkGroupId'; -export const MAPPING_ERROR_ROUTE = '/mapping-error'; - export const ERROR_DETAILS_ROUTE = '/monitor/:monitorId/errors/:errorStateId'; export enum STATUS { @@ -61,10 +59,6 @@ export enum MONITOR_TYPES { BROWSER = 'browser', } -export const ML_JOB_ID = 'high_latency_by_geo'; - -export const ML_MODULE_ID = 'uptime_heartbeat'; - export const UNNAMED_LOCATION = 'Unnamed-location'; export enum CERT_STATUS { @@ -74,15 +68,6 @@ export enum CERT_STATUS { TOO_OLD = 'TOO_OLD', } -export const KQL_SYNTAX_LOCAL_STORAGE = 'xpack.uptime.kql.syntax'; - -export const FILTER_FIELDS = { - TAGS: 'tags', - PORT: 'url.port', - LOCATION: 'observer.geo.name', - TYPE: 'monitor.type', -}; - export const SYNTHETICS_INDEX_PATTERN = 'synthetics-*'; export const LICENSE_NOT_ACTIVE_ERROR = 'License not active'; diff --git a/x-pack/plugins/synthetics/common/constants/uptime_alerts.ts b/x-pack/plugins/synthetics/common/constants/uptime_alerts.ts deleted file mode 100644 index 71f6bd1b183fb..0000000000000 --- a/x-pack/plugins/synthetics/common/constants/uptime_alerts.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ActionGroup } from '@kbn/alerting-plugin/common'; - -export type MonitorStatusActionGroup = - ActionGroup<'xpack.uptime.alerts.actionGroups.monitorStatus'>; -export type TLSLegacyActionGroup = ActionGroup<'xpack.uptime.alerts.actionGroups.tls'>; -export type TLSActionGroup = ActionGroup<'xpack.uptime.alerts.actionGroups.tlsCertificate'>; -export type DurationAnomalyActionGroup = - ActionGroup<'xpack.uptime.alerts.actionGroups.durationAnomaly'>; - -export const MONITOR_STATUS: MonitorStatusActionGroup = { - id: 'xpack.uptime.alerts.actionGroups.monitorStatus', - name: 'Uptime Down Monitor', -}; - -export const TLS_LEGACY: TLSLegacyActionGroup = { - id: 'xpack.uptime.alerts.actionGroups.tls', - name: 'Uptime TLS Alert (Legacy)', -}; - -export const TLS: TLSActionGroup = { - id: 'xpack.uptime.alerts.actionGroups.tlsCertificate', - name: 'Uptime TLS Alert', -}; - -export const DURATION_ANOMALY: DurationAnomalyActionGroup = { - id: 'xpack.uptime.alerts.actionGroups.durationAnomaly', - name: 'Uptime Duration Anomaly', -}; - -export const CLIENT_ALERT_TYPES = { - MONITOR_STATUS: 'xpack.uptime.alerts.monitorStatus', - TLS_LEGACY: 'xpack.uptime.alerts.tls', - TLS: 'xpack.uptime.alerts.tlsCertificate', - DURATION_ANOMALY: 'xpack.uptime.alerts.durationAnomaly', -}; - -export const UPTIME_RULE_TYPES = [ - 'xpack.uptime.alerts.tls', - 'xpack.uptime.alerts.tlsCertificate', - 'xpack.uptime.alerts.monitorStatus', - 'xpack.uptime.alerts.durationAnomaly', -]; diff --git a/x-pack/plugins/synthetics/common/lib/__snapshots__/assert_close_to.test.ts.snap b/x-pack/plugins/synthetics/common/lib/__snapshots__/assert_close_to.test.ts.snap deleted file mode 100644 index 7dfb8c88be1ca..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/__snapshots__/assert_close_to.test.ts.snap +++ /dev/null @@ -1,3 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`assertCloseTo throws an error when expected value is outside of precision range 1`] = `"expected [12500] to be within 100 of 10000"`; diff --git a/x-pack/plugins/synthetics/common/lib/assert_close_to.test.ts b/x-pack/plugins/synthetics/common/lib/assert_close_to.test.ts deleted file mode 100644 index 1a8a2e4cddb3a..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/assert_close_to.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { assertCloseTo } from './assert_close_to'; - -describe('assertCloseTo', () => { - it('does not throw an error when expected value is correct', () => { - assertCloseTo(10000, 10001, 100); - }); - - it('does not throw an error when expected value is under actual, but within precision threshold', () => { - assertCloseTo(10000, 9875, 300); - }); - - it('throws an error when expected value is outside of precision range', () => { - expect(() => assertCloseTo(10000, 12500, 100)).toThrowErrorMatchingSnapshot(); - }); -}); diff --git a/x-pack/plugins/synthetics/common/lib/assert_close_to.ts b/x-pack/plugins/synthetics/common/lib/assert_close_to.ts deleted file mode 100644 index eeadacaa5703a..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/assert_close_to.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const assertCloseTo = (actual: number, expected: number, precision: number) => { - if (Math.abs(expected - actual) > precision) { - throw new Error(`expected [${expected}] to be within ${precision} of ${actual}`); - } - - // if actual is undefined above math condition will be NAN and it will be always false - if (actual === undefined) { - throw new Error(`expected close to [${expected}] but got [${actual}]`); - } -}; diff --git a/x-pack/plugins/synthetics/common/lib/get_histogram_interval.test.ts b/x-pack/plugins/synthetics/common/lib/get_histogram_interval.test.ts deleted file mode 100644 index 5c5f5415ff23e..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/get_histogram_interval.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getHistogramInterval } from './get_histogram_interval'; -import { assertCloseTo } from './assert_close_to'; - -describe('getHistogramInterval', () => { - it('specifies the interval necessary to divide a given timespan into equal buckets, rounded to the nearest integer, expressed in ms', () => { - const interval = getHistogramInterval('now-15m', 'now', 10); - assertCloseTo(interval, 90000, 20); - }); - - it('will supply a default constant value for bucketCount when none is provided', () => { - const interval = getHistogramInterval('now-15m', 'now'); - assertCloseTo(interval, 36000, 20); - }); -}); diff --git a/x-pack/plugins/synthetics/common/lib/get_histogram_interval.ts b/x-pack/plugins/synthetics/common/lib/get_histogram_interval.ts deleted file mode 100644 index 8d44fb594c03a..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/get_histogram_interval.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import DateMath from '@kbn/datemath'; -import { QUERY } from '../constants'; - -export const parseRelativeDate = (dateStr: string, options = {}) => { - // We need this this parsing because if user selects This week or this date - // That represents end date in future, if week or day is still in the middle - // Uptime data can never be collected in future, so we will reset date to now - // in That case. Example case we select this week range will be to='now/w' and from = 'now/w'; - - const parsedDate = DateMath.parse(dateStr, options); - const dateTimestamp = parsedDate?.valueOf() ?? 0; - if (dateTimestamp > Date.now()) { - return DateMath.parse('now'); - } - return parsedDate; -}; - -export const getHistogramInterval = ( - dateRangeStart: string, - dateRangeEnd: string, - bucketCount?: number -): number => { - const from = parseRelativeDate(dateRangeStart); - - // roundUp is required for relative date like now/w to get the end of the week - const to = parseRelativeDate(dateRangeEnd, { roundUp: true }); - if (from === undefined) { - throw Error('Invalid dateRangeStart value'); - } - if (to === undefined) { - throw Error('Invalid dateRangeEnd value'); - } - const interval = Math.round( - (to.valueOf() - from.valueOf()) / (bucketCount || QUERY.DEFAULT_BUCKET_COUNT) - ); - - // Interval can never be zero, if it's 0 we return at least 1ms interval - return interval > 0 ? interval : 1; -}; diff --git a/x-pack/plugins/synthetics/common/lib/index.ts b/x-pack/plugins/synthetics/common/lib/index.ts index 497284f3fa281..c0a2226d59b93 100644 --- a/x-pack/plugins/synthetics/common/lib/index.ts +++ b/x-pack/plugins/synthetics/common/lib/index.ts @@ -7,5 +7,3 @@ export * from './combine_filters_and_user_search'; export * from './stringify_kueries'; - -export { getMLJobId } from './ml'; diff --git a/x-pack/plugins/synthetics/common/lib/ml.test.ts b/x-pack/plugins/synthetics/common/lib/ml.test.ts deleted file mode 100644 index e7f572b12bb47..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/ml.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getMLJobId } from './ml'; - -describe('ML Anomaly API', () => { - it('it generates a lowercase job id', async () => { - const monitorId = 'ABC1334haa'; - - const jobId = getMLJobId(monitorId); - - expect(jobId).toEqual(jobId.toLowerCase()); - }); - - it('should truncate long monitor IDs', () => { - const longAndWeirdMonitorId = - 'https://auto-mmmmxhhhhhccclongAndWeirdMonitorId123yyyyyrereauto-xcmpa-1345555454646'; - - expect(getMLJobId(longAndWeirdMonitorId)).toHaveLength(64); - }); - - it('should remove special characters and replace them with underscore', () => { - const monIdSpecialChars = '/ ? , " < > | * a'; - - const jobId = getMLJobId(monIdSpecialChars); - - const format = /[/?,"<>|*]+/; - - expect(format.test(jobId)).toBe(false); - }); -}); diff --git a/x-pack/plugins/synthetics/common/lib/ml.ts b/x-pack/plugins/synthetics/common/lib/ml.ts deleted file mode 100644 index 8b9c4bf7c5857..0000000000000 --- a/x-pack/plugins/synthetics/common/lib/ml.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ML_JOB_ID } from '../constants'; - -export const getJobPrefix = (monitorId: string) => { - // ML App doesn't support upper case characters in job name - // Also Spaces and the characters / ? , " < > | * are not allowed - // so we will replace all special chars with _ - - const prefix = monitorId.replace(/[^A-Z0-9]+/gi, '_').toLowerCase(); - - // ML Job ID can't be greater than 64 length, so will be substring it, and hope - // At such big length, there is minimum chance of having duplicate monitor id - // Subtracting ML_JOB_ID constant as well - const postfix = '_' + ML_JOB_ID; - - if ((prefix + postfix).length > 64) { - return prefix.substring(0, 64 - postfix.length) + '_'; - } - return prefix + '_'; -}; - -export const getMLJobId = (monitorId: string) => `${getJobPrefix(monitorId)}${ML_JOB_ID}`; diff --git a/x-pack/plugins/synthetics/common/rules/alert_actions.test.ts b/x-pack/plugins/synthetics/common/rules/alert_actions.test.ts index 3315f5059ef86..345342d1f4c62 100644 --- a/x-pack/plugins/synthetics/common/rules/alert_actions.test.ts +++ b/x-pack/plugins/synthetics/common/rules/alert_actions.test.ts @@ -7,237 +7,9 @@ import { populateAlertActions } from './alert_actions'; import { ActionConnector } from './types'; -import { MONITOR_STATUS } from '../constants/uptime_alerts'; import { MONITOR_STATUS as SYNTHETICS_MONITOR_STATUS } from '../constants/synthetics_alerts'; -import { MonitorStatusTranslations } from './legacy_uptime/translations'; import { SyntheticsMonitorStatusTranslations } from './synthetics/translations'; -describe('Legacy Alert Actions factory', () => { - it('generate expected action for pager duty', async () => { - const resp = populateAlertActions({ - groupId: MONITOR_STATUS.id, - defaultActions: [ - { - actionTypeId: '.pagerduty', - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - params: { - dedupKey: 'always-downxpack.uptime.alerts.actionGroups.monitorStatus', - eventAction: 'trigger', - severity: 'error', - summary: MonitorStatusTranslations.defaultActionMessage, - }, - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - }, - ] as unknown as ActionConnector[], - translations: { - defaultActionMessage: MonitorStatusTranslations.defaultActionMessage, - defaultRecoveryMessage: MonitorStatusTranslations.defaultRecoveryMessage, - defaultSubjectMessage: MonitorStatusTranslations.defaultSubjectMessage, - defaultRecoverySubjectMessage: MonitorStatusTranslations.defaultRecoverySubjectMessage, - }, - isLegacy: true, - }); - expect(resp).toEqual([ - { - group: 'recovered', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - dedupKey: expect.any(String), - eventAction: 'resolve', - summary: - 'Alert for monitor {{context.monitorName}} with url {{{context.monitorUrl}}} from {{context.observerLocation}} has recovered', - }, - }, - { - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - dedupKey: expect.any(String), - eventAction: 'trigger', - severity: 'error', - summary: MonitorStatusTranslations.defaultActionMessage, - }, - }, - ]); - }); - - it('generate expected action for email', async () => { - const resp = populateAlertActions({ - groupId: MONITOR_STATUS.id, - defaultActions: [ - { - actionTypeId: '.email', - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - params: { - dedupKey: 'always-downxpack.uptime.alerts.actionGroups.monitorStatus', - eventAction: 'trigger', - severity: 'error', - summary: MonitorStatusTranslations.defaultActionMessage, - }, - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - }, - ] as unknown as ActionConnector[], - translations: { - defaultActionMessage: MonitorStatusTranslations.defaultActionMessage, - defaultRecoveryMessage: MonitorStatusTranslations.defaultRecoveryMessage, - defaultSubjectMessage: MonitorStatusTranslations.defaultSubjectMessage, - defaultRecoverySubjectMessage: MonitorStatusTranslations.defaultRecoverySubjectMessage, - }, - isLegacy: true, - defaultEmail: { - to: ['test@email.com'], - }, - }); - expect(resp).toEqual([ - { - group: 'recovered', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - bcc: [], - cc: [], - kibanaFooterLink: { - path: '', - text: '', - }, - message: - 'Alert for monitor {{context.monitorName}} with url {{{context.monitorUrl}}} from {{context.observerLocation}} has recovered', - messageHTML: null, - subject: - 'Monitor {{context.monitorName}} with url {{{context.monitorUrl}}} has recovered', - to: ['test@email.com'], - }, - }, - { - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - bcc: [], - cc: [], - kibanaFooterLink: { - path: '', - text: '', - }, - message: - 'Monitor {{context.monitorName}} with url {{{context.monitorUrl}}} from {{context.observerLocation}} {{{context.statusMessage}}} The latest error message is {{{context.latestErrorMessage}}}, checked at {{context.checkedAt}}', - messageHTML: null, - subject: 'Monitor {{context.monitorName}} with url {{{context.monitorUrl}}} is down', - to: ['test@email.com'], - }, - }, - ]); - }); - - it('generate expected action for index', async () => { - const resp = populateAlertActions({ - groupId: MONITOR_STATUS.id, - defaultActions: [ - { - actionTypeId: '.index', - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - params: { - dedupKey: 'always-downxpack.uptime.alerts.actionGroups.monitorStatus', - eventAction: 'trigger', - severity: 'error', - summary: MonitorStatusTranslations.defaultActionMessage, - }, - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - }, - ] as unknown as ActionConnector[], - translations: { - defaultActionMessage: MonitorStatusTranslations.defaultActionMessage, - defaultRecoveryMessage: MonitorStatusTranslations.defaultRecoveryMessage, - defaultSubjectMessage: MonitorStatusTranslations.defaultSubjectMessage, - defaultRecoverySubjectMessage: MonitorStatusTranslations.defaultRecoverySubjectMessage, - }, - isLegacy: true, - }); - expect(resp).toEqual([ - { - group: 'recovered', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - documents: [ - { - latestErrorMessage: '', - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - observerLocation: '{{context.observerLocation}}', - statusMessage: - 'Alert for monitor {{context.monitorName}} with url {{{context.monitorUrl}}} from {{context.observerLocation}} has recovered', - }, - ], - indexOverride: null, - }, - }, - { - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - documents: [ - { - latestErrorMessage: '{{{context.latestErrorMessage}}}', - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - observerLocation: '{{context.observerLocation}}', - statusMessage: '{{{context.statusMessage}}}', - }, - ], - indexOverride: null, - }, - }, - ]); - }); - - it('generate expected action for slack action connector', async () => { - const resp = populateAlertActions({ - groupId: MONITOR_STATUS.id, - defaultActions: [ - { - actionTypeId: '.pagerduty', - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - params: { - dedupKey: 'always-downxpack.uptime.alerts.actionGroups.monitorStatus', - eventAction: 'trigger', - severity: 'error', - summary: - 'Monitor {{context.monitorName}} with url {{{context.monitorUrl}}} from {{context.observerLocation}} {{{context.statusMessage}}} The latest error message is {{{context.latestErrorMessage}}}', - }, - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - }, - ] as unknown as ActionConnector[], - translations: { - defaultActionMessage: MonitorStatusTranslations.defaultActionMessage, - defaultRecoveryMessage: MonitorStatusTranslations.defaultRecoveryMessage, - defaultSubjectMessage: MonitorStatusTranslations.defaultSubjectMessage, - defaultRecoverySubjectMessage: MonitorStatusTranslations.defaultRecoverySubjectMessage, - }, - isLegacy: true, - }); - expect(resp).toEqual([ - { - group: 'recovered', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - dedupKey: expect.any(String), - eventAction: 'resolve', - summary: - 'Alert for monitor {{context.monitorName}} with url {{{context.monitorUrl}}} from {{context.observerLocation}} has recovered', - }, - }, - { - group: 'xpack.uptime.alerts.actionGroups.monitorStatus', - id: 'f2a3b195-ed76-499a-805d-82d24d4eeba9', - params: { - dedupKey: expect.any(String), - eventAction: 'trigger', - severity: 'error', - summary: MonitorStatusTranslations.defaultActionMessage, - }, - }, - ]); - }); -}); - describe('Alert Actions factory', () => { it('generate expected action for pager duty', async () => { const resp = populateAlertActions({ diff --git a/x-pack/plugins/synthetics/common/rules/alert_actions.ts b/x-pack/plugins/synthetics/common/rules/alert_actions.ts index 70a2de4d7cf59..02402c8eed944 100644 --- a/x-pack/plugins/synthetics/common/rules/alert_actions.ts +++ b/x-pack/plugins/synthetics/common/rules/alert_actions.ts @@ -44,13 +44,11 @@ export function populateAlertActions({ defaultEmail, groupId, translations, - isLegacy = false, }: { groupId: string; defaultActions: ActionConnector[]; defaultEmail?: DefaultEmail; translations: Translations; - isLegacy?: boolean; }) { const actions: RuleAction[] = []; defaultActions.forEach((aId) => { @@ -58,13 +56,11 @@ export function populateAlertActions({ id: aId.id, group: groupId, params: {}, - frequency: !isLegacy - ? { - notifyWhen: 'onActionGroupChange', - throttle: null, - summary: false, - } - : undefined, + frequency: { + notifyWhen: 'onActionGroupChange', + throttle: null, + summary: false, + }, }; const recoveredAction: RuleAction = { @@ -73,13 +69,11 @@ export function populateAlertActions({ params: { message: translations.defaultRecoveryMessage, }, - frequency: !isLegacy - ? { - notifyWhen: 'onActionGroupChange', - throttle: null, - summary: false, - } - : undefined, + frequency: { + notifyWhen: 'onActionGroupChange', + throttle: null, + summary: false, + }, }; switch (aId.actionTypeId) { @@ -95,8 +89,8 @@ export function populateAlertActions({ actions.push(recoveredAction); break; case INDEX_ACTION_ID: - action.params = getIndexActionParams(translations, false, isLegacy); - recoveredAction.params = getIndexActionParams(translations, true, isLegacy); + action.params = getIndexActionParams(translations, false); + recoveredAction.params = getIndexActionParams(translations, true); actions.push(recoveredAction); break; case SERVICE_NOW_ACTION_ID: @@ -138,41 +132,7 @@ export function populateAlertActions({ return actions; } -function getIndexActionParams( - translations: Translations, - recovery = false, - isLegacy = false -): IndexActionParams { - if (isLegacy && recovery) { - return { - documents: [ - { - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - statusMessage: translations.defaultRecoveryMessage, - latestErrorMessage: '', - observerLocation: '{{context.observerLocation}}', - }, - ], - indexOverride: null, - }; - } - - if (isLegacy) { - return { - documents: [ - { - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - statusMessage: '{{{context.statusMessage}}}', - latestErrorMessage: '{{{context.latestErrorMessage}}}', - observerLocation: '{{context.observerLocation}}', - }, - ], - indexOverride: null, - }; - } - +function getIndexActionParams(translations: Translations, recovery = false): IndexActionParams { if (recovery) { return { documents: [ diff --git a/x-pack/plugins/synthetics/common/rules/legacy_uptime/translations.ts b/x-pack/plugins/synthetics/common/rules/legacy_uptime/translations.ts deleted file mode 100644 index fab705daeb0c0..0000000000000 --- a/x-pack/plugins/synthetics/common/rules/legacy_uptime/translations.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { i18n } from '@kbn/i18n'; - -export const MonitorStatusTranslations = { - defaultActionMessage: i18n.translate( - 'xpack.synthetics.alerts.monitorStatus.defaultActionMessage', - { - defaultMessage: - 'Monitor {monitorName} with url {monitorUrl} from {observerLocation} {statusMessage} The latest error message is {latestErrorMessage}, checked at {checkedAt}', - values: { - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - statusMessage: '{{{context.statusMessage}}}', - latestErrorMessage: '{{{context.latestErrorMessage}}}', - observerLocation: '{{context.observerLocation}}', - checkedAt: '{{context.checkedAt}}', - }, - } - ), - defaultSubjectMessage: i18n.translate( - 'xpack.synthetics.alerts.monitorStatus.defaultSubjectMessage', - { - defaultMessage: 'Monitor {monitorName} with url {monitorUrl} is down', - values: { - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - }, - } - ), - defaultRecoverySubjectMessage: i18n.translate( - 'xpack.synthetics.alerts.monitorStatus.defaultRecoverySubjectMessage', - { - defaultMessage: 'Monitor {monitorName} with url {monitorUrl} has recovered', - values: { - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - }, - } - ), - defaultRecoveryMessage: i18n.translate( - 'xpack.synthetics.alerts.monitorStatus.defaultRecoveryMessage', - { - defaultMessage: - 'Alert for monitor {monitorName} with url {monitorUrl} from {observerLocation} has recovered', - values: { - monitorName: '{{context.monitorName}}', - monitorUrl: '{{{context.monitorUrl}}}', - observerLocation: '{{context.observerLocation}}', - }, - } - ), - name: i18n.translate('xpack.synthetics.alerts.monitorStatus.clientName', { - defaultMessage: 'Uptime monitor status', - }), - description: i18n.translate('xpack.synthetics.alerts.monitorStatus.description', { - defaultMessage: 'Alert when a monitor is down or an availability threshold is breached.', - }), -}; - -export const TlsTranslations = { - defaultActionMessage: i18n.translate('xpack.synthetics.alerts.tls.defaultActionMessage', { - defaultMessage: `Detected TLS certificate {commonName} from issuer {issuer} is {status}. Certificate {summary}`, - values: { - commonName: '{{context.commonName}}', - issuer: '{{context.issuer}}', - summary: '{{context.summary}}', - status: '{{context.status}}', - }, - }), - defaultRecoveryMessage: i18n.translate('xpack.synthetics.alerts.tls.defaultRecoveryMessage', { - defaultMessage: `Alert for TLS certificate {commonName} from issuer {issuer} has recovered`, - values: { - commonName: '{{context.commonName}}', - issuer: '{{context.issuer}}', - }, - }), - name: i18n.translate('xpack.synthetics.alerts.tls.clientName', { - defaultMessage: 'Uptime TLS', - }), - description: i18n.translate('xpack.synthetics.alerts.tls.description', { - defaultMessage: 'Alert when the TLS certificate of an Uptime monitor is about to expire.', - }), -}; - -export const TlsTranslationsLegacy = { - defaultActionMessage: i18n.translate('xpack.synthetics.alerts.tls.legacy.defaultActionMessage', { - defaultMessage: `Detected {count} TLS certificates expiring or becoming too old. -{expiringConditionalOpen} -Expiring cert count: {expiringCount} -Expiring Certificates: {expiringCommonNameAndDate} -{expiringConditionalClose} -{agingConditionalOpen} -Aging cert count: {agingCount} -Aging Certificates: {agingCommonNameAndDate} -{agingConditionalClose} -`, - values: { - count: '{{state.count}}', - expiringCount: '{{state.expiringCount}}', - expiringCommonNameAndDate: '{{state.expiringCommonNameAndDate}}', - expiringConditionalOpen: '{{#state.hasExpired}}', - expiringConditionalClose: '{{/state.hasExpired}}', - agingCount: '{{state.agingCount}}', - agingCommonNameAndDate: '{{state.agingCommonNameAndDate}}', - agingConditionalOpen: '{{#state.hasAging}}', - agingConditionalClose: '{{/state.hasAging}}', - }, - }), - name: i18n.translate('xpack.synthetics.alerts.tls.legacy.clientName', { - defaultMessage: 'Uptime TLS (Legacy)', - }), - description: i18n.translate('xpack.synthetics.alerts.tls.legacy.description', { - defaultMessage: - 'Alert when the TLS certificate of an Uptime monitor is about to expire. This alert will be deprecated in a future version.', - }), -}; - -export const DurationAnomalyTranslations = { - defaultActionMessage: i18n.translate( - 'xpack.synthetics.alerts.durationAnomaly.defaultActionMessage', - { - defaultMessage: `Abnormal ({severity} level) response time detected on {monitor} with url {monitorUrl} at {anomalyStartTimestamp}. Anomaly severity score is {severityScore}. -Response times as high as {slowestAnomalyResponse} have been detected from location {observerLocation}. Expected response time is {expectedResponseTime}.`, - values: { - severity: '{{context.severity}}', - anomalyStartTimestamp: '{{context.anomalyStartTimestamp}}', - monitor: '{{context.monitor}}', - monitorUrl: '{{{context.monitorUrl}}}', - slowestAnomalyResponse: '{{context.slowestAnomalyResponse}}', - expectedResponseTime: '{{context.expectedResponseTime}}', - severityScore: '{{context.severityScore}}', - observerLocation: '{{context.observerLocation}}', - }, - } - ), - defaultRecoveryMessage: i18n.translate( - 'xpack.synthetics.alerts.durationAnomaly.defaultRecoveryMessage', - { - defaultMessage: `Alert for abnormal ({severity} level) response time detected on monitor {monitor} with url {monitorUrl} from location {observerLocation} at {anomalyStartTimestamp} has recovered`, - values: { - severity: '{{context.severity}}', - anomalyStartTimestamp: '{{context.anomalyStartTimestamp}}', - monitor: '{{context.monitor}}', - monitorUrl: '{{{context.monitorUrl}}}', - observerLocation: '{{context.observerLocation}}', - }, - } - ), - name: i18n.translate('xpack.synthetics.alerts.durationAnomaly.clientName', { - defaultMessage: 'Uptime Duration Anomaly', - }), - description: i18n.translate('xpack.synthetics.alerts.durationAnomaly.description', { - defaultMessage: 'Alert when the Uptime monitor duration is anomalous.', - }), -}; diff --git a/x-pack/plugins/synthetics/common/translations.ts b/x-pack/plugins/synthetics/common/translations.ts deleted file mode 100644 index ac2bec7a5506b..0000000000000 --- a/x-pack/plugins/synthetics/common/translations.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { i18n } from '@kbn/i18n'; - -export const VALUE_MUST_BE_GREATER_THAN_ZERO = i18n.translate( - 'xpack.synthetics.settings.invalid.error', - { - defaultMessage: 'Value must be greater than 0.', - } -); - -export const VALUE_MUST_BE_AN_INTEGER = i18n.translate( - 'xpack.synthetics.settings.invalid.nanError', - { - defaultMessage: 'Value must be an integer.', - } -); diff --git a/x-pack/plugins/synthetics/common/types/index.ts b/x-pack/plugins/synthetics/common/types/index.ts index 6a864b700092d..2a70803a43211 100644 --- a/x-pack/plugins/synthetics/common/types/index.ts +++ b/x-pack/plugins/synthetics/common/types/index.ts @@ -5,7 +5,5 @@ * 2.0. */ -export * from './monitor_duration'; export * from './synthetics_monitor'; export * from './monitor_validation'; -export * from './integration_deprecation'; diff --git a/x-pack/plugins/synthetics/common/types/monitor_duration.ts b/x-pack/plugins/synthetics/common/types/monitor_duration.ts deleted file mode 100644 index 253adba03cdcf..0000000000000 --- a/x-pack/plugins/synthetics/common/types/monitor_duration.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/** Represents the average monitor duration ms at a point in time. */ -export interface MonitorDurationAveragePoint { - /** The timeseries value for this point. */ - x: number; - /** The average duration ms for the monitor. */ - y?: number | null; -} - -export interface LocationDurationLine { - name: string; - - line: MonitorDurationAveragePoint[]; -} - -/** The data used to populate the monitor charts. */ -export interface MonitorDurationResult { - /** The average values for the monitor duration. */ - locationDurationLines: LocationDurationLine[]; -} diff --git a/x-pack/plugins/synthetics/common/utils/get_monitor_url.ts b/x-pack/plugins/synthetics/common/utils/get_monitor_url.ts deleted file mode 100644 index 09b02150957d0..0000000000000 --- a/x-pack/plugins/synthetics/common/utils/get_monitor_url.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { stringify } from 'querystring'; - -export const format = ({ - pathname, - query, -}: { - pathname: string; - query: Record; -}): string => { - return `${pathname}?${stringify(query)}`; -}; - -export const getMonitorRouteFromMonitorId = ({ - monitorId, - dateRangeStart, - dateRangeEnd, - filters = {}, -}: { - monitorId: string; - dateRangeStart: string; - dateRangeEnd: string; - filters?: Record; -}) => - format({ - pathname: `/app/uptime/monitor/${btoa(monitorId)}`, - query: { - dateRangeEnd, - dateRangeStart, - ...(Object.keys(filters).length - ? { filters: JSON.stringify(Object.keys(filters).map((key) => [key, filters[key]])) } - : {}), - }, - }); diff --git a/x-pack/plugins/synthetics/common/utils/get_synthetics_monitor_url.ts b/x-pack/plugins/synthetics/common/utils/get_synthetics_monitor_url.ts index 007b0c5812dce..7bd962b0b1640 100644 --- a/x-pack/plugins/synthetics/common/utils/get_synthetics_monitor_url.ts +++ b/x-pack/plugins/synthetics/common/utils/get_synthetics_monitor_url.ts @@ -4,9 +4,11 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { stringify } from 'querystring'; -import { format } from './get_monitor_url'; - +const format = ({ pathname, query }: { pathname: string; query: Record }): string => { + return `${pathname}?${stringify(query)}`; +}; export const getSyntheticsErrorRouteFromMonitorId = ({ configId, stateId, diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/state/settings/api.ts b/x-pack/plugins/synthetics/public/apps/synthetics/state/settings/api.ts index da8855f531ef9..b5430aae3239d 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/state/settings/api.ts +++ b/x-pack/plugins/synthetics/public/apps/synthetics/state/settings/api.ts @@ -19,10 +19,10 @@ import { LocationMonitorsResponse, LocationMonitorsType, } from '../../../../../common/runtime_types'; -import { API_URLS, SYNTHETICS_API_URLS } from '../../../../../common/constants'; +import { SYNTHETICS_API_URLS } from '../../../../../common/constants'; import { LocationMonitor } from '.'; -const apiPath = API_URLS.DYNAMIC_SETTINGS; +const apiPath = SYNTHETICS_API_URLS.DYNAMIC_SETTINGS; interface SaveApiRequest { settings: DynamicSettings; @@ -51,7 +51,7 @@ export const fetchLocationMonitors = async (): Promise => { export type ActionConnector = Omit; export const fetchConnectors = async (): Promise => { - const response = (await apiService.get(API_URLS.RULE_CONNECTORS)) as Array< + const response = (await apiService.get(SYNTHETICS_API_URLS.RULE_CONNECTORS)) as Array< AsApiContract >; return response.map( @@ -76,7 +76,7 @@ export const fetchConnectors = async (): Promise => { }; export const fetchActionTypes = async (): Promise => { - const response = (await apiService.get(API_URLS.CONNECTOR_TYPES, { + const response = (await apiService.get(SYNTHETICS_API_URLS.CONNECTOR_TYPES, { feature_id: 'uptime', })) as Array>; return response.map( diff --git a/x-pack/plugins/synthetics/server/feature.ts b/x-pack/plugins/synthetics/server/feature.ts index ac5bce8a6b790..265def5c3fa01 100644 --- a/x-pack/plugins/synthetics/server/feature.ts +++ b/x-pack/plugins/synthetics/server/feature.ts @@ -10,10 +10,15 @@ import { syntheticsMonitorType, syntheticsParamType } from '../common/types/save import { SYNTHETICS_RULE_TYPES } from '../common/constants/synthetics_alerts'; import { privateLocationsSavedObjectName } from '../common/saved_objects/private_locations'; import { PLUGIN } from '../common/constants/plugin'; -import { UPTIME_RULE_TYPES } from '../common/constants/uptime_alerts'; import { settingsObjectType } from './saved_objects/uptime_settings'; import { syntheticsApiKeyObjectType } from './saved_objects/service_api_key'; +const UPTIME_RULE_TYPES = [ + 'xpack.uptime.alerts.tls', + 'xpack.uptime.alerts.tlsCertificate', + 'xpack.uptime.alerts.monitorStatus', + 'xpack.uptime.alerts.durationAnomaly', +]; export const uptimeFeature = { id: PLUGIN.ID, name: PLUGIN.NAME, diff --git a/x-pack/plugins/synthetics/server/lib.ts b/x-pack/plugins/synthetics/server/lib.ts index 9a7c280d3434e..4dc06bc7b9674 100644 --- a/x-pack/plugins/synthetics/server/lib.ts +++ b/x-pack/plugins/synthetics/server/lib.ts @@ -18,7 +18,7 @@ import { RequestStatus } from '@kbn/inspector-plugin/common'; import { InspectResponse } from '@kbn/observability-plugin/typings/common'; import { enableInspectEsQueries } from '@kbn/observability-plugin/common'; import { getInspectResponse } from '@kbn/observability-shared-plugin/common'; -import { API_URLS } from '../common/constants'; +import { SYNTHETICS_API_URLS } from '../common/constants'; import { SyntheticsServerSetup } from './types'; import { savedObjectsAdapter } from './saved_objects/saved_objects'; @@ -168,7 +168,7 @@ export class UptimeEsClient { async getInspectData(path: string) { const isInspectorEnabled = await this.getInspectEnabled(); const showInspectData = - (isInspectorEnabled || this.isDev) && path !== API_URLS.DYNAMIC_SETTINGS; + (isInspectorEnabled || this.isDev) && path !== SYNTHETICS_API_URLS.DYNAMIC_SETTINGS; if (showInspectData) { return { _inspect: this.inspectableEsQueries }; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 64ffa9ac501f5..442a6ebedd0bd 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -4738,7 +4738,6 @@ "kibana-react.kibanaCodeEditor.ariaLabel": "Éditeur de code", "kibana-react.kibanaCodeEditor.enterKeyLabel": "Entrée", "kibana-react.kibanaCodeEditor.escapeKeyLabel": "Échap", - "reactPackages.mountPointPortal.errorMessage": "Erreur lors du rendu du contenu du portail.", "kibana-react.noDataPage.cantDecide.link": "Consultez la documentation pour en savoir plus.", "kibana-react.noDataPage.elasticAgentCard.description": "Utilisez Elastic Agent pour collecter de manière simple et unifiée les données de vos machines.", "kibana-react.noDataPage.elasticAgentCard.noPermission.description": "Cette intégration n'est pas encore activée. Votre administrateur possède les autorisations requises pour l'activer.", @@ -4752,6 +4751,7 @@ "kibana-react.pageFooter.makeDefaultRouteLink": "Choisir comme page de destination", "kibana-react.solutionNav.collapsibleLabel": "Réduire la navigation latérale", "kibana-react.solutionNav.openLabel": "Ouvrir la navigation latérale", + "reactPackages.mountPointPortal.errorMessage": "Erreur lors du rendu du contenu du portail.", "languageDocumentationPopover.header": "Référence de {language}", "languageDocumentationPopover.tooltip": "Référence de {lang}", "languageDocumentationPopover.searchPlaceholder": "Recherche", @@ -36834,12 +36834,6 @@ "xpack.stackConnectors.xmatters.title": "xMatters", "xpack.stackConnectors.xmatters.unexpectedNullResponseErrorMessage": "réponse nulle inattendue de xmatters", "xpack.synthetics.alertRules.monitorStatus.reasonMessage": "Le moniteur \"{name}\" de {location} est {status}. Vérifié à {checkedAt}.", - "xpack.synthetics.alerts.durationAnomaly.defaultActionMessage": "Temps de réponse anormal (niveau {severity}) détecté sur le {monitor} avec l'URL {monitorUrl} à {anomalyStartTimestamp}. La note de sévérité d'anomalie est {severityScore}.\nDes temps de réponse aussi élevés que {slowestAnomalyResponse} ont été détectés à partir de l'emplacement {observerLocation}. Le temps de réponse attendu est {expectedResponseTime}.", - "xpack.synthetics.alerts.durationAnomaly.defaultRecoveryMessage": "L'alerte pour temps de réponse anormal (niveau {severity}) détecté sur le moniteur {monitor} possédant l'URL {monitorUrl} depuis l'emplacement {observerLocation} à {anomalyStartTimestamp} a été résolue", - "xpack.synthetics.alerts.monitorStatus.defaultActionMessage": "Moniteur {monitorName} avec l'URL {monitorUrl} depuis {observerLocation} {statusMessage} Le dernier message d'erreur est {latestErrorMessage}, vérifié à {checkedAt}", - "xpack.synthetics.alerts.monitorStatus.defaultRecoveryMessage": "L'alerte pour le moniteur {monitorName} avec l'URL {monitorUrl} depuis {observerLocation} a été résolue", - "xpack.synthetics.alerts.monitorStatus.defaultRecoverySubjectMessage": "Le moniteur {monitorName} avec l'URL {monitorUrl} a récupéré", - "xpack.synthetics.alerts.monitorStatus.defaultSubjectMessage": "Le moniteur {monitorName} avec l'URL {monitorUrl} est arrêté", "xpack.synthetics.alerts.monitorStatus.upCheck.reasonWithoutDuration": "le moniteur est de nouveau opérationnel. Il a été exécuté avec succès à {checkedAt}", "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultActionMessage": "\"{monitorName}\" est {status} de {locationName}. - Elastic Synthetics\n\nDétails :\n\n- Nom du moniteur : {monitorName} \n- {monitorUrlLabel} : {monitorUrl} \n- Type du moniteur : {monitorType} \n- Vérifié à : {checkedAt} \n- De : {locationName} \n- Erreur reçue : {lastErrorMessage} \n{linkMessage}", "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultRecoveryMessage": "L’alerte pour \"{monitorName}\" de {locationName} n’est plus active : {recoveryReason}. - Elastic Synthetics\n\nDétails :\n\n- Nom du moniteur : {monitorName} \n- {monitorUrlLabel} : {monitorUrl} \n- Type du moniteur : {monitorType} \n- De : {locationName} \n- Dernière erreur reçue : {lastErrorMessage} \n{linkMessage}", @@ -36847,9 +36841,6 @@ "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultSubjectMessage": "\"{monitorName}\" ({locationName}) est arrêté - Elastic Synthetics", "xpack.synthetics.alerts.syntheticsMonitorTLS.defaultRecoverySubjectMessage": "L’alerte a été résolue pour le certificat {commonName} - Elastic Synthetics", "xpack.synthetics.alerts.syntheticsMonitorTLS.defaultSubjectMessage": "L’alerte a été déclenchée pour le certificat {commonName} - Elastic Synthetics", - "xpack.synthetics.alerts.tls.defaultActionMessage": "Le certificat TLS {commonName} détecté de l'émetteur {issuer} est {status}. Certificat {summary}", - "xpack.synthetics.alerts.tls.defaultRecoveryMessage": "L'alerte pour le certificat TLS {commonName} de l'émetteur {issuer} a été résolue", - "xpack.synthetics.alerts.tls.legacy.defaultActionMessage": "Détection de {count} certificats TLS sur le point d'expirer ou devenant trop anciens.\n{expiringConditionalOpen}\nNombre de certificats sur le point d'expirer : {expiringCount}\nCertificats sur le point d'expirer : {expiringCommonNameAndDate}\n{expiringConditionalClose}\n{agingConditionalOpen}\nNombre de certificats vieillissants : {agingCount}\nCertificats vieillissants : {agingCommonNameAndDate}\n{agingConditionalClose}\n", "xpack.synthetics.certificates.heading": "Certificats TLS ({total})", "xpack.synthetics.certificatesRoute.title": "Certificats | {baseTitle}", "xpack.synthetics.certs.status.ok.label": " pour {okRelativeDate}", @@ -37015,14 +37006,10 @@ "xpack.synthetics.alertRules.monitorStatus.browser.label": "navigateur", "xpack.synthetics.alertRules.monitorStatus.host.label": "Hôte", "xpack.synthetics.alertRules.monitorStatus.unavailableUrlLabel": "(indisponible)", - "xpack.synthetics.alerts.durationAnomaly.clientName": "Anomalie de durée Uptime", - "xpack.synthetics.alerts.durationAnomaly.description": "Alerte lorsque la durée du moniteur Uptime est anormale.", "xpack.synthetics.alerts.monitorStatus.absoluteLink.label": "- Lien", - "xpack.synthetics.alerts.monitorStatus.clientName": "Statut du moniteur Uptime", "xpack.synthetics.alerts.monitorStatus.defaultRecovery.status": "a récupéré", "xpack.synthetics.alerts.monitorStatus.deleteMonitor.reason": "le moniteur a été supprimé", "xpack.synthetics.alerts.monitorStatus.deleteMonitor.status": "a été supprimé", - "xpack.synthetics.alerts.monitorStatus.description": "Alerte lorsqu'un monitoring est arrêté ou qu'un seuil de disponibilité est dépassé.", "xpack.synthetics.alerts.monitorStatus.downLabel": "bas", "xpack.synthetics.alerts.monitorStatus.relativeLink.label": "- Lien relatif", "xpack.synthetics.alerts.monitorStatus.removedLocation.reason": "cet emplacement a été retiré du moniteur", @@ -37032,11 +37019,7 @@ "xpack.synthetics.alerts.settings.addConnector": "Ajouter un connecteur", "xpack.synthetics.alerts.syntheticsMonitorStatus.clientName": "Statut du moniteur", "xpack.synthetics.alerts.syntheticsMonitorStatus.description": "Alerte lorsqu'un moniteur est arrêté.", - "xpack.synthetics.alerts.tls.clientName": "Uptime TLS", "xpack.synthetics.alerts.tls.criteriaExpression.description": "quand", - "xpack.synthetics.alerts.tls.description": "Alerte lorsque le certificat TLS d'un moniteur Uptime est sur le point d'expirer.", - "xpack.synthetics.alerts.tls.legacy.clientName": "Uptime TLS (existant)", - "xpack.synthetics.alerts.tls.legacy.description": "Alerte lorsque le certificat TLS d'un moniteur Uptime est sur le point d'expirer. Cette alerte sera déclassée dans une future version.", "xpack.synthetics.alerts.toggleAlertFlyoutButtonText": "Alertes et règles", "xpack.synthetics.alertsRulesPopover.toggleButton.ariaLabel": "Ouvrir les alertes et le menu des règles", "xpack.synthetics.analyzeDataButtonLabel": "Explorer les données", @@ -37782,8 +37765,6 @@ "xpack.synthetics.settings.enablement.fail": "Échec de l’activation de l’application Synthetics", "xpack.synthetics.settings.error.couldNotSave": "Impossible d'enregistrer les paramètres !", "xpack.synthetics.settings.getParams.failed": "Impossible d’obtenir les paramètres globaux.", - "xpack.synthetics.settings.invalid.error": "La valeur doit être supérieure à 0.", - "xpack.synthetics.settings.invalid.nanError": "La valeur doit être un entier.", "xpack.synthetics.settings.noSpace.error": "Les noms d'index ne doivent pas contenir d'espace", "xpack.synthetics.settings.saveSuccess": "Paramètres enregistrés !", "xpack.synthetics.settings.syncGlobalParams": "Paramètres globaux appliqués correctement à tous les moniteurs", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 8eabc63eb8ea9..ed681f32a13a3 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -4754,7 +4754,6 @@ "kibana-react.kibanaCodeEditor.ariaLabel": "コードエディター", "kibana-react.kibanaCodeEditor.enterKeyLabel": "Enter", "kibana-react.kibanaCodeEditor.escapeKeyLabel": "Esc", - "reactPackages.mountPointPortal.errorMessage": "ポータルコンテンツのレンダリングエラー", "kibana-react.noDataPage.cantDecide.link": "詳細については、ドキュメントをご確認ください。", "kibana-react.noDataPage.elasticAgentCard.description": "Elasticエージェントを使用すると、シンプルで統一された方法でコンピューターからデータを収集するできます。", "kibana-react.noDataPage.elasticAgentCard.noPermission.description": "この統合はまだ有効ではありません。管理者にはオンにするために必要なアクセス権があります。", @@ -4768,6 +4767,7 @@ "kibana-react.pageFooter.makeDefaultRouteLink": "これをランディングページにする", "kibana-react.solutionNav.collapsibleLabel": "サイドナビゲーションを折りたたむ", "kibana-react.solutionNav.openLabel": "サイドナビゲーションを開く", + "reactPackages.mountPointPortal.errorMessage": "ポータルコンテンツのレンダリングエラー", "languageDocumentationPopover.header": "{language}リファレンス", "languageDocumentationPopover.tooltip": "{lang}リファレンス", "languageDocumentationPopover.searchPlaceholder": "検索", @@ -36833,12 +36833,6 @@ "xpack.stackConnectors.xmatters.title": "xMatters", "xpack.stackConnectors.xmatters.unexpectedNullResponseErrorMessage": "xmattersからの予期しないnull応答", "xpack.synthetics.alertRules.monitorStatus.reasonMessage": "{location}からのモニター\"{name}\"は{status}です。確認：{checkedAt}。", - "xpack.synthetics.alerts.durationAnomaly.defaultActionMessage": "{monitor}で{anomalyStartTimestamp}で{monitorUrl}のurlで異常な（{severity}レベル）応答時間を検出しました。異常重要度スコアは{severityScore}です。\n{observerLocation}の位置から{slowestAnomalyResponse}の高い応答時間が検出されています。想定応答時間は{expectedResponseTime}です。", - "xpack.synthetics.alerts.durationAnomaly.defaultRecoveryMessage": "{anomalyStartTimestamp}の{observerLocation}地点から{monitorUrl}のモニター{monitor}で検出された応答時間異常（{severity}レベル）のアラートが回復しました", - "xpack.synthetics.alerts.monitorStatus.defaultActionMessage": "{monitorName} を {observerLocation} から {monitorUrl} のurlで監視する {statusMessage} 最新のエラーメッセージは {latestErrorMessage} で、{checkedAt} で確認", - "xpack.synthetics.alerts.monitorStatus.defaultRecoveryMessage": "{observerLocation}から{monitorUrl}のurlを持つモニター{monitorName}に対するアラートが回復しました", - "xpack.synthetics.alerts.monitorStatus.defaultRecoverySubjectMessage": "URL {monitorUrl}のモニター\"{monitorName}\"が回復しました", - "xpack.synthetics.alerts.monitorStatus.defaultSubjectMessage": "url {monitorUrl}のモニタ{monitorName}がダウンしています", "xpack.synthetics.alerts.monitorStatus.upCheck.reasonWithoutDuration": "モニターが再起動しました。{checkedAt}に正常に実行されました。", "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultActionMessage": "\"{monitorName}\"は{locationName}からの{status}です。- Elastic Synthetics\n\n詳細：\n\n- モニター名：{monitorName} \n- {monitorUrlLabel}: {monitorUrl} \n- モニタータイプ：{monitorType} \n- 確認日時：{checkedAt} \n- 場所：{locationName} \n- 受信したエラー：{lastErrorMessage} \n{linkMessage}", "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultRecoveryMessage": "{locationName}からの\"{monitorName}\"のアラートはアクティブではありません：{recoveryReason}。- Elastic Synthetics\n\n詳細：\n\n- モニター名：{monitorName} \n- {monitorUrlLabel}: {monitorUrl} \n- モニタータイプ：{monitorType} \n- 場所：{locationName} \n- 最後に受信したエラー：{lastErrorMessage} \n{linkMessage}", @@ -36846,9 +36840,6 @@ "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultSubjectMessage": "\"{monitorName}\"（{locationName}）は停止しています - Elastic Synthetics", "xpack.synthetics.alerts.syntheticsMonitorTLS.defaultRecoverySubjectMessage": "証明書{commonName}のアラートが解決しました - Elastic Synthetics", "xpack.synthetics.alerts.syntheticsMonitorTLS.defaultSubjectMessage": "証明書{commonName}のアラートがトリガーされました - Elastic Synthetics", - "xpack.synthetics.alerts.tls.defaultActionMessage": "発行者{issuer}のTLS証明書{commonName}が{status}であることを検出しました。証明書{summary}", - "xpack.synthetics.alerts.tls.defaultRecoveryMessage": "発行者{issuer}のTLS証明書{commonName}のアラートが回復しました", - "xpack.synthetics.alerts.tls.legacy.defaultActionMessage": "期限切れになるか古くなりすぎた{count}個のTLS証明書証明書を検知しました。\n{expiringConditionalOpen}\n有効期限切れになる証明書件数：{expiringCount}\n有効期限切れになる証明書：{expiringCommonNameAndDate}\n{expiringConditionalClose}\n{agingConditionalOpen}\n古い証明書件数：{agingCount}\n古い証明書：{agingCommonNameAndDate}\n{agingConditionalClose}\n", "xpack.synthetics.certificates.heading": "TLS証明書（{total}）", "xpack.synthetics.certificatesRoute.title": "証明書 | {baseTitle}", "xpack.synthetics.certs.status.ok.label": " {okRelativeDate}", @@ -37014,14 +37005,10 @@ "xpack.synthetics.alertRules.monitorStatus.browser.label": "ブラウザー", "xpack.synthetics.alertRules.monitorStatus.host.label": "ホスト", "xpack.synthetics.alertRules.monitorStatus.unavailableUrlLabel": "（使用不可）", - "xpack.synthetics.alerts.durationAnomaly.clientName": "アップタイム期間異常", - "xpack.synthetics.alerts.durationAnomaly.description": "アップタイム監視期間が異常なときにアラートを発行します。", "xpack.synthetics.alerts.monitorStatus.absoluteLink.label": "- リンク", - "xpack.synthetics.alerts.monitorStatus.clientName": "稼働状況の監視ステータス", "xpack.synthetics.alerts.monitorStatus.defaultRecovery.status": "回復しました", "xpack.synthetics.alerts.monitorStatus.deleteMonitor.reason": "モニターが削除されました", "xpack.synthetics.alerts.monitorStatus.deleteMonitor.status": "が削除されました", - "xpack.synthetics.alerts.monitorStatus.description": "監視が停止しているか、可用性しきい値に違反したときにアラートを発行します。", "xpack.synthetics.alerts.monitorStatus.downLabel": "ダウン", "xpack.synthetics.alerts.monitorStatus.relativeLink.label": "- 相対リンク", "xpack.synthetics.alerts.monitorStatus.removedLocation.reason": "この場所はモニターから削除されました", @@ -37031,11 +37018,7 @@ "xpack.synthetics.alerts.settings.addConnector": "コネクターの追加", "xpack.synthetics.alerts.syntheticsMonitorStatus.clientName": "監視ステータス", "xpack.synthetics.alerts.syntheticsMonitorStatus.description": "モニターがダウンしているときにアラートを通知します。", - "xpack.synthetics.alerts.tls.clientName": "アップタイムTLS", "xpack.synthetics.alerts.tls.criteriaExpression.description": "タイミング", - "xpack.synthetics.alerts.tls.description": "アップタイム監視の TLS 証明書の有効期限が近いときにアラートを発行します。", - "xpack.synthetics.alerts.tls.legacy.clientName": "アップタイムTLS（レガシー）", - "xpack.synthetics.alerts.tls.legacy.description": "アップタイム監視の TLS 証明書の有効期限が近いときにアラートを発行します。このアラートは将来のバージョンで廃止予定です。", "xpack.synthetics.alerts.toggleAlertFlyoutButtonText": "アラートとルール", "xpack.synthetics.alertsRulesPopover.toggleButton.ariaLabel": "アラートおよびルールメニューを開く", "xpack.synthetics.analyzeDataButtonLabel": "データの探索", @@ -37781,8 +37764,6 @@ "xpack.synthetics.settings.enablement.fail": "Syntheticsアプリを有効化できませんでした", "xpack.synthetics.settings.error.couldNotSave": "設定を保存できませんでした！", "xpack.synthetics.settings.getParams.failed": "グローバルパラメーターを取得できませんでした。", - "xpack.synthetics.settings.invalid.error": "値は0よりも大きい値でなければなりません。", - "xpack.synthetics.settings.invalid.nanError": "値は整数でなければなりません。", "xpack.synthetics.settings.noSpace.error": "インデックス名にはスペースを使用できません", "xpack.synthetics.settings.saveSuccess": "設定が保存されました。", "xpack.synthetics.settings.syncGlobalParams": "グローバルパラメーターをすべてのモニターに正常に適用しました", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 61295a1cc366b..918538b3edfad 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -4753,7 +4753,6 @@ "kibana-react.kibanaCodeEditor.ariaLabel": "代码编辑器", "kibana-react.kibanaCodeEditor.enterKeyLabel": "Enter", "kibana-react.kibanaCodeEditor.escapeKeyLabel": "Esc", - "reactPackages.mountPointPortal.errorMessage": "呈现门户内容时出错", "kibana-react.noDataPage.cantDecide.link": "请参阅我们的文档以了解更多信息。", "kibana-react.noDataPage.elasticAgentCard.description": "使用 Elastic 代理以简单统一的方式从您的计算机中收集数据。", "kibana-react.noDataPage.elasticAgentCard.noPermission.description": "尚未启用此集成。您的管理员具有打开它所需的权限。", @@ -4767,6 +4766,7 @@ "kibana-react.pageFooter.makeDefaultRouteLink": "将此设为我的登陆页面", "kibana-react.solutionNav.collapsibleLabel": "折叠侧边导航", "kibana-react.solutionNav.openLabel": "打开侧边导航", + "reactPackages.mountPointPortal.errorMessage": "呈现门户内容时出错", "languageDocumentationPopover.header": "{language} 参考", "languageDocumentationPopover.tooltip": "{lang} 参考", "languageDocumentationPopover.searchPlaceholder": "搜索", @@ -36827,12 +36827,6 @@ "xpack.stackConnectors.xmatters.title": "xMatters", "xpack.stackConnectors.xmatters.unexpectedNullResponseErrorMessage": "来自 xmatters 的异常空响应", "xpack.synthetics.alertRules.monitorStatus.reasonMessage": "来自 {location} 的监测“{name}”为 {status}。已于 {checkedAt} 检查。", - "xpack.synthetics.alerts.durationAnomaly.defaultActionMessage": "{anomalyStartTimestamp} 在 url {monitorUrl} 的 {monitor} 上检测到异常（{severity} 级别）响应时间。异常严重性分数为 {severityScore}。\n从位置 {observerLocation} 检测到高达 {slowestAnomalyResponse} 的响应时间。预期响应时间为 {expectedResponseTime}。", - "xpack.synthetics.alerts.durationAnomaly.defaultRecoveryMessage": "{anomalyStartTimestamp} 从位置 {observerLocation} 在 url {monitorUrl} 的监测 {monitor} 上检测到异常（{severity} 级别）响应时间的告警已恢复", - "xpack.synthetics.alerts.monitorStatus.defaultActionMessage": "在 {observerLocation}，URL 为 {monitorUrl} 的监测 {monitorName} 是 {statusMessage} 最新错误消息是 {latestErrorMessage}，已于 {checkedAt} 检查", - "xpack.synthetics.alerts.monitorStatus.defaultRecoveryMessage": "来自 {observerLocation} 且 url 为 {monitorUrl} 的监测 {monitorName} 的告警已恢复", - "xpack.synthetics.alerts.monitorStatus.defaultRecoverySubjectMessage": "URL 为 {monitorUrl} 的监测 {monitorName} 已恢复", - "xpack.synthetics.alerts.monitorStatus.defaultSubjectMessage": "URL 为 {monitorUrl} 的监测 {monitorName} 已关闭", "xpack.synthetics.alerts.monitorStatus.upCheck.reasonWithoutDuration": "监测现已再次启动。它于 {checkedAt}成功运行", "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultActionMessage": "“{monitorName}”为来自 {locationName} 的 {status} 状态 - Elastic Synthetics\n\n详情：\n\n- 监测名称：{monitorName} \n- {monitorUrlLabel}：{monitorUrl} \n- 监测类型：{monitorType} \n- 检查时间：{checkedAt} \n- 来自：{locationName} \n- 收到错误：{lastErrorMessage} \n{linkMessage}", "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultRecoveryMessage": "来自 {locationName} 的“{monitorName}”的告警不再处于活动状态：{recoveryReason} - Elastic Synthetics\n\n详情：\n\n- 监测名称：{monitorName} \n- {monitorUrlLabel}：{monitorUrl} \n- 监测类型：{monitorType} \n- 来自：{locationName} \n- 收到的上一个错误：{lastErrorMessage} \n{linkMessage}", @@ -36840,9 +36834,6 @@ "xpack.synthetics.alerts.syntheticsMonitorStatus.defaultSubjectMessage": "“{monitorName}”({locationName}) 已关闭 - Elastic Synthetics", "xpack.synthetics.alerts.syntheticsMonitorTLS.defaultRecoverySubjectMessage": "告警已解析证书 {commonName} - Elastic Synthetics", "xpack.synthetics.alerts.syntheticsMonitorTLS.defaultSubjectMessage": "已针对证书 {commonName} 触发告警 - Elastic Synthetics", - "xpack.synthetics.alerts.tls.defaultActionMessage": "检测到来自颁发者 {issuer} 的 TLS 证书 {commonName} 的状态为 {status}。证书 {summary}", - "xpack.synthetics.alerts.tls.defaultRecoveryMessage": "来自颁发者 {issuer} 的 TLS 证书 {commonName} 的告警已恢复", - "xpack.synthetics.alerts.tls.legacy.defaultActionMessage": "检测到 {count} 个 TLS 证书即将过期或即将过时。\n{expiringConditionalOpen}\n即将过期的证书计数：{expiringCount}\n即将过期的证书：{expiringCommonNameAndDate}\n{expiringConditionalClose}\n{agingConditionalOpen}\n过时的证书计数：{agingCount}\n过时的证书：{agingCommonNameAndDate}\n{agingConditionalClose}\n", "xpack.synthetics.certificates.heading": "TLS 证书 ({total})", "xpack.synthetics.certificatesRoute.title": "证书 | {baseTitle}", "xpack.synthetics.certs.status.ok.label": " 对于 {okRelativeDate}", @@ -37008,14 +36999,10 @@ "xpack.synthetics.alertRules.monitorStatus.browser.label": "浏览器", "xpack.synthetics.alertRules.monitorStatus.host.label": "主机", "xpack.synthetics.alertRules.monitorStatus.unavailableUrlLabel": "（不可用）", - "xpack.synthetics.alerts.durationAnomaly.clientName": "Uptime 持续时间异常", - "xpack.synthetics.alerts.durationAnomaly.description": "运行时间监测持续时间异常时告警。", "xpack.synthetics.alerts.monitorStatus.absoluteLink.label": "- 链接", - "xpack.synthetics.alerts.monitorStatus.clientName": "运行时间监测状态", "xpack.synthetics.alerts.monitorStatus.defaultRecovery.status": "已恢复", "xpack.synthetics.alerts.monitorStatus.deleteMonitor.reason": "此监测已删除", "xpack.synthetics.alerts.monitorStatus.deleteMonitor.status": "已删除", - "xpack.synthetics.alerts.monitorStatus.description": "监测关闭或超出可用性阈值时告警。", "xpack.synthetics.alerts.monitorStatus.downLabel": "关闭", "xpack.synthetics.alerts.monitorStatus.relativeLink.label": "- 相对链接", "xpack.synthetics.alerts.monitorStatus.removedLocation.reason": "此位置已从监测中移除", @@ -37025,11 +37012,7 @@ "xpack.synthetics.alerts.settings.addConnector": "添加连接器", "xpack.synthetics.alerts.syntheticsMonitorStatus.clientName": "监测状态", "xpack.synthetics.alerts.syntheticsMonitorStatus.description": "监测关闭时告警。", - "xpack.synthetics.alerts.tls.clientName": "Uptime TLS", "xpack.synthetics.alerts.tls.criteriaExpression.description": "当", - "xpack.synthetics.alerts.tls.description": "运行时间监测的 TLS 证书即将过期时告警。", - "xpack.synthetics.alerts.tls.legacy.clientName": "Uptime TLS（旧版）", - "xpack.synthetics.alerts.tls.legacy.description": "运行时间监测的 TLS 证书即将过期时告警。未来的版本将弃用此告警。", "xpack.synthetics.alerts.toggleAlertFlyoutButtonText": "告警和规则", "xpack.synthetics.alertsRulesPopover.toggleButton.ariaLabel": "打开告警和规则菜单", "xpack.synthetics.analyzeDataButtonLabel": "浏览数据", @@ -37775,8 +37758,6 @@ "xpack.synthetics.settings.enablement.fail": "无法启用 Synthetics 应用", "xpack.synthetics.settings.error.couldNotSave": "无法保存设置！", "xpack.synthetics.settings.getParams.failed": "无法获取全局参数。", - "xpack.synthetics.settings.invalid.error": "值必须大于 0。", - "xpack.synthetics.settings.invalid.nanError": "值必须为整数。", "xpack.synthetics.settings.noSpace.error": "索引名称不得包含空格", "xpack.synthetics.settings.saveSuccess": "设置已保存！", "xpack.synthetics.settings.syncGlobalParams": "已成功将全局参数应用到所有监测", diff --git a/x-pack/plugins/synthetics/common/types/integration_deprecation.ts b/x-pack/plugins/uptime/common/index.ts similarity index 72% rename from x-pack/plugins/synthetics/common/types/integration_deprecation.ts rename to x-pack/plugins/uptime/common/index.ts index 951de85c046b6..50725f0c3a90a 100644 --- a/x-pack/plugins/synthetics/common/types/integration_deprecation.ts +++ b/x-pack/plugins/uptime/common/index.ts @@ -5,6 +5,4 @@ * 2.0. */ -export interface SyntheticsHasIntegrationMonitorsResponse { - hasIntegrationMonitors: boolean; -} +export { UPTIME_RULE_TYPES } from './constants/uptime_alerts'; diff --git a/x-pack/test/alerting_api_integration/observability/synthetics_rule.ts b/x-pack/test/alerting_api_integration/observability/synthetics_rule.ts index 21192aa2075f9..888d2a0d7f897 100644 --- a/x-pack/test/alerting_api_integration/observability/synthetics_rule.ts +++ b/x-pack/test/alerting_api_integration/observability/synthetics_rule.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; -import { API_URLS, SYNTHETICS_API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { SYNTHETICS_API_URLS } from '@kbn/synthetics-plugin/common/constants'; import { SanitizedRule } from '@kbn/alerting-plugin/common'; import { omit } from 'lodash'; import { TlsTranslations } from '@kbn/synthetics-plugin/common/rules/synthetics/translations'; @@ -36,7 +36,7 @@ export default function ({ getService }: FtrProviderContext) { it('creates rule when settings are configured', async () => { await supertest - .post(API_URLS.DYNAMIC_SETTINGS) + .post(SYNTHETICS_API_URLS.DYNAMIC_SETTINGS) .set('kbn-xsrf', 'true') .send({ heartbeatIndices: 'heartbeat-8*,heartbeat-7*', @@ -76,7 +76,7 @@ export default function ({ getService }: FtrProviderContext) { it('updates rules when settings are updated', async () => { await supertest - .post(API_URLS.DYNAMIC_SETTINGS) + .post(SYNTHETICS_API_URLS.DYNAMIC_SETTINGS) .set('kbn-xsrf', 'true') .send({ heartbeatIndices: 'heartbeat-8*,heartbeat-7*', diff --git a/x-pack/test/api_integration/apis/uptime/feature_controls.ts b/x-pack/test/api_integration/apis/uptime/feature_controls.ts index 39d7406636353..c9c545cbad242 100644 --- a/x-pack/test/api_integration/apis/uptime/feature_controls.ts +++ b/x-pack/test/api_integration/apis/uptime/feature_controls.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { FtrProviderContext } from '../../ftr_provider_context'; import { PINGS_DATE_RANGE_END, PINGS_DATE_RANGE_START } from './constants'; diff --git a/x-pack/test/api_integration/apis/uptime/get_all_pings.ts b/x-pack/test/api_integration/apis/uptime/get_all_pings.ts index 19b78854a8df3..a7547d61992f2 100644 --- a/x-pack/test/api_integration/apis/uptime/get_all_pings.ts +++ b/x-pack/test/api_integration/apis/uptime/get_all_pings.ts @@ -7,7 +7,7 @@ import moment from 'moment'; import expect from '@kbn/expect'; -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { PINGS_DATE_RANGE_START, PINGS_DATE_RANGE_END } from './constants'; import { FtrProviderContext } from '../../ftr_provider_context'; diff --git a/x-pack/test/api_integration/apis/uptime/rest/dynamic_settings.ts b/x-pack/test/api_integration/apis/uptime/rest/dynamic_settings.ts index 44ab0f694ba5d..8ecdbc9b615da 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/dynamic_settings.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/dynamic_settings.ts @@ -7,8 +7,8 @@ import expect from '@kbn/expect'; import { isRight } from 'fp-ts/lib/Either'; -import { DynamicSettingsCodec, DynamicSettings } from '@kbn/synthetics-plugin/common/runtime_types'; -import { DYNAMIC_SETTINGS_DEFAULTS, API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { DynamicSettingsCodec, DynamicSettings } from '@kbn/uptime-plugin/common/runtime_types'; +import { DYNAMIC_SETTINGS_DEFAULTS, API_URLS } from '@kbn/uptime-plugin/common/constants'; import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService }: FtrProviderContext) { diff --git a/x-pack/test/api_integration/apis/uptime/rest/index_status.ts b/x-pack/test/api_integration/apis/uptime/rest/index_status.ts index d29d66d436bc9..50ff47f04be5d 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/index_status.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/index_status.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_duration.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_duration.ts index c5915d06564df..3a0d066789be7 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_duration.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_duration.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import expect from '@kbn/expect'; import { expectFixtureEql } from './helper/expect_fixture_eql'; import { FtrProviderContext } from '../../../ftr_provider_context'; diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_latest_status.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_latest_status.ts index e5cf59be67233..9e4632f552a90 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_latest_status.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_latest_status.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { expectFixtureEql } from './helper/expect_fixture_eql'; import { FtrProviderContext } from '../../../ftr_provider_context'; diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts index 5bbe7f7312102..0ae73258d7acb 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; import { MonitorSummary } from '@kbn/synthetics-plugin/common/runtime_types'; -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { makeChecksWithStatus } from './helper/make_checks'; diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts index 438211665674a..1f0e04a74db71 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_real_data.ts @@ -11,7 +11,7 @@ import { MonitorSummariesResult, MonitorSummariesResultType, } from '@kbn/synthetics-plugin/common/runtime_types'; -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { FtrProviderContext } from '../../../ftr_provider_context'; interface ExpectedMonitorStatesPage { diff --git a/x-pack/test/api_integration/apis/uptime/rest/ping_histogram.ts b/x-pack/test/api_integration/apis/uptime/rest/ping_histogram.ts index a46eff97826a8..6246a2d049583 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/ping_histogram.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/ping_histogram.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { expectFixtureEql } from './helper/expect_fixture_eql'; import { FtrProviderContext } from '../../../ftr_provider_context'; diff --git a/x-pack/test/api_integration/apis/uptime/rest/ping_list.ts b/x-pack/test/api_integration/apis/uptime/rest/ping_list.ts index 5c179a15b7525..84d4af6f650be 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/ping_list.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/ping_list.ts @@ -8,8 +8,8 @@ import expect from '@kbn/expect'; import { isLeft } from 'fp-ts/lib/Either'; import { PathReporter } from 'io-ts/lib/PathReporter'; -import { PingsResponseType } from '@kbn/synthetics-plugin/common/runtime_types'; -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { PingsResponseType } from '@kbn/uptime-plugin/common/runtime_types'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import { FtrProviderContext } from '../../../ftr_provider_context'; function decodePingsResponseData(response: any) { diff --git a/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts b/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts index 3fa3d458321d3..492dce4841511 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { API_URLS } from '@kbn/synthetics-plugin/common/constants'; +import { API_URLS } from '@kbn/uptime-plugin/common/constants'; import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { makeChecksWithStatus, getChecksDateRange } from './helper/make_checks'; diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/uptime/simple_down_alert.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/uptime/simple_down_alert.ts index 81a0a9b438af7..98a4a09ee2b8a 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/uptime/simple_down_alert.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/uptime/simple_down_alert.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; -import { MonitorStatusTranslations } from '@kbn/synthetics-plugin/common/rules/legacy_uptime/translations'; +import { MonitorStatusTranslations } from '@kbn/uptime-plugin/common/rules/legacy_uptime/translations'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { deleteUptimeSettingsObject } from '../../../../functional/apps/uptime'; From bf5c2703f9794767b8b004304f250f02d346a01c Mon Sep 17 00:00:00 2001 From: Saikat Sarkar <132922331+saikatsarkar056@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:26:34 -0600 Subject: [PATCH 17/33] [ELSER] Remove missing type hints related to the inference config (#162602) This pr is related to issue: https://github.com/elastic/enterprise-search-team/issues/4432 This change involves removing hints related to the inference_config attribute --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- package.json | 2 +- .../field_capabilities/field_caps_response.ts | 4 +-- .../server/rest_api_routes/route_types.ts | 4 +-- .../analytics/fetch_analytics_collection.ts | 2 +- .../server/lib/ml/start_ml_model_download.ts | 1 - x-pack/plugins/lens/public/types.ts | 2 +- yarn.lock | 29 +++++++++++++++---- 7 files changed, 30 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index b54b947092192..7d270243db9ad 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "@elastic/apm-rum-react": "^1.4.3", "@elastic/charts": "59.1.0", "@elastic/datemath": "5.0.3", - "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.8.0-canary.2", + "@elastic/elasticsearch": "npm:@elastic/elasticsearch@8.9.0", "@elastic/ems-client": "8.4.0", "@elastic/eui": "85.0.1", "@elastic/filesaver": "1.1.2", diff --git a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts index ee029f1271297..715fea9beef3b 100644 --- a/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts +++ b/src/plugins/data_views/server/fetcher/lib/field_capabilities/field_caps_response.ts @@ -129,7 +129,7 @@ export function readFieldCapsResponse( if (timeSeriesMetricProp.length === 1 && timeSeriesMetricProp[0] === 'counter') { timeSeriesMetricType = 'counter'; } - // @ts-expect-error MappingTimeSeriesMetricType does not contain 'position' + if (timeSeriesMetricProp.length === 1 && timeSeriesMetricProp[0] === 'position') { timeSeriesMetricType = 'position'; } @@ -148,9 +148,7 @@ export function readFieldCapsResponse( timeSeriesDimension: capsByType[types[0]].time_series_dimension, }; // This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes - // @ts-expect-error MappingTimeSeriesMetricType does not contain 'position' agg.array.push(field); - // @ts-expect-error MappingTimeSeriesMetricType does not contain 'position' agg.hash[fieldName] = field; return agg; }, diff --git a/src/plugins/data_views/server/rest_api_routes/route_types.ts b/src/plugins/data_views/server/rest_api_routes/route_types.ts index 0c872aa5bd53d..8498b6c3b1701 100644 --- a/src/plugins/data_views/server/rest_api_routes/route_types.ts +++ b/src/plugins/data_views/server/rest_api_routes/route_types.ts @@ -97,7 +97,7 @@ export type FieldSpecRestResponse = DataViewFieldBaseRestResponse & { fixedInterval?: string[]; timeZone?: string[]; timeSeriesDimension?: boolean; - timeSeriesMetric?: 'histogram' | 'summary' | 'gauge' | 'counter'; + timeSeriesMetric?: 'histogram' | 'summary' | 'gauge' | 'counter' | 'position'; shortDotsEnable?: boolean; isMapped?: boolean; parentName?: string; @@ -161,7 +161,7 @@ export interface FieldDescriptorRestResponse { metadata_field?: boolean; fixedInterval?: string[]; timeZone?: string[]; - timeSeriesMetric?: 'histogram' | 'summary' | 'counter' | 'gauge'; + timeSeriesMetric?: 'histogram' | 'summary' | 'counter' | 'gauge' | 'position'; timeSeriesDimension?: boolean; conflictDescriptions?: Record; } diff --git a/x-pack/plugins/enterprise_search/server/lib/analytics/fetch_analytics_collection.ts b/x-pack/plugins/enterprise_search/server/lib/analytics/fetch_analytics_collection.ts index 09d04caee0281..34b188038bf4a 100644 --- a/x-pack/plugins/enterprise_search/server/lib/analytics/fetch_analytics_collection.ts +++ b/x-pack/plugins/enterprise_search/server/lib/analytics/fetch_analytics_collection.ts @@ -18,7 +18,7 @@ export const fetchAnalyticsCollections = async ( ): Promise => { try { const collections = await client.asCurrentUser.searchApplication.getBehavioralAnalytics({ - name: query, + name: [query], }); return Object.keys(collections).map((value) => { diff --git a/x-pack/plugins/enterprise_search/server/lib/ml/start_ml_model_download.ts b/x-pack/plugins/enterprise_search/server/lib/ml/start_ml_model_download.ts index c5abfcf1f1035..4d7f3c21e1210 100644 --- a/x-pack/plugins/enterprise_search/server/lib/ml/start_ml_model_download.ts +++ b/x-pack/plugins/enterprise_search/server/lib/ml/start_ml_model_download.ts @@ -50,7 +50,6 @@ export const startMlModelDownload = async ( // we're not downloaded yet - let's initiate that... const putRequest: MlPutTrainedModelRequest = { - // @ts-expect-error @elastic-elasticsearch inference_config can be optional body: { input: { field_names: ['text_field'], diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index d027f5c60558d..efba612203c92 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -93,7 +93,7 @@ export type IndexPatternField = FieldSpec & { * Map of fields which can be used, but may fail partially (ranked lower than others) */ partiallyApplicableFunctions?: Partial>; - timeSeriesMetric?: 'histogram' | 'summary' | 'gauge' | 'counter'; + timeSeriesMetric?: 'histogram' | 'summary' | 'gauge' | 'counter' | 'position'; timeSeriesRollup?: boolean; meta?: boolean; runtime?: boolean; diff --git a/yarn.lock b/yarn.lock index 282453bf33d92..c00213e71e0e6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1517,12 +1517,12 @@ "@elastic/transport" "^8.3.1" tslib "^2.4.0" -"@elastic/elasticsearch@npm:@elastic/elasticsearch-canary@8.8.0-canary.2": - version "8.8.0-canary.2" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch-canary/-/elasticsearch-canary-8.8.0-canary.2.tgz#10b5699541d644797b33c7e24058d2e55367d88d" - integrity sha512-UxH8YUxcsqHXGh4t2PjuL0q03XunF9vCLHPAs9r+fQcaPXpNbEuv9jbNGXv/9TLyeAKYEgcq9Xm0p0Nk/Mh0lQ== +"@elastic/elasticsearch@npm:@elastic/elasticsearch@8.9.0": + version "8.9.0" + resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-8.9.0.tgz#d132021c6c12e4171fe14371609a5c69b535edd4" + integrity sha512-UyolnzjOYTRL2966TYS3IoJP4tQbvak/pmYmbP3JdphD53RjkyVDdxMpTBv+2LcNBRrvYPTzxQbpRW/nGSXA9g== dependencies: - "@elastic/transport" "^8.3.1" + "@elastic/transport" "^8.3.2" tslib "^2.4.0" "@elastic/ems-client@8.4.0": @@ -1715,6 +1715,18 @@ tslib "^2.4.0" undici "^5.5.1" +"@elastic/transport@^8.3.2": + version "8.3.2" + resolved "https://registry.yarnpkg.com/@elastic/transport/-/transport-8.3.2.tgz#295e91f43e3a60a839f998ac3090a83ddb441592" + integrity sha512-ZiBYRVPj6pwYW99fueyNU4notDf7ZPs7Ix+4T1btIJsKJmeaORIItIfs+0O7KV4vV+DcvyMhkY1FXQx7kQOODw== + dependencies: + debug "^4.3.4" + hpagent "^1.0.0" + ms "^2.1.3" + secure-json-parse "^2.4.0" + tslib "^2.4.0" + undici "^5.22.1" + "@emotion/babel-plugin-jsx-pragmatic@^0.2.1": version "0.2.1" resolved "https://registry.yarnpkg.com/@emotion/babel-plugin-jsx-pragmatic/-/babel-plugin-jsx-pragmatic-0.2.1.tgz#01d3306fde73b60d683f78f3bd9f6b2c919b63b6" @@ -28385,6 +28397,13 @@ undici@^5.5.1: dependencies: busboy "^1.6.0" +undici@^5.22.1: + version "5.22.1" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.22.1.tgz#877d512effef2ac8be65e695f3586922e1a57d7b" + integrity sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw== + dependencies: + busboy "^1.6.0" + unfetch@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/unfetch/-/unfetch-4.2.0.tgz#7e21b0ef7d363d8d9af0fb929a5555f6ef97a3be" From 46a073f01b6b63b79f0d8ce3004b89382356624c Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Mon, 31 Jul 2023 17:41:40 +0300 Subject: [PATCH 18/33] Revert "[Remote Clusters] Clean up null properties depending on conection mode" (#162780) Reverts elastic/kibana#162405 --- .../remote_clusters_add.test.ts.snap | 18 ------------ .../add/remote_clusters_add.test.ts | 8 ----- .../helpers/remote_clusters_actions.ts | 22 -------------- .../remote_cluster_form/request_flyout.tsx | 23 ++------------- .../remote_cluster_form/utils.test.ts | 29 ------------------- 5 files changed, 2 insertions(+), 98 deletions(-) delete mode 100644 x-pack/plugins/remote_clusters/__jest__/client_integration/add/__snapshots__/remote_clusters_add.test.ts.snap delete mode 100644 x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/utils.test.ts diff --git a/x-pack/plugins/remote_clusters/__jest__/client_integration/add/__snapshots__/remote_clusters_add.test.ts.snap b/x-pack/plugins/remote_clusters/__jest__/client_integration/add/__snapshots__/remote_clusters_add.test.ts.snap deleted file mode 100644 index f657eba17f2dc..0000000000000 --- a/x-pack/plugins/remote_clusters/__jest__/client_integration/add/__snapshots__/remote_clusters_add.test.ts.snap +++ /dev/null @@ -1,18 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`Create Remote cluster on component mount show request flyout doesnt contain null values 1`] = ` -"PUT _cluster/settings -{ - \\"persistent\\": { - \\"cluster\\": { - \\"remote\\": { - \\"\\": { - \\"skip_unavailable\\": false, - \\"mode\\": \\"sniff\\", - \\"node_connections\\": 3 - } - } - } - } -}" -`; diff --git a/x-pack/plugins/remote_clusters/__jest__/client_integration/add/remote_clusters_add.test.ts b/x-pack/plugins/remote_clusters/__jest__/client_integration/add/remote_clusters_add.test.ts index fc488984da911..75a1656b0daed 100644 --- a/x-pack/plugins/remote_clusters/__jest__/client_integration/add/remote_clusters_add.test.ts +++ b/x-pack/plugins/remote_clusters/__jest__/client_integration/add/remote_clusters_add.test.ts @@ -48,14 +48,6 @@ describe('Create Remote cluster', () => { expect(actions.skipUnavailableSwitch.isChecked()).toBe(true); }); - test('show request flyout doesnt contain null values', async () => { - await actions.showRequest.click(); - const requestBody = actions.showRequest.getESRequestBody(); - - expect(requestBody).not.toContain('null'); - expect(requestBody).toMatchSnapshot(); - }); - describe('on prem', () => { test('should have a toggle to enable "proxy" mode for a remote cluster', () => { expect(actions.connectionModeSwitch.exists()).toBe(true); diff --git a/x-pack/plugins/remote_clusters/__jest__/client_integration/helpers/remote_clusters_actions.ts b/x-pack/plugins/remote_clusters/__jest__/client_integration/helpers/remote_clusters_actions.ts index e34de66fe69c0..3a2d4be3e060d 100644 --- a/x-pack/plugins/remote_clusters/__jest__/client_integration/helpers/remote_clusters_actions.ts +++ b/x-pack/plugins/remote_clusters/__jest__/client_integration/helpers/remote_clusters_actions.ts @@ -53,10 +53,6 @@ export interface RemoteClustersActions { click: () => void; isDisabled: () => boolean; }; - showRequest: { - click: () => void; - getESRequestBody: () => string; - }; getErrorMessages: () => string[]; globalErrorExists: () => boolean; } @@ -174,23 +170,6 @@ export const createRemoteClustersActions = (testBed: TestBed): RemoteClustersAct }; }; - const createShowRequestActions = () => { - const click = () => { - act(() => { - find('remoteClustersRequestButton').simulate('click'); - }); - - component.update(); - }; - - return { - showRequest: { - click, - getESRequestBody: () => find('esRequestBody').text(), - }, - }; - }; - const globalErrorExists = () => exists('remoteClusterFormGlobalError'); const createCloudUrlInputActions = () => { @@ -214,7 +193,6 @@ export const createRemoteClustersActions = (testBed: TestBed): RemoteClustersAct ...createProxyAddressActions(), ...createServerNameActions(), ...createSaveButtonActions(), - ...createShowRequestActions(), getErrorMessages: form.getErrorsMessages, globalErrorExists, }; diff --git a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/request_flyout.tsx b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/request_flyout.tsx index 4cd95a2d389e4..75f45b9a56566 100644 --- a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/request_flyout.tsx +++ b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/request_flyout.tsx @@ -7,7 +7,6 @@ import React, { PureComponent } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; -import { transform, isObject, isEmpty, isNull } from 'lodash'; import { EuiButtonEmpty, @@ -23,20 +22,6 @@ import { import { Cluster, serializeCluster } from '../../../../../common/lib'; -// Remove all null properties from an object -export function removeNullProperties(obj: any) { - return transform(obj, (result: any, value, key) => { - if (isObject(value)) { - result[key] = removeNullProperties(value); - if (isEmpty(result[key])) { - delete result[key]; - } - } else if (!isNull(value)) { - result[key] = value; - } - }); -} - interface Props { close: () => void; cluster: Cluster; @@ -47,11 +32,7 @@ export class RequestFlyout extends PureComponent { const { close, cluster } = this.props; const { name } = cluster; const endpoint = 'PUT _cluster/settings'; - // Given that the request still requires that we send all properties, regardless of whether they - // are null, we need to remove all null properties from the serialized cluster object that we - // render in the flyout. - const serializedCluster = removeNullProperties(serializeCluster(cluster)); - const payload = JSON.stringify(serializedCluster, null, 2); + const payload = JSON.stringify(serializeCluster(cluster), null, 2); const request = `${endpoint}\n${payload}`; return ( @@ -87,7 +68,7 @@ export class RequestFlyout extends PureComponent { - + {request} diff --git a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/utils.test.ts b/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/utils.test.ts deleted file mode 100644 index 263c7ccf47e68..0000000000000 --- a/x-pack/plugins/remote_clusters/public/application/sections/components/remote_cluster_form/utils.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { removeNullProperties } from './request_flyout'; - -describe('Remote cluster form Utils', () => { - test('can remote deeply nested null properties from object', () => { - const obj = { - a: 'a', - b: { - c: 'c', - d: null, - }, - }; - - expect(removeNullProperties(obj)).toStrictEqual({ - ...obj, - b: { - c: 'c', - }, - }); - - expect(removeNullProperties({ a: 'a', b: null })).toStrictEqual({ a: 'a' }); - }); -}); From 90a882399a9fa39b9bcf7f4abf1705a6021e4d90 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:49:40 -0400 Subject: [PATCH 19/33] [Security Solution][Serverless] Remove esArchive data load from Cypress test runner (#162797) ## Summary - Remove esArchive load from Cypress runner (data should be loaded by individual tests and not globally across all tests) --- .../functional/test_suites/security/cypress/runner.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts index 62625e03920dc..a83d8afbaefdc 100644 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts +++ b/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts @@ -14,9 +14,6 @@ export async function SecuritySolutionCypressTestRunner( envVars?: Record ) { const config = getService('config'); - const esArchiver = getService('esArchiver'); - - await esArchiver.load('x-pack/test/security_solution_cypress/es_archives/auditbeat'); return { FORCE_COLOR: '1', From 832aec0a8e2d0055edd70e765b2114a44cc6b29d Mon Sep 17 00:00:00 2001 From: Clint Andrew Hall Date: Mon, 31 Jul 2023 16:51:28 +0200 Subject: [PATCH 20/33] [regression] Fix Storybooks after #161914 (#162757) ## Summary As titled. This PR corrects a pair of mistakes I made before committing #161914 that @dgieselaar identified shortly thereafter. - I had tested Storybook extensively, but after I rebased, I changed the render context, and I forgot to update the `decorator` in Storybook. This meant Emotion styles worked, but the EUI styles were missing. - In addition, when I rebased, I missed the addition of the utils cache that had been added by EUI. - Interestingly, #162365 missed adding the cache `meta` tag to the template. Emotion simply added the styles to the `head`, but it's best to reproduce what we see in Kibana. So I've corrected that, as well. - While creating the PR, I went ahead and addressed [feedback](https://github.com/elastic/kibana/pull/161914#discussion_r1277765276) from @cee-chen on the original PR./ Sorry if anyone was confused by the sudden drop in styles in their Storybooks. Should be resolved now. Thanks! --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- packages/kbn-storybook/src/lib/decorators.tsx | 6 +- packages/kbn-storybook/templates/index.ejs | 102 ++++++++++-------- packages/kbn-storybook/tsconfig.json | 2 +- .../react/kibana_context/styled/README.mdx | 2 + 4 files changed, 62 insertions(+), 50 deletions(-) diff --git a/packages/kbn-storybook/src/lib/decorators.tsx b/packages/kbn-storybook/src/lib/decorators.tsx index c49571df48a2f..270b9662e2210 100644 --- a/packages/kbn-storybook/src/lib/decorators.tsx +++ b/packages/kbn-storybook/src/lib/decorators.tsx @@ -14,7 +14,7 @@ import 'core_styles'; import { BehaviorSubject } from 'rxjs'; import { CoreTheme } from '@kbn/core-theme-browser'; import { I18nStart } from '@kbn/core-i18n-browser'; -import { KibanaRenderContextProvider } from '@kbn/react-kibana-context-render'; +import { KibanaRootContextProvider } from '@kbn/react-kibana-context-root'; const theme$ = new BehaviorSubject({ darkMode: false }); @@ -34,9 +34,9 @@ const KibanaContextDecorator: DecoratorFn = (storyFn, { globals }) => { }, [colorMode]); return ( - + {storyFn()} - + ); }; diff --git a/packages/kbn-storybook/templates/index.ejs b/packages/kbn-storybook/templates/index.ejs index b52fc548b814b..5540031e3b73c 100644 --- a/packages/kbn-storybook/templates/index.ejs +++ b/packages/kbn-storybook/templates/index.ejs @@ -4,64 +4,74 @@ We use this one instead because we want to add the @kbn/ui-shared-deps-* tags here. --> - - - <%= htmlWebpackPlugin.options.title || 'Storybook'%> - <% if (htmlWebpackPlugin.files.favicon) { %> + + + + <%= htmlWebpackPlugin.options.title || 'Storybook' %> + + + <% if (htmlWebpackPlugin.files.favicon) { %> <% } %> - - - - + - - - - - - - + + + - <% if (typeof headHtmlSnippet !== 'undefined') { %> <%= headHtmlSnippet %> <% } %> <% - htmlWebpackPlugin.files.css.forEach(file => { %> - - <% }); %> + + + + + + + - - - - - - - <% if (typeof bodyHtmlSnippet !== 'undefined') { %> - <%= bodyHtmlSnippet %> - <% } %> + <% if (typeof headHtmlSnippet !=='undefined' ) { %> + <%= headHtmlSnippet %> + <% } %> + <% htmlWebpackPlugin.files.css.forEach(file=> { %> + + <% }); %> -
-
+ + + + + - <% if (typeof globals !== 'undefined' && Object.keys(globals).length) { %> - - <% } %> + + <% } %> + + <% htmlWebpackPlugin.files.js.forEach(file=> { %> + + <% }); %> + - <% htmlWebpackPlugin.files.js.forEach(file => { %> - - <% }); %> - diff --git a/packages/kbn-storybook/tsconfig.json b/packages/kbn-storybook/tsconfig.json index 20b509a7b434d..6463962ff65f9 100644 --- a/packages/kbn-storybook/tsconfig.json +++ b/packages/kbn-storybook/tsconfig.json @@ -19,7 +19,7 @@ "@kbn/core-theme-browser", "@kbn/i18n-react", "@kbn/core-i18n-browser", - "@kbn/react-kibana-context-render", + "@kbn/react-kibana-context-root", ], "exclude": [ "target/**/*", diff --git a/packages/react/kibana_context/styled/README.mdx b/packages/react/kibana_context/styled/README.mdx index 335fc7ee69e63..9f9ec22268ceb 100644 --- a/packages/react/kibana_context/styled/README.mdx +++ b/packages/react/kibana_context/styled/README.mdx @@ -13,3 +13,5 @@ Before `emotion` was introduced, some components used `styled-components` to eas It should _not_ be used in new code. +**NOTE:** plugins cannot use `styled-components` and `emotion` at the same time, their Babel plugins conflict. The EUI team has fielded many questions about this. The best solution is to migrate to `emotion` and not use `styled-components`. + From 5369eb32abcabf7fe5a9a33efaefbd2e2ed3aec9 Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Mon, 31 Jul 2023 17:01:23 +0200 Subject: [PATCH 21/33] [APM] Fix throwing appropriate error when user is missing necessary permission (#162466) Closes https://github.com/elastic/kibana/issues/126078 Currently the 2 places where the user can create Agent Key already handles the situation by not displaying the option to user with limited permissions. 1. Observability - Settings 2. Stack Management -> API Keys [Public API](https://www.elastic.co/guide/en/kibana/current/agent-key-api.html) is still available to create Agent Keys. With this change the below would happen ### Request ``` curl -X POST "http://localhost:5601/{basePath}/api/apm/agent_keys" \ -u editor:changeme \ -H "Content-Type: application/json" \ -H "Elastic-Api-Version: 2023-10-31" \ -H "kbn-xsrf: true" \ -d '{ "name": "apm-key", "privileges": ["event:write", "config_agent:read"] }' ``` ### Response ``` { "statusCode": 403, "error": "Forbidden", "message": "editor is missing the following requested privilege(s): config_agent:read, event:write and following cluster privileges - manage_api_key, manage_own_api_key privilege(s). You might try with the superuser, or add the missing APM application privileges to the role of the authenticated user, eg.:\n PUT /_security/role/my_role\n {\n ...\n \"applications\": [{\n \"application\": \"apm\",\n \"privileges\": [\"config_agent:read\",\"event:write\"],\n \"resources\": [*]\n }],\n ...\n }", "attributes": { "data": { "missingPrivileges": [ "config_agent:read", "event:write" ], "missingClusterPrivileges": [ "manage_api_key", "manage_own_api_key" ] }, "_inspect": [ ] } } ``` --- x-pack/plugins/apm/common/privilege_type.ts | 4 ++++ .../routes/agent_keys/create_agent_key.ts | 21 +++++++++++++++++-- .../create_apm_users/authentication.ts | 7 +++++-- .../settings/agent_keys/agent_keys.spec.ts | 4 +++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/apm/common/privilege_type.ts b/x-pack/plugins/apm/common/privilege_type.ts index 3c2a0da3f3f05..4cb7831b42c02 100644 --- a/x-pack/plugins/apm/common/privilege_type.ts +++ b/x-pack/plugins/apm/common/privilege_type.ts @@ -12,6 +12,10 @@ export enum PrivilegeType { AGENT_CONFIG = 'config_agent:read', } +export enum ClusterPrivilegeType { + MANAGE_OWN_API_KEY = 'manage_own_api_key', +} + export const privilegesTypeRt = t.array( t.union([ t.literal(PrivilegeType.EVENT), diff --git a/x-pack/plugins/apm/server/routes/agent_keys/create_agent_key.ts b/x-pack/plugins/apm/server/routes/agent_keys/create_agent_key.ts index 0ceac90227d8b..05560a7fb0dd0 100644 --- a/x-pack/plugins/apm/server/routes/agent_keys/create_agent_key.ts +++ b/x-pack/plugins/apm/server/routes/agent_keys/create_agent_key.ts @@ -8,8 +8,10 @@ import { SecurityCreateApiKeyResponse } from '@elastic/elasticsearch/lib/api/types'; import Boom from '@hapi/boom'; import { ApmPluginRequestHandlerContext } from '../typings'; +import { ClusterPrivilegeType } from '../../../common/privilege_type'; const resource = '*'; +const CLUSTER_PRIVILEGES = [ClusterPrivilegeType.MANAGE_OWN_API_KEY]; export interface CreateAgentKeyResponse { agentKey: SecurityCreateApiKeyResponse; @@ -39,10 +41,12 @@ export async function createAgentKey({ application: userApplicationPrivileges, username, has_all_requested: hasRequiredPrivileges, + cluster: clusterPrivileges, } = await coreContext.elasticsearch.client.asCurrentUser.security.hasPrivileges( { body: { application: [application], + cluster: CLUSTER_PRIVILEGES, }, } ); @@ -54,9 +58,19 @@ export async function createAgentKey({ .filter((x) => !x[1]) .map((x) => x[0]); + const missingClusterPrivileges = Object.keys(clusterPrivileges).filter( + (key) => !clusterPrivileges[key] + ); + const error = `${username} is missing the following requested privilege(s): ${missingPrivileges.join( ', ' - )}.\ + )}${ + missingClusterPrivileges && missingClusterPrivileges.length > 0 + ? ` and following cluster privileges - ${missingClusterPrivileges.join( + ', ' + )} privilege(s)` + : '' + }.\ You might try with the superuser, or add the missing APM application privileges to the role of the authenticated user, eg.: PUT /_security/role/my_role { @@ -68,7 +82,10 @@ export async function createAgentKey({ }], ... }`; - throw Boom.internal(error, { missingPrivileges }, 403); + throw Boom.forbidden(error, { + missingPrivileges, + missingClusterPrivileges, + }); } const body = { diff --git a/x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts b/x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts index fedc98151ce40..0b6fc8bb9947a 100644 --- a/x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts +++ b/x-pack/plugins/apm/server/test_helpers/create_apm_users/authentication.ts @@ -5,7 +5,10 @@ * 2.0. */ -import { PrivilegeType } from '../../../common/privilege_type'; +import { + PrivilegeType, + ClusterPrivilegeType, +} from '../../../common/privilege_type'; export enum ApmUsername { noAccessUser = 'no_access_user', @@ -67,7 +70,7 @@ export const customRoles = { }, [ApmCustomRolename.apmManageOwnAgentKeys]: { elasticsearch: { - cluster: ['manage_own_api_key'], + cluster: [ClusterPrivilegeType.MANAGE_OWN_API_KEY], }, }, [ApmCustomRolename.apmManageOwnAndCreateAgentKeys]: { diff --git a/x-pack/test/apm_api_integration/tests/settings/agent_keys/agent_keys.spec.ts b/x-pack/test/apm_api_integration/tests/settings/agent_keys/agent_keys.spec.ts index 6ec83468ac411..2b8506af33a36 100644 --- a/x-pack/test/apm_api_integration/tests/settings/agent_keys/agent_keys.spec.ts +++ b/x-pack/test/apm_api_integration/tests/settings/agent_keys/agent_keys.spec.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; import { first } from 'lodash'; -import { PrivilegeType } from '@kbn/apm-plugin/common/privilege_type'; +import { PrivilegeType, ClusterPrivilegeType } from '@kbn/apm-plugin/common/privilege_type'; import { ApmUsername } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/authentication'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; import { ApmApiError, ApmApiSupertest } from '../../../common/apm_api_supertest'; @@ -19,6 +19,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const agentKeyName = 'test'; const allApplicationPrivileges = [PrivilegeType.AGENT_CONFIG, PrivilegeType.EVENT]; + const clusterPrivileges = [ClusterPrivilegeType.MANAGE_OWN_API_KEY]; async function createAgentKey(apiClient: ApmApiSupertest, privileges = allApplicationPrivileges) { return await apiClient({ @@ -60,6 +61,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { _inspect: [], data: { missingPrivileges: allApplicationPrivileges, + missingClusterPrivileges: clusterPrivileges, }, }); }); From b0fbe9340ce0206326957f15534c4a633c531b90 Mon Sep 17 00:00:00 2001 From: Philippe Oberti Date: Mon, 31 Jul 2023 17:06:17 +0200 Subject: [PATCH 22/33] [Security Solution] - expandable flyout - code owners (#162792) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12d60292a2dde..d803cb9789714 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1071,6 +1071,7 @@ x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kib /x-pack/plugins/security_solution/public/detections/components/alerts_kpis @elastic/security-threat-hunting-investigations /x-pack/plugins/security_solution/public/detections/components/alerts_table @elastic/security-threat-hunting-investigations /x-pack/plugins/security_solution/public/detections/components/alerts_info @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/flyout @elastic/security-threat-hunting-investigations /x-pack/plugins/security_solution/public/resolver @elastic/security-threat-hunting-investigations /x-pack/plugins/security_solution/public/timelines @elastic/security-threat-hunting-investigations From 07280038655ac256011ee740fce760ca270303b2 Mon Sep 17 00:00:00 2001 From: "Quynh Nguyen (Quinn)" <43350163+qn895@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:24:01 -0500 Subject: [PATCH 23/33] [ML] Add new Data comparison view (#161365) Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 1 + api_docs/kbn_aiops_components.devdocs.json | 2 +- package.json | 1 + packages/deeplinks/ml/deep_links.ts | 1 + tsconfig.base.json | 2 + x-pack/packages/ml/aiops_components/index.ts | 2 + .../src}/document_count_chart/brush_badge.tsx | 9 + .../document_count_chart.tsx | 120 +- .../src}/document_count_chart/index.ts | 2 +- .../src/dual_brush/dual_brush.tsx | 49 + .../src/dual_brush/dual_brush_annotation.tsx | 25 +- .../progress_controls/progress_controls.tsx | 29 + .../ml/aiops_components/tsconfig.json | 4 + x-pack/packages/ml/in_memory_table/README.md | 3 + .../in_memory_table/hooks}/use_table_state.ts | 45 +- x-pack/packages/ml/in_memory_table/index.ts | 9 + .../ml/in_memory_table/jest.config.js | 12 + .../packages/ml/in_memory_table/kibana.jsonc | 5 + .../packages/ml/in_memory_table/package.json | 6 + .../packages/ml/in_memory_table/tsconfig.json | 23 + x-pack/packages/ml/query_utils/index.ts | 3 + .../ml/query_utils/src/get_default_query.ts | 26 + x-pack/packages/ml/query_utils/src/types.ts | 19 + .../packages/ml/random_sampler_utils/index.ts | 1 + .../src/random_sampler_manager.ts | 168 +++ .../public/application/utils/search_utils.ts | 30 +- .../public/application/utils/url_state.ts | 2 +- .../change_point_detection/search_bar.tsx | 2 +- .../document_count_content.tsx | 21 +- .../category_table/category_table.tsx | 2 +- .../document_count_chart.tsx | 6 +- .../log_categorization_page.tsx | 2 +- .../log_rate_analysis_page.tsx | 2 +- .../use_view_in_discover_action.tsx | 2 +- ...se_view_in_log_pattern_analysis_action.tsx | 2 +- .../components/search_panel/search_panel.tsx | 2 +- x-pack/plugins/aiops/tsconfig.json | 1 + .../data_visualizer/common/i18n_constants.ts | 16 + .../data_visualizer/public/api/index.ts | 11 +- .../document_count_content.tsx | 18 +- .../random_sampling_menu/probability_used.tsx | 28 + .../random_sampler_range_slider.tsx | 6 +- .../random_sampling_menu.tsx | 184 +++ .../common/hooks/data_source_context.ts | 24 + .../application/common/hooks/use_data.ts | 113 ++ .../common/hooks/use_document_count_stats.ts | 290 +++++ .../application/common/hooks/use_search.ts | 69 ++ .../common/hooks/use_time_buckets.ts | 24 + .../application/common/util/display_error.ts | 35 + ...or_messages_from_es_shard_failures.test.ts | 56 + ...t_error_messages_from_es_shard_failures.ts | 22 + .../data_comparison_distribution_chart.tsx | 51 + .../data_comparison/charts/no_charts_data.tsx | 29 + .../charts/overlap_distribution_chart.tsx | 64 + .../charts/single_distribution_chart.tsx | 45 + .../application/data_comparison/constants.ts | 1064 +++++++++++++++++ .../data_comparison_app_state.tsx | 101 ++ .../data_comparison_chart_tooltip_body.tsx | 60 + .../data_comparison_overview_table.tsx | 291 +++++ .../data_comparison/data_comparison_page.tsx | 395 ++++++ .../data_comparison_utils.test.ts | 88 ++ .../data_comparison/data_comparison_utils.ts | 82 ++ .../data_comparison/data_comparison_view.tsx | 228 ++++ .../document_count_with_dual_brush.tsx | 133 +++ .../application/data_comparison/index.ts | 15 + .../application/data_comparison/types.ts | 115 ++ .../data_comparison/use_data_drift_result.ts | 891 ++++++++++++++ .../public/application/index.ts | 1 + .../index_data_visualizer_view.tsx | 26 +- .../components/search_panel/search_bar.tsx | 115 ++ .../components/search_panel/search_panel.tsx | 90 +- .../hooks/use_data_visualizer_grid_data.ts | 13 +- .../hooks/use_overall_stats.ts | 29 +- .../index_data_visualizer/locator/locator.ts | 2 +- .../types/combined_query.ts | 7 - .../types/index_data_visualizer_state.ts | 2 +- .../index_data_visualizer/types/storage.ts | 4 +- .../utils/saved_search_utils.ts | 21 +- .../plugins/data_visualizer/public/index.ts | 1 + .../lazy_load_bundle/component_wrapper.tsx | 22 +- .../public/lazy_load_bundle/index.ts | 7 +- .../lazy/{index.ts => index.tsx} | 1 + .../plugins/data_visualizer/public/plugin.ts | 7 +- x-pack/plugins/data_visualizer/tsconfig.json | 9 +- x-pack/plugins/ml/common/constants/locator.ts | 2 + x-pack/plugins/ml/common/constants/search.ts | 7 - x-pack/plugins/ml/common/types/locator.ts | 7 +- .../components/anomalies_table/links_menu.tsx | 2 +- .../components/ml_page/side_nav.tsx | 9 + .../configuration_step_form.tsx | 2 +- .../configuration_step/use_saved_search.ts | 2 +- .../expandable_section_results.tsx | 2 +- .../exploration_query_bar.tsx | 5 +- .../hooks/use_exploration_url_state.ts | 2 +- .../data_comparison/data_comparison_page.tsx | 53 + .../explorer/anomaly_context_menu.tsx | 2 +- .../application/explorer/anomaly_timeline.tsx | 2 +- .../explorer_query_bar/explorer_query_bar.tsx | 2 +- ...d_anomaly_charts_to_dashboard_controls.tsx | 2 +- .../add_swimlane_to_dashboard_controls.tsx | 2 +- .../explorer_charts_container.js | 2 +- .../jobs/new_job/utils/new_job_utils.ts | 2 +- .../space_management/space_management.tsx | 2 +- .../space_management/use_table_state.ts | 46 - .../public/application/routing/breadcrumbs.ts | 8 + .../routes/datavisualizer/data_comparison.tsx | 64 + .../routes/new_job/index_or_search.tsx | 20 + .../routing/routes/trained_models/index.ts | 1 + .../plugins/ml/public/locator/ml_locator.ts | 2 + x-pack/plugins/ml/public/maps/util.ts | 2 +- .../search_deep_links.ts | 12 + x-pack/plugins/ml/tsconfig.json | 1 + .../translations/translations/fr-FR.json | 2 - .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - yarn.lock | 4 + 116 files changed, 5487 insertions(+), 334 deletions(-) rename x-pack/{plugins/aiops/public/components/document_count_content => packages/ml/aiops_components/src}/document_count_chart/brush_badge.tsx (88%) rename x-pack/{plugins/aiops/public/components/document_count_content => packages/ml/aiops_components/src}/document_count_chart/document_count_chart.tsx (76%) rename x-pack/{plugins/aiops/public/components/document_count_content => packages/ml/aiops_components/src}/document_count_chart/index.ts (76%) create mode 100644 x-pack/packages/ml/in_memory_table/README.md rename x-pack/{plugins/aiops/public/components/log_categorization/category_table => packages/ml/in_memory_table/hooks}/use_table_state.ts (51%) create mode 100644 x-pack/packages/ml/in_memory_table/index.ts create mode 100644 x-pack/packages/ml/in_memory_table/jest.config.js create mode 100644 x-pack/packages/ml/in_memory_table/kibana.jsonc create mode 100644 x-pack/packages/ml/in_memory_table/package.json create mode 100644 x-pack/packages/ml/in_memory_table/tsconfig.json create mode 100644 x-pack/packages/ml/query_utils/src/get_default_query.ts create mode 100644 x-pack/packages/ml/query_utils/src/types.ts create mode 100644 x-pack/packages/ml/random_sampler_utils/src/random_sampler_manager.ts create mode 100644 x-pack/plugins/data_visualizer/common/i18n_constants.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/probability_used.tsx rename x-pack/plugins/data_visualizer/public/application/common/components/{document_count_content => random_sampling_menu}/random_sampler_range_slider.tsx (100%) create mode 100644 x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampling_menu.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/common/hooks/data_source_context.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/hooks/use_document_count_stats.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/hooks/use_search.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/hooks/use_time_buckets.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/util/display_error.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.test.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/charts/data_comparison_distribution_chart.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/charts/no_charts_data.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/charts/overlap_distribution_chart.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/charts/single_distribution_chart.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/constants.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_app_state.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_chart_tooltip_body.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_overview_table.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_page.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.test.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_view.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/document_count_with_dual_brush.tsx create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/index.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/types.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/data_comparison/use_data_drift_result.ts create mode 100644 x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx rename x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/{index.ts => index.tsx} (88%) create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/data_comparison/data_comparison_page.tsx delete mode 100644 x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/use_table_state.ts create mode 100644 x-pack/plugins/ml/public/application/routing/routes/datavisualizer/data_comparison.tsx diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d803cb9789714..1570781997115 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -492,6 +492,7 @@ x-pack/packages/ml/data_grid @elastic/ml-ui x-pack/packages/ml/date_picker @elastic/ml-ui x-pack/packages/ml/date_utils @elastic/ml-ui x-pack/packages/ml/error_utils @elastic/ml-ui +x-pack/packages/ml/in_memory_table @elastic/ml-ui x-pack/packages/ml/is_defined @elastic/ml-ui x-pack/packages/ml/is_populated_object @elastic/ml-ui x-pack/packages/ml/kibana_theme @elastic/ml-ui diff --git a/api_docs/kbn_aiops_components.devdocs.json b/api_docs/kbn_aiops_components.devdocs.json index 35b37e8f523f5..31b0e57a6437b 100644 --- a/api_docs/kbn_aiops_components.devdocs.json +++ b/api_docs/kbn_aiops_components.devdocs.json @@ -124,4 +124,4 @@ "misc": [], "objects": [] } -} \ No newline at end of file +} diff --git a/package.json b/package.json index 7d270243db9ad..ef1f3a71d64c0 100644 --- a/package.json +++ b/package.json @@ -508,6 +508,7 @@ "@kbn/ml-date-picker": "link:x-pack/packages/ml/date_picker", "@kbn/ml-date-utils": "link:x-pack/packages/ml/date_utils", "@kbn/ml-error-utils": "link:x-pack/packages/ml/error_utils", + "@kbn/ml-in-memory-table": "link:x-pack/packages/ml/in_memory_table", "@kbn/ml-is-defined": "link:x-pack/packages/ml/is_defined", "@kbn/ml-is-populated-object": "link:x-pack/packages/ml/is_populated_object", "@kbn/ml-kibana-theme": "link:x-pack/packages/ml/kibana_theme", diff --git a/packages/deeplinks/ml/deep_links.ts b/packages/deeplinks/ml/deep_links.ts index d0c79bcec7648..dc6accb275272 100644 --- a/packages/deeplinks/ml/deep_links.ts +++ b/packages/deeplinks/ml/deep_links.ts @@ -15,6 +15,7 @@ export type LinkId = | 'anomalyDetection' | 'anomalyExplorer' | 'singleMetricViewer' + | 'dataComparison' | 'dataFrameAnalytics' | 'resultExplorer' | 'analyticsMap' diff --git a/tsconfig.base.json b/tsconfig.base.json index 0a7dc1821f67e..fc2fbde3ae0d6 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -978,6 +978,8 @@ "@kbn/ml-date-utils/*": ["x-pack/packages/ml/date_utils/*"], "@kbn/ml-error-utils": ["x-pack/packages/ml/error_utils"], "@kbn/ml-error-utils/*": ["x-pack/packages/ml/error_utils/*"], + "@kbn/ml-in-memory-table": ["x-pack/packages/ml/in_memory_table"], + "@kbn/ml-in-memory-table/*": ["x-pack/packages/ml/in_memory_table/*"], "@kbn/ml-is-defined": ["x-pack/packages/ml/is_defined"], "@kbn/ml-is-defined/*": ["x-pack/packages/ml/is_defined/*"], "@kbn/ml-is-populated-object": ["x-pack/packages/ml/is_populated_object"], diff --git a/x-pack/packages/ml/aiops_components/index.ts b/x-pack/packages/ml/aiops_components/index.ts index ec23951d9e8fb..bfe84e260decd 100644 --- a/x-pack/packages/ml/aiops_components/index.ts +++ b/x-pack/packages/ml/aiops_components/index.ts @@ -7,3 +7,5 @@ export { DualBrush, DualBrushAnnotation } from './src/dual_brush'; export { ProgressControls } from './src/progress_controls'; +export { DocumentCountChart } from './src/document_count_chart'; +export type { DocumentCountChartPoint, DocumentCountChartProps } from './src/document_count_chart'; diff --git a/x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/brush_badge.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/brush_badge.tsx similarity index 88% rename from x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/brush_badge.tsx rename to x-pack/packages/ml/aiops_components/src/document_count_chart/brush_badge.tsx index 0f9fa9dbcde97..53563a721026b 100644 --- a/x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/brush_badge.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/brush_badge.tsx @@ -22,6 +22,15 @@ interface BrushBadgeProps { width: number; } +/** + * Badge component + * @param label - label + * @param marginLeft - margin left + * @param timestampFrom - start timestamp + * @param timestampTo - ending timestamp + * @param width - width of badge + * @constructor + */ export const BrushBadge: FC = ({ label, marginLeft, diff --git a/x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx similarity index 76% rename from x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/document_count_chart.tsx rename to x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx index a9e813dbec540..5b488522d23c7 100644 --- a/x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/document_count_chart.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx @@ -23,14 +23,20 @@ import { import { i18n } from '@kbn/i18n'; import { IUiSettingsClient } from '@kbn/core/public'; -import { DualBrush, DualBrushAnnotation } from '@kbn/aiops-components'; import { getSnappedWindowParameters, getWindowParameters } from '@kbn/aiops-utils'; import type { WindowParameters } from '@kbn/aiops-utils'; import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common'; -import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context'; +import { + BarStyleAccessor, + RectAnnotationSpec, +} from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; +import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { BrushBadge } from './brush_badge'; +import { DualBrush, DualBrushAnnotation } from '../..'; declare global { interface Window { @@ -46,27 +52,79 @@ interface TimeFilterRange { to: number; } +/** + * Datum for the bar chart + */ export interface DocumentCountChartPoint { + /** + * Time of bucket + */ time: number | string; + /** + * Number of doc count for that time bucket + */ value: number; } -interface DocumentCountChartProps { - brushSelectionUpdateHandler?: (d: WindowParameters, force: boolean) => void; +/** + * Brush settings + */ +export interface BrushSettings { + /** + * Optional label name for brush + */ + label?: string; + /** + * Optional style for brush + */ + annotationStyle?: RectAnnotationSpec['style']; + /** + * Optional width for brush + */ + badgeWidth?: number; +} + +/** + * Props for document count chart + */ +export interface DocumentCountChartProps { + /** List of Kibana services that are required as dependencies */ + dependencies: { + data: DataPublicPluginStart; + charts: ChartsPluginStart; + fieldFormats: FieldFormatsStart; + uiSettings: IUiSettingsClient; + }; + /** Optional callback function which gets called the brush selection has changed */ + brushSelectionUpdateHandler?: (windowParameters: WindowParameters, force: boolean) => void; + /** Optional width */ width?: number; + /** Data chart points */ chartPoints: DocumentCountChartPoint[]; + /** Data chart points split */ chartPointsSplit?: DocumentCountChartPoint[]; + /** Start time range for the chart */ timeRangeEarliest: number; + /** Ending time range for the chart */ timeRangeLatest: number; + /** Time interval for the document count buckets */ interval: number; + /** Label to name the adjustedChartPointsSplit histogram */ chartPointsSplitLabel: string; + /** Whether or not brush has been reset */ isBrushCleared: boolean; - /* Timestamp for start of initial analysis */ + /** Timestamp for start of initial analysis */ autoAnalysisStart?: number | WindowParameters; + /** Optional style to override bar chart */ + barStyleAccessor?: BarStyleAccessor; /** Optional color override for the default bar color for charts */ barColorOverride?: string; /** Optional color override for the highlighted bar color for charts */ barHighlightColorOverride?: string; + /** Optional settings override for the 'deviation' brush */ + deviationBrush?: BrushSettings; + /** Optional settings override for the 'baseline' brush */ + baselineBrush?: BrushSettings; } const SPEC_ID = 'document_count'; @@ -102,7 +160,29 @@ function getBaselineBadgeOverflow( : 0; } +/** + * Document count chart with draggable brushes to select time ranges + * by default use `Baseline` and `Deviation` for the badge names + * @param dependencies - List of Kibana services that are required as dependencies + * @param brushSelectionUpdateHandler - Optional callback function which gets called the brush selection has changed + * @param width - Optional width + * @param chartPoints - Data chart points + * @param chartPointsSplit - Data chart points split + * @param timeRangeEarliest - Start time range for the chart + * @param timeRangeLatest - Ending time range for the chart + * @param interval - Time interval for the document count buckets + * @param chartPointsSplitLabel - Label to name the adjustedChartPointsSplit histogram + * @param isBrushCleared - Whether or not brush has been reset + * @param autoAnalysisStart - Timestamp for start of initial analysis + * @param barColorOverride - Optional color override for the default bar color for charts + * @param barStyleAccessor - Optional style to override bar chart + * @param barHighlightColorOverride - Optional color override for the highlighted bar color for charts + * @param deviationBrush - Optional settings override for the 'deviation' brush + * @param baselineBrush - Optional settings override for the 'baseline' brush + * @constructor + */ export const DocumentCountChart: FC = ({ + dependencies, brushSelectionUpdateHandler, width, chartPoints, @@ -114,9 +194,12 @@ export const DocumentCountChart: FC = ({ isBrushCleared, autoAnalysisStart, barColorOverride, + barStyleAccessor, barHighlightColorOverride, + deviationBrush = {}, + baselineBrush = {}, }) => { - const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext(); + const { data, uiSettings, fieldFormats, charts } = dependencies; const chartTheme = charts.theme.useChartsTheme(); const chartBaseTheme = charts.theme.useChartsBaseTheme(); @@ -339,22 +422,28 @@ export const DocumentCountChart: FC = ({
= ({ timeZone={timeZone} color={barColor} yNice + styleAccessor={barStyleAccessor} /> )} {adjustedChartPointsSplit?.length && ( @@ -438,11 +528,13 @@ export const DocumentCountChart: FC = ({ id="aiopsBaseline" min={windowParameters.baselineMin} max={windowParameters.baselineMax} + style={baselineBrush.annotationStyle} /> )} diff --git a/x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/index.ts b/x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts similarity index 76% rename from x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/index.ts rename to x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts index 22f39e2ca5d1d..808496f810782 100644 --- a/x-pack/plugins/aiops/public/components/document_count_content/document_count_chart/index.ts +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts @@ -6,4 +6,4 @@ */ export { DocumentCountChart } from './document_count_chart'; -export type { DocumentCountChartPoint } from './document_count_chart'; +export type { DocumentCountChartPoint, DocumentCountChartProps } from './document_count_chart'; diff --git a/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush.tsx b/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush.tsx index 80ba3850044a3..05635405e39d9 100644 --- a/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush.tsx +++ b/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush.tsx @@ -55,22 +55,71 @@ const BRUSH_HANDLE_SIZE = 4; const BRUSH_HANDLE_ROUNDED_CORNER = 2; interface DualBrushProps { + /** + * Min and max numeric timestamps for the two brushes + */ windowParameters: WindowParameters; + /** + * Min timestamp for x domain + */ min: number; + /** + * Max timestamp for x domain + */ max: number; + /** + * Callback function whenever the brush changes + */ onChange?: (windowParameters: WindowParameters, windowPxParameters: WindowParameters) => void; + /** + * Margin left + */ marginLeft: number; + /** + * Nearest timestamps to snap to the brushes to + */ snapTimestamps?: number[]; + /** + * Width + */ width: number; } +/** + * DualBrush React Component + * Dual brush component that overlays the document count chart + * @type {FC} + * @param props - `DualBrushProps` component props + * @returns {React.ReactElement} The DualBrush component. + */ export function DualBrush({ + /** + * Min and max numeric timestamps for the two brushes + */ windowParameters, + /** + * Min timestamp for x domain + */ min, + /** + * Max timestamp for x domain + */ max, + /** + * Callback function whenever the brush changes + */ onChange, + /** + * Margin left + */ marginLeft, + /** + * Nearest timestamps to snap to the brushes to + */ snapTimestamps, + /** + * Width + */ width, }: DualBrushProps) { const d3BrushContainer = useRef(null); diff --git a/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush_annotation.tsx b/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush_annotation.tsx index a689ab340224b..f78bfc78f3cce 100644 --- a/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush_annotation.tsx +++ b/x-pack/packages/ml/aiops_components/src/dual_brush/dual_brush_annotation.tsx @@ -9,14 +9,23 @@ import React, { FC } from 'react'; import { RectAnnotation } from '@elastic/charts'; import { useEuiTheme } from '@elastic/eui'; +import { RectAnnotationSpec } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; interface BrushAnnotationProps { id: string; min: number; max: number; + style?: RectAnnotationSpec['style']; } -export const DualBrushAnnotation: FC = ({ id, min, max }) => { +/** + * DualBrushAnnotation React Component + * Dual brush annotation component that overlays the document count chart + * @type {FC} + * @param props - `BrushAnnotationProps` component props + * @returns {React.ReactElement} The DualBrushAnnotation component. + */ +export const DualBrushAnnotation: FC = ({ id, min, max, style }) => { const { euiTheme } = useEuiTheme(); const { colors } = euiTheme; @@ -34,12 +43,14 @@ export const DualBrushAnnotation: FC = ({ id, min, max }) }, ]} id={`rect_brush_annotation_${id}`} - style={{ - strokeWidth: 0, - stroke: colors.lightShade, - fill: colors.lightShade, - opacity: 0.5, - }} + style={ + style ?? { + strokeWidth: 0, + stroke: colors.lightShade, + fill: colors.lightShade, + opacity: 0.5, + } + } hideTooltips={true} /> ); diff --git a/x-pack/packages/ml/aiops_components/src/progress_controls/progress_controls.tsx b/x-pack/packages/ml/aiops_components/src/progress_controls/progress_controls.tsx index 43e51e9cc4c69..7083282d3609f 100644 --- a/x-pack/packages/ml/aiops_components/src/progress_controls/progress_controls.tsx +++ b/x-pack/packages/ml/aiops_components/src/progress_controls/progress_controls.tsx @@ -26,6 +26,9 @@ import { useAnimatedProgressBarBackground } from './use_animated_progress_bar_ba // TODO Consolidate with duplicate component `CorrelationsProgressControls` in // `x-pack/plugins/apm/public/components/app/correlations/progress_controls.tsx` +/** + * Props for ProgressControlProps + */ interface ProgressControlProps { isBrushCleared: boolean; progress: number; @@ -35,8 +38,32 @@ interface ProgressControlProps { onReset: () => void; isRunning: boolean; shouldRerunAnalysis: boolean; + runAnalysisDisabled?: boolean; } +/** + * ProgressControls React Component + * Component with ability to Run & cancel analysis + * by default use `Baseline` and `Deviation` for the badge name + * @type {FC} + * @param children - List of Kibana services that are required as dependencies + * @param brushSelectionUpdateHandler - Optional callback function which gets called the brush selection has changed + * @param width - Optional width + * @param chartPoints - Data chart points + * @param chartPointsSplit - Data chart points split + * @param timeRangeEarliest - Start time range for the chart + * @param timeRangeLatest - Ending time range for the chart + * @param interval - Time interval for the document count buckets + * @param chartPointsSplitLabel - Label to name the adjustedChartPointsSplit histogram + * @param isBrushCleared - Whether or not brush has been reset + * @param autoAnalysisStart - Timestamp for start of initial analysis + * @param barColorOverride - Optional color override for the default bar color for charts + * @param barStyleAccessor - Optional style to override bar chart + * @param barHighlightColorOverride - Optional color override for the highlighted bar color for charts + * @param deviationBrush - Optional settings override for the 'deviation' brush + * @param baselineBrush - Optional settings override for the 'baseline' brush + * @returns {React.ReactElement} The ProgressControls component. + */ export const ProgressControls: FC = ({ children, isBrushCleared, @@ -47,6 +74,7 @@ export const ProgressControls: FC = ({ onReset, isRunning, shouldRerunAnalysis, + runAnalysisDisabled = false, }) => { const { euiTheme } = useEuiTheme(); const runningProgressBarStyles = useAnimatedProgressBarBackground(euiTheme.colors.success); @@ -57,6 +85,7 @@ export const ProgressControls: FC = ({ {!isRunning && ( (items: T[], initialSortField: string) { +/** + * Returned type for useTableState hook + */ +export interface UseTableState { + /** + * Callback function which gets called whenever the pagination or sorting state of the table changed + */ + onTableChange: EuiInMemoryTable['onTableChange']; + /** + * Pagination object which contains pageIndex, pageSize + */ + pagination: Pagination; + /** + * Sort field and sort direction + */ + sorting: { sort: { field: string; direction: Direction } }; + /** + * setPageIndex setter function which updates page index + */ + setPageIndex: Dispatch>; +} + +/** + * Hook to help with managing the pagination and sorting for EuiInMemoryTable + * @param {TableItem} items - data to show in the table + * @param {string} initialSortField - field name to sort by default + * @param {string} initialSortDirection - default to 'asc' + */ +export function useTableState( + items: T[], + initialSortField: string, + initialSortDirection: 'asc' | 'desc' = 'asc' +) { const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(10); const [sortField, setSortField] = useState(initialSortField); - const [sortDirection, setSortDirection] = useState('asc'); + const [sortDirection, setSortDirection] = useState(initialSortDirection); const onTableChange: EuiInMemoryTable['onTableChange'] = ({ page = { index: 0, size: 10 }, @@ -42,5 +74,10 @@ export function useTableState(items: T[], initialSortField: string) { }, }; - return { onTableChange, pagination, sorting, setPageIndex }; + return { + onTableChange, + pagination, + sorting, + setPageIndex, + }; } diff --git a/x-pack/packages/ml/in_memory_table/index.ts b/x-pack/packages/ml/in_memory_table/index.ts new file mode 100644 index 0000000000000..2975582a4298b --- /dev/null +++ b/x-pack/packages/ml/in_memory_table/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { useTableState } from './hooks/use_table_state'; +export type { UseTableState } from './hooks/use_table_state'; diff --git a/x-pack/packages/ml/in_memory_table/jest.config.js b/x-pack/packages/ml/in_memory_table/jest.config.js new file mode 100644 index 0000000000000..a0dd7192dac50 --- /dev/null +++ b/x-pack/packages/ml/in_memory_table/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../../..', + roots: ['/x-pack/packages/ml/in_memory_table'], +}; diff --git a/x-pack/packages/ml/in_memory_table/kibana.jsonc b/x-pack/packages/ml/in_memory_table/kibana.jsonc new file mode 100644 index 0000000000000..cb50985bc6159 --- /dev/null +++ b/x-pack/packages/ml/in_memory_table/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/ml-in-memory-table", + "owner": "@elastic/ml-ui" +} diff --git a/x-pack/packages/ml/in_memory_table/package.json b/x-pack/packages/ml/in_memory_table/package.json new file mode 100644 index 0000000000000..54cf41c738653 --- /dev/null +++ b/x-pack/packages/ml/in_memory_table/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/ml-in-memory-table", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} \ No newline at end of file diff --git a/x-pack/packages/ml/in_memory_table/tsconfig.json b/x-pack/packages/ml/in_memory_table/tsconfig.json new file mode 100644 index 0000000000000..67fc862a62f30 --- /dev/null +++ b/x-pack/packages/ml/in_memory_table/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react", + "@emotion/react/types/css-prop", + "@testing-library/jest-dom", + "@testing-library/react" + ] + }, + "include": [ + "**/*.ts", + "**/*.tsx", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + ] +} diff --git a/x-pack/packages/ml/query_utils/index.ts b/x-pack/packages/ml/query_utils/index.ts index c5aa2901b3613..b7dde76982130 100644 --- a/x-pack/packages/ml/query_utils/index.ts +++ b/x-pack/packages/ml/query_utils/index.ts @@ -9,3 +9,6 @@ export { addExcludeFrozenToQuery } from './src/add_exclude_frozen_to_query'; export { buildBaseFilterCriteria } from './src/build_base_filter_criteria'; export { ES_CLIENT_TOTAL_HITS_RELATION } from './src/es_client_total_hits_relation'; export { getSafeAggregationName } from './src/get_safe_aggregation_name'; +export { SEARCH_QUERY_LANGUAGE } from './src/types'; +export type { SearchQueryLanguage } from './src/types'; +export { getDefaultDSLQuery } from './src/get_default_query'; diff --git a/x-pack/packages/ml/query_utils/src/get_default_query.ts b/x-pack/packages/ml/query_utils/src/get_default_query.ts new file mode 100644 index 0000000000000..f17746ddd2929 --- /dev/null +++ b/x-pack/packages/ml/query_utils/src/get_default_query.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep } from 'lodash'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +const DEFAULT_QUERY = { + bool: { + must: [ + { + match_all: {}, + }, + ], + }, +}; + +/** + * Default DSL query which matches all the results + */ +export function getDefaultDSLQuery(): QueryDslQueryContainer { + return cloneDeep(DEFAULT_QUERY); +} diff --git a/x-pack/packages/ml/query_utils/src/types.ts b/x-pack/packages/ml/query_utils/src/types.ts new file mode 100644 index 0000000000000..ce4a92a073138 --- /dev/null +++ b/x-pack/packages/ml/query_utils/src/types.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Constant for kuery and lucene string + */ +export const SEARCH_QUERY_LANGUAGE = { + KUERY: 'kuery', + LUCENE: 'lucene', +} as const; + +/** + * Type for SearchQueryLanguage + */ +export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE]; diff --git a/x-pack/packages/ml/random_sampler_utils/index.ts b/x-pack/packages/ml/random_sampler_utils/index.ts index 941df2408ae82..5b73c4d732f89 100644 --- a/x-pack/packages/ml/random_sampler_utils/index.ts +++ b/x-pack/packages/ml/random_sampler_utils/index.ts @@ -10,3 +10,4 @@ export { createRandomSamplerWrapper, type RandomSamplerWrapper, } from './src/random_sampler_wrapper'; +export * from './src/random_sampler_manager'; diff --git a/x-pack/packages/ml/random_sampler_utils/src/random_sampler_manager.ts b/x-pack/packages/ml/random_sampler_utils/src/random_sampler_manager.ts new file mode 100644 index 0000000000000..364fbdfc4b8a0 --- /dev/null +++ b/x-pack/packages/ml/random_sampler_utils/src/random_sampler_manager.ts @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { BehaviorSubject } from 'rxjs'; +import { createRandomSamplerWrapper } from './random_sampler_wrapper'; + +/** + * List of default probabilities to use for random sampler + */ +export const RANDOM_SAMPLER_PROBABILITIES = [ + 0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, +].map((n) => n * 100); + +/** + * Default recommended minimum probability for default sampling + */ +export const MIN_SAMPLER_PROBABILITY = 0.00001; + +/** + * Default step minimum probability for default sampling + */ +export const RANDOM_SAMPLER_STEP = MIN_SAMPLER_PROBABILITY * 100; + +/** + * Default probability to use + */ +export const DEFAULT_PROBABILITY = 0.001; + +/** + * Default options for random sampler + */ +export const RANDOM_SAMPLER_OPTION = { + ON_AUTOMATIC: 'on_automatic', + ON_MANUAL: 'on_manual', + OFF: 'off', +} as const; + +/** + * Default option for random sampler type + */ +export type RandomSamplerOption = typeof RANDOM_SAMPLER_OPTION[keyof typeof RANDOM_SAMPLER_OPTION]; + +/** + * Type for the random sampler probability + */ +export type RandomSamplerProbability = number | null; + +/** + * Class that helps manage random sampling settings + * Automatically calculates the probability if only total doc count is provided + * Else, use the probability that was explicitly set + */ +export class RandomSampler { + private docCount$ = new BehaviorSubject(0); + private mode$ = new BehaviorSubject(RANDOM_SAMPLER_OPTION.ON_AUTOMATIC); + private probability$ = new BehaviorSubject(DEFAULT_PROBABILITY); + private setRandomSamplerModeInStorage: (mode: RandomSamplerOption) => void; + private setRandomSamplerProbabilityInStorage: (prob: RandomSamplerProbability) => void; + + /** + * Initial values + * @param {RandomSamplerOption} randomSamplerMode - random sampler mode + * @param setRandomSamplerMode - callback to be called when random sampler mode is set + * @param randomSamplerProbability - initial value for random sampler probability + * @param setRandomSamplerProbability - initial setter for random sampler probability + */ + constructor( + randomSamplerMode: RandomSamplerOption, + setRandomSamplerMode: (mode: RandomSamplerOption) => void, + randomSamplerProbability: RandomSamplerProbability, + setRandomSamplerProbability: (prob: RandomSamplerProbability) => void + ) { + this.mode$.next(randomSamplerMode); + this.setRandomSamplerModeInStorage = setRandomSamplerMode; + this.probability$.next(randomSamplerProbability); + this.setRandomSamplerProbabilityInStorage = setRandomSamplerProbability; + } + + /** + * Set total doc count + * If probability is not explicitly set, this doc count is used for calculating the suggested probability for sampling + * @param docCount - total document count + */ + setDocCount(docCount: number) { + return this.docCount$.next(docCount); + } + + /** + * Get doc count + */ + getDocCount() { + return this.docCount$.getValue(); + } + /** + * Set and save in storage what mode of random sampling to use + * @param {RandomSamplerOption} mode - mode to use when wrapping/unwrapping random sampling aggs + */ + public setMode(mode: RandomSamplerOption) { + this.setRandomSamplerModeInStorage(mode); + return this.mode$.next(mode); + } + + /** + * Observable to get currently set mode of random sampling + */ + public getMode$() { + return this.mode$.asObservable(); + } + + /** + * Helper to get currently set mode of random sampling + */ + public getMode() { + return this.mode$.getValue(); + } + + /** + * Helper to set the probability to use for random sampling requests + * @param {RandomSamplerProbability} probability - numeric value 0 < probability < 1 to use for random sampling + */ + public setProbability(probability: RandomSamplerProbability) { + this.setRandomSamplerProbabilityInStorage(probability); + return this.probability$.next(probability); + } + + /** + * Observability to get the probability to use for random sampling requests + */ + public getProbability$() { + return this.probability$.asObservable(); + } + + /** + * Observability to get the probability to use for random sampling requests + */ + public getProbability() { + return this.probability$.getValue(); + } + + /** + * Helper to return factory to extend any ES aggregations with the random sampling probability + * Returns wrapper = {wrap, unwrap} + * Where {wrap} extends the ES aggregations with the random sampling probability + * And {unwrap} accesses the original ES aggregations directly + */ + public createRandomSamplerWrapper() { + const mode = this.getMode(); + const probability = this.getProbability(); + + let prob = {}; + if (mode === RANDOM_SAMPLER_OPTION.ON_MANUAL) { + prob = { probability }; + } else if (mode === RANDOM_SAMPLER_OPTION.OFF) { + prob = { probability: 1 }; + } + + const wrapper = createRandomSamplerWrapper({ + ...prob, + totalNumDocs: this.getDocCount(), + }); + this.setProbability(wrapper.probability); + return wrapper; + } +} diff --git a/x-pack/plugins/aiops/public/application/utils/search_utils.ts b/x-pack/plugins/aiops/public/application/utils/search_utils.ts index 4f6e0508c83d7..99a9dfda4c642 100644 --- a/x-pack/plugins/aiops/public/application/utils/search_utils.ts +++ b/x-pack/plugins/aiops/public/application/utils/search_utils.ts @@ -26,27 +26,11 @@ import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWith import type { DataView } from '@kbn/data-views-plugin/public'; import type { SimpleSavedObject } from '@kbn/core/public'; import { isPopulatedObject } from '@kbn/ml-is-populated-object'; - -const DEFAULT_QUERY = { - bool: { - must: [ - { - match_all: {}, - }, - ], - }, -}; - -export const SEARCH_QUERY_LANGUAGE = { - KUERY: 'kuery', - LUCENE: 'lucene', -} as const; - -export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE]; - -export function getDefaultQuery() { - return cloneDeep(DEFAULT_QUERY); -} +import { + getDefaultDSLQuery, + type SearchQueryLanguage, + SEARCH_QUERY_LANGUAGE, +} from '@kbn/ml-query-utils'; export type SavedSearchSavedObject = SimpleSavedObject; @@ -94,7 +78,7 @@ export function createMergedEsQuery( dataView?: DataView, uiSettings?: IUiSettingsClient ) { - let combinedQuery: QueryDslQueryContainer = getDefaultQuery(); + let combinedQuery: QueryDslQueryContainer = getDefaultDSLQuery(); // FIXME: Add support for AggregateQuery type #150091 if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { @@ -171,7 +155,7 @@ export function getEsQueryFromSavedSearch({ // Flattened query from search source may contain a clause that narrows the time range // which might interfere with global time pickers so we need to remove const savedQuery = - cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery(); + cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultDSLQuery(); const timeField = savedSearch.searchSource.getField('index')?.timeFieldName; if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) { diff --git a/x-pack/plugins/aiops/public/application/utils/url_state.ts b/x-pack/plugins/aiops/public/application/utils/url_state.ts index f174733ea11bf..22a32a3d610ed 100644 --- a/x-pack/plugins/aiops/public/application/utils/url_state.ts +++ b/x-pack/plugins/aiops/public/application/utils/url_state.ts @@ -9,7 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { Filter, Query } from '@kbn/es-query'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from './search_utils'; +import { SEARCH_QUERY_LANGUAGE, type SearchQueryLanguage } from '@kbn/ml-query-utils'; const defaultSearchQuery = { match_all: {}, diff --git a/x-pack/plugins/aiops/public/components/change_point_detection/search_bar.tsx b/x-pack/plugins/aiops/public/components/change_point_detection/search_bar.tsx index 020c5da876b2a..69537c1ef26da 100644 --- a/x-pack/plugins/aiops/public/components/change_point_detection/search_bar.tsx +++ b/x-pack/plugins/aiops/public/components/change_point_detection/search_bar.tsx @@ -10,7 +10,7 @@ import { i18n } from '@kbn/i18n'; import { type Filter, fromKueryExpression, type Query } from '@kbn/es-query'; import { type SearchBarOwnProps } from '@kbn/unified-search-plugin/public/search_bar'; import { EuiSpacer, EuiTextColor } from '@elastic/eui'; -import { SEARCH_QUERY_LANGUAGE } from '../../application/utils/search_utils'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { useDataSource } from '../../hooks/use_data_source'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; diff --git a/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx b/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx index 01fa56b1470d9..cc6e147f5ec00 100644 --- a/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/aiops/public/components/document_count_content/document_count_content/document_count_content.tsx @@ -11,9 +11,14 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import type { WindowParameters } from '@kbn/aiops-utils'; +import { + BarStyleAccessor, + RectAnnotationSpec, +} from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; +import { DocumentCountChart, type DocumentCountChartPoint } from '@kbn/aiops-components'; +import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context'; import { DocumentCountStats } from '../../../get_document_stats'; -import { DocumentCountChart, DocumentCountChartPoint } from '../document_count_chart'; import { TotalCountHeader } from '../total_count_header'; export interface DocumentCountContentProps { @@ -29,6 +34,13 @@ export interface DocumentCountContentProps { barColorOverride?: string; /** Optional color override for the highlighted bar color for charts */ barHighlightColorOverride?: string; + windowParameters?: WindowParameters; + incomingInitialAnalysisStart?: number | WindowParameters; + baselineLabel?: string; + deviationLabel?: string; + barStyleAccessor?: BarStyleAccessor; + baselineAnnotationStyle?: RectAnnotationSpec['style']; + deviationAnnotationStyle?: RectAnnotationSpec['style']; } export const DocumentCountContent: FC = ({ @@ -42,7 +54,12 @@ export const DocumentCountContent: FC = ({ initialAnalysisStart, barColorOverride, barHighlightColorOverride, + windowParameters, + incomingInitialAnalysisStart, + ...docCountChartProps }) => { + const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext(); + const bucketTimestamps = Object.keys(documentCountStats?.buckets ?? {}).map((time) => +time); const splitBucketTimestamps = Object.keys(documentCountStatsSplit?.buckets ?? {}).map( (time) => +time @@ -84,6 +101,7 @@ export const DocumentCountContent: FC = ({ {documentCountStats.interval !== undefined && ( = ({ autoAnalysisStart={initialAnalysisStart} barColorOverride={barColorOverride} barHighlightColorOverride={barHighlightColorOverride} + {...docCountChartProps} /> )} diff --git a/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx b/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx index 103e5c7d846df..1dc30c04253bc 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx @@ -21,12 +21,12 @@ import { import { DataViewField } from '@kbn/data-views-plugin/common'; import { Filter } from '@kbn/es-query'; +import { useTableState } from '@kbn/ml-in-memory-table'; import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links'; import { MiniHistogram } from '../../mini_histogram'; import { useEuiTheme } from '../../../hooks/use_eui_theme'; import type { LogCategorizationAppState } from '../../../application/utils/url_state'; import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request'; -import { useTableState } from './use_table_state'; import { getLabels } from './labels'; import { TableHeader } from './table_header'; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx b/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx index 7e02eabf01dbc..d34861b924ccb 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx @@ -8,7 +8,8 @@ import React, { FC, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; -import { DocumentCountChart as DocumentCountChartRoot } from '../document_count_content/document_count_chart'; +import { DocumentCountChart as DocumentCountChartRoot } from '@kbn/aiops-components'; +import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; import { TotalCountHeader } from '../document_count_content/total_count_header'; import type { Category, SparkLinesPerCategory } from './use_categorize_request'; import type { EventRate } from './use_categorize_request'; @@ -31,6 +32,8 @@ export const DocumentCountChart: FC = ({ selectedCategory, documentCountStats, }) => { + const { data, uiSettings, fieldFormats, charts } = useAiopsAppContext(); + const chartPointsSplitLabel = i18n.translate( 'xpack.aiops.logCategorization.chartPointsSplitLabel', { @@ -69,6 +72,7 @@ export const DocumentCountChart: FC = ({ <> FileDataVisualizerSpec> { const modules = await lazyLoadModules(); @@ -17,3 +21,8 @@ export async function getIndexDataVisualizerComponent(): Promise<() => IndexData const modules = await lazyLoadModules(); return () => modules.IndexDataVisualizer; } + +export async function getDataComparisonComponent(): Promise<() => DataComparisonSpec> { + const modules = await lazyLoadModules(); + return () => modules.DataComparison; +} diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx index cb4f55b49cb41..911a6ba9ea731 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/document_count_content.tsx @@ -20,7 +20,6 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; -import { isDefined } from '@kbn/ml-is-defined'; import type { DocumentCountChartPoint } from './document_count_chart'; import { RandomSamplerOption, @@ -30,7 +29,8 @@ import { import { TotalCountHeader } from './total_count_header'; import type { DocumentCountStats } from '../../../../../common/types/field_stats'; import { DocumentCountChart } from './document_count_chart'; -import { RandomSamplerRangeSlider } from './random_sampler_range_slider'; +import { RandomSamplerRangeSlider } from '../random_sampling_menu/random_sampler_range_slider'; +import { ProbabilityUsedMessage } from '../random_sampling_menu/probability_used'; export interface Props { documentCountStats?: DocumentCountStats; @@ -42,20 +42,6 @@ export interface Props { loading: boolean; } -const ProbabilityUsedMessage = ({ samplingProbability }: Pick) => { - return isDefined(samplingProbability) ? ( -
- - - -
- ) : null; -}; - const CalculatingProbabilityMessage = (
diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/probability_used.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/probability_used.tsx new file mode 100644 index 0000000000000..e36b59791a2d9 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/probability_used.tsx @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isDefined } from '@kbn/ml-is-defined'; +import { EuiSpacer } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import React from 'react'; +import { Props } from '../document_count_content/document_count_content'; + +export const ProbabilityUsedMessage = ({ + samplingProbability, +}: Pick) => { + return isDefined(samplingProbability) ? ( +
+ + + +
+ ) : null; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/random_sampler_range_slider.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampler_range_slider.tsx similarity index 100% rename from x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/random_sampler_range_slider.tsx rename to x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampler_range_slider.tsx index 1ccf5b16c411e..926dd0ed80af1 100644 --- a/x-pack/plugins/data_visualizer/public/application/common/components/document_count_content/random_sampler_range_slider.tsx +++ b/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampler_range_slider.tsx @@ -5,12 +5,12 @@ * 2.0. */ +import React, { useState } from 'react'; +import { isDefined } from '@kbn/ml-is-defined'; import { EuiButton, EuiFlexItem, EuiFormRow, EuiRange, EuiSpacer } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { isDefined } from '@kbn/ml-is-defined'; -import { FormattedMessage } from '@kbn/i18n-react'; -import React, { useState } from 'react'; import { roundToDecimalPlace } from '@kbn/ml-number-utils'; +import { FormattedMessage } from '@kbn/i18n-react'; import { MIN_SAMPLER_PROBABILITY, RANDOM_SAMPLER_PROBABILITIES, diff --git a/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampling_menu.tsx b/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampling_menu.tsx new file mode 100644 index 0000000000000..41b0413662012 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/components/random_sampling_menu/random_sampling_menu.tsx @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useCallback, useMemo, useState } from 'react'; +import useObservable from 'react-use/lib/useObservable'; +import { i18n } from '@kbn/i18n'; +import { + EuiButtonEmpty, + EuiCallOut, + EuiFlexItem, + EuiFormRow, + EuiPanel, + EuiPopover, + EuiSelect, + EuiSpacer, +} from '@elastic/eui'; +import { RandomSampler } from '@kbn/ml-random-sampler-utils'; +import { RandomSamplerRangeSlider } from './random_sampler_range_slider'; +import { + MIN_SAMPLER_PROBABILITY, + RANDOM_SAMPLER_OPTION, + RANDOM_SAMPLER_SELECT_OPTIONS, + RandomSamplerOption, +} from '../../../index_data_visualizer/constants/random_sampler'; +import { ProbabilityUsedMessage } from './probability_used'; + +interface Props { + randomSampler: RandomSampler; + reload: () => void; +} + +export const SamplingMenu: FC = ({ randomSampler, reload }) => { + const [showSamplingOptionsPopover, setShowSamplingOptionsPopover] = useState(false); + + const samplingProbability = useObservable( + randomSampler.getProbability$(), + randomSampler.getProbability() + ); + + const setSamplingProbability = useCallback( + (probability: number | null) => { + randomSampler.setProbability(probability); + reload(); + }, + [reload, randomSampler] + ); + + const randomSamplerPreference = useObservable(randomSampler.getMode$(), randomSampler.getMode()); + + const setRandomSamplerPreference = useCallback( + (nextPref: RandomSamplerOption) => { + if (nextPref === RANDOM_SAMPLER_OPTION.ON_MANUAL) { + // By default, when switching to manual, restore previously chosen probability + // else, default to 0.001% + const savedRandomSamplerProbability = randomSampler.getProbability(); + randomSampler.setProbability( + savedRandomSamplerProbability && + savedRandomSamplerProbability > 0 && + savedRandomSamplerProbability <= 0.5 + ? savedRandomSamplerProbability + : MIN_SAMPLER_PROBABILITY + ); + } + randomSampler.setMode(nextPref); + reload(); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [setSamplingProbability, randomSampler] + ); + + const { calloutInfoMessage, buttonText } = useMemo(() => { + switch (randomSamplerPreference) { + case RANDOM_SAMPLER_OPTION.OFF: + return { + calloutInfoMessage: i18n.translate( + 'xpack.dataVisualizer.randomSamplerSettingsPopUp.offCallout.message', + { + defaultMessage: + 'Random sampling can be turned on to increase the speed of analysis, although some accuracy will be lost.', + } + ), + buttonText: i18n.translate( + 'xpack.dataVisualizer.randomSamplerSettingsPopUp.offCallout.button', + { + defaultMessage: 'No sampling', + } + ), + }; + case RANDOM_SAMPLER_OPTION.ON_AUTOMATIC: + return { + calloutInfoMessage: i18n.translate( + 'xpack.dataVisualizer.randomSamplerSettingsPopUp.onAutomaticCallout.message', + { + defaultMessage: + 'The view will use random sampler aggregations. The probability is automatically set to balance accuracy and speed.', + } + ), + buttonText: i18n.translate( + 'xpack.dataVisualizer.randomSamplerSettingsPopUp.onAutomaticCallout.button', + { + defaultMessage: 'Auto sampling', + } + ), + }; + + case RANDOM_SAMPLER_OPTION.ON_MANUAL: + default: + return { + calloutInfoMessage: i18n.translate( + 'xpack.dataVisualizer.randomSamplerSettingsPopUp.onManualCallout.message', + { + defaultMessage: + 'The view will use random sampler aggregations. A lower percentage probability increases performance, but some accuracy is lost.', + } + ), + buttonText: i18n.translate( + 'xpack.dataVisualizer.randomSamplerSettingsPopUp.onManualCallout.button', + { + defaultMessage: 'Manual sampling', + } + ), + }; + } + }, [randomSamplerPreference]); + + return ( + setShowSamplingOptionsPopover(!showSamplingOptionsPopover)} + iconSide="right" + iconType="arrowDown" + > + {buttonText} + + } + isOpen={showSamplingOptionsPopover} + closePopover={() => setShowSamplingOptionsPopover(false)} + panelPaddingSize="none" + anchorPosition="downLeft" + > + + + + + + + + setRandomSamplerPreference(e.target.value as RandomSamplerOption)} + /> + + + {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_MANUAL ? ( + + ) : null} + + {randomSamplerPreference === RANDOM_SAMPLER_OPTION.ON_AUTOMATIC ? ( + + ) : null} + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/hooks/data_source_context.ts b/x-pack/plugins/data_visualizer/public/application/common/hooks/data_source_context.ts new file mode 100644 index 0000000000000..fcb627bf925ba --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/hooks/data_source_context.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createContext, useContext } from 'react'; +import type { DataView } from '@kbn/data-views-plugin/common'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; + +export const DataSourceContext = createContext<{ + dataView: DataView | never; + savedSearch: SavedSearch | null; +}>({ + get dataView(): never { + throw new Error('DataSourceContext is not implemented'); + }, + savedSearch: null, +}); + +export function useDataSource() { + return useContext(DataSourceContext); +} diff --git a/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts new file mode 100644 index 0000000000000..9e7e3f9633291 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_data.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataView } from '@kbn/data-views-plugin/common'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { Dictionary } from '@kbn/ml-url-state'; +import { Moment } from 'moment'; +import { useExecutionContext } from '@kbn/kibana-react-plugin/public'; +import { useEffect, useMemo, useState } from 'react'; +import { mlTimefilterRefresh$, useTimefilter } from '@kbn/ml-date-picker'; +import { merge } from 'rxjs'; +import { RandomSampler } from '@kbn/ml-random-sampler-utils'; +import { + DocumentStatsSearchStrategyParams, + useDocumentCountStats, +} from './use_document_count_stats'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import { useTimeBuckets } from './use_time_buckets'; + +const DEFAULT_BAR_TARGET = 75; + +export const useData = ( + selectedDataView: DataView, + contextId: string, + searchQuery: estypes.QueryDslQueryContainer, + randomSampler: RandomSampler, + onUpdate?: (params: Dictionary) => void, + barTarget: number = DEFAULT_BAR_TARGET, + timeRange?: { min: Moment; max: Moment } +) => { + const { + services: { executionContext }, + } = useDataVisualizerKibana(); + + useExecutionContext(executionContext, { + name: 'data_visualizer', + type: 'application', + id: contextId, + }); + + const [lastRefresh, setLastRefresh] = useState(0); + + const _timeBuckets = useTimeBuckets(); + const timefilter = useTimefilter({ + timeRangeSelector: selectedDataView?.timeFieldName !== undefined, + autoRefreshSelector: true, + }); + + const docCountRequestParams: DocumentStatsSearchStrategyParams | undefined = useMemo(() => { + const timefilterActiveBounds = timeRange ?? timefilter.getActiveBounds(); + if (timefilterActiveBounds !== undefined) { + _timeBuckets.setInterval('auto'); + _timeBuckets.setBounds(timefilterActiveBounds); + _timeBuckets.setBarTarget(barTarget); + return { + earliest: timefilterActiveBounds.min?.valueOf(), + latest: timefilterActiveBounds.max?.valueOf(), + intervalMs: _timeBuckets.getInterval()?.asMilliseconds(), + index: selectedDataView.getIndexPattern(), + searchQuery, + timeFieldName: selectedDataView.timeFieldName, + runtimeFieldMap: selectedDataView.getRuntimeMappings(), + }; + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [lastRefresh, JSON.stringify({ searchQuery, timeRange })]); + + const documentStats = useDocumentCountStats(docCountRequestParams, lastRefresh, randomSampler); + + useEffect(() => { + const timefilterUpdateSubscription = merge( + timefilter.getAutoRefreshFetch$(), + timefilter.getTimeUpdate$(), + mlTimefilterRefresh$ + ).subscribe(() => { + if (onUpdate) { + onUpdate({ + time: timefilter.getTime(), + refreshInterval: timefilter.getRefreshInterval(), + }); + setLastRefresh(Date.now()); + } + }); + + // This listens just for an initial update of the timefilter to be switched on. + const timefilterEnabledSubscription = timefilter.getEnabledUpdated$().subscribe(() => { + if (docCountRequestParams === undefined) { + setLastRefresh(Date.now()); + } + }); + + return () => { + timefilterUpdateSubscription.unsubscribe(); + timefilterEnabledSubscription.unsubscribe(); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return { + documentStats, + timefilter, + /** Start timestamp filter */ + earliest: docCountRequestParams?.earliest, + /** End timestamp filter */ + latest: docCountRequestParams?.latest, + intervalMs: docCountRequestParams?.intervalMs, + forceRefresh: () => setLastRefresh(Date.now()), + }; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/hooks/use_document_count_stats.ts b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_document_count_stats.ts new file mode 100644 index 0000000000000..6514b1f98a292 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_document_count_stats.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { stringHash } from '@kbn/ml-string-hash'; +import { extractErrorProperties } from '@kbn/ml-error-utils'; +import { Query } from '@kbn/es-query'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { SignificantTerm } from '@kbn/ml-agg-utils'; +import { + createRandomSamplerWrapper, + RandomSampler, + RandomSamplerWrapper, +} from '@kbn/ml-random-sampler-utils'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { each, get } from 'lodash'; +import { lastValueFrom } from 'rxjs'; +import { buildBaseFilterCriteria } from '@kbn/ml-query-utils'; +import useObservable from 'react-use/lib/useObservable'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import { displayError } from '../util/display_error'; + +export const RANDOM_SAMPLER_SEED = 3867418; + +export interface DocumentStats { + sampleProbability: number; + totalCount: number; + documentCountStats?: DocumentCountStats; + documentCountStatsCompare?: DocumentCountStats; +} + +export interface DocumentCountStats { + interval?: number; + buckets?: { [key: string]: number }; + timeRangeEarliest?: number; + timeRangeLatest?: number; + totalCount: number; +} + +export interface DocumentStatsSearchStrategyParams { + earliest?: number; + latest?: number; + intervalMs?: number; + index: string; + searchQuery: Query['query']; + timeFieldName?: string; + runtimeFieldMap?: estypes.MappingRuntimeFields; + fieldsToFetch?: string[]; + selectedSignificantTerm?: SignificantTerm; + includeSelectedSignificantTerm?: boolean; + trackTotalHits?: boolean; +} + +export const getDocumentCountStatsRequest = ( + params: DocumentStatsSearchStrategyParams, + randomSamplerWrapper?: RandomSamplerWrapper, + skipAggs = false +) => { + const { + index, + timeFieldName, + earliest: earliestMs, + latest: latestMs, + runtimeFieldMap, + searchQuery, + intervalMs, + fieldsToFetch, + trackTotalHits, + } = params; + + const size = 0; + const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, searchQuery); + + const rawAggs: Record = { + eventRate: { + date_histogram: { + field: timeFieldName, + fixed_interval: `${intervalMs}ms`, + min_doc_count: 0, + ...(earliestMs !== undefined && latestMs !== undefined + ? { + extended_bounds: { + min: earliestMs, + max: latestMs, + }, + } + : {}), + }, + }, + }; + + const aggs = randomSamplerWrapper ? randomSamplerWrapper.wrap(rawAggs) : rawAggs; + + const searchBody = { + query: { + bool: { + filter: filterCriteria, + }, + }, + ...(!fieldsToFetch && + !skipAggs && + timeFieldName !== undefined && + intervalMs !== undefined && + intervalMs > 0 + ? { aggs } + : {}), + ...(isPopulatedObject(runtimeFieldMap) ? { runtime_mappings: runtimeFieldMap } : {}), + track_total_hits: trackTotalHits === true, + size, + }; + return { + index, + body: searchBody, + }; +}; + +export const processDocumentCountStats = ( + body: estypes.SearchResponse | undefined, + params: DocumentStatsSearchStrategyParams, + randomSamplerWrapper?: RandomSamplerWrapper +): DocumentCountStats | undefined => { + if (!body) return undefined; + + const totalCount = (body.hits.total as estypes.SearchTotalHits).value ?? body.hits.total ?? 0; + + if ( + params.intervalMs === undefined || + params.earliest === undefined || + params.latest === undefined + ) { + return { + totalCount, + }; + } + const buckets: { [key: string]: number } = {}; + const dataByTimeBucket: Array<{ key: string; doc_count: number }> = get( + randomSamplerWrapper && body.aggregations !== undefined + ? randomSamplerWrapper.unwrap(body.aggregations) + : body.aggregations, + ['eventRate', 'buckets'], + [] + ); + each(dataByTimeBucket, (dataForTime) => { + const time = dataForTime.key; + buckets[time] = dataForTime.doc_count; + }); + + return { + interval: params.intervalMs, + buckets, + timeRangeEarliest: params.earliest, + timeRangeLatest: params.latest, + totalCount, + }; +}; + +export interface DocumentStatsSearchStrategyParams { + earliest?: number; + latest?: number; + intervalMs?: number; + index: string; + searchQuery: Query['query']; + timeFieldName?: string; + runtimeFieldMap?: estypes.MappingRuntimeFields; + fieldsToFetch?: string[]; + selectedSignificantTerm?: SignificantTerm; + includeSelectedSignificantTerm?: boolean; + trackTotalHits?: boolean; +} + +export function useDocumentCountStats( + searchParams: TParams | undefined, + lastRefresh: number, + randomSampler: RandomSampler +): DocumentStats { + const { + data, + notifications: { toasts }, + } = useDataVisualizerKibana().services; + + const abortCtrl = useRef(new AbortController()); + + const [documentStats, setDocumentStats] = useState({ + sampleProbability: 1, + totalCount: 0, + }); + + const [documentStatsCache, setDocumentStatsCache] = useState>({}); + const samplingProbability = useObservable( + randomSampler.getProbability$(), + randomSampler.getProbability() + ); + + const fetchDocumentCountData = useCallback(async () => { + if (!searchParams) return; + + const cacheKey = stringHash( + `${JSON.stringify(searchParams)}-${randomSampler.getProbability()}` + ); + + if (documentStatsCache[cacheKey]) { + setDocumentStats(documentStatsCache[cacheKey]); + return; + } + + try { + abortCtrl.current = new AbortController(); + + const totalHitsParams = { + ...searchParams, + selectedSignificantTerm: undefined, + trackTotalHits: true, + }; + + const totalHitsResp = await lastValueFrom( + data.search.search( + { + params: getDocumentCountStatsRequest(totalHitsParams, undefined, true), + }, + { abortSignal: abortCtrl.current.signal } + ) + ); + const totalHitsStats = processDocumentCountStats(totalHitsResp?.rawResponse, searchParams); + const totalCount = totalHitsStats?.totalCount ?? 0; + + if (randomSampler) { + randomSampler.setDocCount(totalCount); + } + const randomSamplerWrapper = randomSampler + ? randomSampler.createRandomSamplerWrapper() + : createRandomSamplerWrapper({ + totalNumDocs: totalCount, + seed: RANDOM_SAMPLER_SEED, + }); + const resp = await lastValueFrom( + data.search.search( + { + params: getDocumentCountStatsRequest( + { ...searchParams, trackTotalHits: false }, + randomSamplerWrapper + ), + }, + { abortSignal: abortCtrl.current.signal } + ) + ); + + const documentCountStats = processDocumentCountStats( + resp?.rawResponse, + searchParams, + randomSamplerWrapper + ); + + const newStats: DocumentStats = { + sampleProbability: randomSamplerWrapper.probability, + documentCountStats, + totalCount, + }; + + setDocumentStatsCache({ + ...documentStatsCache, + [cacheKey]: newStats, + }); + } catch (error) { + // An `AbortError` gets triggered when a user cancels a request by navigating away, we need to ignore these errors. + if (error.name !== 'AbortError') { + displayError(toasts, searchParams!.index, extractErrorProperties(error)); + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [data?.search, documentStatsCache, searchParams, toasts, samplingProbability]); + + useEffect( + function getDocumentCountData() { + fetchDocumentCountData(); + return () => abortCtrl.current.abort(); + }, + [fetchDocumentCountData, lastRefresh, samplingProbability] + ); + + // Clear the document count stats cache when the outer page (date picker/search bar) triggers a refresh. + useEffect(() => { + setDocumentStatsCache({}); + }, [lastRefresh]); + + return documentStats; +} diff --git a/x-pack/plugins/data_visualizer/public/application/common/hooks/use_search.ts b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_search.ts new file mode 100644 index 0000000000000..0dc825dd295f8 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_search.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { DataView } from '@kbn/data-views-plugin/public'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; + +import { useEffect } from 'react'; +import { FilterStateStore } from '@kbn/es-query'; +import { getEsQueryFromSavedSearch } from '../../index_data_visualizer/utils/saved_search_utils'; +import { useDataVisualizerKibana } from '../../kibana_context'; +import type { BasicAppState } from '../../data_comparison/types'; + +export const useSearch = ( + { dataView, savedSearch }: { dataView: DataView; savedSearch: SavedSearch | null | undefined }, + appState: BasicAppState, + readOnly: boolean = false +) => { + const { + uiSettings, + data: { + query: { filterManager }, + }, + } = useDataVisualizerKibana().services; + + useEffect( + function clearFiltersOnLeave() { + return () => { + // We want to clear all filters that have not been pinned globally + // when navigating to other pages + filterManager + .getFilters() + .filter((f) => f.$state?.store === FilterStateStore.APP_STATE) + .forEach((f) => filterManager.removeFilter(f)); + }; + }, + [filterManager] + ); + + const searchData = getEsQueryFromSavedSearch({ + dataView, + uiSettings, + savedSearch, + filterManager, + }); + + if (searchData === undefined || (appState && appState.searchString !== '')) { + if (appState?.filters && readOnly === false) { + const globalFilters = filterManager?.getGlobalFilters(); + + if (filterManager) filterManager.setFilters(appState.filters); + if (globalFilters) filterManager?.addFilters(globalFilters); + } + return { + searchQuery: appState?.searchQuery, + searchString: appState?.searchString, + searchQueryLanguage: appState?.searchQueryLanguage, + }; + } else { + return { + searchQuery: searchData.searchQuery, + searchString: searchData.searchString, + searchQueryLanguage: searchData.queryLanguage, + }; + } +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/hooks/use_time_buckets.ts b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_time_buckets.ts new file mode 100644 index 0000000000000..0c7999dba9adc --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/hooks/use_time_buckets.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useMemo } from 'react'; +import { UI_SETTINGS } from '@kbn/data-service'; +import { TimeBuckets } from '../../../../common/services/time_buckets'; +import { useDataVisualizerKibana } from '../../kibana_context'; + +export const useTimeBuckets = () => { + const { uiSettings } = useDataVisualizerKibana().services; + + return useMemo(() => { + return new TimeBuckets({ + [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/display_error.ts b/x-pack/plugins/data_visualizer/public/application/common/util/display_error.ts new file mode 100644 index 0000000000000..c36bb0f03e51d --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/util/display_error.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToastsStart } from '@kbn/core-notifications-browser'; +import { i18n } from '@kbn/i18n'; + +export function displayError(toastNotifications: ToastsStart, index: string, err: any) { + if (err.statusCode === 500) { + toastNotifications.addError(err, { + title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', { + defaultMessage: + 'Error loading data in index {index}. {message}. ' + + 'The request may have timed out. Try using a smaller sample size or narrowing the time range.', + values: { + index, + message: err.error ?? err.message, + }, + }), + }); + } else { + toastNotifications.addError(err, { + title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', { + defaultMessage: 'Error loading data in index {index}. {message}.', + values: { + index, + message: err.error ?? err.message, + }, + }), + }); + } +} diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.test.ts b/x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.test.ts new file mode 100644 index 0000000000000..fa060ee60079e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getErrorMessagesFromEsShardFailures } from './get_error_messages_from_es_shard_failures'; + +describe('getErrorMessagesFromEsShardFailures', () => { + test('returns extracted reasons if _shard.failures exist', () => { + const reason = + '[parent] Data too large, data for [] would be [1059268784/1010.1mb], which is larger than the limit of [1020054732/972.7mb], real usage: [940288176/896.7mb], new bytes reserved: [118980608/113.4mb], usages [inflight_requests=60008/58.6kb, model_inference=0/0b, eql_sequence=0/0b, fielddata=245141854/233.7mb, request=162398296/154.8mb]'; + const resp = { + took: 37, + timed_out: false, + _shards: { + total: 2, + successful: 1, + skipped: 1, + failed: 1, + failures: [ + { + shard: 0, + index: 'apm-7.2.0-span-2019.10.31', + node: 'PEyAKEkKQFql88n4oXyYMw', + reason: { + type: 'circuit_breaking_exception', + reason, + bytes_wanted: 1059268784, + bytes_limit: 1020054732, + durability: 'PERMANENT', + }, + }, + ], + }, + hits: { + total: 0, + max_score: 0, + hits: [], + }, + }; + expect(getErrorMessagesFromEsShardFailures(resp)).toEqual([reason]); + }); + + test('returns empty array if _shard.failures not defined', () => { + const resp = { + took: 37, + timed_out: false, + }; + expect(getErrorMessagesFromEsShardFailures(resp)).toEqual([]); + expect(getErrorMessagesFromEsShardFailures(null)).toEqual([]); + expect(getErrorMessagesFromEsShardFailures(undefined)).toEqual([]); + expect(getErrorMessagesFromEsShardFailures('')).toEqual([]); + }); +}); diff --git a/x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.ts b/x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.ts new file mode 100644 index 0000000000000..d0e39eae1fd50 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/common/util/get_error_messages_from_es_shard_failures.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { isDefined } from '@kbn/ml-is-defined'; + +export const getErrorMessagesFromEsShardFailures = (arg?: unknown): string[] => { + if (isPopulatedObject(arg, ['_shards'])) { + return (arg._shards.failures ?? []) + .map((failure) => + isPopulatedObject(failure, ['reason']) && failure.reason.reason + ? failure.reason.reason + : undefined + ) + .filter(isDefined); + } + return []; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/data_comparison_distribution_chart.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/data_comparison_distribution_chart.tsx new file mode 100644 index 0000000000000..96a4876d5df11 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/data_comparison_distribution_chart.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Axis, BarSeries, Chart, Position, ScaleType, Settings, Tooltip } from '@elastic/charts'; +import React from 'react'; +import { NoChartsData } from './no_charts_data'; +import { ComparisonHistogram } from '../types'; +import { DataComparisonChartTooltipBody } from '../data_comparison_chart_tooltip_body'; +import { COMPARISON_LABEL, DATA_COMPARISON_TYPE } from '../constants'; + +export const DataComparisonDistributionChart = ({ + featureName, + fieldType, + data, + colors, +}: { + featureName: string; + fieldType: string; + data: ComparisonHistogram[]; + colors: { referenceColor: string; productionColor: string }; +}) => { + if (data.length === 0) return ; + return ( + + + + + Number(d).toFixed(2)} /> + { + const key = identifier.seriesKeys[0]; + return key === COMPARISON_LABEL ? colors.productionColor : colors.referenceColor; + }} + /> + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/no_charts_data.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/no_charts_data.tsx new file mode 100644 index 0000000000000..1c35f700f52f2 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/no_charts_data.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FormattedMessage } from '@kbn/i18n-react'; +import { EuiText, type EuiTextProps } from '@elastic/eui'; +import React from 'react'; + +export const NoChartsData = ({ textAlign }: { textAlign?: EuiTextProps['textAlign'] }) => { + return ( + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/overlap_distribution_chart.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/overlap_distribution_chart.tsx new file mode 100644 index 0000000000000..34f6a797831c0 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/overlap_distribution_chart.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AreaSeries, Chart, CurveType, ScaleType, Settings, Tooltip } from '@elastic/charts'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { NoChartsData } from './no_charts_data'; +import type { ComparisonHistogram, DataComparisonField } from '../types'; +import { DataComparisonChartTooltipBody } from '../data_comparison_chart_tooltip_body'; +import { COMPARISON_LABEL, DATA_COMPARISON_TYPE, REFERENCE_LABEL } from '../constants'; + +export const OverlapDistributionComparison = ({ + data, + colors, + fieldType, + fieldName, +}: { + data: ComparisonHistogram[]; + colors: { referenceColor: string; productionColor: string }; + fieldType?: DataComparisonField['type']; + fieldName?: DataComparisonField['field']; +}) => { + if (data.length === 0) return ; + + return ( + + + + + { + const key = identifier.seriesKeys[0]; + return key === COMPARISON_LABEL ? colors.productionColor : colors.referenceColor; + }} + /> + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/single_distribution_chart.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/single_distribution_chart.tsx new file mode 100644 index 0000000000000..22796f371cb55 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/charts/single_distribution_chart.tsx @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SeriesColorAccessor } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; +import { BarSeries, Chart, ScaleType, Settings } from '@elastic/charts'; +import React from 'react'; +import { NoChartsData } from './no_charts_data'; +import { DATA_COMPARISON_TYPE } from '../constants'; +import { DataComparisonField, Histogram } from '../types'; + +export const SingleDistributionChart = ({ + data, + color, + fieldType, + name, +}: { + data: Histogram[]; + name: string; + color?: SeriesColorAccessor; + fieldType?: DataComparisonField['type']; +}) => { + if (data.length === 0) return ; + + return ( + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/constants.ts b/x-pack/plugins/data_visualizer/public/application/data_comparison/constants.ts new file mode 100644 index 0000000000000..e64e19cc736fd --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/constants.ts @@ -0,0 +1,1064 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const DATA_COMPARISON_TYPE = { + NUMERIC: 'numeric', + CATEGORICAL: 'categorical', + UNSUPPORTED: 'unsupported', +} as const; + +export const NUMERIC_TYPE_LABEL = i18n.translate( + 'xpack.dataVisualizer.dataComparison.numericLabel', + { + defaultMessage: 'Numeric', + } +); +export const CATEGORICAL_TYPE_LABEL = i18n.translate( + 'xpack.dataVisualizer.dataComparison.categoricalLabel', + { + defaultMessage: 'Categorical', + } +); + +export const UNSUPPORTED_LABEL = i18n.translate( + 'xpack.dataVisualizer.dataComparison.UnsupportedLabel', + { + defaultMessage: 'Unsupported', + } +); + +export const REFERENCE_LABEL = i18n.translate( + 'xpack.dataVisualizer.dataComparison.referenceLabel', + { + defaultMessage: 'Reference', + } +); + +export const COMPARISON_LABEL = i18n.translate( + 'xpack.dataVisualizer.dataComparison.productionLabel', + { + defaultMessage: 'Comparison', + } +); + +export const DATA_COMPARISON_TYPE_LABEL = { + [DATA_COMPARISON_TYPE.NUMERIC]: NUMERIC_TYPE_LABEL, + [DATA_COMPARISON_TYPE.CATEGORICAL]: CATEGORICAL_TYPE_LABEL, + [DATA_COMPARISON_TYPE.UNSUPPORTED]: UNSUPPORTED_LABEL, +} as const; + +export const DRIFT_P_VALUE_THRESHOLD = 0.05; +/** + * Table generated from following python code + df = range(1,100) + + # levels of significance + significance_levels = np.concatenate((np.logspace(-6, -3, 3), np.linspace(0.01, 0.99, 99))) + + # create the table + table = [] + for d in df: + row = [] + for l in significance_levels: + row.append(round(stats.chi2.ppf(1 - l, d), 2)) + table.append(row) + + critical_value_table = np.array(table) + * until we find a low size replacement for doing chi2test in js + */ +export const CRITICAL_VALUES_TABLE = [ + [ + 23.93, 17.32, 10.83, 6.63, 5.41, 4.71, 4.22, 3.84, 3.54, 3.28, 3.06, 2.87, 2.71, 2.55, 2.42, + 2.29, 2.18, 2.07, 1.97, 1.88, 1.8, 1.72, 1.64, 1.57, 1.5, 1.44, 1.38, 1.32, 1.27, 1.22, 1.17, + 1.12, 1.07, 1.03, 0.99, 0.95, 0.91, 0.87, 0.84, 0.8, 0.77, 0.74, 0.71, 0.68, 0.65, 0.62, 0.6, + 0.57, 0.55, 0.52, 0.5, 0.48, 0.45, 0.43, 0.41, 0.39, 0.38, 0.36, 0.34, 0.32, 0.31, 0.29, 0.27, + 0.26, 0.25, 0.23, 0.22, 0.21, 0.19, 0.18, 0.17, 0.16, 0.15, 0.14, 0.13, 0.12, 0.11, 0.1, 0.09, + 0.09, 0.08, 0.07, 0.06, 0.06, 0.05, 0.05, 0.04, 0.04, 0.03, 0.03, 0.02, 0.02, 0.02, 0.01, 0.01, + 0.01, 0.01, 0.0, 0.0, 0.0, 0.0, 0.0, + ], + [ + 27.63, 20.72, 13.82, 9.21, 7.82, 7.01, 6.44, 5.99, 5.63, 5.32, 5.05, 4.82, 4.61, 4.41, 4.24, + 4.08, 3.93, 3.79, 3.67, 3.54, 3.43, 3.32, 3.22, 3.12, 3.03, 2.94, 2.85, 2.77, 2.69, 2.62, 2.55, + 2.48, 2.41, 2.34, 2.28, 2.22, 2.16, 2.1, 2.04, 1.99, 1.94, 1.88, 1.83, 1.78, 1.74, 1.69, 1.64, + 1.6, 1.55, 1.51, 1.47, 1.43, 1.39, 1.35, 1.31, 1.27, 1.23, 1.2, 1.16, 1.12, 1.09, 1.06, 1.02, + 0.99, 0.96, 0.92, 0.89, 0.86, 0.83, 0.8, 0.77, 0.74, 0.71, 0.68, 0.66, 0.63, 0.6, 0.58, 0.55, + 0.52, 0.5, 0.47, 0.45, 0.42, 0.4, 0.37, 0.35, 0.33, 0.3, 0.28, 0.26, 0.23, 0.21, 0.19, 0.17, + 0.15, 0.12, 0.1, 0.08, 0.06, 0.04, 0.02, + ], + [ + 30.66, 23.51, 16.27, 11.34, 9.84, 8.95, 8.31, 7.81, 7.41, 7.06, 6.76, 6.49, 6.25, 6.03, 5.83, + 5.65, 5.48, 5.32, 5.17, 5.02, 4.89, 4.76, 4.64, 4.53, 4.41, 4.31, 4.21, 4.11, 4.01, 3.92, 3.83, + 3.75, 3.66, 3.58, 3.51, 3.43, 3.36, 3.28, 3.21, 3.14, 3.08, 3.01, 2.95, 2.88, 2.82, 2.76, 2.7, + 2.64, 2.59, 2.53, 2.47, 2.42, 2.37, 2.31, 2.26, 2.21, 2.16, 2.11, 2.06, 2.01, 1.96, 1.92, 1.87, + 1.82, 1.78, 1.73, 1.69, 1.64, 1.6, 1.55, 1.51, 1.47, 1.42, 1.38, 1.34, 1.3, 1.25, 1.21, 1.17, + 1.13, 1.09, 1.05, 1.01, 0.96, 0.92, 0.88, 0.84, 0.8, 0.76, 0.71, 0.67, 0.63, 0.58, 0.54, 0.49, + 0.45, 0.4, 0.35, 0.3, 0.25, 0.18, 0.11, + ], + [ + 33.38, 26.0, 18.47, 13.28, 11.67, 10.71, 10.03, 9.49, 9.04, 8.67, 8.34, 8.04, 7.78, 7.54, 7.32, + 7.11, 6.92, 6.74, 6.58, 6.42, 6.27, 6.13, 5.99, 5.86, 5.73, 5.61, 5.5, 5.39, 5.28, 5.17, 5.07, + 4.97, 4.88, 4.79, 4.7, 4.61, 4.52, 4.44, 4.36, 4.28, 4.2, 4.12, 4.04, 3.97, 3.9, 3.83, 3.76, + 3.69, 3.62, 3.55, 3.49, 3.42, 3.36, 3.29, 3.23, 3.17, 3.11, 3.05, 2.99, 2.93, 2.87, 2.81, 2.75, + 2.7, 2.64, 2.58, 2.53, 2.47, 2.41, 2.36, 2.3, 2.25, 2.19, 2.14, 2.09, 2.03, 1.98, 1.92, 1.87, + 1.81, 1.76, 1.7, 1.65, 1.59, 1.54, 1.48, 1.42, 1.37, 1.31, 1.25, 1.19, 1.13, 1.06, 1.0, 0.93, + 0.86, 0.79, 0.71, 0.63, 0.54, 0.43, 0.3, + ], + [ + 35.89, 28.31, 20.52, 15.09, 13.39, 12.37, 11.64, 11.07, 10.6, 10.19, 9.84, 9.52, 9.24, 8.98, + 8.74, 8.52, 8.31, 8.12, 7.93, 7.76, 7.6, 7.44, 7.29, 7.15, 7.01, 6.88, 6.75, 6.63, 6.51, 6.39, + 6.28, 6.17, 6.06, 5.96, 5.86, 5.76, 5.67, 5.57, 5.48, 5.39, 5.3, 5.22, 5.13, 5.05, 4.97, 4.89, + 4.81, 4.73, 4.65, 4.57, 4.5, 4.42, 4.35, 4.28, 4.21, 4.14, 4.07, 4.0, 3.93, 3.86, 3.79, 3.72, + 3.66, 3.59, 3.52, 3.46, 3.39, 3.33, 3.26, 3.19, 3.13, 3.06, 3.0, 2.94, 2.87, 2.81, 2.74, 2.67, + 2.61, 2.54, 2.48, 2.41, 2.34, 2.27, 2.21, 2.14, 2.07, 1.99, 1.92, 1.85, 1.77, 1.69, 1.61, 1.53, + 1.44, 1.35, 1.25, 1.15, 1.03, 0.9, 0.75, 0.55, + ], + [ + 38.26, 30.5, 22.46, 16.81, 15.03, 13.97, 13.2, 12.59, 12.09, 11.66, 11.28, 10.95, 10.64, 10.37, + 10.11, 9.88, 9.65, 9.45, 9.25, 9.06, 8.89, 8.72, 8.56, 8.4, 8.26, 8.11, 7.97, 7.84, 7.71, 7.59, + 7.46, 7.35, 7.23, 7.12, 7.01, 6.9, 6.8, 6.69, 6.59, 6.5, 6.4, 6.3, 6.21, 6.12, 6.03, 5.94, 5.85, + 5.77, 5.68, 5.6, 5.51, 5.43, 5.35, 5.27, 5.19, 5.11, 5.03, 4.95, 4.87, 4.8, 4.72, 4.65, 4.57, + 4.5, 4.42, 4.35, 4.27, 4.2, 4.12, 4.05, 3.98, 3.9, 3.83, 3.75, 3.68, 3.6, 3.53, 3.45, 3.38, 3.3, + 3.23, 3.15, 3.07, 2.99, 2.91, 2.83, 2.75, 2.66, 2.57, 2.49, 2.4, 2.3, 2.2, 2.1, 2.0, 1.88, 1.76, + 1.64, 1.49, 1.33, 1.13, 0.87, + ], + [ + 40.52, 32.59, 24.32, 18.48, 16.62, 15.51, 14.7, 14.07, 13.54, 13.09, 12.69, 12.34, 12.02, 11.72, + 11.45, 11.2, 10.97, 10.75, 10.54, 10.34, 10.15, 9.97, 9.8, 9.64, 9.48, 9.33, 9.18, 9.04, 8.9, + 8.76, 8.63, 8.51, 8.38, 8.26, 8.14, 8.03, 7.92, 7.81, 7.7, 7.59, 7.49, 7.38, 7.28, 7.18, 7.09, + 6.99, 6.89, 6.8, 6.71, 6.62, 6.52, 6.43, 6.35, 6.26, 6.17, 6.08, 6.0, 5.91, 5.83, 5.74, 5.66, + 5.58, 5.49, 5.41, 5.33, 5.25, 5.16, 5.08, 5.0, 4.92, 4.84, 4.75, 4.67, 4.59, 4.51, 4.42, 4.34, + 4.25, 4.17, 4.08, 4.0, 3.91, 3.82, 3.73, 3.64, 3.55, 3.45, 3.36, 3.26, 3.16, 3.05, 2.95, 2.83, + 2.72, 2.59, 2.46, 2.32, 2.17, 2.0, 1.8, 1.56, 1.24, + ], + [ + 42.7, 34.6, 26.12, 20.09, 18.17, 17.01, 16.17, 15.51, 14.96, 14.48, 14.07, 13.7, 13.36, 13.05, + 12.77, 12.51, 12.26, 12.03, 11.81, 11.6, 11.4, 11.21, 11.03, 10.86, 10.69, 10.53, 10.37, 10.22, + 10.07, 9.93, 9.79, 9.66, 9.52, 9.4, 9.27, 9.15, 9.03, 8.91, 8.79, 8.68, 8.57, 8.46, 8.35, 8.24, + 8.14, 8.04, 7.93, 7.83, 7.73, 7.63, 7.54, 7.44, 7.34, 7.25, 7.16, 7.06, 6.97, 6.88, 6.78, 6.69, + 6.6, 6.51, 6.42, 6.33, 6.24, 6.15, 6.06, 5.98, 5.89, 5.8, 5.71, 5.62, 5.53, 5.44, 5.35, 5.26, + 5.16, 5.07, 4.98, 4.88, 4.79, 4.69, 4.59, 4.49, 4.39, 4.29, 4.19, 4.08, 3.97, 3.85, 3.74, 3.62, + 3.49, 3.36, 3.22, 3.07, 2.91, 2.73, 2.54, 2.31, 2.03, 1.65, + ], + [ + 44.81, 36.55, 27.88, 21.67, 19.68, 18.48, 17.61, 16.92, 16.35, 15.85, 15.42, 15.03, 14.68, + 14.36, 14.07, 13.79, 13.53, 13.29, 13.06, 12.84, 12.63, 12.43, 12.24, 12.06, 11.88, 11.71, + 11.55, 11.39, 11.23, 11.08, 10.94, 10.8, 10.66, 10.52, 10.39, 10.26, 10.13, 10.01, 9.88, 9.76, + 9.64, 9.53, 9.41, 9.3, 9.19, 9.08, 8.97, 8.86, 8.76, 8.65, 8.55, 8.44, 8.34, 8.24, 8.14, 8.04, + 7.94, 7.84, 7.75, 7.65, 7.55, 7.45, 7.36, 7.26, 7.16, 7.07, 6.97, 6.88, 6.78, 6.68, 6.59, 6.49, + 6.39, 6.3, 6.2, 6.1, 6.0, 5.9, 5.8, 5.7, 5.59, 5.49, 5.38, 5.27, 5.16, 5.05, 4.93, 4.82, 4.7, + 4.57, 4.44, 4.31, 4.17, 4.02, 3.87, 3.7, 3.52, 3.33, 3.1, 2.85, 2.53, 2.09, + ], + [ + 46.86, 38.45, 29.59, 23.21, 21.16, 19.92, 19.02, 18.31, 17.71, 17.2, 16.75, 16.35, 15.99, 15.65, + 15.34, 15.06, 14.79, 14.53, 14.29, 14.07, 13.85, 13.64, 13.44, 13.25, 13.07, 12.89, 12.72, + 12.55, 12.39, 12.23, 12.08, 11.93, 11.78, 11.64, 11.5, 11.36, 11.23, 11.1, 10.97, 10.84, 10.72, + 10.59, 10.47, 10.35, 10.24, 10.12, 10.01, 9.89, 9.78, 9.67, 9.56, 9.45, 9.34, 9.23, 9.13, 9.02, + 8.92, 8.81, 8.71, 8.6, 8.5, 8.4, 8.3, 8.19, 8.09, 7.99, 7.89, 7.78, 7.68, 7.58, 7.47, 7.37, + 7.27, 7.16, 7.06, 6.95, 6.84, 6.74, 6.63, 6.52, 6.41, 6.29, 6.18, 6.06, 5.94, 5.82, 5.7, 5.57, + 5.44, 5.3, 5.16, 5.02, 4.87, 4.7, 4.54, 4.35, 4.16, 3.94, 3.7, 3.41, 3.06, 2.56, + ], + [ + 48.87, 40.31, 31.26, 24.72, 22.62, 21.34, 20.41, 19.68, 19.06, 18.53, 18.07, 17.65, 17.28, + 16.93, 16.61, 16.31, 16.03, 15.77, 15.52, 15.28, 15.05, 14.84, 14.63, 14.43, 14.24, 14.05, + 13.87, 13.7, 13.53, 13.37, 13.21, 13.05, 12.9, 12.75, 12.6, 12.46, 12.32, 12.18, 12.05, 11.92, + 11.79, 11.66, 11.53, 11.4, 11.28, 11.16, 11.04, 10.92, 10.8, 10.69, 10.57, 10.45, 10.34, 10.23, + 10.12, 10.0, 9.89, 9.78, 9.67, 9.56, 9.45, 9.35, 9.24, 9.13, 9.02, 8.91, 8.8, 8.7, 8.59, 8.48, + 8.37, 8.26, 8.15, 8.04, 7.93, 7.81, 7.7, 7.58, 7.47, 7.35, 7.23, 7.11, 6.99, 6.86, 6.74, 6.61, + 6.47, 6.34, 6.2, 6.05, 5.9, 5.74, 5.58, 5.4, 5.22, 5.02, 4.81, 4.57, 4.31, 4.0, 3.61, 3.05, + ], + [ + 50.83, 42.13, 32.91, 26.22, 24.05, 22.74, 21.79, 21.03, 20.39, 19.85, 19.37, 18.94, 18.55, + 18.19, 17.86, 17.55, 17.26, 16.99, 16.73, 16.49, 16.25, 16.03, 15.81, 15.61, 15.41, 15.21, + 15.03, 14.85, 14.67, 14.5, 14.33, 14.17, 14.01, 13.86, 13.7, 13.56, 13.41, 13.27, 13.13, 12.99, + 12.85, 12.72, 12.58, 12.45, 12.32, 12.2, 12.07, 11.95, 11.82, 11.7, 11.58, 11.46, 11.34, 11.22, + 11.1, 10.99, 10.87, 10.76, 10.64, 10.53, 10.41, 10.3, 10.18, 10.07, 9.95, 9.84, 9.73, 9.61, 9.5, + 9.38, 9.27, 9.15, 9.03, 8.92, 8.8, 8.68, 8.56, 8.44, 8.32, 8.19, 8.07, 7.94, 7.81, 7.67, 7.54, + 7.4, 7.26, 7.11, 6.96, 6.81, 6.65, 6.48, 6.3, 6.12, 5.92, 5.71, 5.48, 5.23, 4.94, 4.6, 4.18, + 3.57, + ], + [ + 52.75, 43.92, 34.53, 27.69, 25.47, 24.12, 23.14, 22.36, 21.71, 21.15, 20.66, 20.21, 19.81, + 19.44, 19.1, 18.78, 18.48, 18.2, 17.94, 17.68, 17.44, 17.21, 16.98, 16.77, 16.56, 16.36, 16.17, + 15.98, 15.8, 15.62, 15.45, 15.28, 15.12, 14.96, 14.8, 14.65, 14.49, 14.35, 14.2, 14.05, 13.91, + 13.77, 13.64, 13.5, 13.37, 13.23, 13.1, 12.97, 12.84, 12.72, 12.59, 12.46, 12.34, 12.22, 12.09, + 11.97, 11.85, 11.73, 11.61, 11.49, 11.37, 11.25, 11.13, 11.01, 10.89, 10.77, 10.65, 10.53, + 10.41, 10.29, 10.17, 10.05, 9.93, 9.8, 9.68, 9.55, 9.43, 9.3, 9.17, 9.04, 8.91, 8.77, 8.63, + 8.49, 8.35, 8.2, 8.05, 7.9, 7.74, 7.58, 7.41, 7.23, 7.04, 6.84, 6.63, 6.41, 6.16, 5.89, 5.58, + 5.22, 4.77, 4.11, + ], + [ + 54.64, 45.68, 36.12, 29.14, 26.87, 25.49, 24.49, 23.68, 23.02, 22.44, 21.93, 21.48, 21.06, + 20.68, 20.33, 20.0, 19.7, 19.41, 19.13, 18.87, 18.62, 18.38, 18.15, 17.93, 17.72, 17.51, 17.31, + 17.12, 16.93, 16.75, 16.57, 16.39, 16.22, 16.06, 15.89, 15.73, 15.58, 15.42, 15.27, 15.12, + 14.97, 14.83, 14.69, 14.54, 14.4, 14.27, 14.13, 14.0, 13.86, 13.73, 13.6, 13.47, 13.34, 13.21, + 13.08, 12.96, 12.83, 12.7, 12.58, 12.45, 12.33, 12.2, 12.08, 11.95, 11.83, 11.7, 11.58, 11.45, + 11.33, 11.2, 11.08, 10.95, 10.82, 10.69, 10.56, 10.43, 10.3, 10.17, 10.03, 9.89, 9.75, 9.61, + 9.47, 9.32, 9.17, 9.02, 8.86, 8.7, 8.53, 8.36, 8.18, 7.99, 7.79, 7.58, 7.36, 7.12, 6.86, 6.57, + 6.24, 5.86, 5.37, 4.66, + ], + [ + 56.49, 47.41, 37.7, 30.58, 28.26, 26.85, 25.82, 25.0, 24.31, 23.72, 23.2, 22.73, 22.31, 21.92, + 21.56, 21.22, 20.9, 20.6, 20.32, 20.05, 19.79, 19.55, 19.31, 19.08, 18.86, 18.65, 18.44, 18.25, + 18.05, 17.86, 17.68, 17.5, 17.32, 17.15, 16.98, 16.82, 16.65, 16.49, 16.34, 16.18, 16.03, 15.88, + 15.73, 15.59, 15.44, 15.3, 15.16, 15.02, 14.88, 14.74, 14.61, 14.47, 14.34, 14.21, 14.07, 13.94, + 13.81, 13.68, 13.55, 13.42, 13.29, 13.16, 13.03, 12.9, 12.77, 12.64, 12.51, 12.38, 12.25, 12.12, + 11.99, 11.85, 11.72, 11.59, 11.45, 11.31, 11.18, 11.04, 10.9, 10.75, 10.61, 10.46, 10.31, 10.15, + 10.0, 9.83, 9.67, 9.5, 9.32, 9.14, 8.95, 8.75, 8.55, 8.33, 8.09, 7.84, 7.57, 7.26, 6.91, 6.5, + 5.98, 5.23, + ], + [ + 58.32, 49.12, 39.25, 32.0, 29.63, 28.19, 27.14, 26.3, 25.59, 24.99, 24.46, 23.98, 23.54, 23.14, + 22.77, 22.42, 22.1, 21.79, 21.5, 21.23, 20.96, 20.71, 20.47, 20.23, 20.0, 19.79, 19.57, 19.37, + 19.17, 18.97, 18.78, 18.6, 18.42, 18.24, 18.07, 17.9, 17.73, 17.56, 17.4, 17.24, 17.09, 16.93, + 16.78, 16.63, 16.48, 16.33, 16.19, 16.04, 15.9, 15.76, 15.62, 15.48, 15.34, 15.2, 15.06, 14.93, + 14.79, 14.66, 14.52, 14.39, 14.25, 14.12, 13.98, 13.85, 13.71, 13.58, 13.44, 13.31, 13.17, + 13.04, 12.9, 12.76, 12.62, 12.48, 12.34, 12.2, 12.06, 11.91, 11.76, 11.62, 11.46, 11.31, 11.15, + 10.99, 10.83, 10.66, 10.49, 10.31, 10.13, 9.94, 9.74, 9.53, 9.31, 9.08, 8.84, 8.57, 8.28, 7.96, + 7.6, 7.16, 6.61, 5.81, + ], + [ + 60.13, 50.81, 40.79, 33.41, 31.0, 29.52, 28.44, 27.59, 26.87, 26.25, 25.71, 25.21, 24.77, 24.36, + 23.98, 23.62, 23.29, 22.98, 22.68, 22.4, 22.12, 21.86, 21.61, 21.37, 21.14, 20.92, 20.7, 20.49, + 20.28, 20.08, 19.89, 19.7, 19.51, 19.33, 19.15, 18.97, 18.8, 18.63, 18.47, 18.3, 18.14, 17.98, + 17.82, 17.67, 17.52, 17.36, 17.21, 17.06, 16.92, 16.77, 16.63, 16.48, 16.34, 16.2, 16.05, 15.91, + 15.77, 15.63, 15.49, 15.35, 15.21, 15.08, 14.94, 14.8, 14.66, 14.52, 14.38, 14.24, 14.1, 13.96, + 13.82, 13.67, 13.53, 13.39, 13.24, 13.09, 12.94, 12.79, 12.64, 12.48, 12.33, 12.17, 12.0, 11.84, + 11.66, 11.49, 11.31, 11.12, 10.93, 10.74, 10.53, 10.31, 10.09, 9.84, 9.59, 9.31, 9.01, 8.67, + 8.29, 7.83, 7.26, 6.41, + ], + [ + 61.91, 52.47, 42.31, 34.81, 32.35, 30.84, 29.75, 28.87, 28.14, 27.5, 26.95, 26.45, 25.99, 25.57, + 25.18, 24.82, 24.48, 24.16, 23.85, 23.56, 23.28, 23.02, 22.76, 22.51, 22.28, 22.04, 21.82, 21.6, + 21.39, 21.19, 20.99, 20.79, 20.6, 20.41, 20.23, 20.05, 19.87, 19.7, 19.53, 19.36, 19.19, 19.03, + 18.87, 18.71, 18.55, 18.39, 18.24, 18.09, 17.93, 17.78, 17.63, 17.49, 17.34, 17.19, 17.05, 16.9, + 16.76, 16.61, 16.47, 16.32, 16.18, 16.04, 15.89, 15.75, 15.61, 15.46, 15.32, 15.17, 15.03, + 14.88, 14.74, 14.59, 14.44, 14.29, 14.14, 13.99, 13.83, 13.68, 13.52, 13.36, 13.19, 13.03, + 12.86, 12.68, 12.51, 12.33, 12.14, 11.95, 11.75, 11.54, 11.33, 11.1, 10.86, 10.61, 10.35, 10.06, + 9.74, 9.39, 8.99, 8.51, 7.91, 7.01, + ], + [ + 63.68, 54.12, 43.82, 36.19, 33.69, 32.16, 31.04, 30.14, 29.4, 28.75, 28.18, 27.67, 27.2, 26.78, + 26.38, 26.01, 25.66, 25.33, 25.02, 24.72, 24.43, 24.16, 23.9, 23.65, 23.4, 23.17, 22.94, 22.72, + 22.5, 22.29, 22.09, 21.89, 21.69, 21.5, 21.31, 21.12, 20.94, 20.76, 20.59, 20.42, 20.24, 20.08, + 19.91, 19.75, 19.58, 19.42, 19.26, 19.11, 18.95, 18.8, 18.64, 18.49, 18.34, 18.19, 18.04, 17.89, + 17.74, 17.59, 17.44, 17.29, 17.15, 17.0, 16.85, 16.7, 16.55, 16.41, 16.26, 16.11, 15.96, 15.81, + 15.66, 15.51, 15.35, 15.2, 15.04, 14.88, 14.72, 14.56, 14.4, 14.23, 14.06, 13.89, 13.72, 13.54, + 13.35, 13.17, 12.97, 12.77, 12.57, 12.35, 12.13, 11.9, 11.65, 11.39, 11.11, 10.81, 10.48, 10.12, + 9.7, 9.2, 8.57, 7.63, + ], + [ + 65.42, 55.76, 45.31, 37.57, 35.02, 33.46, 32.32, 31.41, 30.65, 29.99, 29.41, 28.89, 28.41, + 27.97, 27.57, 27.19, 26.83, 26.5, 26.18, 25.87, 25.58, 25.31, 25.04, 24.78, 24.53, 24.29, 24.05, + 23.83, 23.61, 23.39, 23.18, 22.98, 22.77, 22.58, 22.38, 22.2, 22.01, 21.83, 21.65, 21.47, 21.29, + 21.12, 20.95, 20.78, 20.62, 20.45, 20.29, 20.13, 19.97, 19.81, 19.65, 19.49, 19.34, 19.18, + 19.03, 18.87, 18.72, 18.57, 18.42, 18.26, 18.11, 17.96, 17.81, 17.66, 17.5, 17.35, 17.2, 17.05, + 16.89, 16.74, 16.58, 16.42, 16.27, 16.11, 15.95, 15.78, 15.62, 15.45, 15.28, 15.11, 14.94, + 14.76, 14.58, 14.39, 14.2, 14.01, 13.81, 13.6, 13.39, 13.17, 12.94, 12.7, 12.44, 12.17, 11.88, + 11.57, 11.23, 10.85, 10.42, 9.9, 9.24, 8.26, + ], + [ + 67.15, 57.37, 46.8, 38.93, 36.34, 34.76, 33.6, 32.67, 31.89, 31.22, 30.63, 30.1, 29.62, 29.17, + 28.76, 28.37, 28.01, 27.66, 27.34, 27.03, 26.73, 26.44, 26.17, 25.91, 25.65, 25.41, 25.17, + 24.93, 24.71, 24.49, 24.27, 24.06, 23.86, 23.66, 23.46, 23.27, 23.07, 22.89, 22.7, 22.52, 22.34, + 22.17, 21.99, 21.82, 21.65, 21.48, 21.31, 21.15, 20.98, 20.82, 20.66, 20.5, 20.34, 20.18, 20.02, + 19.86, 19.71, 19.55, 19.39, 19.24, 19.08, 18.92, 18.77, 18.61, 18.46, 18.3, 18.14, 17.98, 17.83, + 17.67, 17.51, 17.34, 17.18, 17.02, 16.85, 16.69, 16.52, 16.34, 16.17, 15.99, 15.81, 15.63, + 15.44, 15.25, 15.06, 14.86, 14.65, 14.44, 14.22, 13.99, 13.75, 13.5, 13.24, 12.96, 12.66, 12.34, + 11.99, 11.59, 11.14, 10.6, 9.91, 8.9, + ], + [ + 68.86, 58.98, 48.27, 40.29, 37.66, 36.05, 34.87, 33.92, 33.13, 32.45, 31.85, 31.31, 30.81, + 30.36, 29.94, 29.54, 29.17, 28.82, 28.49, 28.17, 27.87, 27.58, 27.3, 27.03, 26.77, 26.52, 26.28, + 26.04, 25.81, 25.58, 25.36, 25.15, 24.94, 24.73, 24.53, 24.33, 24.14, 23.95, 23.76, 23.57, + 23.39, 23.21, 23.03, 22.85, 22.68, 22.51, 22.34, 22.17, 22.0, 21.83, 21.67, 21.5, 21.34, 21.17, + 21.01, 20.85, 20.69, 20.53, 20.37, 20.21, 20.05, 19.89, 19.73, 19.57, 19.41, 19.25, 19.09, + 18.92, 18.76, 18.6, 18.43, 18.27, 18.1, 17.93, 17.76, 17.59, 17.42, 17.24, 17.06, 16.88, 16.69, + 16.51, 16.31, 16.12, 15.92, 15.71, 15.5, 15.28, 15.05, 14.82, 14.57, 14.31, 14.04, 13.75, 13.45, + 13.11, 12.75, 12.34, 11.87, 11.31, 10.6, 9.54, + ], + [ + 70.55, 60.57, 49.73, 41.64, 38.97, 37.33, 36.13, 35.17, 34.37, 33.68, 33.06, 32.51, 32.01, + 31.54, 31.11, 30.71, 30.34, 29.98, 29.64, 29.32, 29.01, 28.71, 28.43, 28.15, 27.89, 27.63, + 27.38, 27.14, 26.91, 26.68, 26.45, 26.23, 26.02, 25.81, 25.6, 25.4, 25.2, 25.01, 24.81, 24.62, + 24.44, 24.25, 24.07, 23.89, 23.71, 23.53, 23.36, 23.19, 23.01, 22.84, 22.67, 22.5, 22.34, 22.17, + 22.0, 21.84, 21.67, 21.51, 21.35, 21.18, 21.02, 20.85, 20.69, 20.53, 20.36, 20.2, 20.03, 19.87, + 19.7, 19.53, 19.36, 19.19, 19.02, 18.85, 18.67, 18.5, 18.32, 18.14, 17.95, 17.77, 17.58, 17.38, + 17.19, 16.98, 16.78, 16.57, 16.35, 16.12, 15.89, 15.65, 15.39, 15.13, 14.85, 14.55, 14.23, + 13.89, 13.51, 13.09, 12.61, 12.03, 11.29, 10.2, + ], + [ + 72.23, 62.14, 51.18, 42.98, 40.27, 38.61, 37.39, 36.42, 35.6, 34.89, 34.27, 33.71, 33.2, 32.73, + 32.29, 31.88, 31.5, 31.13, 30.79, 30.46, 30.14, 29.84, 29.55, 29.27, 29.0, 28.74, 28.49, 28.24, + 28.0, 27.77, 27.54, 27.31, 27.1, 26.88, 26.67, 26.47, 26.26, 26.06, 25.87, 25.67, 25.48, 25.29, + 25.11, 24.92, 24.74, 24.56, 24.38, 24.2, 24.03, 23.85, 23.68, 23.51, 23.34, 23.17, 23.0, 22.83, + 22.66, 22.49, 22.32, 22.16, 21.99, 21.82, 21.65, 21.48, 21.32, 21.15, 20.98, 20.81, 20.64, + 20.47, 20.29, 20.12, 19.94, 19.77, 19.59, 19.41, 19.22, 19.04, 18.85, 18.66, 18.46, 18.26, + 18.06, 17.85, 17.64, 17.42, 17.2, 16.97, 16.73, 16.48, 16.22, 15.95, 15.66, 15.35, 15.03, 14.67, + 14.28, 13.85, 13.35, 12.75, 11.99, 10.86, + ], + [ + 73.89, 63.71, 52.62, 44.31, 41.57, 39.88, 38.64, 37.65, 36.82, 36.11, 35.47, 34.9, 34.38, 33.9, + 33.46, 33.04, 32.65, 32.28, 31.93, 31.6, 31.28, 30.97, 30.68, 30.39, 30.12, 29.85, 29.59, 29.34, + 29.09, 28.86, 28.62, 28.39, 28.17, 27.95, 27.74, 27.53, 27.32, 27.12, 26.92, 26.72, 26.53, + 26.33, 26.14, 25.96, 25.77, 25.59, 25.4, 25.22, 25.04, 24.86, 24.69, 24.51, 24.34, 24.16, 23.99, + 23.82, 23.64, 23.47, 23.3, 23.13, 22.96, 22.79, 22.62, 22.44, 22.27, 22.1, 21.93, 21.75, 21.58, + 21.4, 21.22, 21.05, 20.87, 20.69, 20.5, 20.32, 20.13, 19.94, 19.75, 19.55, 19.35, 19.15, 18.94, + 18.73, 18.51, 18.29, 18.06, 17.82, 17.57, 17.32, 17.05, 16.77, 16.47, 16.16, 15.82, 15.46, + 15.06, 14.61, 14.1, 13.48, 12.7, 11.52, + ], + [ + 75.55, 65.26, 54.05, 45.64, 42.86, 41.15, 39.89, 38.89, 38.04, 37.32, 36.67, 36.09, 35.56, + 35.08, 34.62, 34.2, 33.81, 33.43, 33.07, 32.73, 32.41, 32.09, 31.79, 31.5, 31.22, 30.95, 30.69, + 30.43, 30.19, 29.94, 29.71, 29.47, 29.25, 29.02, 28.81, 28.59, 28.38, 28.17, 27.97, 27.77, + 27.57, 27.37, 27.18, 26.99, 26.8, 26.61, 26.42, 26.24, 26.06, 25.87, 25.69, 25.51, 25.34, 25.16, + 24.98, 24.81, 24.63, 24.45, 24.28, 24.1, 23.93, 23.75, 23.58, 23.4, 23.23, 23.05, 22.88, 22.7, + 22.52, 22.34, 22.16, 21.98, 21.79, 21.61, 21.42, 21.23, 21.04, 20.84, 20.65, 20.45, 20.24, + 20.03, 19.82, 19.6, 19.38, 19.15, 18.91, 18.67, 18.42, 18.16, 17.88, 17.6, 17.29, 16.97, 16.62, + 16.25, 15.84, 15.38, 14.85, 14.22, 13.41, 12.2, + ], + [ + 77.19, 66.81, 55.48, 46.96, 44.14, 42.41, 41.13, 40.11, 39.26, 38.52, 37.87, 37.28, 36.74, + 36.25, 35.79, 35.36, 34.96, 34.57, 34.21, 33.87, 33.53, 33.22, 32.91, 32.62, 32.33, 32.06, + 31.79, 31.53, 31.27, 31.03, 30.79, 30.55, 30.32, 30.09, 29.87, 29.65, 29.44, 29.23, 29.02, + 28.81, 28.61, 28.41, 28.21, 28.02, 27.83, 27.63, 27.44, 27.26, 27.07, 26.89, 26.7, 26.52, 26.34, + 26.16, 25.97, 25.79, 25.62, 25.44, 25.26, 25.08, 24.9, 24.72, 24.54, 24.37, 24.19, 24.01, 23.83, + 23.64, 23.46, 23.28, 23.09, 22.91, 22.72, 22.53, 22.34, 22.14, 21.95, 21.75, 21.55, 21.34, + 21.13, 20.92, 20.7, 20.48, 20.25, 20.02, 19.78, 19.53, 19.27, 19.0, 18.72, 18.42, 18.11, 17.78, + 17.43, 17.05, 16.62, 16.15, 15.61, 14.96, 14.13, 12.88, + ], + [ + 78.82, 68.34, 56.89, 48.28, 45.42, 43.66, 42.37, 41.34, 40.47, 39.72, 39.06, 38.46, 37.92, + 37.41, 36.95, 36.51, 36.1, 35.71, 35.35, 35.0, 34.66, 34.34, 34.03, 33.73, 33.44, 33.16, 32.89, + 32.62, 32.36, 32.11, 31.87, 31.63, 31.39, 31.16, 30.93, 30.71, 30.49, 30.28, 30.07, 29.86, + 29.65, 29.45, 29.25, 29.05, 28.85, 28.66, 28.47, 28.27, 28.08, 27.9, 27.71, 27.52, 27.34, 27.15, + 26.97, 26.78, 26.6, 26.42, 26.24, 26.06, 25.87, 25.69, 25.51, 25.33, 25.14, 24.96, 24.78, 24.59, + 24.4, 24.22, 24.03, 23.84, 23.65, 23.45, 23.26, 23.06, 22.86, 22.66, 22.45, 22.24, 22.03, 21.81, + 21.59, 21.36, 21.13, 20.89, 20.64, 20.39, 20.12, 19.85, 19.56, 19.26, 18.94, 18.6, 18.24, 17.84, + 17.41, 16.93, 16.37, 15.7, 14.85, 13.56, + ], + [ + 80.44, 69.87, 58.3, 49.59, 46.69, 44.91, 43.6, 42.56, 41.68, 40.92, 40.25, 39.64, 39.09, 38.58, + 38.11, 37.66, 37.25, 36.85, 36.48, 36.12, 35.78, 35.45, 35.14, 34.84, 34.54, 34.26, 33.98, + 33.71, 33.45, 33.19, 32.94, 32.7, 32.46, 32.23, 32.0, 31.77, 31.55, 31.33, 31.12, 30.9, 30.69, + 30.49, 30.28, 30.08, 29.88, 29.68, 29.49, 29.29, 29.1, 28.91, 28.71, 28.52, 28.34, 28.15, 27.96, + 27.77, 27.59, 27.4, 27.22, 27.03, 26.85, 26.66, 26.48, 26.29, 26.1, 25.92, 25.73, 25.54, 25.35, + 25.16, 24.97, 24.77, 24.58, 24.38, 24.18, 23.98, 23.77, 23.57, 23.36, 23.14, 22.92, 22.7, 22.48, + 22.24, 22.0, 21.76, 21.51, 21.25, 20.98, 20.7, 20.4, 20.09, 19.77, 19.42, 19.05, 18.65, 18.2, + 17.71, 17.14, 16.45, 15.57, 14.26, + ], + [ + 82.04, 71.38, 59.7, 50.89, 47.96, 46.16, 44.83, 43.77, 42.88, 42.11, 41.43, 40.82, 40.26, 39.74, + 39.26, 38.81, 38.39, 37.99, 37.61, 37.25, 36.9, 36.57, 36.25, 35.94, 35.64, 35.35, 35.07, 34.8, + 34.53, 34.27, 34.02, 33.77, 33.53, 33.29, 33.06, 32.83, 32.6, 32.38, 32.16, 31.95, 31.73, 31.52, + 31.32, 31.11, 30.91, 30.71, 30.51, 30.31, 30.11, 29.92, 29.72, 29.53, 29.34, 29.14, 28.95, + 28.76, 28.57, 28.39, 28.2, 28.01, 27.82, 27.63, 27.44, 27.25, 27.06, 26.87, 26.68, 26.49, 26.29, + 26.1, 25.9, 25.71, 25.51, 25.31, 25.1, 24.9, 24.69, 24.48, 24.26, 24.04, 23.82, 23.6, 23.36, + 23.13, 22.88, 22.63, 22.38, 22.11, 21.83, 21.55, 21.25, 20.93, 20.6, 20.25, 19.87, 19.45, 19.0, + 18.49, 17.91, 17.21, 16.31, 14.95, + ], + [ + 83.64, 72.89, 61.1, 52.19, 49.23, 47.4, 46.06, 44.99, 44.08, 43.3, 42.61, 41.99, 41.42, 40.9, + 40.41, 39.96, 39.53, 39.12, 38.74, 38.37, 38.02, 37.68, 37.36, 37.05, 36.74, 36.45, 36.16, + 35.89, 35.62, 35.35, 35.1, 34.84, 34.6, 34.36, 34.12, 33.89, 33.66, 33.43, 33.21, 32.99, 32.77, + 32.56, 32.35, 32.14, 31.93, 31.73, 31.52, 31.32, 31.12, 30.92, 30.73, 30.53, 30.34, 30.14, + 29.95, 29.75, 29.56, 29.37, 29.18, 28.99, 28.79, 28.6, 28.41, 28.22, 28.02, 27.83, 27.63, 27.44, + 27.24, 27.04, 26.84, 26.64, 26.44, 26.23, 26.03, 25.82, 25.61, 25.39, 25.17, 24.95, 24.72, + 24.49, 24.26, 24.01, 23.77, 23.51, 23.25, 22.98, 22.69, 22.4, 22.1, 21.77, 21.43, 21.07, 20.68, + 20.26, 19.8, 19.28, 18.68, 17.97, 17.04, 15.66, + ], + [ + 85.23, 74.39, 62.49, 53.49, 50.49, 48.64, 47.28, 46.19, 45.28, 44.49, 43.79, 43.16, 42.58, + 42.05, 41.56, 41.1, 40.67, 40.26, 39.87, 39.49, 39.14, 38.8, 38.47, 38.15, 37.84, 37.54, 37.25, + 36.97, 36.7, 36.43, 36.17, 35.92, 35.66, 35.42, 35.18, 34.94, 34.71, 34.48, 34.25, 34.03, 33.81, + 33.6, 33.38, 33.17, 32.96, 32.75, 32.54, 32.34, 32.14, 31.93, 31.73, 31.53, 31.34, 31.14, 30.94, + 30.74, 30.55, 30.35, 30.16, 29.96, 29.77, 29.57, 29.38, 29.18, 28.98, 28.79, 28.59, 28.39, + 28.19, 27.99, 27.78, 27.58, 27.37, 27.16, 26.95, 26.74, 26.52, 26.3, 26.08, 25.85, 25.62, 25.39, + 25.15, 24.9, 24.65, 24.39, 24.12, 23.84, 23.56, 23.26, 22.95, 22.62, 22.27, 21.9, 21.51, 21.08, + 20.6, 20.07, 19.46, 18.73, 17.78, 16.36, + ], + [ + 86.81, 75.88, 63.87, 54.78, 51.74, 49.88, 48.5, 47.4, 46.48, 45.68, 44.97, 44.33, 43.75, 43.21, + 42.71, 42.24, 41.8, 41.39, 40.99, 40.61, 40.25, 39.91, 39.57, 39.25, 38.94, 38.64, 38.34, 38.06, + 37.78, 37.51, 37.24, 36.98, 36.73, 36.48, 36.24, 36.0, 35.76, 35.53, 35.3, 35.07, 34.85, 34.63, + 34.41, 34.2, 33.98, 33.77, 33.56, 33.36, 33.15, 32.94, 32.74, 32.54, 32.34, 32.13, 31.93, 31.74, + 31.54, 31.34, 31.14, 30.94, 30.74, 30.54, 30.34, 30.15, 29.95, 29.74, 29.54, 29.34, 29.14, + 28.93, 28.73, 28.52, 28.31, 28.09, 27.88, 27.66, 27.44, 27.22, 26.99, 26.76, 26.53, 26.29, + 26.04, 25.79, 25.53, 25.27, 25.0, 24.71, 24.42, 24.12, 23.8, 23.46, 23.11, 22.73, 22.33, 21.89, + 21.41, 20.87, 20.24, 19.49, 18.53, 17.07, + ], + [ + 88.38, 77.37, 65.25, 56.06, 53.0, 51.11, 49.72, 48.6, 47.67, 46.86, 46.14, 45.49, 44.9, 44.36, + 43.85, 43.38, 42.94, 42.51, 42.11, 41.73, 41.37, 41.01, 40.68, 40.35, 40.03, 39.73, 39.43, + 39.14, 38.86, 38.58, 38.32, 38.05, 37.8, 37.54, 37.3, 37.05, 36.81, 36.58, 36.34, 36.11, 35.89, + 35.66, 35.44, 35.23, 35.01, 34.79, 34.58, 34.37, 34.16, 33.95, 33.75, 33.54, 33.34, 33.13, + 32.93, 32.73, 32.52, 32.32, 32.12, 31.92, 31.72, 31.52, 31.31, 31.11, 30.91, 30.7, 30.5, 30.29, + 30.09, 29.88, 29.67, 29.46, 29.24, 29.03, 28.81, 28.59, 28.36, 28.14, 27.91, 27.67, 27.43, + 27.19, 26.94, 26.68, 26.42, 26.15, 25.87, 25.59, 25.29, 24.98, 24.65, 24.31, 23.95, 23.57, + 23.16, 22.71, 22.22, 21.66, 21.03, 20.26, 19.28, 17.79, + ], + [ + 89.95, 78.84, 66.62, 57.34, 54.24, 52.34, 50.93, 49.8, 48.86, 48.04, 47.31, 46.66, 46.06, 45.51, + 45.0, 44.52, 44.07, 43.64, 43.23, 42.85, 42.48, 42.12, 41.78, 41.45, 41.13, 40.82, 40.52, 40.22, + 39.94, 39.66, 39.39, 39.12, 38.86, 38.6, 38.35, 38.11, 37.86, 37.62, 37.39, 37.15, 36.93, 36.7, + 36.47, 36.25, 36.03, 35.82, 35.6, 35.39, 35.17, 34.96, 34.75, 34.54, 34.34, 34.13, 33.92, 33.72, + 33.51, 33.31, 33.1, 32.9, 32.69, 32.49, 32.28, 32.08, 31.87, 31.66, 31.45, 31.25, 31.04, 30.82, + 30.61, 30.4, 30.18, 29.96, 29.74, 29.51, 29.28, 29.05, 28.82, 28.58, 28.34, 28.09, 27.84, 27.58, + 27.31, 27.04, 26.75, 26.46, 26.16, 25.84, 25.51, 25.16, 24.8, 24.41, 23.99, 23.53, 23.03, 22.47, + 21.82, 21.03, 20.03, 18.51, + ], + [ + 91.5, 80.31, 67.99, 58.62, 55.49, 53.56, 52.14, 51.0, 50.04, 49.21, 48.48, 47.82, 47.21, 46.66, + 46.14, 45.65, 45.2, 44.76, 44.35, 43.96, 43.59, 43.23, 42.88, 42.54, 42.22, 41.91, 41.6, 41.3, + 41.01, 40.73, 40.46, 40.19, 39.92, 39.66, 39.41, 39.16, 38.91, 38.67, 38.43, 38.19, 37.96, + 37.73, 37.5, 37.28, 37.06, 36.84, 36.62, 36.4, 36.19, 35.97, 35.76, 35.55, 35.34, 35.13, 34.92, + 34.71, 34.5, 34.29, 34.08, 33.88, 33.67, 33.46, 33.25, 33.04, 32.83, 32.62, 32.41, 32.2, 31.99, + 31.77, 31.55, 31.34, 31.12, 30.89, 30.67, 30.44, 30.21, 29.97, 29.73, 29.49, 29.24, 28.99, + 28.73, 28.47, 28.2, 27.92, 27.63, 27.34, 27.03, 26.71, 26.37, 26.02, 25.64, 25.25, 24.82, 24.35, + 23.84, 23.27, 22.61, 21.81, 20.78, 19.23, + ], + [ + 93.05, 81.78, 69.35, 59.89, 56.73, 54.78, 53.34, 52.19, 51.23, 50.39, 49.64, 48.97, 48.36, 47.8, + 47.28, 46.79, 46.32, 45.89, 45.47, 45.07, 44.69, 44.33, 43.98, 43.64, 43.31, 42.99, 42.68, + 42.38, 42.09, 41.8, 41.53, 41.25, 40.98, 40.72, 40.46, 40.21, 39.96, 39.71, 39.47, 39.23, 39.0, + 38.77, 38.53, 38.31, 38.08, 37.86, 37.64, 37.42, 37.2, 36.98, 36.76, 36.55, 36.34, 36.12, 35.91, + 35.7, 35.49, 35.28, 35.07, 34.85, 34.64, 34.43, 34.22, 34.01, 33.8, 33.58, 33.37, 33.15, 32.94, + 32.72, 32.5, 32.28, 32.05, 31.83, 31.6, 31.37, 31.13, 30.89, 30.65, 30.4, 30.15, 29.9, 29.64, + 29.37, 29.09, 28.81, 28.52, 28.21, 27.9, 27.57, 27.23, 26.87, 26.49, 26.09, 25.65, 25.18, 24.66, + 24.07, 23.4, 22.59, 21.54, 19.96, + ], + [ + 94.59, 83.24, 70.7, 61.16, 57.97, 56.0, 54.55, 53.38, 52.41, 51.56, 50.81, 50.13, 49.51, 48.94, + 48.41, 47.92, 47.45, 47.01, 46.59, 46.19, 45.8, 45.43, 45.08, 44.73, 44.4, 44.08, 43.77, 43.46, + 43.17, 42.88, 42.59, 42.32, 42.05, 41.78, 41.52, 41.26, 41.01, 40.76, 40.51, 40.27, 40.03, 39.8, + 39.56, 39.33, 39.1, 38.88, 38.65, 38.43, 38.21, 37.99, 37.77, 37.55, 37.34, 37.12, 36.9, 36.69, + 36.48, 36.26, 36.05, 35.83, 35.62, 35.41, 35.19, 34.98, 34.76, 34.54, 34.33, 34.11, 33.89, + 33.67, 33.44, 33.22, 32.99, 32.76, 32.53, 32.29, 32.06, 31.81, 31.57, 31.32, 31.06, 30.8, 30.54, + 30.26, 29.99, 29.7, 29.4, 29.09, 28.77, 28.44, 28.09, 27.73, 27.34, 26.93, 26.49, 26.01, 25.48, + 24.88, 24.2, 23.37, 22.3, 20.69, + ], + [ + 96.13, 84.69, 72.05, 62.43, 59.2, 57.22, 55.75, 54.57, 53.58, 52.73, 51.97, 51.28, 50.66, 50.08, + 49.55, 49.05, 48.57, 48.13, 47.7, 47.29, 46.91, 46.53, 46.17, 45.83, 45.49, 45.16, 44.85, 44.54, + 44.24, 43.95, 43.66, 43.38, 43.11, 42.84, 42.57, 42.31, 42.06, 41.8, 41.56, 41.31, 41.07, 40.83, + 40.59, 40.36, 40.13, 39.9, 39.67, 39.44, 39.22, 39.0, 38.78, 38.56, 38.34, 38.12, 37.9, 37.68, + 37.46, 37.25, 37.03, 36.81, 36.6, 36.38, 36.16, 35.94, 35.73, 35.51, 35.29, 35.06, 34.84, 34.62, + 34.39, 34.16, 33.93, 33.7, 33.46, 33.22, 32.98, 32.74, 32.49, 32.23, 31.97, 31.71, 31.44, 31.16, + 30.88, 30.59, 30.29, 29.97, 29.65, 29.31, 28.96, 28.59, 28.2, 27.78, 27.33, 26.84, 26.3, 25.7, + 25.0, 24.16, 23.07, 21.43, + ], + [ + 97.65, 86.14, 73.4, 63.69, 60.44, 58.43, 56.95, 55.76, 54.76, 53.9, 53.13, 52.44, 51.81, 51.22, + 50.68, 50.17, 49.7, 49.24, 48.81, 48.4, 48.01, 47.63, 47.27, 46.92, 46.58, 46.25, 45.93, 45.62, + 45.31, 45.02, 44.73, 44.44, 44.16, 43.89, 43.62, 43.36, 43.1, 42.85, 42.6, 42.35, 42.1, 41.86, + 41.62, 41.39, 41.15, 40.92, 40.69, 40.46, 40.23, 40.01, 39.78, 39.56, 39.34, 39.11, 38.89, + 38.67, 38.45, 38.23, 38.01, 37.79, 37.57, 37.35, 37.13, 36.91, 36.69, 36.47, 36.25, 36.02, + 35.79, 35.57, 35.34, 35.11, 34.87, 34.64, 34.4, 34.15, 33.91, 33.66, 33.41, 33.15, 32.89, 32.62, + 32.34, 32.06, 31.78, 31.48, 31.17, 30.86, 30.53, 30.18, 29.83, 29.45, 29.05, 28.63, 28.17, + 27.67, 27.12, 26.51, 25.8, 24.94, 23.84, 22.16, + ], + [ + 99.17, 87.58, 74.74, 64.95, 61.67, 59.64, 58.14, 56.94, 55.93, 55.06, 54.29, 53.59, 52.95, + 52.36, 51.81, 51.3, 50.82, 50.36, 49.93, 49.51, 49.11, 48.73, 48.36, 48.01, 47.66, 47.33, 47.01, + 46.69, 46.38, 46.08, 45.79, 45.5, 45.22, 44.95, 44.68, 44.41, 44.15, 43.89, 43.64, 43.39, 43.14, + 42.89, 42.65, 42.41, 42.17, 41.94, 41.7, 41.47, 41.24, 41.01, 40.79, 40.56, 40.34, 40.11, 39.89, + 39.66, 39.44, 39.22, 39.0, 38.77, 38.55, 38.33, 38.11, 37.88, 37.66, 37.43, 37.21, 36.98, 36.75, + 36.52, 36.28, 36.05, 35.81, 35.57, 35.33, 35.09, 34.84, 34.58, 34.33, 34.07, 33.8, 33.53, 33.25, + 32.97, 32.67, 32.37, 32.06, 31.74, 31.41, 31.06, 30.69, 30.31, 29.91, 29.48, 29.01, 28.51, + 27.95, 27.33, 26.6, 25.73, 24.61, 22.91, + ], + [ + 100.69, 89.01, 76.08, 66.21, 62.89, 60.85, 59.33, 58.12, 57.11, 56.22, 55.44, 54.73, 54.09, + 53.5, 52.94, 52.42, 51.94, 51.47, 51.04, 50.62, 50.21, 49.83, 49.46, 49.1, 48.75, 48.41, 48.08, + 47.77, 47.46, 47.15, 46.86, 46.57, 46.28, 46.0, 45.73, 45.46, 45.19, 44.93, 44.68, 44.42, 44.17, + 43.92, 43.68, 43.44, 43.2, 42.96, 42.72, 42.49, 42.25, 42.02, 41.79, 41.56, 41.34, 41.11, 40.88, + 40.66, 40.43, 40.2, 39.98, 39.75, 39.53, 39.3, 39.08, 38.85, 38.62, 38.39, 38.17, 37.93, 37.7, + 37.47, 37.23, 37.0, 36.75, 36.51, 36.27, 36.02, 35.77, 35.51, 35.25, 34.98, 34.71, 34.44, 34.16, + 33.87, 33.57, 33.27, 32.95, 32.63, 32.29, 31.93, 31.56, 31.18, 30.77, 30.33, 29.86, 29.34, + 28.78, 28.14, 27.41, 26.53, 25.38, 23.65, + ], + [ + 102.2, 90.44, 77.42, 67.46, 64.12, 62.05, 60.53, 59.3, 58.28, 57.38, 56.59, 55.88, 55.23, 54.63, + 54.07, 53.55, 53.05, 52.59, 52.14, 51.72, 51.31, 50.92, 50.55, 50.19, 49.83, 49.49, 49.16, + 48.84, 48.53, 48.22, 47.92, 47.63, 47.34, 47.06, 46.78, 46.51, 46.24, 45.98, 45.72, 45.46, 45.2, + 44.95, 44.71, 44.46, 44.22, 43.98, 43.74, 43.5, 43.26, 43.03, 42.8, 42.57, 42.34, 42.11, 41.88, + 41.65, 41.42, 41.19, 40.96, 40.74, 40.51, 40.28, 40.05, 39.82, 39.59, 39.36, 39.13, 38.89, + 38.66, 38.42, 38.18, 37.94, 37.7, 37.45, 37.2, 36.95, 36.7, 36.44, 36.17, 35.9, 35.63, 35.35, + 35.07, 34.77, 34.47, 34.16, 33.84, 33.51, 33.17, 32.81, 32.44, 32.04, 31.63, 31.18, 30.7, 30.18, + 29.61, 28.96, 28.22, 27.32, 26.16, 24.4, + ], + [ + 103.7, 91.87, 78.75, 68.71, 65.34, 63.25, 61.71, 60.48, 59.44, 58.54, 57.75, 57.03, 56.37, + 55.76, 55.2, 54.67, 54.17, 53.7, 53.25, 52.82, 52.41, 52.02, 51.64, 51.27, 50.92, 50.57, 50.24, + 49.91, 49.6, 49.29, 48.98, 48.69, 48.4, 48.11, 47.83, 47.56, 47.28, 47.02, 46.75, 46.49, 46.24, + 45.98, 45.73, 45.49, 45.24, 45.0, 44.75, 44.51, 44.28, 44.04, 43.8, 43.57, 43.34, 43.1, 42.87, + 42.64, 42.41, 42.18, 41.95, 41.72, 41.49, 41.25, 41.02, 40.79, 40.56, 40.32, 40.09, 39.85, + 39.61, 39.37, 39.13, 38.89, 38.64, 38.39, 38.14, 37.88, 37.63, 37.36, 37.1, 36.82, 36.55, 36.26, + 35.97, 35.68, 35.37, 35.06, 34.74, 34.4, 34.05, 33.69, 33.31, 32.91, 32.49, 32.04, 31.55, 31.02, + 30.44, 29.79, 29.03, 28.12, 26.94, 25.15, + ], + [ + 105.2, 93.29, 80.08, 69.96, 66.56, 64.45, 62.9, 61.66, 60.61, 59.7, 58.9, 58.17, 57.51, 56.89, + 56.32, 55.79, 55.29, 54.81, 54.36, 53.92, 53.51, 53.11, 52.73, 52.36, 52.0, 51.65, 51.31, 50.98, + 50.66, 50.35, 50.05, 49.75, 49.45, 49.16, 48.88, 48.6, 48.33, 48.06, 47.79, 47.53, 47.27, 47.01, + 46.76, 46.51, 46.26, 46.01, 45.77, 45.53, 45.29, 45.05, 44.81, 44.57, 44.34, 44.1, 43.87, 43.63, + 43.4, 43.16, 42.93, 42.7, 42.46, 42.23, 42.0, 41.76, 41.52, 41.29, 41.05, 40.81, 40.57, 40.33, + 40.08, 39.83, 39.58, 39.33, 39.08, 38.82, 38.56, 38.29, 38.02, 37.75, 37.46, 37.18, 36.88, + 36.58, 36.28, 35.96, 35.63, 35.29, 34.94, 34.57, 34.18, 33.78, 33.35, 32.89, 32.4, 31.87, 31.28, + 30.61, 29.84, 28.92, 27.72, 25.9, + ], + [ + 106.69, 94.71, 81.4, 71.2, 67.77, 65.65, 64.09, 62.83, 61.77, 60.86, 60.04, 59.31, 58.64, 58.02, + 57.45, 56.91, 56.4, 55.92, 55.46, 55.03, 54.61, 54.21, 53.82, 53.44, 53.08, 52.73, 52.39, 52.06, + 51.73, 51.42, 51.11, 50.8, 50.51, 50.22, 49.93, 49.65, 49.37, 49.1, 48.83, 48.56, 48.3, 48.04, + 47.79, 47.53, 47.28, 47.03, 46.79, 46.54, 46.3, 46.05, 45.81, 45.57, 45.34, 45.1, 44.86, 44.62, + 44.39, 44.15, 43.91, 43.68, 43.44, 43.21, 42.97, 42.73, 42.49, 42.25, 42.01, 41.77, 41.52, + 41.28, 41.03, 40.78, 40.53, 40.27, 40.02, 39.75, 39.49, 39.22, 38.95, 38.67, 38.38, 38.09, 37.8, + 37.49, 37.18, 36.86, 36.52, 36.18, 35.82, 35.45, 35.06, 34.65, 34.22, 33.75, 33.25, 32.71, + 32.11, 31.44, 30.66, 29.72, 28.5, 26.66, + ], + [ + 108.18, 96.12, 82.72, 72.44, 68.99, 66.85, 65.27, 64.0, 62.94, 62.01, 61.19, 60.45, 59.77, + 59.15, 58.57, 58.03, 57.51, 57.03, 56.57, 56.13, 55.7, 55.3, 54.91, 54.53, 54.16, 53.81, 53.46, + 53.13, 52.8, 52.48, 52.17, 51.86, 51.56, 51.27, 50.98, 50.69, 50.41, 50.14, 49.87, 49.6, 49.33, + 49.07, 48.81, 48.56, 48.3, 48.05, 47.8, 47.55, 47.31, 47.06, 46.82, 46.58, 46.34, 46.09, 45.85, + 45.62, 45.38, 45.14, 44.9, 44.66, 44.42, 44.18, 43.94, 43.7, 43.46, 43.22, 42.97, 42.73, 42.48, + 42.23, 41.98, 41.73, 41.47, 41.22, 40.95, 40.69, 40.42, 40.15, 39.87, 39.59, 39.3, 39.01, 38.71, + 38.4, 38.08, 37.76, 37.42, 37.07, 36.71, 36.33, 35.94, 35.52, 35.08, 34.61, 34.11, 33.56, 32.95, + 32.27, 31.48, 30.53, 29.29, 27.42, + ], + [ + 109.66, 97.53, 84.04, 73.68, 70.2, 68.04, 66.45, 65.17, 64.1, 63.16, 62.34, 61.59, 60.91, 60.28, + 59.69, 59.14, 58.63, 58.14, 57.67, 57.22, 56.8, 56.39, 55.99, 55.61, 55.24, 54.88, 54.54, 54.2, + 53.87, 53.54, 53.23, 52.92, 52.62, 52.32, 52.03, 51.74, 51.46, 51.18, 50.9, 50.63, 50.37, 50.1, + 49.84, 49.58, 49.32, 49.07, 48.82, 48.57, 48.32, 48.07, 47.82, 47.58, 47.34, 47.09, 46.85, + 46.61, 46.37, 46.12, 45.88, 45.64, 45.4, 45.16, 44.92, 44.67, 44.43, 44.18, 43.94, 43.69, 43.44, + 43.19, 42.93, 42.68, 42.42, 42.16, 41.89, 41.63, 41.36, 41.08, 40.8, 40.51, 40.22, 39.92, 39.62, + 39.31, 38.99, 38.66, 38.32, 37.96, 37.6, 37.22, 36.82, 36.4, 35.95, 35.47, 34.96, 34.4, 33.79, + 33.1, 32.3, 31.33, 30.08, 28.18, + ], + [ + 111.14, 98.93, 85.35, 74.92, 71.41, 69.23, 67.63, 66.34, 65.25, 64.31, 63.48, 62.73, 62.04, + 61.4, 60.81, 60.26, 59.74, 59.24, 58.77, 58.32, 57.89, 57.48, 57.08, 56.69, 56.32, 55.96, 55.61, + 55.27, 54.93, 54.61, 54.29, 53.98, 53.67, 53.37, 53.07, 52.79, 52.5, 52.22, 51.94, 51.67, 51.4, + 51.13, 50.87, 50.6, 50.34, 50.09, 49.83, 49.58, 49.33, 49.08, 48.83, 48.58, 48.33, 48.09, 47.84, + 47.6, 47.36, 47.11, 46.87, 46.62, 46.38, 46.13, 45.89, 45.64, 45.4, 45.15, 44.9, 44.65, 44.4, + 44.14, 43.89, 43.63, 43.37, 43.1, 42.83, 42.56, 42.29, 42.01, 41.73, 41.44, 41.14, 40.84, 40.53, + 40.22, 39.89, 39.56, 39.22, 38.86, 38.49, 38.1, 37.7, 37.27, 36.82, 36.34, 35.82, 35.25, 34.63, + 33.93, 33.12, 32.14, 30.87, 28.94, + ], + [ + 112.61, 100.33, 86.66, 76.15, 72.61, 70.42, 68.8, 67.5, 66.41, 65.46, 64.62, 63.86, 63.17, + 62.53, 61.93, 61.37, 60.84, 60.35, 59.87, 59.42, 58.98, 58.57, 58.16, 57.78, 57.4, 57.03, 56.68, + 56.33, 56.0, 55.67, 55.35, 55.03, 54.72, 54.42, 54.12, 53.83, 53.54, 53.26, 52.98, 52.7, 52.43, + 52.16, 51.89, 51.63, 51.37, 51.11, 50.85, 50.59, 50.34, 50.09, 49.83, 49.58, 49.33, 49.09, + 48.84, 48.59, 48.35, 48.1, 47.85, 47.61, 47.36, 47.11, 46.86, 46.62, 46.37, 46.12, 45.86, 45.61, + 45.35, 45.1, 44.84, 44.58, 44.31, 44.05, 43.78, 43.5, 43.22, 42.94, 42.66, 42.36, 42.07, 41.76, + 41.45, 41.13, 40.8, 40.46, 40.12, 39.75, 39.38, 38.99, 38.58, 38.15, 37.69, 37.2, 36.68, 36.11, + 35.47, 34.76, 33.94, 32.95, 31.66, 29.71, + ], + [ + 114.08, 101.73, 87.97, 77.39, 73.82, 71.61, 69.98, 68.67, 67.57, 66.61, 65.76, 65.0, 64.3, + 63.65, 63.05, 62.48, 61.95, 61.45, 60.97, 60.51, 60.08, 59.65, 59.25, 58.86, 58.48, 58.11, + 57.75, 57.4, 57.06, 56.73, 56.4, 56.09, 55.78, 55.47, 55.17, 54.87, 54.58, 54.3, 54.01, 53.73, + 53.46, 53.19, 52.92, 52.65, 52.39, 52.12, 51.86, 51.6, 51.35, 51.09, 50.84, 50.59, 50.33, 50.08, + 49.83, 49.58, 49.34, 49.09, 48.84, 48.59, 48.34, 48.09, 47.84, 47.59, 47.34, 47.08, 46.83, + 46.57, 46.31, 46.05, 45.79, 45.53, 45.26, 44.99, 44.72, 44.44, 44.16, 43.87, 43.58, 43.29, + 42.99, 42.68, 42.36, 42.04, 41.71, 41.37, 41.02, 40.65, 40.27, 39.87, 39.46, 39.02, 38.56, + 38.07, 37.54, 36.96, 36.32, 35.6, 34.77, 33.76, 32.46, 30.48, + ], + [ + 115.54, 103.12, 89.27, 78.62, 75.02, 72.8, 71.15, 69.83, 68.72, 67.76, 66.9, 66.13, 65.42, + 64.77, 64.16, 63.6, 63.06, 62.55, 62.07, 61.61, 61.17, 60.74, 60.33, 59.94, 59.55, 59.18, 58.82, + 58.47, 58.13, 57.79, 57.46, 57.14, 56.83, 56.52, 56.22, 55.92, 55.62, 55.33, 55.05, 54.77, + 54.49, 54.21, 53.94, 53.67, 53.41, 53.14, 52.88, 52.62, 52.36, 52.1, 51.84, 51.59, 51.33, 51.08, + 50.83, 50.58, 50.33, 50.07, 49.82, 49.57, 49.32, 49.07, 48.81, 48.56, 48.3, 48.05, 47.79, 47.53, + 47.27, 47.01, 46.75, 46.48, 46.21, 45.94, 45.66, 45.38, 45.1, 44.81, 44.51, 44.22, 43.91, 43.6, + 43.28, 42.95, 42.62, 42.27, 41.92, 41.55, 41.16, 40.76, 40.34, 39.9, 39.43, 38.93, 38.4, 37.81, + 37.16, 36.44, 35.59, 34.58, 33.26, 31.25, + ], + [ + 117.0, 104.5, 90.57, 79.84, 76.22, 73.98, 72.32, 70.99, 69.87, 68.9, 68.04, 67.26, 66.55, 65.89, + 65.28, 64.71, 64.17, 63.65, 63.17, 62.7, 62.26, 61.83, 61.41, 61.02, 60.63, 60.25, 59.89, 59.53, + 59.19, 58.85, 58.52, 58.2, 57.88, 57.57, 57.26, 56.96, 56.66, 56.37, 56.08, 55.8, 55.52, 55.24, + 54.97, 54.69, 54.43, 54.16, 53.89, 53.63, 53.37, 53.11, 52.85, 52.59, 52.33, 52.08, 51.82, + 51.57, 51.32, 51.06, 50.81, 50.55, 50.3, 50.04, 49.79, 49.53, 49.27, 49.02, 48.76, 48.5, 48.23, + 47.97, 47.7, 47.43, 47.16, 46.88, 46.6, 46.32, 46.03, 45.74, 45.44, 45.14, 44.84, 44.52, 44.2, + 43.87, 43.53, 43.18, 42.82, 42.45, 42.06, 41.65, 41.23, 40.78, 40.31, 39.8, 39.26, 38.67, 38.01, + 37.28, 36.42, 35.39, 34.06, 32.02, + ], + [ + 118.45, 105.89, 91.87, 81.07, 77.42, 75.16, 73.49, 72.15, 71.02, 70.04, 69.18, 68.39, 67.67, + 67.01, 66.39, 65.82, 65.27, 64.76, 64.26, 63.79, 63.35, 62.91, 62.5, 62.09, 61.7, 61.33, 60.96, + 60.6, 60.25, 59.91, 59.58, 59.25, 58.93, 58.62, 58.31, 58.0, 57.7, 57.41, 57.12, 56.83, 56.55, + 56.27, 55.99, 55.72, 55.45, 55.18, 54.91, 54.64, 54.38, 54.12, 53.85, 53.59, 53.33, 53.08, + 52.82, 52.56, 52.31, 52.05, 51.79, 51.54, 51.28, 51.02, 50.76, 50.5, 50.24, 49.98, 49.72, 49.46, + 49.19, 48.92, 48.65, 48.38, 48.11, 47.83, 47.55, 47.26, 46.97, 46.68, 46.38, 46.07, 45.76, + 45.44, 45.12, 44.78, 44.44, 44.09, 43.72, 43.34, 42.95, 42.54, 42.11, 41.66, 41.18, 40.67, + 40.12, 39.52, 38.86, 38.12, 37.25, 36.21, 34.86, 32.79, + ], + [ + 119.9, 107.27, 93.17, 82.29, 78.62, 76.34, 74.66, 73.31, 72.17, 71.19, 70.31, 69.52, 68.8, + 68.13, 67.51, 66.92, 66.38, 65.86, 65.36, 64.89, 64.43, 64.0, 63.58, 63.17, 62.78, 62.4, 62.03, + 61.66, 61.31, 60.97, 60.63, 60.3, 59.98, 59.66, 59.35, 59.05, 58.74, 58.45, 58.15, 57.86, 57.58, + 57.3, 57.02, 56.74, 56.46, 56.19, 55.92, 55.65, 55.39, 55.12, 54.86, 54.6, 54.33, 54.07, 53.81, + 53.55, 53.3, 53.04, 52.78, 52.52, 52.26, 52.0, 51.74, 51.48, 51.22, 50.95, 50.69, 50.42, 50.15, + 49.88, 49.61, 49.33, 49.06, 48.77, 48.49, 48.2, 47.91, 47.61, 47.31, 47.0, 46.69, 46.36, 46.04, + 45.7, 45.35, 45.0, 44.63, 44.24, 43.85, 43.43, 43.0, 42.54, 42.06, 41.54, 40.99, 40.38, 39.71, + 38.96, 38.08, 37.03, 35.66, 33.57, + ], + [ + 121.35, 108.65, 94.46, 83.51, 79.81, 77.52, 75.83, 74.47, 73.32, 72.33, 71.44, 70.65, 69.92, + 69.25, 68.62, 68.03, 67.48, 66.95, 66.45, 65.98, 65.52, 65.08, 64.66, 64.25, 63.85, 63.47, + 63.09, 62.73, 62.37, 62.03, 61.69, 61.36, 61.03, 60.71, 60.4, 60.09, 59.78, 59.48, 59.19, 58.9, + 58.61, 58.32, 58.04, 57.76, 57.48, 57.21, 56.94, 56.67, 56.4, 56.13, 55.86, 55.6, 55.33, 55.07, + 54.81, 54.55, 54.29, 54.02, 53.76, 53.5, 53.24, 52.98, 52.71, 52.45, 52.19, 51.92, 51.65, 51.38, + 51.11, 50.84, 50.56, 50.29, 50.01, 49.72, 49.43, 49.14, 48.85, 48.55, 48.24, 47.93, 47.61, + 47.29, 46.96, 46.61, 46.26, 45.9, 45.53, 45.15, 44.74, 44.33, 43.89, 43.43, 42.94, 42.42, 41.85, + 41.24, 40.56, 39.8, 38.92, 37.85, 36.46, 34.35, + ], + [ + 122.79, 110.02, 95.75, 84.73, 81.01, 78.7, 76.99, 75.62, 74.47, 73.47, 72.58, 71.77, 71.04, + 70.36, 69.73, 69.14, 68.58, 68.05, 67.55, 67.07, 66.61, 66.16, 65.74, 65.32, 64.93, 64.54, + 64.16, 63.79, 63.44, 63.09, 62.74, 62.41, 62.08, 61.76, 61.44, 61.13, 60.82, 60.52, 60.22, + 59.93, 59.64, 59.35, 59.06, 58.78, 58.5, 58.23, 57.95, 57.68, 57.41, 57.14, 56.87, 56.6, 56.33, + 56.07, 55.8, 55.54, 55.28, 55.01, 54.75, 54.49, 54.22, 53.96, 53.69, 53.42, 53.16, 52.89, 52.62, + 52.35, 52.07, 51.8, 51.52, 51.24, 50.96, 50.67, 50.38, 50.08, 49.79, 49.48, 49.17, 48.86, 48.54, + 48.21, 47.88, 47.53, 47.18, 46.81, 46.44, 46.05, 45.64, 45.22, 44.78, 44.31, 43.82, 43.29, + 42.72, 42.1, 41.42, 40.65, 39.75, 38.67, 37.27, 35.13, + ], + [ + 124.23, 111.39, 97.04, 85.95, 82.2, 79.88, 78.16, 76.78, 75.62, 74.61, 73.71, 72.9, 72.16, + 71.48, 70.84, 70.24, 69.68, 69.15, 68.64, 68.16, 67.69, 67.25, 66.82, 66.4, 66.0, 65.61, 65.23, + 64.86, 64.5, 64.14, 63.8, 63.46, 63.13, 62.8, 62.48, 62.17, 61.86, 61.56, 61.26, 60.96, 60.67, + 60.38, 60.09, 59.8, 59.52, 59.24, 58.96, 58.69, 58.42, 58.14, 57.87, 57.6, 57.33, 57.07, 56.8, + 56.53, 56.27, 56.0, 55.73, 55.47, 55.2, 54.93, 54.67, 54.4, 54.13, 53.86, 53.59, 53.31, 53.03, + 52.76, 52.48, 52.19, 51.91, 51.62, 51.32, 51.03, 50.73, 50.42, 50.11, 49.79, 49.47, 49.14, 48.8, + 48.45, 48.09, 47.72, 47.34, 46.95, 46.54, 46.11, 45.67, 45.2, 44.7, 44.16, 43.59, 42.96, 42.27, + 41.49, 40.59, 39.5, 38.08, 35.91, + ], + [ + 125.66, 112.76, 98.32, 87.17, 83.39, 81.05, 79.32, 77.93, 76.76, 75.74, 74.84, 74.02, 73.28, + 72.59, 71.95, 71.35, 70.78, 70.25, 69.73, 69.25, 68.78, 68.33, 67.89, 67.48, 67.07, 66.68, + 66.29, 65.92, 65.56, 65.2, 64.85, 64.51, 64.18, 63.85, 63.53, 63.21, 62.9, 62.59, 62.29, 61.99, + 61.69, 61.4, 61.11, 60.82, 60.54, 60.26, 59.98, 59.7, 59.43, 59.15, 58.88, 58.61, 58.33, 58.06, + 57.8, 57.53, 57.26, 56.99, 56.72, 56.45, 56.18, 55.91, 55.64, 55.37, 55.1, 54.83, 54.55, 54.27, + 54.0, 53.72, 53.43, 53.15, 52.86, 52.57, 52.27, 51.97, 51.67, 51.36, 51.04, 50.72, 50.39, 50.06, + 49.72, 49.37, 49.01, 48.64, 48.25, 47.85, 47.44, 47.01, 46.56, 46.08, 45.58, 45.04, 44.46, + 43.83, 43.13, 42.34, 41.43, 40.32, 38.89, 36.7, + ], + [ + 127.1, 114.12, 99.61, 88.38, 84.58, 82.23, 80.48, 79.08, 77.9, 76.88, 75.97, 75.15, 74.4, 73.7, + 73.06, 72.45, 71.88, 71.34, 70.83, 70.33, 69.86, 69.41, 68.97, 68.55, 68.14, 67.74, 67.36, + 66.98, 66.61, 66.26, 65.91, 65.56, 65.23, 64.9, 64.57, 64.25, 63.94, 63.63, 63.32, 63.02, 62.72, + 62.43, 62.13, 61.85, 61.56, 61.27, 60.99, 60.71, 60.43, 60.16, 59.88, 59.61, 59.33, 59.06, + 58.79, 58.52, 58.25, 57.98, 57.71, 57.44, 57.16, 56.89, 56.62, 56.35, 56.07, 55.8, 55.52, 55.24, + 54.96, 54.67, 54.39, 54.1, 53.81, 53.51, 53.22, 52.91, 52.61, 52.29, 51.98, 51.65, 51.32, 50.99, + 50.64, 50.29, 49.92, 49.55, 49.16, 48.76, 48.34, 47.91, 47.45, 46.97, 46.46, 45.91, 45.33, + 44.69, 43.98, 43.19, 42.27, 41.15, 39.7, 37.48, + ], + [ + 128.52, 115.49, 100.89, 89.59, 85.77, 83.4, 81.64, 80.23, 79.04, 78.01, 77.1, 76.27, 75.51, + 74.82, 74.17, 73.56, 72.98, 72.44, 71.92, 71.42, 70.95, 70.49, 70.05, 69.62, 69.21, 68.81, + 68.42, 68.04, 67.67, 67.31, 66.96, 66.61, 66.27, 65.94, 65.61, 65.29, 64.98, 64.66, 64.35, + 64.05, 63.75, 63.45, 63.16, 62.87, 62.58, 62.29, 62.01, 61.72, 61.44, 61.16, 60.89, 60.61, + 60.33, 60.06, 59.79, 59.51, 59.24, 58.97, 58.69, 58.42, 58.15, 57.87, 57.6, 57.32, 57.04, 56.77, + 56.49, 56.2, 55.92, 55.63, 55.35, 55.06, 54.76, 54.46, 54.16, 53.86, 53.55, 53.23, 52.91, 52.59, + 52.25, 51.91, 51.56, 51.21, 50.84, 50.46, 50.07, 49.66, 49.24, 48.8, 48.34, 47.86, 47.34, 46.79, + 46.2, 45.56, 44.84, 44.04, 43.11, 41.98, 40.51, 38.27, + ], + [ + 129.95, 116.85, 102.17, 90.8, 86.95, 84.57, 82.8, 81.38, 80.19, 79.15, 78.22, 77.39, 76.63, + 75.93, 75.27, 74.66, 74.08, 73.53, 73.01, 72.51, 72.03, 71.57, 71.13, 70.7, 70.28, 69.88, 69.49, + 69.1, 68.73, 68.37, 68.01, 67.66, 67.32, 66.99, 66.66, 66.33, 66.01, 65.7, 65.39, 65.08, 64.78, + 64.48, 64.18, 63.89, 63.6, 63.31, 63.02, 62.74, 62.45, 62.17, 61.89, 61.61, 61.33, 61.06, 60.78, + 60.51, 60.23, 59.95, 59.68, 59.4, 59.13, 58.85, 58.57, 58.3, 58.02, 57.74, 57.45, 57.17, 56.88, + 56.59, 56.3, 56.01, 55.71, 55.41, 55.11, 54.8, 54.49, 54.17, 53.85, 53.52, 53.18, 52.84, 52.49, + 52.13, 51.76, 51.37, 50.98, 50.57, 50.14, 49.7, 49.24, 48.75, 48.23, 47.67, 47.07, 46.42, 45.7, + 44.89, 43.95, 42.81, 41.33, 39.06, + ], + [ + 131.37, 118.2, 103.44, 92.01, 88.14, 85.74, 83.96, 82.53, 81.33, 80.28, 79.35, 78.51, 77.75, + 77.04, 76.38, 75.76, 75.18, 74.62, 74.1, 73.59, 73.11, 72.65, 72.2, 71.77, 71.35, 70.94, 70.55, + 70.16, 69.79, 69.42, 69.06, 68.71, 68.37, 68.03, 67.7, 67.37, 67.05, 66.73, 66.42, 66.11, 65.8, + 65.5, 65.2, 64.91, 64.61, 64.32, 64.03, 63.75, 63.46, 63.18, 62.9, 62.61, 62.33, 62.06, 61.78, + 61.5, 61.22, 60.94, 60.67, 60.39, 60.11, 59.83, 59.55, 59.27, 58.99, 58.71, 58.42, 58.13, 57.85, + 57.56, 57.26, 56.97, 56.67, 56.36, 56.06, 55.75, 55.43, 55.11, 54.78, 54.45, 54.11, 53.77, + 53.41, 53.05, 52.67, 52.29, 51.89, 51.48, 51.05, 50.6, 50.13, 49.64, 49.11, 48.55, 47.95, 47.29, + 46.56, 45.74, 44.79, 43.64, 42.14, 39.86, + ], + [ + 132.79, 119.55, 104.72, 93.22, 89.32, 86.9, 85.11, 83.68, 82.46, 81.41, 80.48, 79.63, 78.86, + 78.15, 77.48, 76.86, 76.27, 75.72, 75.19, 74.68, 74.19, 73.73, 73.28, 72.84, 72.42, 72.01, + 71.61, 71.23, 70.85, 70.48, 70.12, 69.76, 69.42, 69.08, 68.74, 68.41, 68.09, 67.77, 67.45, + 67.14, 66.83, 66.53, 66.23, 65.93, 65.63, 65.34, 65.05, 64.76, 64.47, 64.18, 63.9, 63.62, 63.33, + 63.05, 62.77, 62.49, 62.21, 61.93, 61.65, 61.37, 61.09, 60.81, 60.53, 60.25, 59.96, 59.68, + 59.39, 59.1, 58.81, 58.52, 58.22, 57.92, 57.62, 57.31, 57.0, 56.69, 56.37, 56.05, 55.72, 55.39, + 55.04, 54.69, 54.34, 53.97, 53.59, 53.2, 52.8, 52.38, 51.95, 51.5, 51.03, 50.53, 50.0, 49.43, + 48.82, 48.16, 47.42, 46.59, 45.63, 44.47, 42.96, 40.65, + ], + [ + 134.2, 120.9, 105.99, 94.42, 90.5, 88.07, 86.27, 84.82, 83.6, 82.54, 81.6, 80.75, 79.97, 79.26, + 78.59, 77.96, 77.37, 76.81, 76.27, 75.76, 75.27, 74.8, 74.35, 73.91, 73.49, 73.08, 72.68, 72.28, + 71.9, 71.53, 71.17, 70.81, 70.46, 70.12, 69.78, 69.45, 69.12, 68.8, 68.48, 68.17, 67.86, 67.55, + 67.25, 66.95, 66.65, 66.35, 66.06, 65.77, 65.48, 65.19, 64.9, 64.62, 64.33, 64.05, 63.77, 63.49, + 63.2, 62.92, 62.64, 62.36, 62.07, 61.79, 61.51, 61.22, 60.93, 60.65, 60.36, 60.07, 59.77, 59.48, + 59.18, 58.88, 58.57, 58.27, 57.95, 57.64, 57.32, 56.99, 56.66, 56.32, 55.98, 55.62, 55.26, + 54.89, 54.51, 54.12, 53.71, 53.29, 52.86, 52.4, 51.92, 51.42, 50.88, 50.31, 49.7, 49.03, 48.29, + 47.45, 46.48, 45.31, 43.78, 41.44, + ], + [ + 135.61, 122.25, 107.26, 95.63, 91.68, 89.23, 87.42, 85.96, 84.74, 83.67, 82.72, 81.87, 81.09, + 80.36, 79.69, 79.06, 78.46, 77.9, 77.36, 76.85, 76.35, 75.88, 75.42, 74.98, 74.56, 74.14, 73.74, + 73.34, 72.96, 72.59, 72.22, 71.86, 71.51, 71.16, 70.82, 70.49, 70.16, 69.83, 69.51, 69.2, 68.89, + 68.58, 68.27, 67.97, 67.67, 67.37, 67.07, 66.78, 66.49, 66.2, 65.91, 65.62, 65.33, 65.05, 64.76, + 64.48, 64.19, 63.91, 63.63, 63.34, 63.06, 62.77, 62.48, 62.2, 61.91, 61.62, 61.33, 61.03, 60.74, + 60.44, 60.14, 59.83, 59.53, 59.22, 58.9, 58.58, 58.26, 57.93, 57.6, 57.26, 56.91, 56.55, 56.19, + 55.81, 55.43, 55.03, 54.63, 54.2, 53.76, 53.3, 52.82, 52.31, 51.77, 51.19, 50.57, 49.9, 49.15, + 48.31, 47.33, 46.14, 44.6, 42.24, + ], + [ + 137.02, 123.6, 108.53, 96.83, 92.86, 90.4, 88.57, 87.11, 85.87, 84.8, 83.85, 82.98, 82.2, 81.47, + 80.79, 80.16, 79.56, 78.99, 78.45, 77.93, 77.43, 76.96, 76.5, 76.05, 75.62, 75.21, 74.8, 74.4, + 74.02, 73.64, 73.27, 72.91, 72.55, 72.21, 71.86, 71.53, 71.2, 70.87, 70.55, 70.23, 69.91, 69.6, + 69.29, 68.99, 68.69, 68.39, 68.09, 67.79, 67.5, 67.2, 66.91, 66.62, 66.33, 66.05, 65.76, 65.47, + 65.19, 64.9, 64.61, 64.33, 64.04, 63.75, 63.46, 63.17, 62.88, 62.59, 62.29, 62.0, 61.7, 61.4, + 61.1, 60.79, 60.48, 60.17, 59.85, 59.53, 59.2, 58.87, 58.53, 58.19, 57.84, 57.48, 57.11, 56.74, + 56.35, 55.95, 55.54, 55.11, 54.67, 54.2, 53.72, 53.2, 52.66, 52.08, 51.45, 50.77, 50.01, 49.16, + 48.17, 46.98, 45.42, 43.04, + ], + [ + 138.43, 124.94, 109.79, 98.03, 94.04, 91.56, 89.73, 88.25, 87.01, 85.93, 84.97, 84.1, 83.31, + 82.58, 81.89, 81.25, 80.65, 80.08, 79.53, 79.01, 78.51, 78.03, 77.57, 77.12, 76.69, 76.27, + 75.86, 75.46, 75.07, 74.69, 74.32, 73.96, 73.6, 73.25, 72.9, 72.57, 72.23, 71.9, 71.58, 71.26, + 70.94, 70.63, 70.32, 70.01, 69.7, 69.4, 69.1, 68.8, 68.51, 68.21, 67.92, 67.63, 67.33, 67.04, + 66.75, 66.47, 66.18, 65.89, 65.6, 65.31, 65.02, 64.73, 64.44, 64.15, 63.85, 63.56, 63.26, 62.97, + 62.66, 62.36, 62.06, 61.75, 61.44, 61.12, 60.8, 60.48, 60.15, 59.81, 59.47, 59.13, 58.77, 58.41, + 58.04, 57.66, 57.27, 56.87, 56.45, 56.02, 55.57, 55.11, 54.61, 54.1, 53.55, 52.96, 52.33, 51.64, + 50.88, 50.02, 49.02, 47.82, 46.24, 43.84, + ], + [ + 139.83, 126.28, 111.06, 99.23, 95.21, 92.72, 90.88, 89.39, 88.14, 87.05, 86.09, 85.22, 84.42, + 83.68, 82.99, 82.35, 81.74, 81.17, 80.62, 80.09, 79.59, 79.11, 78.64, 78.19, 77.76, 77.33, + 76.92, 76.52, 76.13, 75.75, 75.37, 75.0, 74.64, 74.29, 73.94, 73.6, 73.27, 72.93, 72.61, 72.28, + 71.97, 71.65, 71.34, 71.03, 70.72, 70.42, 70.11, 69.81, 69.51, 69.22, 68.92, 68.63, 68.33, + 68.04, 67.75, 67.46, 67.17, 66.88, 66.59, 66.29, 66.0, 65.71, 65.42, 65.12, 64.83, 64.53, 64.23, + 63.93, 63.63, 63.32, 63.02, 62.71, 62.39, 62.07, 61.75, 61.42, 61.09, 60.76, 60.41, 60.06, + 59.71, 59.34, 58.97, 58.59, 58.19, 57.79, 57.37, 56.93, 56.48, 56.01, 55.51, 54.99, 54.44, + 53.85, 53.21, 52.51, 51.75, 50.88, 49.87, 48.66, 47.07, 44.64, + ], + [ + 141.23, 127.62, 112.32, 100.43, 96.39, 93.88, 92.02, 90.53, 89.27, 88.18, 87.21, 86.33, 85.53, + 84.79, 84.09, 83.45, 82.83, 82.26, 81.7, 81.18, 80.67, 80.18, 79.71, 79.26, 78.82, 78.4, 77.98, + 77.58, 77.18, 76.8, 76.42, 76.05, 75.69, 75.33, 74.98, 74.64, 74.3, 73.97, 73.64, 73.31, 72.99, + 72.67, 72.36, 72.05, 71.74, 71.43, 71.13, 70.82, 70.52, 70.22, 69.93, 69.63, 69.33, 69.04, + 68.75, 68.45, 68.16, 67.87, 67.57, 67.28, 66.99, 66.69, 66.4, 66.1, 65.8, 65.5, 65.2, 64.9, + 64.59, 64.29, 63.98, 63.66, 63.35, 63.03, 62.7, 62.37, 62.04, 61.7, 61.35, 61.0, 60.64, 60.27, + 59.9, 59.51, 59.11, 58.71, 58.28, 57.84, 57.39, 56.91, 56.41, 55.89, 55.33, 54.73, 54.09, 53.39, + 52.61, 51.74, 50.72, 49.5, 47.89, 45.44, + ], + [ + 142.63, 128.95, 113.58, 101.62, 97.56, 95.04, 93.17, 91.67, 90.4, 89.3, 88.33, 87.44, 86.64, + 85.89, 85.19, 84.54, 83.93, 83.34, 82.79, 82.26, 81.75, 81.26, 80.79, 80.33, 79.89, 79.46, + 79.04, 78.63, 78.24, 77.85, 77.47, 77.1, 76.73, 76.38, 76.02, 75.68, 75.34, 75.0, 74.67, 74.34, + 74.02, 73.7, 73.38, 73.07, 72.75, 72.45, 72.14, 71.83, 71.53, 71.23, 70.93, 70.63, 70.33, 70.04, + 69.74, 69.45, 69.15, 68.86, 68.56, 68.26, 67.97, 67.67, 67.37, 67.08, 66.78, 66.47, 66.17, + 65.87, 65.56, 65.25, 64.94, 64.62, 64.3, 63.98, 63.65, 63.32, 62.98, 62.64, 62.29, 61.94, 61.58, + 61.21, 60.83, 60.44, 60.04, 59.62, 59.2, 58.76, 58.3, 57.82, 57.31, 56.78, 56.22, 55.62, 54.97, + 54.26, 53.48, 52.6, 51.58, 50.34, 48.72, 46.25, + ], + [ + 144.02, 130.29, 114.84, 102.82, 98.73, 96.2, 94.32, 92.81, 91.54, 90.43, 89.44, 88.56, 87.74, + 86.99, 86.29, 85.64, 85.02, 84.43, 83.87, 83.34, 82.82, 82.33, 81.86, 81.4, 80.95, 80.52, 80.1, + 79.69, 79.29, 78.9, 78.52, 78.14, 77.78, 77.42, 77.06, 76.71, 76.37, 76.03, 75.7, 75.37, 75.04, + 74.72, 74.4, 74.08, 73.77, 73.46, 73.15, 72.84, 72.54, 72.24, 71.93, 71.63, 71.33, 71.04, 70.74, + 70.44, 70.14, 69.85, 69.55, 69.25, 68.95, 68.65, 68.35, 68.05, 67.75, 67.45, 67.14, 66.83, + 66.52, 66.21, 65.9, 65.58, 65.26, 64.93, 64.6, 64.27, 63.93, 63.58, 63.23, 62.88, 62.51, 62.14, + 61.76, 61.36, 60.96, 60.54, 60.11, 59.67, 59.21, 58.72, 58.22, 57.68, 57.11, 56.51, 55.85, + 55.14, 54.35, 53.46, 52.43, 51.18, 49.55, 47.05, + ], + [ + 145.41, 131.62, 116.09, 104.01, 99.9, 97.35, 95.46, 93.95, 92.66, 91.55, 90.56, 89.67, 88.85, + 88.09, 87.39, 86.73, 86.11, 85.52, 84.95, 84.42, 83.9, 83.41, 82.93, 82.46, 82.02, 81.58, 81.16, + 80.75, 80.34, 79.95, 79.57, 79.19, 78.82, 78.46, 78.1, 77.75, 77.41, 77.06, 76.73, 76.4, 76.07, + 75.74, 75.42, 75.1, 74.79, 74.47, 74.16, 73.86, 73.55, 73.24, 72.94, 72.64, 72.33, 72.03, 71.73, + 71.43, 71.13, 70.83, 70.54, 70.24, 69.94, 69.63, 69.33, 69.03, 68.72, 68.42, 68.11, 67.8, 67.49, + 67.18, 66.86, 66.54, 66.21, 65.89, 65.55, 65.22, 64.88, 64.53, 64.17, 63.81, 63.45, 63.07, + 62.69, 62.29, 61.88, 61.47, 61.03, 60.58, 60.12, 59.63, 59.12, 58.58, 58.01, 57.4, 56.74, 56.02, + 55.22, 54.33, 53.28, 52.02, 50.38, 47.86, + ], + [ + 146.8, 132.94, 117.35, 105.2, 101.07, 98.51, 96.61, 95.08, 93.79, 92.67, 91.68, 90.78, 89.96, + 89.2, 88.49, 87.82, 87.2, 86.6, 86.04, 85.5, 84.98, 84.48, 84.0, 83.53, 83.08, 82.64, 82.22, + 81.8, 81.4, 81.0, 80.62, 80.24, 79.86, 79.5, 79.14, 78.79, 78.44, 78.1, 77.76, 77.42, 77.09, + 76.77, 76.44, 76.12, 75.8, 75.49, 75.18, 74.87, 74.56, 74.25, 73.94, 73.64, 73.33, 73.03, 72.73, + 72.43, 72.13, 71.82, 71.52, 71.22, 70.92, 70.62, 70.31, 70.01, 69.7, 69.39, 69.08, 68.77, 68.46, + 68.14, 67.82, 67.5, 67.17, 66.84, 66.51, 66.17, 65.82, 65.47, 65.12, 64.75, 64.38, 64.0, 63.62, + 63.22, 62.81, 62.39, 61.95, 61.5, 61.03, 60.54, 60.02, 59.48, 58.9, 58.28, 57.62, 56.9, 56.09, + 55.19, 54.14, 52.87, 51.21, 48.67, + ], + [ + 148.19, 134.27, 118.6, 106.39, 102.24, 99.66, 97.75, 96.22, 94.92, 93.79, 92.79, 91.89, 91.06, + 90.3, 89.58, 88.92, 88.29, 87.69, 87.12, 86.57, 86.05, 85.55, 85.07, 84.6, 84.14, 83.7, 83.28, + 82.86, 82.45, 82.05, 81.66, 81.28, 80.91, 80.54, 80.18, 79.82, 79.47, 79.13, 78.79, 78.45, + 78.12, 77.79, 77.46, 77.14, 76.82, 76.5, 76.19, 75.88, 75.56, 75.26, 74.95, 74.64, 74.33, 74.03, + 73.73, 73.42, 73.12, 72.81, 72.51, 72.21, 71.9, 71.6, 71.29, 70.98, 70.67, 70.36, 70.05, 69.74, + 69.42, 69.1, 68.78, 68.46, 68.13, 67.79, 67.46, 67.12, 66.77, 66.42, 66.06, 65.69, 65.32, 64.94, + 64.55, 64.15, 63.73, 63.31, 62.87, 62.41, 61.94, 61.44, 60.92, 60.38, 59.79, 59.17, 58.5, 57.77, + 56.97, 56.05, 55.0, 53.71, 52.04, 49.48, + ], + [ + 149.57, 135.6, 119.85, 107.58, 103.41, 100.82, 98.9, 97.35, 96.05, 94.92, 93.91, 93.0, 92.17, + 91.4, 90.68, 90.01, 89.37, 88.77, 88.2, 87.65, 87.13, 86.62, 86.13, 85.66, 85.21, 84.76, 84.33, + 83.91, 83.5, 83.1, 82.71, 82.33, 81.95, 81.58, 81.22, 80.86, 80.51, 80.16, 79.82, 79.48, 79.14, + 78.81, 78.48, 78.16, 77.84, 77.52, 77.2, 76.89, 76.57, 76.26, 75.95, 75.64, 75.33, 75.03, 74.72, + 74.41, 74.11, 73.8, 73.5, 73.19, 72.89, 72.58, 72.27, 71.96, 71.65, 71.34, 71.02, 70.71, 70.39, + 70.07, 69.74, 69.42, 69.08, 68.75, 68.41, 68.07, 67.72, 67.36, 67.0, 66.63, 66.26, 65.87, 65.48, + 65.07, 64.66, 64.23, 63.79, 63.33, 62.85, 62.35, 61.83, 61.28, 60.69, 60.06, 59.39, 58.65, + 57.84, 56.92, 55.85, 54.56, 52.87, 50.29, + ], + [ + 150.95, 136.92, 121.1, 108.77, 104.58, 101.97, 100.04, 98.48, 97.17, 96.03, 95.02, 94.11, 93.27, + 92.5, 91.78, 91.1, 90.46, 89.86, 89.28, 88.73, 88.2, 87.69, 87.2, 86.73, 86.27, 85.82, 85.39, + 84.97, 84.56, 84.15, 83.76, 83.37, 82.99, 82.62, 82.26, 81.9, 81.54, 81.19, 80.85, 80.51, 80.17, + 79.84, 79.51, 79.18, 78.85, 78.53, 78.21, 77.9, 77.58, 77.27, 76.96, 76.64, 76.33, 76.03, 75.72, + 75.41, 75.1, 74.79, 74.49, 74.18, 73.87, 73.56, 73.25, 72.94, 72.62, 72.31, 71.99, 71.68, 71.35, + 71.03, 70.7, 70.37, 70.04, 69.7, 69.36, 69.02, 68.66, 68.31, 67.94, 67.57, 67.19, 66.81, 66.41, + 66.0, 65.58, 65.15, 64.71, 64.24, 63.76, 63.26, 62.73, 62.18, 61.59, 60.96, 60.28, 59.53, 58.71, + 57.79, 56.71, 55.41, 53.71, 51.1, + ], + [ + 152.33, 138.24, 122.35, 109.96, 105.74, 103.12, 101.18, 99.62, 98.3, 97.15, 96.14, 95.22, 94.37, + 93.6, 92.87, 92.19, 91.55, 90.94, 90.36, 89.81, 89.28, 88.76, 88.27, 87.79, 87.33, 86.88, 86.45, + 86.02, 85.61, 85.2, 84.81, 84.42, 84.04, 83.66, 83.29, 82.93, 82.57, 82.22, 81.88, 81.53, 81.19, + 80.86, 80.53, 80.2, 79.87, 79.55, 79.23, 78.91, 78.59, 78.27, 77.96, 77.65, 77.33, 77.02, 76.71, + 76.4, 76.09, 75.78, 75.47, 75.16, 74.85, 74.54, 74.23, 73.91, 73.6, 73.28, 72.96, 72.64, 72.32, + 72.0, 71.67, 71.33, 71.0, 70.66, 70.32, 69.97, 69.61, 69.25, 68.89, 68.51, 68.13, 67.74, 67.34, + 66.93, 66.51, 66.07, 65.63, 65.16, 64.67, 64.17, 63.64, 63.08, 62.48, 61.85, 61.16, 60.42, + 59.59, 58.65, 57.57, 56.26, 54.54, 51.91, + ], + [ + 153.71, 139.55, 123.59, 111.14, 106.91, 104.27, 102.32, 100.75, 99.42, 98.27, 97.25, 96.32, + 95.48, 94.69, 93.96, 93.28, 92.64, 92.02, 91.44, 90.88, 90.35, 89.83, 89.34, 88.86, 88.39, + 87.94, 87.5, 87.08, 86.66, 86.25, 85.85, 85.46, 85.08, 84.7, 84.33, 83.97, 83.61, 83.25, 82.9, + 82.56, 82.22, 81.88, 81.55, 81.22, 80.89, 80.56, 80.24, 79.92, 79.6, 79.28, 78.96, 78.65, 78.33, + 78.02, 77.71, 77.4, 77.08, 76.77, 76.46, 76.15, 75.84, 75.52, 75.21, 74.89, 74.58, 74.26, 73.94, + 73.61, 73.29, 72.96, 72.63, 72.3, 71.96, 71.62, 71.27, 70.92, 70.56, 70.2, 69.83, 69.45, 69.07, + 68.68, 68.27, 67.86, 67.44, 67.0, 66.55, 66.08, 65.59, 65.08, 64.54, 63.98, 63.38, 62.74, 62.05, + 61.3, 60.46, 59.52, 58.43, 57.1, 55.38, 52.72, + ], + [ + 155.08, 140.87, 124.84, 112.33, 108.07, 105.42, 103.46, 101.88, 100.55, 99.39, 98.36, 97.43, + 96.58, 95.79, 95.06, 94.37, 93.72, 93.11, 92.52, 91.96, 91.42, 90.9, 90.41, 89.92, 89.46, 89.0, + 88.56, 88.13, 87.71, 87.3, 86.9, 86.51, 86.12, 85.74, 85.37, 85.0, 84.64, 84.28, 83.93, 83.59, + 83.24, 82.9, 82.57, 82.23, 81.9, 81.58, 81.25, 80.93, 80.61, 80.29, 79.97, 79.65, 79.33, 79.02, + 78.7, 78.39, 78.08, 77.76, 77.45, 77.13, 76.82, 76.5, 76.19, 75.87, 75.55, 75.23, 74.91, 74.58, + 74.26, 73.93, 73.59, 73.26, 72.92, 72.57, 72.22, 71.87, 71.51, 71.14, 70.77, 70.39, 70.01, + 69.61, 69.21, 68.79, 68.36, 67.92, 67.47, 66.99, 66.5, 65.99, 65.45, 64.88, 64.28, 63.63, 62.94, + 62.18, 61.34, 60.39, 59.29, 57.96, 56.21, 53.54, + ], + [ + 156.45, 142.18, 126.08, 113.51, 109.23, 106.57, 104.6, 103.01, 101.67, 100.51, 99.47, 98.54, + 97.68, 96.89, 96.15, 95.46, 94.81, 94.19, 93.6, 93.03, 92.49, 91.97, 91.47, 90.99, 90.52, 90.06, + 89.62, 89.18, 88.76, 88.35, 87.95, 87.55, 87.16, 86.78, 86.41, 86.04, 85.67, 85.31, 84.96, + 84.61, 84.27, 83.92, 83.59, 83.25, 82.92, 82.59, 82.26, 81.94, 81.61, 81.29, 80.97, 80.65, + 80.33, 80.02, 79.7, 79.38, 79.07, 78.75, 78.44, 78.12, 77.8, 77.49, 77.17, 76.85, 76.53, 76.2, + 75.88, 75.55, 75.22, 74.89, 74.55, 74.22, 73.87, 73.53, 73.18, 72.82, 72.46, 72.09, 71.72, + 71.34, 70.95, 70.55, 70.14, 69.72, 69.29, 68.85, 68.39, 67.91, 67.42, 66.9, 66.36, 65.78, 65.18, + 64.53, 63.83, 63.06, 62.22, 61.26, 60.15, 58.81, 57.05, 54.36, + ], + [ + 157.82, 143.5, 127.32, 114.69, 110.39, 107.72, 105.73, 104.14, 102.79, 101.62, 100.58, 99.64, + 98.78, 97.98, 97.24, 96.55, 95.89, 95.27, 94.68, 94.11, 93.57, 93.04, 92.54, 92.05, 91.58, + 91.12, 90.67, 90.24, 89.81, 89.4, 88.99, 88.59, 88.2, 87.82, 87.44, 87.07, 86.71, 86.34, 85.99, + 85.64, 85.29, 84.95, 84.61, 84.27, 83.93, 83.6, 83.27, 82.95, 82.62, 82.3, 81.98, 81.65, 81.33, + 81.02, 80.7, 80.38, 80.06, 79.74, 79.43, 79.11, 78.79, 78.47, 78.15, 77.83, 77.5, 77.18, 76.85, + 76.52, 76.19, 75.86, 75.52, 75.18, 74.83, 74.48, 74.13, 73.77, 73.41, 73.04, 72.66, 72.28, + 71.89, 71.48, 71.07, 70.65, 70.22, 69.77, 69.31, 68.83, 68.33, 67.81, 67.26, 66.69, 66.08, + 65.42, 64.72, 63.95, 63.09, 62.13, 61.01, 59.66, 57.89, 55.17, + ], + [ + 159.19, 144.81, 128.56, 115.88, 111.55, 108.87, 106.87, 105.27, 103.91, 102.74, 101.69, 100.75, + 99.88, 99.08, 98.34, 97.64, 96.98, 96.35, 95.75, 95.18, 94.64, 94.11, 93.6, 93.11, 92.64, 92.18, + 91.73, 91.29, 90.86, 90.44, 90.04, 89.64, 89.24, 88.86, 88.48, 88.11, 87.74, 87.37, 87.02, + 86.66, 86.31, 85.97, 85.63, 85.29, 84.95, 84.62, 84.29, 83.96, 83.63, 83.3, 82.98, 82.66, 82.33, + 82.01, 81.69, 81.37, 81.05, 80.73, 80.41, 80.09, 79.77, 79.45, 79.13, 78.8, 78.48, 78.15, 77.82, + 77.49, 77.16, 76.82, 76.48, 76.14, 75.79, 75.44, 75.08, 74.72, 74.36, 73.99, 73.61, 73.22, + 72.83, 72.42, 72.01, 71.58, 71.15, 70.7, 70.23, 69.75, 69.25, 68.72, 68.17, 67.59, 66.98, 66.32, + 65.61, 64.83, 63.97, 63.0, 61.88, 60.51, 58.73, 55.99, + ], + [ + 160.55, 146.11, 129.8, 117.06, 112.71, 110.01, 108.01, 106.39, 105.03, 103.85, 102.8, 101.85, + 100.98, 100.18, 99.43, 98.72, 98.06, 97.43, 96.83, 96.26, 95.71, 95.18, 94.67, 94.18, 93.7, + 93.23, 92.78, 92.34, 91.91, 91.49, 91.08, 90.68, 90.28, 89.9, 89.51, 89.14, 88.77, 88.4, 88.04, + 87.69, 87.34, 86.99, 86.65, 86.3, 85.97, 85.63, 85.3, 84.97, 84.64, 84.31, 83.98, 83.66, 83.33, + 83.01, 82.69, 82.37, 82.05, 81.72, 81.4, 81.08, 80.76, 80.43, 80.11, 79.78, 79.46, 79.13, 78.79, + 78.46, 78.13, 77.79, 77.45, 77.1, 76.75, 76.4, 76.04, 75.68, 75.31, 74.93, 74.55, 74.16, 73.77, + 73.36, 72.94, 72.52, 72.08, 71.62, 71.15, 70.67, 70.16, 69.63, 69.08, 68.5, 67.88, 67.21, 66.5, + 65.72, 64.85, 63.88, 62.74, 61.37, 59.57, 56.81, + ], + [ + 161.92, 147.42, 131.04, 118.24, 113.87, 111.16, 109.14, 107.52, 106.15, 104.97, 103.91, 102.95, + 102.08, 101.27, 100.52, 99.81, 99.14, 98.51, 97.91, 97.33, 96.78, 96.25, 95.73, 95.24, 94.76, + 94.29, 93.84, 93.39, 92.96, 92.54, 92.13, 91.72, 91.32, 90.93, 90.55, 90.17, 89.8, 89.43, 89.07, + 88.71, 88.36, 88.01, 87.67, 87.32, 86.98, 86.64, 86.31, 85.98, 85.64, 85.31, 84.99, 84.66, + 84.33, 84.01, 83.69, 83.36, 83.04, 82.71, 82.39, 82.07, 81.74, 81.42, 81.09, 80.76, 80.43, 80.1, + 79.77, 79.43, 79.09, 78.75, 78.41, 78.06, 77.71, 77.35, 76.99, 76.63, 76.26, 75.88, 75.5, 75.11, + 74.71, 74.3, 73.88, 73.45, 73.01, 72.55, 72.08, 71.59, 71.08, 70.55, 69.99, 69.4, 68.78, 68.11, + 67.39, 66.6, 65.73, 64.75, 63.61, 62.22, 60.41, 57.63, + ], + [ + 163.28, 148.73, 132.28, 119.41, 115.03, 112.3, 110.28, 108.65, 107.27, 106.08, 105.02, 104.06, + 103.18, 102.36, 101.61, 100.9, 100.23, 99.59, 98.98, 98.41, 97.85, 97.31, 96.8, 96.3, 95.82, + 95.35, 94.89, 94.45, 94.01, 93.59, 93.17, 92.76, 92.37, 91.97, 91.59, 91.21, 90.83, 90.46, 90.1, + 89.74, 89.38, 89.03, 88.68, 88.34, 88.0, 87.66, 87.32, 86.99, 86.65, 86.32, 85.99, 85.66, 85.33, + 85.01, 84.68, 84.36, 84.03, 83.7, 83.38, 83.05, 82.73, 82.4, 82.07, 81.74, 81.41, 81.07, 80.74, + 80.4, 80.06, 79.72, 79.37, 79.02, 78.67, 78.31, 77.95, 77.58, 77.21, 76.83, 76.44, 76.05, 75.65, + 75.23, 74.81, 74.38, 73.94, 73.48, 73.0, 72.51, 72.0, 71.46, 70.9, 70.31, 69.68, 69.01, 68.28, + 67.49, 66.61, 65.62, 64.47, 63.08, 61.25, 58.46, + ], + [ + 164.64, 150.03, 133.51, 120.59, 116.18, 113.44, 111.41, 109.77, 108.39, 107.19, 106.12, 105.16, + 104.28, 103.46, 102.7, 101.98, 101.31, 100.67, 100.06, 99.48, 98.92, 98.38, 97.86, 97.36, 96.88, + 96.4, 95.94, 95.5, 95.06, 94.63, 94.22, 93.81, 93.41, 93.01, 92.62, 92.24, 91.86, 91.49, 91.13, + 90.77, 90.41, 90.05, 89.7, 89.36, 89.01, 88.67, 88.33, 87.99, 87.66, 87.33, 86.99, 86.66, 86.33, + 86.01, 85.68, 85.35, 85.02, 84.69, 84.37, 84.04, 83.71, 83.38, 83.05, 82.72, 82.38, 82.05, + 81.71, 81.37, 81.03, 80.69, 80.34, 79.99, 79.63, 79.27, 78.91, 78.54, 78.16, 77.78, 77.39, + 76.99, 76.59, 76.17, 75.75, 75.31, 74.87, 74.4, 73.93, 73.43, 72.91, 72.38, 71.81, 71.21, 70.58, + 69.9, 69.17, 68.38, 67.49, 66.5, 65.34, 63.93, 62.1, 59.28, + ], + [ + 165.99, 151.33, 134.75, 121.77, 117.34, 114.59, 112.54, 110.9, 109.51, 108.3, 107.23, 106.26, + 105.37, 104.55, 103.79, 103.07, 102.39, 101.75, 101.14, 100.55, 99.99, 99.45, 98.93, 98.42, + 97.93, 97.46, 97.0, 96.55, 96.11, 95.68, 95.26, 94.85, 94.44, 94.05, 93.66, 93.27, 92.9, 92.52, + 92.15, 91.79, 91.43, 91.08, 90.72, 90.37, 90.03, 89.68, 89.34, 89.0, 88.67, 88.33, 88.0, 87.67, + 87.33, 87.0, 86.67, 86.34, 86.01, 85.69, 85.36, 85.03, 84.69, 84.36, 84.03, 83.7, 83.36, 83.02, + 82.69, 82.34, 82.0, 81.65, 81.3, 80.95, 80.59, 80.23, 79.86, 79.49, 79.11, 78.73, 78.33, 77.94, + 77.53, 77.11, 76.69, 76.25, 75.8, 75.33, 74.85, 74.35, 73.83, 73.29, 72.72, 72.12, 71.48, 70.8, + 70.07, 69.27, 68.38, 67.37, 66.21, 64.79, 62.94, 60.1, + ], + [ + 167.35, 152.63, 135.98, 122.94, 118.49, 115.73, 113.67, 112.02, 110.63, 109.41, 108.34, 107.36, + 106.47, 105.64, 104.88, 104.15, 103.47, 102.83, 102.21, 101.62, 101.06, 100.51, 99.99, 99.48, + 98.99, 98.52, 98.05, 97.6, 97.16, 96.73, 96.3, 95.89, 95.48, 95.09, 94.69, 94.31, 93.93, 93.55, + 93.18, 92.82, 92.45, 92.1, 91.74, 91.39, 91.04, 90.7, 90.35, 90.01, 89.68, 89.34, 89.0, 88.67, + 88.33, 88.0, 87.67, 87.34, 87.01, 86.68, 86.34, 86.01, 85.68, 85.35, 85.01, 84.68, 84.34, 84.0, + 83.66, 83.31, 82.97, 82.62, 82.27, 81.91, 81.55, 81.19, 80.82, 80.44, 80.06, 79.68, 79.28, + 78.88, 78.47, 78.05, 77.62, 77.18, 76.73, 76.26, 75.78, 75.27, 74.75, 74.21, 73.63, 73.03, + 72.39, 71.7, 70.96, 70.15, 69.26, 68.25, 67.07, 65.65, 63.79, 60.93, + ], + [ + 168.7, 153.93, 137.21, 124.12, 119.65, 116.87, 114.81, 113.15, 111.74, 110.53, 109.44, 108.46, + 107.57, 106.74, 105.96, 105.24, 104.55, 103.9, 103.29, 102.69, 102.13, 101.58, 101.05, 100.54, + 100.05, 99.57, 99.1, 98.65, 98.21, 97.77, 97.35, 96.93, 96.52, 96.12, 95.73, 95.34, 94.96, + 94.58, 94.21, 93.84, 93.48, 93.12, 92.76, 92.41, 92.06, 91.71, 91.37, 91.02, 90.68, 90.34, + 90.01, 89.67, 89.33, 89.0, 88.67, 88.33, 88.0, 87.67, 87.33, 87.0, 86.66, 86.33, 85.99, 85.65, + 85.32, 84.97, 84.63, 84.29, 83.94, 83.59, 83.23, 82.87, 82.51, 82.14, 81.77, 81.4, 81.01, 80.62, + 80.23, 79.82, 79.41, 78.99, 78.56, 78.11, 77.66, 77.19, 76.7, 76.2, 75.67, 75.12, 74.54, 73.94, + 73.29, 72.6, 71.86, 71.04, 70.14, 69.13, 67.94, 66.51, 64.63, 61.75, + ], + [ + 170.05, 155.22, 138.44, 125.29, 120.8, 118.01, 115.94, 114.27, 112.86, 111.64, 110.55, 109.56, + 108.66, 107.83, 107.05, 106.32, 105.63, 104.98, 104.36, 103.77, 103.19, 102.65, 102.12, 101.6, + 101.11, 100.63, 100.16, 99.7, 99.25, 98.82, 98.39, 97.97, 97.56, 97.16, 96.76, 96.37, 95.99, + 95.61, 95.24, 94.87, 94.5, 94.14, 93.78, 93.43, 93.07, 92.72, 92.38, 92.03, 91.69, 91.35, 91.01, + 90.67, 90.33, 90.0, 89.66, 89.33, 88.99, 88.66, 88.32, 87.99, 87.65, 87.31, 86.97, 86.63, 86.29, + 85.95, 85.6, 85.26, 84.91, 84.55, 84.2, 83.84, 83.47, 83.1, 82.73, 82.35, 81.97, 81.57, 81.18, + 80.77, 80.35, 79.93, 79.5, 79.05, 78.59, 78.12, 77.63, 77.12, 76.59, 76.04, 75.46, 74.84, 74.2, + 73.5, 72.75, 71.93, 71.03, 70.0, 68.81, 67.37, 65.48, 62.58, + ], + [ + 171.4, 156.52, 139.67, 126.46, 121.95, 119.15, 117.07, 115.39, 113.98, 112.74, 111.65, 110.66, + 109.76, 108.92, 108.14, 107.41, 106.71, 106.06, 105.43, 104.84, 104.26, 103.71, 103.18, 102.66, + 102.17, 101.68, 101.21, 100.75, 100.3, 99.86, 99.43, 99.01, 98.6, 98.2, 97.8, 97.41, 97.02, + 96.64, 96.26, 95.89, 95.52, 95.16, 94.8, 94.44, 94.09, 93.74, 93.39, 93.04, 92.7, 92.35, 92.01, + 91.67, 91.33, 91.0, 90.66, 90.32, 89.98, 89.65, 89.31, 88.97, 88.63, 88.3, 87.95, 87.61, 87.27, + 86.92, 86.58, 86.23, 85.88, 85.52, 85.16, 84.8, 84.43, 84.06, 83.69, 83.31, 82.92, 82.52, 82.12, + 81.71, 81.3, 80.87, 80.43, 79.98, 79.52, 79.05, 78.55, 78.04, 77.51, 76.95, 76.37, 75.75, 75.1, + 74.4, 73.65, 72.82, 71.91, 70.88, 69.68, 68.23, 66.33, 63.41, + ], + [ + 172.75, 157.81, 140.89, 127.63, 123.1, 120.29, 118.19, 116.51, 115.09, 113.85, 112.75, 111.76, + 110.85, 110.01, 109.22, 108.49, 107.79, 107.13, 106.51, 105.91, 105.33, 104.78, 104.24, 103.72, + 103.22, 102.74, 102.26, 101.8, 101.35, 100.91, 100.48, 100.06, 99.64, 99.23, 98.83, 98.44, + 98.05, 97.67, 97.29, 96.91, 96.55, 96.18, 95.82, 95.46, 95.1, 94.75, 94.4, 94.05, 93.7, 93.36, + 93.02, 92.68, 92.33, 91.99, 91.65, 91.32, 90.98, 90.64, 90.3, 89.96, 89.62, 89.28, 88.94, 88.59, + 88.25, 87.9, 87.55, 87.2, 86.85, 86.49, 86.13, 85.76, 85.39, 85.02, 84.64, 84.26, 83.87, 83.47, + 83.07, 82.66, 82.24, 81.81, 81.37, 80.92, 80.45, 79.97, 79.48, 78.96, 78.43, 77.87, 77.28, + 76.66, 76.01, 75.3, 74.54, 73.72, 72.8, 71.76, 70.55, 69.09, 67.18, 64.24, + ], + [ + 174.09, 159.1, 142.12, 128.8, 124.26, 121.42, 119.32, 117.63, 116.2, 114.96, 113.86, 112.86, + 111.94, 111.1, 110.31, 109.57, 108.87, 108.21, 107.58, 106.98, 106.4, 105.84, 105.3, 104.78, + 104.28, 103.79, 103.31, 102.85, 102.4, 101.95, 101.52, 101.1, 100.68, 100.27, 99.87, 99.47, + 99.08, 98.69, 98.31, 97.94, 97.57, 97.2, 96.84, 96.48, 96.12, 95.76, 95.41, 95.06, 94.71, 94.37, + 94.02, 93.68, 93.33, 92.99, 92.65, 92.31, 91.97, 91.63, 91.29, 90.95, 90.6, 90.26, 89.92, 89.57, + 89.22, 88.88, 88.52, 88.17, 87.81, 87.46, 87.09, 86.73, 86.36, 85.98, 85.6, 85.21, 84.82, 84.42, + 84.02, 83.61, 83.18, 82.75, 82.31, 81.85, 81.39, 80.9, 80.41, 79.89, 79.35, 78.79, 78.2, 77.57, + 76.91, 76.2, 75.44, 74.61, 73.68, 72.64, 71.43, 69.95, 68.03, 65.07, + ], + [ + 175.44, 160.39, 143.34, 129.97, 125.4, 122.56, 120.45, 118.75, 117.32, 116.07, 114.96, 113.96, + 113.04, 112.19, 111.4, 110.65, 109.95, 109.29, 108.65, 108.05, 107.46, 106.9, 106.36, 105.84, + 105.34, 104.84, 104.37, 103.9, 103.44, 103.0, 102.56, 102.14, 101.72, 101.31, 100.9, 100.5, + 100.11, 99.72, 99.34, 98.96, 98.59, 98.22, 97.85, 97.49, 97.13, 96.78, 96.42, 96.07, 95.72, + 95.37, 95.02, 94.68, 94.33, 93.99, 93.65, 93.3, 92.96, 92.62, 92.28, 91.93, 91.59, 91.24, 90.9, + 90.55, 90.2, 89.85, 89.5, 89.14, 88.78, 88.42, 88.06, 87.69, 87.32, 86.94, 86.56, 86.17, 85.78, + 85.38, 84.97, 84.55, 84.13, 83.69, 83.25, 82.79, 82.32, 81.84, 81.33, 80.81, 80.27, 79.71, + 79.11, 78.48, 77.82, 77.11, 76.34, 75.5, 74.57, 73.52, 72.3, 70.82, 68.88, 65.9, + ], + [ + 176.78, 161.68, 144.57, 131.14, 126.55, 123.7, 121.58, 119.87, 118.43, 117.18, 116.06, 115.05, + 114.13, 113.28, 112.48, 111.74, 111.03, 110.36, 109.72, 109.12, 108.53, 107.97, 107.43, 106.9, + 106.39, 105.9, 105.42, 104.95, 104.49, 104.04, 103.61, 103.18, 102.76, 102.34, 101.94, 101.53, + 101.14, 100.75, 100.37, 99.99, 99.61, 99.24, 98.87, 98.51, 98.15, 97.79, 97.43, 97.08, 96.73, + 96.38, 96.03, 95.68, 95.33, 94.99, 94.64, 94.3, 93.95, 93.61, 93.27, 92.92, 92.58, 92.23, 91.88, + 91.53, 91.18, 90.83, 90.47, 90.12, 89.75, 89.39, 89.02, 88.65, 88.28, 87.9, 87.52, 87.13, 86.73, + 86.33, 85.92, 85.5, 85.07, 84.63, 84.19, 83.73, 83.25, 82.77, 82.26, 81.74, 81.19, 80.62, 80.03, + 79.39, 78.73, 78.01, 77.24, 76.39, 75.46, 74.4, 73.17, 71.68, 69.73, 66.73, + ], + [ + 178.12, 162.97, 145.79, 132.31, 127.7, 124.83, 122.7, 120.99, 119.54, 118.28, 117.16, 116.15, + 115.22, 114.37, 113.57, 112.82, 112.11, 111.44, 110.8, 110.18, 109.6, 109.03, 108.49, 107.96, + 107.45, 106.95, 106.47, 106.0, 105.54, 105.09, 104.65, 104.22, 103.79, 103.38, 102.97, 102.57, + 102.17, 101.78, 101.39, 101.01, 100.63, 100.26, 99.89, 99.53, 99.16, 98.8, 98.44, 98.09, 97.73, + 97.38, 97.03, 96.68, 96.33, 95.99, 95.64, 95.29, 94.95, 94.6, 94.25, 93.91, 93.56, 93.21, 92.86, + 92.51, 92.16, 91.8, 91.45, 91.09, 90.73, 90.36, 89.99, 89.62, 89.24, 88.86, 88.47, 88.08, 87.68, + 87.28, 86.87, 86.44, 86.02, 85.58, 85.13, 84.66, 84.19, 83.7, 83.19, 82.66, 82.11, 81.54, 80.94, + 80.31, 79.63, 78.91, 78.13, 77.29, 76.35, 75.28, 74.05, 72.54, 70.58, 67.56, + ], + [ + 179.46, 164.25, 147.01, 133.48, 128.85, 125.97, 123.83, 122.11, 120.65, 119.39, 118.26, 117.25, + 116.32, 115.45, 114.65, 113.9, 113.19, 112.51, 111.87, 111.25, 110.66, 110.09, 109.55, 109.02, + 108.5, 108.0, 107.52, 107.05, 106.58, 106.13, 105.69, 105.26, 104.83, 104.41, 104.0, 103.6, + 103.2, 102.81, 102.42, 102.04, 101.66, 101.28, 100.91, 100.54, 100.18, 99.81, 99.45, 99.1, + 98.74, 98.39, 98.04, 97.68, 97.33, 96.98, 96.64, 96.29, 95.94, 95.59, 95.24, 94.9, 94.55, 94.2, + 93.84, 93.49, 93.14, 92.78, 92.42, 92.06, 91.7, 91.33, 90.96, 90.58, 90.2, 89.82, 89.43, 89.04, + 88.64, 88.23, 87.81, 87.39, 86.96, 86.52, 86.07, 85.6, 85.12, 84.63, 84.12, 83.59, 83.04, 82.46, + 81.86, 81.22, 80.54, 79.82, 79.03, 78.18, 77.23, 76.16, 74.92, 73.41, 71.43, 68.4, + ], + [ + 180.79, 165.54, 148.23, 134.64, 130.0, 127.1, 124.95, 123.23, 121.77, 120.49, 119.36, 118.34, + 117.41, 116.54, 115.74, 114.98, 114.26, 113.59, 112.94, 112.32, 111.73, 111.16, 110.61, 110.07, + 109.56, 109.06, 108.57, 108.09, 107.63, 107.18, 106.73, 106.3, 105.87, 105.45, 105.04, 104.63, + 104.23, 103.83, 103.44, 103.06, 102.68, 102.3, 101.93, 101.56, 101.19, 100.83, 100.47, 100.11, + 99.75, 99.39, 99.04, 98.69, 98.33, 97.98, 97.63, 97.28, 96.93, 96.58, 96.23, 95.88, 95.53, + 95.18, 94.83, 94.47, 94.11, 93.76, 93.4, 93.03, 92.67, 92.3, 91.92, 91.55, 91.17, 90.78, 90.39, + 89.99, 89.59, 89.18, 88.76, 88.34, 87.9, 87.46, 87.01, 86.54, 86.06, 85.56, 85.05, 84.51, 83.96, + 83.38, 82.77, 82.13, 81.45, 80.72, 79.93, 79.07, 78.12, 77.05, 75.79, 74.28, 72.29, 69.23, + ], +]; +export const SIGNIFICANCE_LEVELS = [ + 0.0, 0.0, 0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.11, 0.12, 0.13, 0.14, + 0.15, 0.16, 0.17, 0.18, 0.19, 0.2, 0.21, 0.22, 0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3, + 0.31, 0.32, 0.33, 0.34, 0.35, 0.36, 0.37, 0.38, 0.39, 0.4, 0.41, 0.42, 0.43, 0.44, 0.45, 0.46, + 0.47, 0.48, 0.49, 0.5, 0.51, 0.52, 0.53, 0.54, 0.55, 0.56, 0.57, 0.58, 0.59, 0.6, 0.61, 0.62, + 0.63, 0.64, 0.65, 0.66, 0.67, 0.68, 0.69, 0.7, 0.71, 0.72, 0.73, 0.74, 0.75, 0.76, 0.77, 0.78, + 0.79, 0.8, 0.81, 0.82, 0.83, 0.84, 0.85, 0.86, 0.87, 0.88, 0.89, 0.9, 0.91, 0.92, 0.93, 0.94, + 0.95, 0.96, 0.97, 0.98, 0.99, +]; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_app_state.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_app_state.tsx new file mode 100644 index 0000000000000..faf36e18725cd --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_app_state.tsx @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { pick } from 'lodash'; + +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; +import type { DataView } from '@kbn/data-views-plugin/public'; +import { StorageContextProvider } from '@kbn/ml-local-storage'; +import { UrlStateProvider } from '@kbn/ml-url-state'; +import { Storage } from '@kbn/kibana-utils-plugin/public'; +import { DatePickerContextProvider } from '@kbn/ml-date-picker'; +import { UI_SETTINGS } from '@kbn/data-plugin/common'; +import { + KibanaContextProvider, + KibanaThemeProvider, + toMountPoint, + wrapWithTheme, +} from '@kbn/kibana-react-plugin/public'; + +import { DV_STORAGE_KEYS } from '../index_data_visualizer/types/storage'; +import { getCoreStart, getPluginsStart } from '../../kibana_services'; +import { DataComparisonPage } from './data_comparison_page'; +import { DataSourceContext } from '../common/hooks/data_source_context'; + +const localStorage = new Storage(window.localStorage); + +export interface DataComparisonDetectionAppStateProps { + /** The data view to analyze. */ + dataView: DataView; + /** The saved search to analyze. */ + savedSearch: SavedSearch | null; +} + +export type DataComparisonSpec = typeof DataComparisonDetectionAppState; + +export const DataComparisonDetectionAppState: FC = ({ + dataView, + savedSearch, +}) => { + if (!(dataView || savedSearch)) { + throw Error('No data view or saved search available.'); + } + + const coreStart = getCoreStart(); + const { + data, + maps, + embeddable, + discover, + share, + security, + fileUpload, + lens, + dataViewFieldEditor, + uiActions, + charts, + unifiedSearch, + } = getPluginsStart(); + const services = { + data, + maps, + embeddable, + discover, + share, + security, + fileUpload, + lens, + dataViewFieldEditor, + uiActions, + charts, + unifiedSearch, + ...coreStart, + }; + const datePickerDeps = { + ...pick(services, ['data', 'http', 'notifications', 'theme', 'uiSettings']), + toMountPoint, + wrapWithTheme, + uiSettingsKeys: UI_SETTINGS, + }; + + return ( + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_chart_tooltip_body.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_chart_tooltip_body.tsx new file mode 100644 index 0000000000000..b867239884f7e --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_chart_tooltip_body.tsx @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + TooltipCellStyle, + TooltipSpec, + TooltipTable, + TooltipTableBody, + TooltipTableCell, + TooltipTableColorCell, + TooltipTableFooter, + TooltipTableHeader, + TooltipTableRow, +} from '@elastic/charts'; +import React from 'react'; + +const style: TooltipCellStyle = { textAlign: 'right' }; +export const DataComparisonChartTooltipBody: TooltipSpec['body'] = ({ items }) => { + return ( + + + + {} + + Count + Percent + + + + {items.map(({ label, datum, seriesIdentifier: { key }, color }) => ( + + {} + {label} + {datum.doc_count} + {`${(datum.percentage * 100).toFixed( + 1 + )}`} + + ))} + + + + {} + Diff + + + {items[1].datum.doc_count - items[0].datum.doc_count} + + + {`${((items[1].datum.percentage - items[0].datum.percentage) * 100).toFixed(1)}%`} + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_overview_table.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_overview_table.tsx new file mode 100644 index 0000000000000..46cd0510443f8 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_overview_table.tsx @@ -0,0 +1,291 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UseTableState } from '@kbn/ml-in-memory-table'; +import React, { ReactNode, useMemo, useState } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiBasicTableColumn, + EuiButtonIcon, + EuiIcon, + EuiInMemoryTable, + EuiScreenReaderOnly, + EuiTableFieldDataColumnType, + EuiToolTip, +} from '@elastic/eui'; +import { FieldTypeIcon } from '../common/components/field_type_icon'; +import { COLLAPSE_ROW, EXPAND_ROW } from '../../../common/i18n_constants'; +import { COMPARISON_LABEL, REFERENCE_LABEL } from './constants'; +import { useCurrentEuiTheme } from '../common/hooks/use_current_eui_theme'; +import { DataComparisonField, Feature, FETCH_STATUS } from './types'; +import { formatSignificanceLevel } from './data_comparison_utils'; +import { SingleDistributionChart } from './charts/single_distribution_chart'; +import { OverlapDistributionComparison } from './charts/overlap_distribution_chart'; +import { DataComparisonDistributionChart } from './charts/data_comparison_distribution_chart'; + +const dataComparisonYesLabel = i18n.translate( + 'xpack.dataVisualizer.dataComparison.fieldTypeYesLabel', + { + defaultMessage: 'Yes', + } +); +const dataComparisonNoLabel = i18n.translate( + 'xpack.dataVisualizer.dataComparison.driftDetectedNoLabel', + { + defaultMessage: 'No', + } +); + +export const DataComparisonOverviewTable = ({ + data, + onTableChange, + pagination, + sorting, + status, +}: { + data: Feature[]; + status: FETCH_STATUS; +} & UseTableState) => { + const euiTheme = useCurrentEuiTheme(); + const colors = { + referenceColor: euiTheme.euiColorVis2, + productionColor: euiTheme.euiColorVis1, + }; + const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( + {} + ); + + const referenceDistributionLabel = i18n.translate( + 'xpack.dataVisualizer.dataComparison.dataComparisonDistributionLabel', + { + defaultMessage: '{label} distribution', + values: { label: REFERENCE_LABEL }, + } + ); + const comparisonDistributionLabel = i18n.translate( + 'xpack.dataVisualizer.dataComparison.dataComparisonDistributionLabel', + { + defaultMessage: '{label} distribution', + values: { label: COMPARISON_LABEL }, + } + ); + + const columns: Array> = [ + { + align: 'left', + width: '40px', + isExpander: true, + name: ( + + {EXPAND_ROW} + + ), + render: (item: Feature) => { + const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap }; + + return ( + toggleDetails(item)} + aria-label={itemIdToExpandedRowMapValues[item.featureName] ? COLLAPSE_ROW : EXPAND_ROW} + iconType={itemIdToExpandedRowMapValues[item.featureName] ? 'arrowDown' : 'arrowRight'} + /> + ); + }, + }, + + { + field: 'featureName', + name: i18n.translate('xpack.dataVisualizer.dataComparison.fieldNameLabel', { + defaultMessage: 'Name', + }), + 'data-test-subj': 'mlDataComparisonOverviewTableFeatureName', + sortable: true, + textOnly: true, + }, + { + field: 'secondaryType', + name: i18n.translate('xpack.dataVisualizer.dataComparison.fieldTypeLabel', { + defaultMessage: 'Type', + }), + 'data-test-subj': 'mlDataComparisonOverviewTableFeatureType', + sortable: true, + textOnly: true, + render: (secondaryType: DataComparisonField['secondaryType']) => { + return ; + }, + }, + { + field: 'driftDetected', + name: i18n.translate('xpack.dataVisualizer.dataComparison.driftDetectedLabel', { + defaultMessage: 'Drift detected', + }), + 'data-test-subj': 'mlDataComparisonOverviewTableDriftDetected', + sortable: true, + textOnly: true, + render: (driftDetected: boolean) => { + return {driftDetected ? dataComparisonYesLabel : dataComparisonNoLabel}; + }, + }, + { + field: 'similarityTestPValue', + name: ( + + + {i18n.translate('xpack.dataVisualizer.dataComparison.pValueLabel', { + defaultMessage: 'Similarity p-value', + })} + + + + ), + 'data-test-subj': 'mlDataComparisonOverviewTableSimilarityTestPValue', + sortable: true, + textOnly: true, + render: (similarityTestPValue: number) => { + return {formatSignificanceLevel(similarityTestPValue)}; + }, + }, + { + field: 'referenceHistogram', + name: referenceDistributionLabel, + 'data-test-subj': 'mlDataComparisonOverviewTableReferenceDistribution', + sortable: false, + render: (referenceHistogram: Feature['referenceHistogram'], item) => { + return ( +
+ +
+ ); + }, + }, + { + field: 'productionHistogram', + name: comparisonDistributionLabel, + 'data-test-subj': 'mlDataComparisonOverviewTableDataComparisonDistributionChart', + sortable: false, + render: (productionDistribution: Feature['productionHistogram'], item) => { + return ( +
+ +
+ ); + }, + }, + { + field: 'comparisonDistribution', + name: 'Comparison', + 'data-test-subj': 'mlDataComparisonOverviewTableDataComparisonDistributionChart', + sortable: false, + render: (comparisonDistribution: Feature['comparisonDistribution'], item) => { + return ( +
+ +
+ ); + }, + }, + ]; + + const getRowProps = (item: Feature) => { + return { + 'data-test-subj': `mlDataComparisonOverviewTableRow row-${item.featureName}`, + className: 'mlDataComparisonOverviewTableRow', + onClick: () => {}, + }; + }; + + const getCellProps = (item: Feature, column: EuiTableFieldDataColumnType) => { + const { field } = column; + return { + className: 'mlDataComparisonOverviewTableCell', + 'data-test-subj': `mlDataComparisonOverviewTableCell row-${item.featureName}-column-${String( + field + )}`, + textOnly: true, + }; + }; + + const toggleDetails = (item: Feature) => { + const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap }; + + if (itemIdToExpandedRowMapValues[item.featureName]) { + delete itemIdToExpandedRowMapValues[item.featureName]; + } else { + const { featureName, comparisonDistribution } = item; + itemIdToExpandedRowMapValues[item.featureName] = ( +
+ +
+ ); + } + setItemIdToExpandedRowMap(itemIdToExpandedRowMapValues); + }; + + const tableMessage = useMemo(() => { + switch (status) { + case FETCH_STATUS.NOT_INITIATED: + return i18n.translate('xpack.dataVisualizer.dataComparison.dataComparisonRunAnalysisMsg', { + defaultMessage: 'Run analysis to compare reference and comparison data', + }); + case FETCH_STATUS.LOADING: + return i18n.translate('xpack.dataVisualizer.dataComparison.dataComparisonLoadingMsg', { + defaultMessage: 'Analyzing', + }); + default: + return undefined; + } + }, [status]); + + return ( + + tableCaption={i18n.translate( + 'xpack.dataVisualizer.dataComparison.dataComparisonTableCaption', + { + defaultMessage: 'Data comparison overview', + } + )} + items={data} + rowHeader="featureName" + columns={columns} + rowProps={getRowProps} + cellProps={getCellProps} + itemId="featureName" + itemIdToExpandedRowMap={itemIdToExpandedRowMap} + isExpandable={true} + sorting={sorting} + onChange={onTableChange} + pagination={pagination} + loading={status === FETCH_STATUS.LOADING} + message={tableMessage} + /> + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_page.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_page.tsx new file mode 100644 index 0000000000000..801e3c1da7a4b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_page.tsx @@ -0,0 +1,395 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useCallback, useEffect, useState, FC, useMemo } from 'react'; + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPageBody, + EuiPageSection, + EuiPanel, + EuiSpacer, + EuiPageHeader, + EuiCallOut, +} from '@elastic/eui'; + +import type { WindowParameters } from '@kbn/aiops-utils'; +import type { Filter, Query } from '@kbn/es-query'; +import { useUrlState, usePageUrlState } from '@kbn/ml-url-state'; +import type { DataSeriesDatum } from '@elastic/charts/dist/chart_types/xy_chart/utils/series'; +import { useStorage } from '@kbn/ml-local-storage'; +import { + DatePickerWrapper, + FROZEN_TIER_PREFERENCE, + FullTimeRangeSelector, + FullTimeRangeSelectorProps, + useTimefilter, +} from '@kbn/ml-date-picker'; +import moment from 'moment'; +import { css } from '@emotion/react'; +import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { i18n } from '@kbn/i18n'; +import { RANDOM_SAMPLER_OPTION, RandomSampler } from '@kbn/ml-random-sampler-utils'; +import { MIN_SAMPLER_PROBABILITY } from '../index_data_visualizer/constants/random_sampler'; +import { useData } from '../common/hooks/use_data'; +import { + DV_FROZEN_TIER_PREFERENCE, + DV_RANDOM_SAMPLER_P_VALUE, + DV_RANDOM_SAMPLER_PREFERENCE, + DVKey, + DVStorageMapped, +} from '../index_data_visualizer/types/storage'; +import { useCurrentEuiTheme } from '../common/hooks/use_current_eui_theme'; +import { DataComparisonFullAppState, getDefaultDataComparisonState } from './types'; +import { useDataSource } from '../common/hooks/data_source_context'; +import { useDataVisualizerKibana } from '../kibana_context'; +import { DataComparisonView } from './data_comparison_view'; +import { COMPARISON_LABEL, REFERENCE_LABEL } from './constants'; +import { SearchPanelContent } from '../index_data_visualizer/components/search_panel/search_bar'; +import { useSearch } from '../common/hooks/use_search'; +import { DocumentCountWithDualBrush } from './document_count_with_dual_brush'; + +const dataViewTitleHeader = css({ + minWidth: '300px', +}); + +export const PageHeader: FC = () => { + const [, setGlobalState] = useUrlState('_g'); + const { dataView } = useDataSource(); + + const [frozenDataPreference, setFrozenDataPreference] = useStorage< + DVKey, + DVStorageMapped + >( + DV_FROZEN_TIER_PREFERENCE, + // By default we will exclude frozen data tier + FROZEN_TIER_PREFERENCE.EXCLUDE + ); + + const timefilter = useTimefilter({ + timeRangeSelector: dataView.timeFieldName !== undefined, + autoRefreshSelector: true, + }); + + const updateTimeState: FullTimeRangeSelectorProps['callback'] = useCallback( + (update) => { + setGlobalState({ + time: { + from: moment(update.start.epoch).toISOString(), + to: moment(update.end.epoch).toISOString(), + }, + }); + }, + [setGlobalState] + ); + + const hasValidTimeField = useMemo( + () => dataView.timeFieldName !== undefined && dataView.timeFieldName !== '', + [dataView.timeFieldName] + ); + + return ( + {dataView.getName()}
} + rightSideItems={[ + + {hasValidTimeField ? ( + + + + ) : null} + + , + ]} + /> + ); +}; + +export const DataComparisonPage: FC = () => { + const { + services: { data: dataService }, + } = useDataVisualizerKibana(); + const { dataView, savedSearch } = useDataSource(); + + const [dataComparisonListState, setAiopsListState] = usePageUrlState<{ + pageKey: 'DV_DATA_COMP'; + pageUrlState: DataComparisonFullAppState; + }>('DV_DATA_COMP', getDefaultDataComparisonState()); + + const [randomSamplerMode, setRandomSamplerMode] = useStorage< + DVKey, + DVStorageMapped + >(DV_RANDOM_SAMPLER_PREFERENCE, RANDOM_SAMPLER_OPTION.ON_AUTOMATIC); + + const [randomSamplerProbability, setRandomSamplerProbability] = useStorage< + DVKey, + DVStorageMapped + >(DV_RANDOM_SAMPLER_P_VALUE, MIN_SAMPLER_PROBABILITY); + const [lastRefresh, setLastRefresh] = useState(0); + + const forceRefresh = useCallback(() => setLastRefresh(Date.now()), [setLastRefresh]); + + const randomSampler = useMemo( + () => + new RandomSampler( + randomSamplerMode, + setRandomSamplerMode, + randomSamplerProbability, + setRandomSamplerProbability + ), + // eslint-disable-next-line react-hooks/exhaustive-deps + [] + ); + + const [globalState, setGlobalState] = useUrlState('_g'); + + const [selectedSavedSearch, setSelectedSavedSearch] = useState(savedSearch); + + useEffect(() => { + if (savedSearch) { + setSelectedSavedSearch(savedSearch); + } + }, [savedSearch]); + + const setSearchParams = useCallback( + (searchParams: { + searchQuery: estypes.QueryDslQueryContainer; + searchString: Query['query']; + queryLanguage: SearchQueryLanguage; + filters: Filter[]; + }) => { + // When the user loads a saved search and then clears or modifies the query + // we should remove the saved search and replace it with the index pattern id + if (selectedSavedSearch !== null) { + setSelectedSavedSearch(null); + } + + setAiopsListState({ + ...dataComparisonListState, + searchQuery: searchParams.searchQuery, + searchString: searchParams.searchString, + searchQueryLanguage: searchParams.queryLanguage, + filters: searchParams.filters, + }); + }, + [selectedSavedSearch, dataComparisonListState, setAiopsListState] + ); + + const { searchQueryLanguage, searchString, searchQuery } = useSearch( + { dataView, savedSearch }, + dataComparisonListState + ); + + const { documentStats, timefilter } = useData( + dataView, + 'data_drift', + searchQuery, + randomSampler, + setGlobalState, + undefined + ); + + const { sampleProbability, totalCount, documentCountStats, documentCountStatsCompare } = + documentStats; + + useEffect(() => { + randomSampler.setDocCount(totalCount); + }, [totalCount, randomSampler]); + + useEffect(() => { + if (globalState?.time !== undefined) { + timefilter.setTime({ + from: globalState.time.from, + to: globalState.time.to, + }); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [JSON.stringify(globalState?.time), timefilter]); + + useEffect(() => { + if (globalState?.refreshInterval !== undefined) { + timefilter.setRefreshInterval(globalState.refreshInterval); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [JSON.stringify(globalState?.refreshInterval), timefilter]); + + useEffect(() => { + // Update data query manager if input string is updated + dataService?.query.queryString.setQuery({ + query: searchString ?? '', + language: searchQueryLanguage, + }); + }, [dataService, searchQueryLanguage, searchString]); + + const euiTheme = useCurrentEuiTheme(); + const colors = { + referenceColor: euiTheme.euiColorVis2, + productionColor: euiTheme.euiColorVis1, + }; + + const [windowParameters, setWindowParameters] = useState(); + const [initialAnalysisStart, setInitialAnalysisStart] = useState< + number | WindowParameters | undefined + >(); + const [isBrushCleared, setIsBrushCleared] = useState(true); + + function brushSelectionUpdate(d: WindowParameters, force: boolean) { + if (!isBrushCleared || force) { + setWindowParameters(d); + } + if (force) { + setIsBrushCleared(false); + } + } + + function clearSelection() { + setWindowParameters(undefined); + setIsBrushCleared(true); + setInitialAnalysisStart(undefined); + } + + const barStyleAccessor = useCallback( + (datum: DataSeriesDatum) => { + if (!windowParameters) return null; + + const start = datum.x; + const end = + (typeof datum.x === 'string' ? parseInt(datum.x, 10) : datum.x) + + (documentCountStats?.interval ?? 0); + + if (start >= windowParameters.baselineMin && end <= windowParameters.baselineMax) { + return colors.referenceColor; + } + if (start >= windowParameters.deviationMin && end <= windowParameters.deviationMax) { + return colors.productionColor; + } + + return null; + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [JSON.stringify({ windowParameters, colors })] + ); + + return ( + + + + + + + + + {documentCountStats !== undefined && ( + + + + + + )} + + + + {!dataView?.isTimeBased() ? ( + +

+ {i18n.translate( + 'xpack.dataVisualizer.dataComparisonTimeSeriesWarning.description', + { + defaultMessage: 'Data comparison only runs over time-based indices.', + } + )} +

+ + ) : ( + + )} + + + + + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.test.ts b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.test.ts new file mode 100644 index 0000000000000..08f0305e4126a --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { computeChi2PValue } from './data_comparison_utils'; +import { Histogram } from './types'; + +describe('computeChi2PValue()', () => { + test('should return close to 1 if datasets are both empty or nearly identical', () => { + const referenceTerms: Histogram[] = [ + { + key: 'ap-northwest-1', + doc_count: 40348, + percentage: 0.2712470588235294, + }, + { + key: 'us-east-1', + doc_count: 15134, + percentage: 0.10174117647058824, + }, + { + key: 'eu-central-1', + doc_count: 12614, + percentage: 0.0848, + }, + { + key: 'sa-east-1', + doc_count: 80654, + percentage: 0.5422117647058824, + }, + ]; + const productionTerms: Histogram[] = [ + { + key: 'ap-northwest-1', + doc_count: 40320, + percentage: 0.2609691846654714, + }, + { + key: 'us-east-1', + doc_count: 15127, + percentage: 0.09790875139966732, + }, + { + key: 'eu-central-1', + doc_count: 12614, + percentage: 0.08164348450819088, + }, + { + key: 'sa-east-1', + doc_count: 86440, + percentage: 0.5594785794266703, + }, + ]; + expect(computeChi2PValue([], [])).toStrictEqual(1); + expect(computeChi2PValue(referenceTerms, productionTerms)).toStrictEqual(0.99); + }); + + test('should return close to 0 if datasets differ', () => { + const referenceTerms: Histogram[] = [ + { + key: 'jackson', + doc_count: 1, + percentage: 1, + }, + { + key: 'yahya', + doc_count: 0, + percentage: 0, + }, + ]; + const productionTerms: Histogram[] = [ + { + key: 'jackson', + doc_count: 0, + percentage: 0, + }, + { + key: 'yahya', + doc_count: 1, + percentage: 1, + }, + ]; + expect(computeChi2PValue(referenceTerms, productionTerms)).toStrictEqual(0); + }); +}); diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.ts b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.ts new file mode 100644 index 0000000000000..10ad33e87cd18 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_utils.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CRITICAL_VALUES_TABLE, SIGNIFICANCE_LEVELS } from './constants'; +import { Histogram } from './types'; + +const criticalTableLookup = (chi2Statistic: number, df: number) => { + if (df < 1) return 1; + if (!Number.isInteger(df)) throw Error('Degrees of freedom must be a valid integer'); + + // Get the row index + const rowIndex: number = df - 1; + + // Get the column index + let minDiff: number = Math.abs(CRITICAL_VALUES_TABLE[rowIndex][0] - chi2Statistic); + let columnIndex: number = 0; + for (let j = 1; j < CRITICAL_VALUES_TABLE[rowIndex].length; j++) { + const diff: number = Math.abs(CRITICAL_VALUES_TABLE[rowIndex][j] - chi2Statistic); + if (diff < minDiff) { + minDiff = diff; + columnIndex = j; + } + } + + const significanceLevel: number = SIGNIFICANCE_LEVELS[columnIndex]; + return significanceLevel; +}; + +/** + * Compute the p-value for how similar the datasets are. + * Returned value ranges from 0 to 1, with 1 meaning the datasets are identical. + * @param normalizedBaselineTerms + * @param normalizedDriftedTerms + */ +export const computeChi2PValue = ( + normalizedBaselineTerms: Histogram[], + normalizedDriftedTerms: Histogram[] +) => { + // Get all unique keys from both arrays + const allKeys: string[] = Array.from( + new Set([ + ...normalizedBaselineTerms.map((term) => term.key.toString()), + ...normalizedDriftedTerms.map((term) => term.key.toString()), + ]) + ).slice(0, 100); + + // Calculate the chi-squared statistic and degrees of freedom + let chiSquared: number = 0; + const degreesOfFreedom: number = allKeys.length - 1; + + if (degreesOfFreedom === 0) return 1; + + allKeys.forEach((key) => { + const baselineTerm = normalizedBaselineTerms.find((term) => term.key === key); + const driftedTerm = normalizedDriftedTerms.find((term) => term.key === key); + + const observed: number = driftedTerm?.percentage ?? 0; + const expected: number = baselineTerm?.percentage ?? 0; + chiSquared += Math.pow(observed - expected, 2) / (expected > 0 ? expected : 1e-6); // Prevent divide by zero + }); + + return criticalTableLookup(chiSquared, degreesOfFreedom); +}; + +/** + * formatSignificanceLevel + * @param significanceLevel + */ +export const formatSignificanceLevel = (significanceLevel: number) => { + if (typeof significanceLevel !== 'number' || isNaN(significanceLevel)) return ''; + if (significanceLevel < 1e-6) { + return '< 0.000001'; + } else if (significanceLevel < 0.01) { + return significanceLevel.toExponential(0); + } else { + return significanceLevel.toFixed(2); + } +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_view.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_view.tsx new file mode 100644 index 0000000000000..31134832ec56a --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/data_comparison_view.tsx @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React, { useCallback, useMemo, useState } from 'react'; +import { EuiEmptyPrompt, EuiFlexItem, EuiFormRow, EuiSwitch, EuiSpacer } from '@elastic/eui'; +import type { DataView } from '@kbn/data-views-plugin/public'; +import type { WindowParameters } from '@kbn/aiops-utils'; +import { i18n } from '@kbn/i18n'; +import type { Query } from '@kbn/es-query'; +import { ProgressControls } from '@kbn/aiops-components'; +import { isEqual } from 'lodash'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { EuiSwitchEvent } from '@elastic/eui/src/components/form/switch/switch'; +import { useTableState } from '@kbn/ml-in-memory-table'; +import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { RandomSampler } from '@kbn/ml-random-sampler-utils'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { kbnTypeToSupportedType } from '../common/util/field_types_utils'; +import { getDataComparisonType, useFetchDataComparisonResult } from './use_data_drift_result'; +import type { DataComparisonField, Feature, TimeRange } from './types'; +import { DataComparisonOverviewTable } from './data_comparison_overview_table'; + +const showOnlyDriftedFieldsOptionLabel = i18n.translate( + 'xpack.dataVisualizer.dataComparison.showOnlyDriftedFieldsOptionLabel', + { defaultMessage: 'Show only fields with drifted data' } +); + +interface DataComparisonViewProps { + windowParameters?: WindowParameters; + dataView: DataView; + searchString: Query['query']; + searchQuery: QueryDslQueryContainer; + searchQueryLanguage: SearchQueryLanguage; + isBrushCleared: boolean; + runAnalysisDisabled?: boolean; + onReset: () => void; + lastRefresh: number; + forceRefresh: () => void; + randomSampler: RandomSampler; +} +// Data drift view +export const DataComparisonView = ({ + windowParameters, + dataView, + searchString, + searchQuery, + searchQueryLanguage, + onReset, + isBrushCleared, + lastRefresh, + forceRefresh, + randomSampler, +}: DataComparisonViewProps) => { + const [showDataComparisonOnly, setShowDataComparisonOnly] = useState(false); + + const [currentAnalysisWindowParameters, setCurrentAnalysisWindowParameters] = useState< + WindowParameters | undefined + >(windowParameters); + + const [fetchInfo, setFetchIno] = useState< + | { + fields: DataComparisonField[]; + currentDataView: DataView; + timeRanges?: { reference: TimeRange; production: TimeRange }; + } + | undefined + >(); + + const onRefresh = useCallback(() => { + setCurrentAnalysisWindowParameters(windowParameters); + const mergedFields: DataComparisonField[] = []; + if (dataView) { + mergedFields.push( + ...dataView.fields + .filter( + (f) => + f.aggregatable === true && + // @ts-ignore metadata does exist + f.spec.metadata_field! !== true && + getDataComparisonType(f.type) !== 'unsupported' && + mergedFields.findIndex((merged) => merged.field === f.name) === -1 + ) + .map((f) => ({ + field: f.name, + type: getDataComparisonType(f.type), + secondaryType: kbnTypeToSupportedType(f), + displayName: f.displayName, + })) + ); + } + setFetchIno({ + fields: mergedFields, + currentDataView: dataView, + ...(windowParameters + ? { + timeRanges: { + reference: { + start: windowParameters.baselineMin, + end: windowParameters.baselineMax, + }, + production: { + start: windowParameters.deviationMin, + end: windowParameters.deviationMax, + }, + }, + } + : {}), + }); + if (forceRefresh) { + forceRefresh(); + } + }, [dataView, windowParameters, forceRefresh]); + + const { result, cancelRequest } = useFetchDataComparisonResult({ + ...fetchInfo, + lastRefresh, + randomSampler, + searchString, + searchQueryLanguage, + searchQuery, + }); + + const filteredData = useMemo(() => { + if (!result?.data) return []; + + switch (showDataComparisonOnly) { + case true: + return result.data.filter((d) => d.driftDetected === true); + default: + return result.data; + } + }, [result.data, showDataComparisonOnly]); + + const { onTableChange, pagination, sorting, setPageIndex } = useTableState( + filteredData, + 'driftDetected', + 'desc' + ); + + const shouldRerunAnalysis = useMemo( + () => + currentAnalysisWindowParameters !== undefined && + !isEqual(currentAnalysisWindowParameters, windowParameters), + [currentAnalysisWindowParameters, windowParameters] + ); + + const onShowDataComparisonOnlyToggle = (e: EuiSwitchEvent) => { + setShowDataComparisonOnly(e.target.checked); + setPageIndex(0); + }; + + return windowParameters === undefined ? ( + + + + } + titleSize="xs" + body={ +

+ +

+ } + data-test-subj="dataVisualizerNoWindowParametersEmptyPrompt" + /> + ) : ( +
+ 0 && result.loaded < 1} + onRefresh={onRefresh} + onCancel={cancelRequest} + shouldRerunAnalysis={shouldRerunAnalysis} + runAnalysisDisabled={!dataView || !windowParameters} + > + + + + + + + + + {result.error ? ( + {result.error}} + titleSize="xs" + body={{result.errorBody}} + /> + ) : ( + + )} +
+ ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/document_count_with_dual_brush.tsx b/x-pack/plugins/data_visualizer/public/application/data_comparison/document_count_with_dual_brush.tsx new file mode 100644 index 0000000000000..cc4267be0087b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/document_count_with_dual_brush.tsx @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { WindowParameters } from '@kbn/aiops-utils'; +import React, { FC } from 'react'; +import { DocumentCountChart, type DocumentCountChartPoint } from '@kbn/aiops-components'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import type { DocumentCountChartProps } from '@kbn/aiops-components'; +import { RandomSampler } from '@kbn/ml-random-sampler-utils'; +import { useDataVisualizerKibana } from '../kibana_context'; +import { DocumentCountStats } from '../../../common/types/field_stats'; +import { TotalCountHeader } from '../common/components/document_count_content/total_count_header'; +import { SamplingMenu } from '../common/components/random_sampling_menu/random_sampling_menu'; + +export interface DocumentCountContentProps + extends Omit< + DocumentCountChartProps, + | 'dependencies' + | 'chartPoints' + | 'timeRangeEarliest' + | 'timeRangeLatest' + | 'interval' + | 'chartPointsSplitLabel' + > { + brushSelectionUpdateHandler: (d: WindowParameters, force: boolean) => void; + documentCountStats?: DocumentCountStats; + documentCountStatsSplit?: DocumentCountStats; + documentCountStatsSplitLabel?: string; + isBrushCleared: boolean; + totalCount: number; + sampleProbability: number; + initialAnalysisStart?: number | WindowParameters; + /** Optional color override for the default bar color for charts */ + barColorOverride?: string; + /** Optional color override for the highlighted bar color for charts */ + barHighlightColorOverride?: string; + windowParameters?: WindowParameters; + incomingInitialAnalysisStart?: number | WindowParameters; + randomSampler: RandomSampler; + reload: () => void; + approximate: boolean; +} + +export const DocumentCountWithDualBrush: FC = ({ + randomSampler, + reload, + brushSelectionUpdateHandler, + documentCountStats, + documentCountStatsSplit, + documentCountStatsSplitLabel = '', + isBrushCleared, + totalCount, + sampleProbability, + initialAnalysisStart, + barColorOverride, + barHighlightColorOverride, + windowParameters, + incomingInitialAnalysisStart, + approximate, + ...docCountChartProps +}) => { + const { + services: { data, uiSettings, fieldFormats, charts }, + } = useDataVisualizerKibana(); + + const bucketTimestamps = Object.keys(documentCountStats?.buckets ?? {}).map((time) => +time); + const splitBucketTimestamps = Object.keys(documentCountStatsSplit?.buckets ?? {}).map( + (time) => +time + ); + const timeRangeEarliest = Math.min(...[...bucketTimestamps, ...splitBucketTimestamps]); + const timeRangeLatest = Math.max(...[...bucketTimestamps, ...splitBucketTimestamps]); + + if ( + documentCountStats === undefined || + documentCountStats.buckets === undefined || + timeRangeEarliest === undefined || + timeRangeLatest === undefined + ) { + return totalCount !== undefined ? : null; + } + + const chartPoints: DocumentCountChartPoint[] = Object.entries(documentCountStats.buckets).map( + ([time, value]) => ({ + time: +time, + value, + }) + ); + + let chartPointsSplit: DocumentCountChartPoint[] | undefined; + if (documentCountStatsSplit?.buckets !== undefined) { + chartPointsSplit = Object.entries(documentCountStatsSplit?.buckets).map(([time, value]) => ({ + time: +time, + value, + })); + } + + return ( + + + + + + + + + + + {documentCountStats.interval !== undefined && ( + + + + )} + + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/index.ts b/x-pack/plugins/data_visualizer/public/application/data_comparison/index.ts new file mode 100644 index 0000000000000..887f640da0d1a --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + DataComparisonDetectionAppState, + type DataComparisonSpec, +} from './data_comparison_app_state'; +export { type DataComparisonSpec }; +// required for dynamic import using React.lazy() +// eslint-disable-next-line import/no-default-export +export default DataComparisonDetectionAppState; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/types.ts b/x-pack/plugins/data_visualizer/public/application/data_comparison/types.ts new file mode 100644 index 0000000000000..46272c045d51c --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/types.ts @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import { Filter, Query } from '@kbn/es-query'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { DATA_COMPARISON_TYPE } from './constants'; + +export interface DataComparisonAppState { + searchString?: Query['query']; + searchQuery?: estypes.QueryDslQueryContainer; + searchQueryLanguage: SearchQueryLanguage; + filters?: Filter[]; +} + +export type DataComparisonFullAppState = Required; +export type BasicAppState = DataComparisonFullAppState; + +const defaultSearchQuery = { + match_all: {}, +}; + +export const getDefaultDataComparisonState = ( + overrides?: Partial +): DataComparisonFullAppState => ({ + searchString: '', + searchQuery: defaultSearchQuery, + searchQueryLanguage: SEARCH_QUERY_LANGUAGE.KUERY, + filters: [], + ...overrides, +}); + +export interface Histogram { + doc_count: number; + key: string | number; + percentage?: number; +} + +export interface ComparisonHistogram extends Histogram { + g: string; +} + +// Show the overview table +export interface Feature { + featureName: string; + fieldType: DataComparisonField['type']; + driftDetected: boolean; + similarityTestPValue: number; + productionHistogram: Histogram[]; + referenceHistogram: Histogram[]; + comparisonDistribution: ComparisonHistogram[]; +} + +export interface DataComparisonField { + field: string; + type: DataComparisonType; + secondaryType: string; + displayName: string; +} +export enum FETCH_STATUS { + LOADING = 'loading', + SUCCESS = 'success', + FAILURE = 'failure', + NOT_INITIATED = 'not_initiated', +} + +export interface Result { + status: FETCH_STATUS; + data?: T; + error?: string; + errorBody?: string; +} + +export interface TimeRange { + start: string | number; + end: string | number; +} + +export interface Range { + min: number; + max: number; + interval: number; +} + +export interface NumericDriftData { + type: 'numeric'; + pValue: number; + range?: Range; + referenceHistogram: Histogram[]; + productionHistogram: Histogram[]; + secondaryType: string; +} +export interface CategoricalDriftData { + type: 'categorical'; + driftedTerms: Histogram[]; + driftedSumOtherDocCount: number; + baselineTerms: Histogram[]; + baselineSumOtherDocCount: number; + secondaryType: string; +} + +export const isNumericDriftData = (arg: any): arg is NumericDriftData => { + return isPopulatedObject(arg, ['type']) && arg.type === DATA_COMPARISON_TYPE.NUMERIC; +}; + +export const isCategoricalDriftData = (arg: any): arg is CategoricalDriftData => { + return isPopulatedObject(arg, ['type']) && arg.type === DATA_COMPARISON_TYPE.CATEGORICAL; +}; + +export type DataComparisonType = typeof DATA_COMPARISON_TYPE[keyof typeof DATA_COMPARISON_TYPE]; diff --git a/x-pack/plugins/data_visualizer/public/application/data_comparison/use_data_drift_result.ts b/x-pack/plugins/data_visualizer/public/application/data_comparison/use_data_drift_result.ts new file mode 100644 index 0000000000000..a8e145c3fee1b --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/data_comparison/use_data_drift_result.ts @@ -0,0 +1,891 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import type { IKibanaSearchRequest } from '@kbn/data-plugin/common'; +import { lastValueFrom } from 'rxjs'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { DataView } from '@kbn/data-views-plugin/public'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; +import type { Query } from '@kbn/data-plugin/common'; +import { chunk, cloneDeep, flatten } from 'lodash'; +import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { getDefaultDSLQuery } from '@kbn/ml-query-utils'; +import { i18n } from '@kbn/i18n'; +import { RandomSampler, RandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; +import { extractErrorMessage } from '@kbn/ml-error-utils'; +import { AggregationsAggregate } from '@elastic/elasticsearch/lib/api/types'; +import { QueryDslBoolQuery } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { isDefined } from '@kbn/ml-is-defined'; +import { useDataVisualizerKibana } from '../kibana_context'; +import { + REFERENCE_LABEL, + COMPARISON_LABEL, + DRIFT_P_VALUE_THRESHOLD, + DATA_COMPARISON_TYPE, +} from './constants'; + +import { + Histogram, + NumericDriftData, + CategoricalDriftData, + Range, + FETCH_STATUS, + Result, + isNumericDriftData, + Feature, + DataComparisonField, + TimeRange, +} from './types'; +import { computeChi2PValue } from './data_comparison_utils'; + +export const getDataComparisonType = (kibanaType: string): DataComparisonField['type'] => { + switch (kibanaType) { + case 'number': + return DATA_COMPARISON_TYPE.NUMERIC; + case 'boolean': + case 'string': + return DATA_COMPARISON_TYPE.CATEGORICAL; + default: + return DATA_COMPARISON_TYPE.UNSUPPORTED; + } +}; + +type UseDataSearch = ReturnType; + +export const useDataSearch = () => { + const { data } = useDataVisualizerKibana().services; + + return useCallback( + async (esSearchRequestParams: IKibanaSearchRequest['params'], abortSignal?: AbortSignal) => { + try { + const { rawResponse: resp } = await lastValueFrom( + data.search.search( + { + params: esSearchRequestParams, + }, + { abortSignal } + ) + ); + + return resp; + } catch (error) { + if (error.name === 'AbortError') { + // ignore abort errors + } else { + throw Error(error); + } + } + }, + [data] + ); +}; + +const percents = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95]; + +const normalizeHistogram = (histogram: Histogram[]): Histogram[] => { + // Compute a total doc_count for all terms + const totalDocCount: number = histogram.reduce((acc, term) => acc + term.doc_count, 0); + // Iterate over the original array and update the doc_count of each term in the new array + return histogram.map((term) => ({ + ...term, + percentage: totalDocCount > 0 ? term.doc_count / totalDocCount : 0, + })); +}; + +const normalizeTerms = ( + terms: Histogram[], + keys: Array<{ key: string; relative_drift: number }>, + totalDocCount: number +): { normalizedTerms: Histogram[]; totalDocCount: number } => { + // Create a new array of terms with the same keys as the given array + const normalizedTerms: Array = keys.map((term) => ({ + ...term, + doc_count: 0, + percentage: 0, + })); + + // Iterate over the original array and update the doc_count of each term in the new array + terms.forEach((term) => { + const index: number = keys.findIndex((k) => k.key === term.key.toString()); + if (index !== -1) { + normalizedTerms[index].doc_count = term.doc_count; + normalizedTerms[index].percentage = term.doc_count / totalDocCount; + } + }); + + return { + normalizedTerms, + totalDocCount, + }; +}; + +const processDataComparisonResult = ( + result: Record +): Feature[] => { + return Object.entries(result).map(([featureName, data]) => { + if (isNumericDriftData(data)) { + // normalize data.referenceHistogram and data.productionHistogram to use frequencies instead of counts + const referenceHistogram: Histogram[] = normalizeHistogram(data.referenceHistogram); + const productionHistogram: Histogram[] = normalizeHistogram(data.productionHistogram); + + return { + featureName, + secondaryType: data.secondaryType, + fieldType: DATA_COMPARISON_TYPE.NUMERIC, + driftDetected: data.pValue < DRIFT_P_VALUE_THRESHOLD, + similarityTestPValue: data.pValue, + referenceHistogram: referenceHistogram ?? [], + productionHistogram: productionHistogram ?? [], + comparisonDistribution: [ + ...referenceHistogram.map((h) => ({ ...h, g: REFERENCE_LABEL })), + ...productionHistogram.map((h) => ({ ...h, g: COMPARISON_LABEL })), + ], + }; + } + + // normalize data.baselineTerms and data.driftedTerms to have same keys + // Get all unique keys from both arrays + const allKeys: string[] = Array.from( + new Set([ + ...data.baselineTerms.map((term) => term.key.toString()), + ...data.driftedTerms.map((term) => term.key.toString()), + ]) + ); + + // Compute a total doc_count for all terms + const referenceTotalDocCount: number = data.baselineTerms.reduce( + (acc, term) => acc + term.doc_count, + data.baselineSumOtherDocCount + ); + const productionTotalDocCount: number = data.driftedTerms.reduce( + (acc, term) => acc + term.doc_count, + data.driftedSumOtherDocCount + ); + + // Sort the categories (allKeys) by the following metric: Math.abs(productionDocCount-referenceDocCount)/referenceDocCount + const sortedKeys = allKeys + .map((k) => { + const key = k.toString(); + const baselineTerm = data.baselineTerms.find((t) => t.key === key); + const driftedTerm = data.driftedTerms.find((t) => t.key === key); + if (baselineTerm && driftedTerm) { + const referencePercentage = baselineTerm.doc_count / referenceTotalDocCount; + const productionPercentage = driftedTerm.doc_count / productionTotalDocCount; + return { + key, + relative_drift: + Math.abs(productionPercentage - referencePercentage) / referencePercentage, + }; + } + return { + key, + relative_drift: 0, + }; + }) + .sort((s1, s2) => s2.relative_drift - s1.relative_drift); + + // Normalize the baseline and drifted terms arrays + const { normalizedTerms: normalizedBaselineTerms } = normalizeTerms( + data.baselineTerms, + sortedKeys, + referenceTotalDocCount + ); + const { normalizedTerms: normalizedDriftedTerms } = normalizeTerms( + data.driftedTerms, + sortedKeys, + productionTotalDocCount + ); + + const pValue: number = computeChi2PValue(normalizedBaselineTerms, normalizedDriftedTerms); + return { + featureName, + secondaryType: data.secondaryType, + fieldType: DATA_COMPARISON_TYPE.CATEGORICAL, + driftDetected: pValue < DRIFT_P_VALUE_THRESHOLD, + similarityTestPValue: pValue, + referenceHistogram: normalizedBaselineTerms ?? [], + productionHistogram: normalizedDriftedTerms ?? [], + comparisonDistribution: [ + ...normalizedBaselineTerms.map((h) => ({ ...h, g: REFERENCE_LABEL })), + ...normalizedDriftedTerms.map((h) => ({ ...h, g: COMPARISON_LABEL })), + ], + }; + }); +}; + +const getDataComparisonQuery = ({ + runtimeFields, + searchQuery, + datetimeField, + timeRange, +}: { + runtimeFields: MappingRuntimeFields; + searchQuery?: estypes.QueryDslQueryContainer; + datetimeField?: string; + timeRange?: TimeRange; +}): NonNullable => { + let rangeFilter; + if (timeRange && datetimeField !== undefined && isPopulatedObject(timeRange, ['start', 'end'])) { + rangeFilter = { + range: { + [datetimeField]: { + gte: timeRange.start, + lte: timeRange.end, + format: 'epoch_millis', + }, + }, + }; + } + + const query = cloneDeep( + !searchQuery || isPopulatedObject(searchQuery, ['match_all']) + ? getDefaultDSLQuery() + : searchQuery + ); + + if (rangeFilter && isPopulatedObject(query, ['bool'])) { + if (Array.isArray(query.bool.filter)) { + // @ts-expect-error gte and lte can be numeric + query.bool.filter.push(rangeFilter); + } else { + // @ts-expect-error gte and lte can be numeric + query.bool.filter = [rangeFilter]; + } + } + + const refDataQuery: NonNullable = { + query, + }; + if (runtimeFields) { + refDataQuery.runtime_mappings = runtimeFields; + } + return refDataQuery; +}; + +const fetchReferenceBaselineData = async ({ + baseRequest, + fields, + randomSamplerWrapper, + dataSearch, + signal, +}: { + baseRequest: EsRequestParams; + dataSearch: UseDataSearch; + fields: DataComparisonField[]; + randomSamplerWrapper: RandomSamplerWrapper; + signal: AbortSignal; +}) => { + const baselineRequest = { ...baseRequest }; + const baselineRequestAggs: Record = {}; + + // for each field with type "numeric", add a percentiles agg to the request + for (const { field, type } of fields) { + // if the field is numeric, add a percentiles and stats aggregations to the request + if (type === DATA_COMPARISON_TYPE.NUMERIC) { + baselineRequestAggs[`${field}_percentiles`] = { + percentiles: { + field, + percents, + }, + }; + baselineRequestAggs[`${field}_stats`] = { + stats: { + field, + }, + }; + } + // if the field is categorical, add a terms aggregation to the request + if (type === DATA_COMPARISON_TYPE.CATEGORICAL) { + baselineRequestAggs[`${field}_terms`] = { + terms: { + field, + size: 100, // also DFA can potentially handle problems with 100 categories, for visualization purposes we will use top 10 + }, + }; + } + } + + const baselineResponse = await dataSearch( + { + ...baselineRequest, + body: { ...baselineRequest.body, aggs: randomSamplerWrapper.wrap(baselineRequestAggs) }, + }, + signal + ); + + return baselineResponse; +}; + +const fetchComparisonDriftedData = async ({ + dataSearch, + fields, + baselineResponseAggs, + baseRequest, + randomSamplerWrapper, + signal, +}: { + baseRequest: EsRequestParams; + dataSearch: UseDataSearch; + fields: DataComparisonField[]; + randomSamplerWrapper: RandomSamplerWrapper; + signal: AbortSignal; + baselineResponseAggs: object; +}) => { + const driftedRequest = { ...baseRequest }; + const driftedRequestAggs: Record = {}; + + for (const { field, type } of fields) { + if ( + isPopulatedObject(baselineResponseAggs, [`${field}_percentiles`]) && + type === DATA_COMPARISON_TYPE.NUMERIC + ) { + // create ranges based on percentiles + const percentiles = Object.values( + (baselineResponseAggs[`${field}_percentiles`] as Record).values + ); + const ranges: Array<{ from?: number; to?: number }> = []; + percentiles.forEach((val: number, idx) => { + if (idx === 0) { + ranges.push({ to: val }); + } else if (idx === percentiles.length - 1) { + ranges.push({ from: val }); + } else { + ranges.push({ from: percentiles[idx - 1], to: val }); + } + }); + // add range and bucket_count_ks_test to the request + driftedRequestAggs[`${field}_ranges`] = { + range: { + field, + ranges, + }, + }; + driftedRequestAggs[`${field}_ks_test`] = { + bucket_count_ks_test: { + buckets_path: `${field}_ranges>_count`, + alternative: ['two_sided'], + }, + }; + // add stats aggregation to the request + driftedRequestAggs[`${field}_stats`] = { + stats: { + field, + }, + }; + } + // if feature is categoric perform terms aggregation + if (type === DATA_COMPARISON_TYPE.CATEGORICAL) { + driftedRequestAggs[`${field}_terms`] = { + terms: { + field, + size: 100, // also DFA can potentially handle problems with 100 categories, for visualization purposes we will use top 10 + }, + }; + } + } + + const driftedResp = await dataSearch( + { + ...driftedRequest, + body: { ...driftedRequest.body, aggs: randomSamplerWrapper.wrap(driftedRequestAggs) }, + }, + signal + ); + return driftedResp; +}; + +const fetchHistogramData = async ({ + dataSearch, + fields, + driftedRespAggs, + baselineResponseAggs, + baseRequest, + randomSamplerWrapper, + signal, +}: { + baseRequest: EsRequestParams; + dataSearch: UseDataSearch; + fields: DataComparisonField[]; + randomSamplerWrapper: RandomSamplerWrapper; + signal: AbortSignal; + baselineResponseAggs: Record; + driftedRespAggs: Record; +}) => { + const histogramRequestAggs: Record = {}; + const fieldRange: { [field: string]: Range } = {}; + + for (const { field, type } of fields) { + // add histogram aggregation with min and max from baseline + if ( + type === DATA_COMPARISON_TYPE.NUMERIC && + baselineResponseAggs[`${field}_stats`] && + driftedRespAggs[`${field}_stats`] + ) { + const numBins = 10; + const min = Math.min( + baselineResponseAggs[`${field}_stats`].min!, + driftedRespAggs[`${field}_stats`].min! + ); + const max = Math.max( + baselineResponseAggs[`${field}_stats`].max!, + driftedRespAggs[`${field}_stats`].max! + ); + const interval = (max - min) / numBins; + + if (interval === 0) { + continue; + } + const offset = min; + fieldRange[field] = { min, max, interval }; + histogramRequestAggs[`${field}_histogram`] = { + histogram: { + field, + interval, + offset, + extended_bounds: { + min, + max, + }, + }, + }; + } + } + if (isPopulatedObject(histogramRequestAggs)) { + const histogramRequest = { + ...baseRequest, + body: { + ...baseRequest.body, + aggs: randomSamplerWrapper.wrap(histogramRequestAggs), + }, + }; + + return dataSearch(histogramRequest, signal); + } +}; + +const isFulfilled = ( + input: PromiseSettledResult> +): input is PromiseFulfilledResult> => input.status === 'fulfilled'; +const isRejected = (input: PromiseSettledResult>): input is PromiseRejectedResult => + input.status === 'rejected'; + +type EsRequestParams = NonNullable< + IKibanaSearchRequest>['params'] +>; + +interface ReturnedError { + error?: string; + errorBody?: string; +} + +function isReturnedError(arg: unknown): arg is ReturnedError { + return isPopulatedObject(arg, ['error']); +} + +/** + * Help split one big request into multiple requests (with max of 30 fields/request) + * to avoid too big of a data payload + * Returns a merged + * @param fields - list of fields to split + * @param randomSamplerWrapper - helper from randomSampler to pack and unpack 'sample' path from esResponse.aggregations + * @param asyncFetchFn - callback function with the divided fields + */ +export const fetchInParallelChunks = async < + ReturnedRespFromFetchFn extends { aggregations: Record } +>({ + fields, + randomSamplerWrapper, + asyncFetchFn, + errorMsg, +}: { + fields: DataComparisonField[]; + randomSamplerWrapper: RandomSamplerWrapper; + asyncFetchFn: (chunkedFields: DataComparisonField[]) => Promise; + errorMsg?: string; +}): Promise => { + const { unwrap } = randomSamplerWrapper; + const results = await Promise.allSettled( + chunk(fields, 30).map((chunkedFields: DataComparisonField[]) => asyncFetchFn(chunkedFields)) + ); + + const mergedResults = results + .filter(isFulfilled) + .filter((r) => r.value) + .map((r) => { + try { + return unwrap(r?.value.aggregations); + } catch (e) { + return undefined; + } + }) + .filter(isDefined); + + if (mergedResults.length === 0) { + const error = results.find(isRejected); + if (error) { + // eslint-disable-next-line no-console + console.error(error); + return { + error: errorMsg ?? 'An error occurred fetching data comparison data', + errorBody: error.reason.message, + }; + } + } + + const baselineResponseAggs = flatten(mergedResults).reduce( + (prev, acc) => ({ ...acc, ...prev }), + {} + ); + return baselineResponseAggs; +}; + +const initialState = { + data: undefined, + status: FETCH_STATUS.NOT_INITIATED, + error: undefined, + errorBody: undefined, +}; +export const useFetchDataComparisonResult = ( + { + fields, + currentDataView, + timeRanges, + searchQuery, + searchString, + lastRefresh, + randomSampler, + }: { + lastRefresh: number; + randomSampler?: RandomSampler; + fields?: DataComparisonField[]; + currentDataView?: DataView; + timeRanges?: { reference: TimeRange; production: TimeRange }; + searchQuery?: estypes.QueryDslQueryContainer; + searchString?: Query['query']; + searchQueryLanguage?: SearchQueryLanguage; + } = { lastRefresh: 0 } +) => { + const dataSearch = useDataSearch(); + const [result, setResult] = useState>(initialState); + const [loaded, setLoaded] = useState(0); + const [progressMessage, setProgressMessage] = useState(); + const abortController = useRef(new AbortController()); + + const cancelRequest = useCallback(() => { + abortController.current.abort(); + abortController.current = new AbortController(); + setResult(initialState); + setProgressMessage(undefined); + setLoaded(0); + }, []); + + useEffect( + () => { + const doFetchEsRequest = async function () { + if (!randomSampler) return; + + const randomSamplerWrapper = randomSampler.createRandomSamplerWrapper(); + + setLoaded(0); + setResult({ + data: undefined, + status: FETCH_STATUS.NOT_INITIATED, + error: undefined, + }); + + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.started', { + defaultMessage: `Ready to fetch data for comparison.`, + }) + ); + + const signal = abortController.current.signal; + if (!fields || !currentDataView) return; + + setResult({ data: undefined, status: FETCH_STATUS.LOADING, error: undefined }); + + // Place holder for when there might be difference data views in the future + const referenceIndex = currentDataView?.getIndexPattern(); + const productionIndex = referenceIndex; + + const runtimeFields = currentDataView?.getRuntimeMappings(); + + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedFields', { + defaultMessage: `Loaded fields from index '{referenceIndex}' to analyze.`, + values: { referenceIndex }, + }) + ); + const refDataQuery = getDataComparisonQuery({ + searchQuery, + datetimeField: currentDataView?.timeFieldName, + runtimeFields, + timeRange: timeRanges?.reference, + }); + + try { + const fieldsCount = fields.length; + + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadingReference', { + defaultMessage: `Loading reference data for {fieldsCount} fields.`, + values: { fieldsCount }, + }) + ); + + const baselineRequest: EsRequestParams = { + index: referenceIndex, + body: { + size: 0, + aggs: {} as Record, + ...refDataQuery, + }, + }; + + const baselineResponseAggs = await fetchInParallelChunks({ + fields, + randomSamplerWrapper, + + asyncFetchFn: (chunkedFields) => + fetchReferenceBaselineData({ + dataSearch, + baseRequest: baselineRequest, + fields: chunkedFields, + randomSamplerWrapper, + signal, + }), + }); + + if (isReturnedError(baselineResponseAggs)) { + setResult({ + data: undefined, + status: FETCH_STATUS.FAILURE, + error: baselineResponseAggs.error, + errorBody: baselineResponseAggs.errorBody, + }); + return; + } + + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedReference', { + defaultMessage: `Loaded reference data.`, + }) + ); + setLoaded(0.25); + + const prodDataQuery = getDataComparisonQuery({ + searchQuery, + datetimeField: currentDataView?.timeFieldName, + runtimeFields, + timeRange: timeRanges?.production, + }); + + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadingComparison', { + defaultMessage: `Loading comparison data for {fieldsCount} fields.`, + values: { fieldsCount }, + }) + ); + + const driftedRequest: EsRequestParams = { + index: productionIndex, + body: { + size: 0, + aggs: {} as Record, + ...prodDataQuery, + }, + }; + const driftedRespAggs = await fetchInParallelChunks({ + fields, + randomSamplerWrapper, + + asyncFetchFn: (chunkedFields: DataComparisonField[]) => + fetchComparisonDriftedData({ + dataSearch, + baseRequest: driftedRequest, + baselineResponseAggs, + fields: chunkedFields, + randomSamplerWrapper, + signal, + }), + }); + if (isReturnedError(driftedRespAggs)) { + setResult({ + data: undefined, + status: FETCH_STATUS.FAILURE, + error: driftedRespAggs.error, + errorBody: driftedRespAggs.errorBody, + }); + return; + } + + setLoaded(0.5); + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedComparison', { + defaultMessage: `Loaded comparison data. Now loading histogram data.`, + }) + ); + + const referenceHistogramRequest: EsRequestParams = { + index: referenceIndex, + body: { + size: 0, + aggs: {} as Record, + ...refDataQuery, + }, + }; + + const referenceHistogramRespAggs = await fetchInParallelChunks({ + fields, + randomSamplerWrapper, + + asyncFetchFn: (chunkedFields: DataComparisonField[]) => + fetchHistogramData({ + dataSearch, + baseRequest: referenceHistogramRequest, + baselineResponseAggs, + driftedRespAggs, + fields: chunkedFields, + randomSamplerWrapper, + signal, + }), + }); + + if (isReturnedError(referenceHistogramRespAggs)) { + setResult({ + data: undefined, + status: FETCH_STATUS.FAILURE, + error: referenceHistogramRespAggs.error, + errorBody: referenceHistogramRespAggs.errorBody, + }); + return; + } + + setLoaded(0.75); + setProgressMessage( + i18n.translate( + 'xpack.dataVisualizer.dataComparison.progress.loadedReferenceHistogram', + { + defaultMessage: `Loaded histogram data for reference data set.`, + } + ) + ); + + const productionHistogramRequest: EsRequestParams = { + index: productionIndex, + body: { + size: 0, + aggs: {} as Record, + ...prodDataQuery, + }, + }; + + const productionHistogramRespAggs = await fetchInParallelChunks({ + fields, + randomSamplerWrapper, + + asyncFetchFn: (chunkedFields: DataComparisonField[]) => + fetchHistogramData({ + dataSearch, + baseRequest: productionHistogramRequest, + baselineResponseAggs, + driftedRespAggs, + fields: chunkedFields, + randomSamplerWrapper, + signal, + }), + }); + + if (isReturnedError(productionHistogramRespAggs)) { + setResult({ + data: undefined, + status: FETCH_STATUS.FAILURE, + error: productionHistogramRespAggs.error, + errorBody: productionHistogramRespAggs.errorBody, + }); + return; + } + + const data: Record = {}; + for (const { field, type, secondaryType } of fields) { + if ( + type === DATA_COMPARISON_TYPE.NUMERIC && + driftedRespAggs[`${field}_ks_test`] && + referenceHistogramRespAggs[`${field}_histogram`] && + productionHistogramRespAggs[`${field}_histogram`] + ) { + data[field] = { + secondaryType, + type: DATA_COMPARISON_TYPE.NUMERIC, + pValue: driftedRespAggs[`${field}_ks_test`].two_sided, + referenceHistogram: referenceHistogramRespAggs[`${field}_histogram`].buckets, + productionHistogram: productionHistogramRespAggs[`${field}_histogram`].buckets, + }; + } + if ( + type === DATA_COMPARISON_TYPE.CATEGORICAL && + driftedRespAggs[`${field}_terms`] && + baselineResponseAggs[`${field}_terms`] + ) { + data[field] = { + secondaryType, + type: DATA_COMPARISON_TYPE.CATEGORICAL, + driftedTerms: driftedRespAggs[`${field}_terms`].buckets ?? [], + driftedSumOtherDocCount: driftedRespAggs[`${field}_terms`].sum_other_doc_count, + baselineTerms: baselineResponseAggs[`${field}_terms`].buckets ?? [], + baselineSumOtherDocCount: + baselineResponseAggs[`${field}_terms`].sum_other_doc_count, + }; + } + } + + setProgressMessage( + i18n.translate('xpack.dataVisualizer.dataComparison.progress.loadedHistogramData', { + defaultMessage: `Loaded histogram data for comparison data set.`, + }) + ); + + setResult({ + data: processDataComparisonResult(data), + status: FETCH_STATUS.SUCCESS, + }); + setLoaded(1); + } catch (e) { + // eslint-disable-next-line no-console + console.error(e); + setResult({ + data: undefined, + status: FETCH_STATUS.FAILURE, + error: 'An error occurred while fetching data comparison data', + errorBody: extractErrorMessage(e), + }); + } + }; + + doFetchEsRequest(); + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [ + dataSearch, + // eslint-disable-next-line react-hooks/exhaustive-deps + JSON.stringify({ + fields, + timeRanges, + currentDataView: currentDataView?.id, + searchString, + lastRefresh, + }), + ] + ); + const dataComparisonResult = useMemo( + () => ({ result: { ...result, loaded, progressMessage }, cancelRequest }), + [result, loaded, progressMessage, cancelRequest] + ); + return dataComparisonResult; +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index.ts b/x-pack/plugins/data_visualizer/public/application/index.ts index 7e84bd84adf6f..8a89702d5ea95 100644 --- a/x-pack/plugins/data_visualizer/public/application/index.ts +++ b/x-pack/plugins/data_visualizer/public/application/index.ts @@ -12,3 +12,4 @@ export type { IndexDataVisualizerViewProps, } from './index_data_visualizer'; export { IndexDataVisualizer } from './index_data_visualizer'; +export type { DataComparisonSpec } from './data_comparison'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx index 6d763b7fbbb93..0c420e7edc453 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/index_data_visualizer_view/index_data_visualizer_view.tsx @@ -24,7 +24,6 @@ import { EuiTitle, } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; import { Filter, FilterStateStore, Query } from '@kbn/es-query'; import { generateFilters } from '@kbn/data-plugin/public'; import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; @@ -37,6 +36,7 @@ import { import { useStorage } from '@kbn/ml-local-storage'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils'; import { kbnTypeToSupportedType } from '../../../common/util/field_types_utils'; import { useCurrentEuiTheme } from '../../../common/hooks/use_current_eui_theme'; import { @@ -59,7 +59,6 @@ import { DataVisualizerIndexBasedAppState, DataVisualizerIndexBasedPageUrlState, } from '../../types/index_data_visualizer_state'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../../types/combined_query'; import { useDataVisualizerKibana } from '../../../kibana_context'; import { FieldCountPanel } from '../../../common/components/field_count_panel'; import { DocumentCountContent } from '../../../common/components/document_count_content'; @@ -173,8 +172,7 @@ export const IndexDataVisualizerView: FC = (dataVi ); const { services } = useDataVisualizerKibana(); - const { notifications, uiSettings, data } = services; - const { toasts } = notifications; + const { uiSettings, data } = services; const [dataVisualizerListState, setDataVisualizerListState] = usePageUrlState( @@ -189,26 +187,6 @@ export const IndexDataVisualizerView: FC = (dataVi const { currentDataView, currentSessionId, getAdditionalLinks } = dataVisualizerProps; - useEffect(() => { - if (!currentDataView.isTimeBased()) { - toasts.addWarning({ - title: i18n.translate( - 'xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationTitle', - { - defaultMessage: 'The data view {dataViewTitle} is not based on a time series', - values: { dataViewTitle: currentDataView.title }, - } - ), - text: i18n.translate( - 'xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationDescription', - { - defaultMessage: 'Anomaly detection only runs over time-based indices', - } - ), - }); - } - }, [currentDataView, toasts]); - const dataViewFields: DataViewField[] = currentDataView.fields; const fieldTypes = useMemo(() => { diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx new file mode 100644 index 0000000000000..9819d3f9d9e91 --- /dev/null +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_bar.tsx @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Filter, Query, TimeRange } from '@kbn/es-query'; +import { i18n } from '@kbn/i18n'; +import React, { useEffect, useState } from 'react'; +import { isDefined } from '@kbn/ml-is-defined'; +import { DataView } from '@kbn/data-views-plugin/common'; +import { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { createMergedEsQuery } from '../../utils/saved_search_utils'; +import { useDataVisualizerKibana } from '../../../kibana_context'; + +export const SearchPanelContent = ({ + searchQuery, + searchString, + searchQueryLanguage, + dataView, + setSearchParams, +}: { + dataView: DataView; + searchQuery: Query['query']; + searchString: Query['query']; + searchQueryLanguage: SearchQueryLanguage; + setSearchParams({ + searchQuery, + searchString, + queryLanguage, + filters, + }: { + searchQuery: Query['query']; + searchString: Query['query']; + queryLanguage: SearchQueryLanguage; + filters: Filter[]; + }): void; +}) => { + const { + services: { + uiSettings, + notifications: { toasts }, + data: { query: queryManager }, + unifiedSearch: { + ui: { SearchBar }, + }, + }, + } = useDataVisualizerKibana(); + // The internal state of the input query bar updated on every key stroke. + const [searchInput, setSearchInput] = useState({ + query: searchString || '', + language: searchQueryLanguage, + }); + + useEffect(() => { + setSearchInput({ + query: searchString || '', + language: searchQueryLanguage, + }); + }, [searchQueryLanguage, searchString, queryManager.filterManager]); + + const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => { + const mergedQuery = isDefined(query) ? query : searchInput; + const mergedFilters = isDefined(filters) ? filters : queryManager.filterManager.getFilters(); + try { + if (mergedFilters) { + queryManager.filterManager.setFilters(mergedFilters); + } + + const combinedQuery = createMergedEsQuery( + mergedQuery, + queryManager.filterManager.getFilters() ?? [], + dataView, + uiSettings + ); + + setSearchParams({ + searchQuery: combinedQuery, + searchString: mergedQuery.query, + queryLanguage: mergedQuery.language as SearchQueryLanguage, + filters: mergedFilters, + }); + } catch (e) { + console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console + toasts.addError(e, { + title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', { + defaultMessage: 'Invalid syntax', + }), + }); + } + }; + + return ( + + searchHandler({ query: params.query }) + } + onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })} + indexPatterns={[dataView]} + placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', { + defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")', + })} + displayStyle={'inPage'} + isClearable={true} + customSubmitButton={
} + /> + ); +}; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx index 7a4b97f4bf5a5..2edb4280c9b07 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/components/search_panel/search_panel.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { FC, useEffect, useState } from 'react'; +import React, { FC } from 'react'; import { css } from '@emotion/react'; import { useEuiBreakpoint, @@ -14,16 +14,12 @@ import { EuiFlexGroup, EuiSpacer, } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; import { Query, Filter } from '@kbn/es-query'; -import type { TimeRange } from '@kbn/es-query'; import { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import { isDefined } from '@kbn/ml-is-defined'; +import { SearchQueryLanguage } from '@kbn/ml-query-utils'; +import { SearchPanelContent } from './search_bar'; import { DataVisualizerFieldNamesFilter } from './field_name_filter'; import { DataVisualizerFieldTypeFilter } from './field_type_filter'; -import { SearchQueryLanguage } from '../../types/combined_query'; -import { useDataVisualizerKibana } from '../../../kibana_context'; -import { createMergedEsQuery } from '../../utils/saved_search_utils'; import { OverallStats } from '../../types/overall_stats'; interface Props { @@ -55,6 +51,7 @@ interface Props { export const SearchPanel: FC = ({ dataView, searchString, + searchQuery, searchQueryLanguage, overallStats, indexedFieldTypes, @@ -65,60 +62,6 @@ export const SearchPanel: FC = ({ setSearchParams, showEmptyFields, }) => { - const { - services: { - uiSettings, - notifications: { toasts }, - data: { query: queryManager }, - unifiedSearch: { - ui: { SearchBar }, - }, - }, - } = useDataVisualizerKibana(); - // The internal state of the input query bar updated on every key stroke. - const [searchInput, setSearchInput] = useState({ - query: searchString || '', - language: searchQueryLanguage, - }); - - useEffect(() => { - setSearchInput({ - query: searchString || '', - language: searchQueryLanguage, - }); - }, [searchQueryLanguage, searchString, queryManager.filterManager]); - - const searchHandler = ({ query, filters }: { query?: Query; filters?: Filter[] }) => { - const mergedQuery = isDefined(query) ? query : searchInput; - const mergedFilters = isDefined(filters) ? filters : queryManager.filterManager.getFilters(); - try { - if (mergedFilters) { - queryManager.filterManager.setFilters(mergedFilters); - } - - const combinedQuery = createMergedEsQuery( - mergedQuery, - queryManager.filterManager.getFilters() ?? [], - dataView, - uiSettings - ); - - setSearchParams({ - searchQuery: combinedQuery, - searchString: mergedQuery.query, - queryLanguage: mergedQuery.language as SearchQueryLanguage, - filters: mergedFilters, - }); - } catch (e) { - console.log('Invalid syntax', JSON.stringify(e, null, 2)); // eslint-disable-line no-console - toasts.addError(e, { - title: i18n.translate('xpack.dataVisualizer.searchPanel.invalidSyntax', { - defaultMessage: 'Invalid syntax', - }), - }); - } - }; - const dvSearchPanelControls = css({ marginLeft: '0px !important', paddingLeft: '0px !important', @@ -135,7 +78,6 @@ export const SearchPanel: FC = ({ flexDirection: 'column', }, }); - const dvSearchBar = css({ [useEuiBreakpoint(['xs', 's', 'm', 'l'])]: { minWidth: `max(100%, 300px)`, @@ -152,24 +94,12 @@ export const SearchPanel: FC = ({ responsive={false} > - - searchHandler({ query: params.query }) - } - onFiltersUpdated={(filters: Filter[]) => searchHandler({ filters })} - indexPatterns={[dataView]} - placeholder={i18n.translate('xpack.dataVisualizer.searchPanel.queryBarPlaceholderText', { - defaultMessage: 'Search… (e.g. status:200 AND extension:"PHP")', - })} - displayStyle={'inPage'} - isClearable={true} - customSubmitButton={
} + diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts index b948cab5ee1ed..87e88e3555b18 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_data_visualizer_grid_data.ts @@ -10,7 +10,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { merge } from 'rxjs'; import type { EuiTableActionsColumnType } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { type DataViewField, UI_SETTINGS } from '@kbn/data-plugin/common'; +import { type DataViewField } from '@kbn/data-plugin/common'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import seedrandom from 'seedrandom'; import type { SamplingOption } from '@kbn/discover-plugin/public/application/main/components/field_stats_table/field_stats_table'; @@ -19,6 +19,7 @@ import { mlTimefilterRefresh$, useTimefilter } from '@kbn/ml-date-picker'; import useObservable from 'react-use/lib/useObservable'; import type { KibanaExecutionContext } from '@kbn/core-execution-context-common'; import { useExecutionContext } from '@kbn/kibana-react-plugin/public'; +import { useTimeBuckets } from '../../common/hooks/use_time_buckets'; import { DATA_VISUALIZER_GRID_EMBEDDABLE_TYPE } from '../embeddables/grid_embeddable/constants'; import { filterFields } from '../../common/components/fields_stats_grid/filter_fields'; import type { RandomSamplerOption } from '../constants/random_sampler'; @@ -26,7 +27,6 @@ import type { DataVisualizerIndexBasedAppState } from '../types/index_data_visua import { useDataVisualizerKibana } from '../../kibana_context'; import { getEsQueryFromSavedSearch } from '../utils/saved_search_utils'; import type { MetricFieldsStats } from '../../common/components/stats_table/components/field_count_stats'; -import { TimeBuckets } from '../../../../common/services/time_buckets'; import type { FieldVisConfig } from '../../common/components/stats_table/types'; import { NON_AGGREGATABLE_FIELD_TYPES, @@ -168,14 +168,7 @@ export const useDataVisualizerGridData = ( lastRefresh, ]); - const _timeBuckets = useMemo(() => { - return new TimeBuckets({ - [UI_SETTINGS.HISTOGRAM_MAX_BARS]: uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), - [UI_SETTINGS.HISTOGRAM_BAR_TARGET]: uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), - dateFormat: uiSettings.get('dateFormat'), - 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), - }); - }, [uiSettings]); + const _timeBuckets = useTimeBuckets(); const timefilter = useTimefilter({ timeRangeSelector: currentDataView?.timeFieldName !== undefined, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts index 3419b1674fddc..ba07a4581223a 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/hooks/use_overall_stats.ts @@ -8,8 +8,6 @@ import { useCallback, useEffect, useState, useRef, useMemo, useReducer } from 'react'; import { from, Subscription, Observable } from 'rxjs'; import { mergeMap, last, map, toArray } from 'rxjs/operators'; -import { i18n } from '@kbn/i18n'; -import type { ToastsStart } from '@kbn/core/public'; import { chunk } from 'lodash'; import type { IKibanaSearchRequest, @@ -38,6 +36,7 @@ import { import { getDocumentCountStats } from '../search_strategy/requests/get_document_stats'; import { getInitialProgress, getReducer } from '../progress_utils'; import { MAX_CONCURRENT_REQUESTS } from '../constants/index_data_visualizer_viewer'; +import { displayError } from '../../common/util/display_error'; /** * Helper function to run forkJoin @@ -63,32 +62,6 @@ export function rateLimitingForkJoin( ); } -function displayError(toastNotifications: ToastsStart, index: string, err: any) { - if (err.statusCode === 500) { - toastNotifications.addError(err, { - title: i18n.translate('xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage', { - defaultMessage: - 'Error loading data in index {index}. {message}. ' + - 'The request may have timed out. Try using a smaller sample size or narrowing the time range.', - values: { - index, - message: err.error ?? err.message, - }, - }), - }); - } else { - toastNotifications.addError(err, { - title: i18n.translate('xpack.dataVisualizer.index.errorLoadingDataMessage', { - defaultMessage: 'Error loading data in index {index}. {message}.', - values: { - index, - message: err.error ?? err.message, - }, - }), - }); - } -} - export function useOverallStats( searchStrategyParams: TParams | undefined, lastRefresh: number, diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts index 4b53f815a986c..6d2fffd09ac3b 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/locator/locator.ts @@ -12,7 +12,7 @@ import type { RefreshInterval } from '@kbn/data-plugin/common'; import { LocatorDefinition, LocatorPublic } from '@kbn/share-plugin/common'; import { GlobalQueryStateFromUrl } from '@kbn/data-plugin/public'; import { type Dictionary, isRisonSerializationRequired } from '@kbn/ml-url-state'; -import { SearchQueryLanguage } from '../types/combined_query'; +import { SearchQueryLanguage } from '@kbn/ml-query-utils'; export const DATA_VISUALIZER_APP_LOCATOR = 'DATA_VISUALIZER_APP_LOCATOR'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts index 13590505a5d1a..c79d3b7e52147 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/combined_query.ts @@ -7,13 +7,6 @@ import { Query } from '@kbn/es-query'; -export const SEARCH_QUERY_LANGUAGE = { - KUERY: 'kuery', - LUCENE: 'lucene', -} as const; - -export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE]; - export interface CombinedQuery { searchString: Query['query']; searchQueryLanguage: string; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts index 5b0bb046f2596..508d6e0015446 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/index_data_visualizer_state.ts @@ -7,8 +7,8 @@ import type { Filter } from '@kbn/es-query'; import type { Query } from '@kbn/data-plugin/common/query'; +import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; import type { RandomSamplerOption } from '../constants/random_sampler'; -import type { SearchQueryLanguage } from './combined_query'; import type { DATA_VISUALIZER_INDEX_VIEWER } from '../constants/index_data_visualizer_viewer'; diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/storage.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/storage.ts index 372fa2a738a53..1ece302b8859e 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/storage.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/types/storage.ts @@ -7,7 +7,7 @@ import { type FrozenTierPreference } from '@kbn/ml-date-picker'; -import { RandomSamplerOption } from '../constants/random_sampler'; +import { type RandomSamplerOption } from '../constants/random_sampler'; export const DV_FROZEN_TIER_PREFERENCE = 'dataVisualizer.frozenDataTierPreference'; export const DV_RANDOM_SAMPLER_PREFERENCE = 'dataVisualizer.randomSamplerPreference'; @@ -26,7 +26,7 @@ export type DVStorageMapped = T extends typeof DV_FROZEN_TIER_P : T extends typeof DV_RANDOM_SAMPLER_PREFERENCE ? RandomSamplerOption | undefined : T extends typeof DV_RANDOM_SAMPLER_P_VALUE - ? number | undefined + ? number | null : null; export const DV_STORAGE_KEYS = [ diff --git a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts index c3473c170b370..6961c5d822a67 100644 --- a/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts +++ b/x-pack/plugins/data_visualizer/public/application/index_data_visualizer/utils/saved_search_utils.ts @@ -23,23 +23,10 @@ import { DataView } from '@kbn/data-views-plugin/public'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; import { getEsQueryConfig, isQuery, SearchSource } from '@kbn/data-plugin/common'; import { FilterManager, mapAndFlattenFilters } from '@kbn/data-plugin/public'; -import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '../types/combined_query'; +import { getDefaultDSLQuery } from '@kbn/ml-query-utils'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils'; import { isSavedSearchSavedObject, SavedSearchSavedObject } from '../../../../common/types'; -const DEFAULT_QUERY = { - bool: { - must: [ - { - match_all: {}, - }, - ], - }, -}; - -export function getDefaultQuery() { - return cloneDeep(DEFAULT_QUERY); -} - /** * Parse the stringified searchSourceJSON * from a saved search or saved search object @@ -82,7 +69,7 @@ export function createMergedEsQuery( dataView?: DataView, uiSettings?: IUiSettingsClient ) { - let combinedQuery: QueryDslQueryContainer = getDefaultQuery(); + let combinedQuery = getDefaultDSLQuery() as QueryDslQueryContainer; if (isQuery(query) && query.language === SEARCH_QUERY_LANGUAGE.KUERY) { const ast = fromKueryExpression(query.query); @@ -158,7 +145,7 @@ export function getEsQueryFromSavedSearch({ // Flattened query from search source may contain a clause that narrows the time range // which might interfere with global time pickers so we need to remove const savedQuery = - cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultQuery(); + cloneDeep(savedSearch.searchSource.getSearchRequestBody()?.query) ?? getDefaultDSLQuery(); const timeField = savedSearch.searchSource.getField('index')?.timeFieldName; if (Array.isArray(savedQuery.bool.filter) && timeField !== undefined) { diff --git a/x-pack/plugins/data_visualizer/public/index.ts b/x-pack/plugins/data_visualizer/public/index.ts index 7c4208106cb75..1312419797bb1 100644 --- a/x-pack/plugins/data_visualizer/public/index.ts +++ b/x-pack/plugins/data_visualizer/public/index.ts @@ -17,6 +17,7 @@ export type { FileDataVisualizerSpec, IndexDataVisualizerSpec, IndexDataVisualizerViewProps, + DataComparisonSpec, } from './application'; export type { GetAdditionalLinksParams, diff --git a/x-pack/plugins/data_visualizer/public/lazy_load_bundle/component_wrapper.tsx b/x-pack/plugins/data_visualizer/public/lazy_load_bundle/component_wrapper.tsx index 028c0a402039d..cbd897ac20811 100644 --- a/x-pack/plugins/data_visualizer/public/lazy_load_bundle/component_wrapper.tsx +++ b/x-pack/plugins/data_visualizer/public/lazy_load_bundle/component_wrapper.tsx @@ -5,7 +5,15 @@ * 2.0. */ -import React, { FC } from 'react'; +import React, { FC, Suspense } from 'react'; +import { EuiErrorBoundary, EuiSkeletonText } from '@elastic/eui'; +import type { DataComparisonDetectionAppStateProps } from '../application/data_comparison/data_comparison_app_state'; + +const LazyWrapper: FC = ({ children }) => ( + + }>{children} + +); const FileDataVisualizerComponent = React.lazy( () => import('../application/file_data_visualizer/file_data_visualizer') @@ -18,3 +26,15 @@ export const FileDataVisualizerWrapper: FC = () => { ); }; + +const DataComparisonLazy = React.lazy(() => import('../application/data_comparison')); + +/** + * Lazy-wrapped ExplainLogRateSpikesAppState React component + * @param {ExplainLogRateSpikesAppStateProps} props - properties specifying the data on which to run the analysis. + */ +export const DataComparison: FC = (props) => ( + + + +); diff --git a/x-pack/plugins/data_visualizer/public/lazy_load_bundle/index.ts b/x-pack/plugins/data_visualizer/public/lazy_load_bundle/index.ts index 0eddbbc248ed9..ba7cb83f78403 100644 --- a/x-pack/plugins/data_visualizer/public/lazy_load_bundle/index.ts +++ b/x-pack/plugins/data_visualizer/public/lazy_load_bundle/index.ts @@ -6,7 +6,11 @@ */ import { HttpSetup } from '@kbn/core/public'; -import type { FileDataVisualizerSpec, IndexDataVisualizerSpec } from '../application'; +import type { + DataComparisonSpec, + FileDataVisualizerSpec, + IndexDataVisualizerSpec, +} from '../application'; import { getCoreStart } from '../kibana_services'; let loadModulesPromise: Promise; @@ -14,6 +18,7 @@ let loadModulesPromise: Promise; interface LazyLoadedModules { FileDataVisualizer: FileDataVisualizerSpec; IndexDataVisualizer: IndexDataVisualizerSpec; + DataComparison: DataComparisonSpec; getHttp: () => HttpSetup; } diff --git a/x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.ts b/x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.tsx similarity index 88% rename from x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.ts rename to x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.tsx index 0001f912bff0d..181e15edc5fd3 100644 --- a/x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.ts +++ b/x-pack/plugins/data_visualizer/public/lazy_load_bundle/lazy/index.tsx @@ -7,3 +7,4 @@ export type { FileDataVisualizerSpec, IndexDataVisualizerSpec } from '../../application'; export { FileDataVisualizer, IndexDataVisualizer } from '../../application'; +export { DataComparison } from '../component_wrapper'; diff --git a/x-pack/plugins/data_visualizer/public/plugin.ts b/x-pack/plugins/data_visualizer/public/plugin.ts index 08885dee8c7e8..38ebe18091e03 100644 --- a/x-pack/plugins/data_visualizer/public/plugin.ts +++ b/x-pack/plugins/data_visualizer/public/plugin.ts @@ -24,7 +24,11 @@ import type { LensPublicStart } from '@kbn/lens-plugin/public'; import type { IndexPatternFieldEditorStart } from '@kbn/data-view-field-editor-plugin/public'; import { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { UiActionsStart } from '@kbn/ui-actions-plugin/public'; -import { getFileDataVisualizerComponent, getIndexDataVisualizerComponent } from './api'; +import { + getDataComparisonComponent, + getFileDataVisualizerComponent, + getIndexDataVisualizerComponent, +} from './api'; import { getMaxBytesFormatted } from './application/common/util/get_max_bytes'; import { registerHomeAddData, registerHomeFeatureCatalogue } from './register_home'; import { registerEmbeddables } from './application/index_data_visualizer/embeddables'; @@ -85,6 +89,7 @@ export class DataVisualizerPlugin return { getFileDataVisualizerComponent, getIndexDataVisualizerComponent, + getDataComparisonComponent, getMaxBytesFormatted, }; } diff --git a/x-pack/plugins/data_visualizer/tsconfig.json b/x-pack/plugins/data_visualizer/tsconfig.json index bfedf95733de5..75dd55b83e274 100644 --- a/x-pack/plugins/data_visualizer/tsconfig.json +++ b/x-pack/plugins/data_visualizer/tsconfig.json @@ -13,6 +13,8 @@ ], "kbn_references": [ "@kbn/ace", + "@kbn/aiops-components", + "@kbn/aiops-utils", "@kbn/charts-plugin", "@kbn/cloud-chat-plugin", "@kbn/cloud-plugin", @@ -50,6 +52,7 @@ "@kbn/ml-data-grid", "@kbn/ml-error-utils", "@kbn/ml-kibana-theme", + "@kbn/ml-in-memory-table", "@kbn/react-field", "@kbn/rison", "@kbn/saved-search-plugin", @@ -60,7 +63,11 @@ "@kbn/unified-search-plugin", "@kbn/usage-collection-plugin", "@kbn/utility-types", - "@kbn/unified-field-list" + "@kbn/unified-field-list", + "@kbn/ml-string-hash", + "@kbn/ml-random-sampler-utils", + "@kbn/data-service", + "@kbn/core-notifications-browser" ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/ml/common/constants/locator.ts b/x-pack/plugins/ml/common/constants/locator.ts index 99d3affa9dcb3..265366a33ce41 100644 --- a/x-pack/plugins/ml/common/constants/locator.ts +++ b/x-pack/plugins/ml/common/constants/locator.ts @@ -15,6 +15,8 @@ export const ML_PAGES = { DATA_FRAME_ANALYTICS_SOURCE_SELECTION: 'data_frame_analytics/source_selection', DATA_FRAME_ANALYTICS_CREATE_JOB: 'data_frame_analytics/new_job', TRAINED_MODELS_MANAGE: 'trained_models', + DATA_COMPARISON_INDEX_SELECT: 'data_comparison_index_select', + DATA_COMPARISON: 'data_comparison', NODES: 'nodes', MEMORY_USAGE: 'memory_usage', DATA_FRAME_ANALYTICS_EXPLORATION: 'data_frame_analytics/exploration', diff --git a/x-pack/plugins/ml/common/constants/search.ts b/x-pack/plugins/ml/common/constants/search.ts index 8ff9b022c274f..a5ff9c497843e 100644 --- a/x-pack/plugins/ml/common/constants/search.ts +++ b/x-pack/plugins/ml/common/constants/search.ts @@ -7,10 +7,3 @@ export const ANNOTATIONS_TABLE_DEFAULT_QUERY_SIZE = 500; export const ANOMALIES_TABLE_DEFAULT_QUERY_SIZE = 500; - -export const SEARCH_QUERY_LANGUAGE = { - KUERY: 'kuery', - LUCENE: 'lucene', -} as const; - -export type SearchQueryLanguage = typeof SEARCH_QUERY_LANGUAGE[keyof typeof SEARCH_QUERY_LANGUAGE]; diff --git a/x-pack/plugins/ml/common/types/locator.ts b/x-pack/plugins/ml/common/types/locator.ts index 6f4a50837f7ef..31de127c56044 100644 --- a/x-pack/plugins/ml/common/types/locator.ts +++ b/x-pack/plugins/ml/common/types/locator.ts @@ -8,10 +8,10 @@ import type { SerializableRecord } from '@kbn/utility-types'; import type { LocatorPublic } from '@kbn/share-plugin/public'; import type { RefreshInterval, TimeRange } from '@kbn/data-plugin/common/query'; -import type { InfluencersFilterQuery } from '@kbn/ml-anomaly-utils'; import type { DataFrameAnalysisConfigType } from '@kbn/ml-data-frame-analytics-utils'; +import type { InfluencersFilterQuery } from '@kbn/ml-anomaly-utils'; +import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; import type { JobId } from './anomaly_detection_jobs/job'; -import type { SearchQueryLanguage } from '../constants/search'; import type { ListingPageUrlState } from './common'; import { ML_PAGES } from '../constants/locator'; @@ -58,6 +58,8 @@ export type MlGenericUrlState = MLPageState< | typeof ML_PAGES.FILTER_LISTS_MANAGE | typeof ML_PAGES.FILTER_LISTS_NEW | typeof ML_PAGES.SETTINGS + | typeof ML_PAGES.DATA_COMPARISON + | typeof ML_PAGES.DATA_COMPARISON_INDEX_SELECT | typeof ML_PAGES.DATA_VISUALIZER | typeof ML_PAGES.DATA_VISUALIZER_FILE | typeof ML_PAGES.DATA_VISUALIZER_INDEX_SELECT @@ -70,7 +72,6 @@ export type MlGenericUrlState = MLPageState< | typeof ML_PAGES.AIOPS_CHANGE_POINT_DETECTION, MlGenericUrlPageState | undefined >; - export interface AnomalyDetectionQueryState { jobId?: JobId | string[]; groupIds?: string[]; diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx b/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx index 30e3686f7e123..d65dd8955b8bc 100644 --- a/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx +++ b/x-pack/plugins/ml/public/application/components/anomalies_table/links_menu.tsx @@ -30,6 +30,7 @@ import { type MlAnomaliesTableRecord, } from '@kbn/ml-anomaly-utils'; import { formatHumanReadableDateTimeSeconds, timeFormatter } from '@kbn/ml-date-utils'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { mlJobService } from '../../services/job_service'; import { getDataViewIdFromName } from '../../util/index_utils'; import { getInitialAnomaliesLayers, getInitialSourceIndexFieldLayers } from '../../../maps/util'; @@ -38,7 +39,6 @@ import { ml } from '../../services/ml_api_service'; import { escapeKueryForFieldValuePair, replaceStringTokens } from '../../util/string_utils'; import { getUrlForRecord, openCustomUrlWindow } from '../../util/custom_url_utils'; import { ML_APP_LOCATOR, ML_PAGES } from '../../../../common/constants/locator'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search'; // @ts-ignore import { escapeDoubleQuotes, diff --git a/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx b/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx index 8c8e284005196..a74dd6d061aad 100644 --- a/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx +++ b/x-pack/plugins/ml/public/application/components/ml_page/side_nav.tsx @@ -234,6 +234,15 @@ export function useSideNavItems(activeRoute: MlRoute | undefined) { disabled: false, testSubj: 'mlMainTab indexDataVisualizer', }, + { + id: 'data_comparison', + pathId: ML_PAGES.DATA_COMPARISON_INDEX_SELECT, + name: i18n.translate('xpack.ml.navMenu.dataComparisonText', { + defaultMessage: 'Data Comparison', + }), + disabled: disableLinks, + testSubj: 'mlMainTab dataComparison', + }, ], }, ]; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index eed5d060d74de..c574a6f661085 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -34,6 +34,7 @@ import { TRAINING_PERCENT_MAX, } from '@kbn/ml-data-frame-analytics-utils'; import { DataGrid } from '@kbn/ml-data-grid'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { useMlKibana } from '../../../../../contexts/kibana'; import { EuiComboBoxWithFieldStats, @@ -62,7 +63,6 @@ import { fetchExplainData } from '../shared'; import { useIndexData } from '../../hooks'; import { ExplorationQueryBar } from '../../../analytics_exploration/components/exploration_query_bar'; import { useSavedSearch, SavedSearchQuery } from './use_saved_search'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search'; import { ExplorationQueryBarProps } from '../../../analytics_exploration/components/exploration_query_bar/exploration_query_bar'; import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts index ef0bc4beac776..bb6867b62aac0 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/use_saved_search.ts @@ -15,9 +15,9 @@ import { Query, toElasticsearchQuery, } from '@kbn/es-query'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { useMlKibana } from '../../../../../contexts/kibana'; import { useDataSource } from '../../../../../contexts/ml'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search'; // `undefined` is used for a non-initialized state // `null` is set if no saved search is used diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx index ecd4cc02b0a79..4e89a2a0833a1 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section_results.tsx @@ -43,7 +43,7 @@ import { } from '@kbn/ml-data-frame-analytics-utils'; import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../../../../common/constants/search'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { useColorRange, ColorRangeLegend } from '../../../../../components/color_range_legend'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx index 80f0f3dff0850..74ca6037f8a90 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx @@ -18,11 +18,8 @@ import type { Query } from '@kbn/es-query'; import { QueryStringInput } from '@kbn/unified-search-plugin/public'; import { QueryErrorMessage } from '@kbn/ml-error-utils'; +import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from '@kbn/ml-query-utils'; import { Dictionary } from '../../../../../../../common/types/common'; -import { - SEARCH_QUERY_LANGUAGE, - SearchQueryLanguage, -} from '../../../../../../../common/constants/search'; import { removeFilterFromQueryString } from '../../../../../explorer/explorer_utils'; import { useMlKibana } from '../../../../../contexts/kibana'; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/hooks/use_exploration_url_state.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/hooks/use_exploration_url_state.ts index 443b4c2298928..d0f66bcc63954 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/hooks/use_exploration_url_state.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/hooks/use_exploration_url_state.ts @@ -7,9 +7,9 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object'; import { usePageUrlState } from '@kbn/ml-url-state'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { ML_PAGES } from '../../../../../../common/constants/locator'; import { ExplorationPageUrlState } from '../../../../../../common/types/locator'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../../../common/constants/search'; export function getDefaultExplorationPageUrlState( overrides?: Partial diff --git a/x-pack/plugins/ml/public/application/datavisualizer/data_comparison/data_comparison_page.tsx b/x-pack/plugins/ml/public/application/datavisualizer/data_comparison/data_comparison_page.tsx new file mode 100644 index 0000000000000..8b016d17cc938 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/data_comparison/data_comparison_page.tsx @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useEffect, useState } from 'react'; +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import type { DataComparisonSpec } from '@kbn/data-visualizer-plugin/public'; +import { useMlKibana } from '../../contexts/kibana'; +import { useDataSource } from '../../contexts/ml'; +import { MlPageHeader } from '../../components/page_header'; +import { TechnicalPreviewBadge } from '../../components/technical_preview_badge'; + +export const DataComparisonPage: FC = () => { + const { + services: { dataVisualizer }, + } = useMlKibana(); + + const [DataComparisonView, setDataComparisonView] = useState(null); + + useEffect(() => { + if (dataVisualizer !== undefined) { + const { getDataComparisonComponent } = dataVisualizer; + getDataComparisonComponent().then(setDataComparisonView); + } + }, [dataVisualizer]); + + const { selectedDataView: dataView, selectedSavedSearch: savedSearch } = useDataSource(); + + return ( + <> + + + + + + + + + + + {dataView && DataComparisonView ? ( + + ) : null} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/explorer/anomaly_context_menu.tsx b/x-pack/plugins/ml/public/application/explorer/anomaly_context_menu.tsx index f077d0f3dfd7f..314221ca38773 100644 --- a/x-pack/plugins/ml/public/application/explorer/anomaly_context_menu.tsx +++ b/x-pack/plugins/ml/public/application/explorer/anomaly_context_menu.tsx @@ -21,9 +21,9 @@ import useObservable from 'react-use/lib/useObservable'; import type { Query, TimeRange } from '@kbn/es-query'; import { isDefined } from '@kbn/ml-is-defined'; import { useTimeRangeUpdates } from '@kbn/ml-date-picker'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { useAnomalyExplorerContext } from './anomaly_explorer_context'; import { escapeKueryForFieldValuePair } from '../util/string_utils'; -import { SEARCH_QUERY_LANGUAGE } from '../../../common/constants/search'; import { useCasesModal } from '../contexts/kibana/use_cases_modal'; import { DEFAULT_MAX_SERIES_TO_PLOT } from '../services/anomaly_explorer_charts_service'; import { ANOMALY_EXPLORER_CHARTS_EMBEDDABLE_TYPE } from '../../embeddables'; diff --git a/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx b/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx index 8f3b8395e8dde..0ba024a9c5f4f 100644 --- a/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx +++ b/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx @@ -30,7 +30,7 @@ import type { Query } from '@kbn/es-query'; import { formatHumanReadableDateTime } from '@kbn/ml-date-utils'; import { isDefined } from '@kbn/ml-is-defined'; import { useTimeRangeUpdates } from '@kbn/ml-date-picker'; -import { SEARCH_QUERY_LANGUAGE } from '../../../common/constants/search'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { useCasesModal } from '../contexts/kibana/use_cases_modal'; import { ANOMALY_SWIMLANE_EMBEDDABLE_TYPE } from '../..'; import { diff --git a/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx b/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx index 412c9e8f3aa7d..aa5b752b1f4f8 100644 --- a/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx +++ b/x-pack/plugins/ml/public/application/explorer/components/explorer_query_bar/explorer_query_bar.tsx @@ -14,7 +14,7 @@ import { QueryStringInput } from '@kbn/unified-search-plugin/public'; import type { DataView } from '@kbn/data-views-plugin/common'; import type { QueryErrorMessage } from '@kbn/ml-error-utils'; import type { InfluencersFilterQuery } from '@kbn/ml-anomaly-utils'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../../common/constants/search'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { useAnomalyExplorerContext } from '../../anomaly_explorer_context'; import { useMlKibana } from '../../../contexts/kibana'; diff --git a/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_anomaly_charts_to_dashboard_controls.tsx b/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_anomaly_charts_to_dashboard_controls.tsx index 429138874238d..9ca48863c1670 100644 --- a/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_anomaly_charts_to_dashboard_controls.tsx +++ b/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_anomaly_charts_to_dashboard_controls.tsx @@ -10,10 +10,10 @@ import { EuiFieldNumber, EuiFormRow, htmlIdGenerator } from '@elastic/eui'; import type { Query } from '@kbn/es-query'; import useObservable from 'react-use/lib/useObservable'; import { isDefined } from '@kbn/ml-is-defined'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { getSelectionInfluencers } from '../explorer_utils'; import { useAnomalyExplorerContext } from '../anomaly_explorer_context'; import { escapeKueryForFieldValuePair } from '../../util/string_utils'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search'; import { useDashboardTable } from './use_dashboards_table'; import { AddToDashboardControl } from './add_to_dashboard_controls'; import { useAddToDashboardActions } from './use_add_to_dashboard_actions'; diff --git a/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_swimlane_to_dashboard_controls.tsx b/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_swimlane_to_dashboard_controls.tsx index 93fa7e908f35a..4dcfdbf826b1d 100644 --- a/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_swimlane_to_dashboard_controls.tsx +++ b/x-pack/plugins/ml/public/application/explorer/dashboard_controls/add_swimlane_to_dashboard_controls.tsx @@ -17,7 +17,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; import { DashboardAttributes } from '@kbn/dashboard-plugin/common'; import type { Query } from '@kbn/es-query'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { getDefaultSwimlanePanelTitle } from '../../../embeddables/anomaly_swimlane/anomaly_swimlane_embeddable'; import { SWIMLANE_TYPE, SwimlaneType } from '../explorer_constants'; import { JobId } from '../../../../common/types/anomaly_detection_jobs'; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js index eb419c1f09370..4c9773e1674a8 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js @@ -29,7 +29,7 @@ import { ExplorerChartSingleMetric } from './explorer_chart_single_metric'; import { ExplorerChartLabel } from './components/explorer_chart_label'; import { CHART_TYPE } from '../explorer_constants'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../common/constants/search'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { MlTooltipComponent } from '../../components/chart_tooltip'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts b/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts index 025905335f1e5..32fd17c9d1f10 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts @@ -19,7 +19,7 @@ import type { Filter } from '@kbn/es-query'; import type { IUiSettingsClient } from '@kbn/core/public'; import { getEsQueryConfig } from '@kbn/data-plugin/public'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; -import { SEARCH_QUERY_LANGUAGE } from '../../../../../common/constants/search'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import { getQueryFromSavedSearchObject } from '../../../util/index_utils'; // Provider for creating the items used for searching and job creation. diff --git a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/space_management.tsx b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/space_management.tsx index 69ea8f717a51f..2564f7c55da1e 100644 --- a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/space_management.tsx +++ b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/space_management.tsx @@ -20,6 +20,7 @@ import { } from '@elastic/eui'; import type { SpacesPluginStart } from '@kbn/spaces-plugin/public'; +import { useTableState } from '@kbn/ml-in-memory-table'; import type { JobType, MlSavedObjectType } from '../../../../../../../common/types/saved_objects'; import type { ManagementListResponse, @@ -29,7 +30,6 @@ import { useManagementApiService } from '../../../../../services/ml_api_service/ import { getColumns } from './columns'; import { MLSavedObjectsSpacesList } from '../../../../../components/ml_saved_objects_spaces_list'; import { getFilters } from './filters'; -import { useTableState } from './use_table_state'; interface Props { spacesApi?: SpacesPluginStart; diff --git a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/use_table_state.ts b/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/use_table_state.ts deleted file mode 100644 index c67fe7d28fa9b..0000000000000 --- a/x-pack/plugins/ml/public/application/management/jobs_list/components/jobs_list_page/space_management/use_table_state.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useState } from 'react'; -import { EuiInMemoryTable, Direction, Pagination } from '@elastic/eui'; - -export function useTableState(items: T[], initialSortField: string) { - const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(initialSortField); - const [sortDirection, setSortDirection] = useState('asc'); - - const onTableChange: EuiInMemoryTable['onTableChange'] = ({ - page = { index: 0, size: 10 }, - sort = { field: sortField, direction: sortDirection }, - }) => { - const { index, size } = page; - setPageIndex(index); - setPageSize(size); - - const { field, direction } = sort; - setSortField(field as string); - setSortDirection(direction as Direction); - }; - - const pagination: Pagination = { - pageIndex, - pageSize, - totalItemCount: (items ?? []).length, - pageSizeOptions: [10, 20, 50], - showPerPageOptions: true, - }; - - const sorting = { - sort: { - field: sortField, - direction: sortDirection, - }, - }; - - return { onTableChange, pagination, sorting, setPageIndex }; -} diff --git a/x-pack/plugins/ml/public/application/routing/breadcrumbs.ts b/x-pack/plugins/ml/public/application/routing/breadcrumbs.ts index 0ff6c61ec8c53..2824b835f914a 100644 --- a/x-pack/plugins/ml/public/application/routing/breadcrumbs.ts +++ b/x-pack/plugins/ml/public/application/routing/breadcrumbs.ts @@ -120,12 +120,20 @@ export const FILTER_LISTS_BREADCRUMB: ChromeBreadcrumb = Object.freeze({ href: '/settings/filter_lists', }); +export const DATA_COMPARISON_BREADCRUMB: ChromeBreadcrumb = Object.freeze({ + text: i18n.translate('xpack.ml.settings.breadcrumbs.dataComparisonLabel', { + defaultMessage: 'Data comparison', + }), + href: '/data_comparison_index_select', +}); + const breadcrumbs = { ML_BREADCRUMB, SETTINGS_BREADCRUMB, ANOMALY_DETECTION_BREADCRUMB, DATA_FRAME_ANALYTICS_BREADCRUMB, TRAINED_MODELS, + DATA_COMPARISON_BREADCRUMB, DATA_VISUALIZER_BREADCRUMB, AIOPS_BREADCRUMB_LOG_RATE_ANALYSIS, AIOPS_BREADCRUMB_LOG_PATTERN_ANALYSIS, diff --git a/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/data_comparison.tsx b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/data_comparison.tsx new file mode 100644 index 0000000000000..b8efd6b7df529 --- /dev/null +++ b/x-pack/plugins/ml/public/application/routing/routes/datavisualizer/data_comparison.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; +import { DataSourceContextProvider } from '../../../contexts/ml'; +import { DataComparisonPage } from '../../../datavisualizer/data_comparison/data_comparison_page'; +import { ML_PAGES } from '../../../../locator'; +import { NavigateToPath } from '../../../contexts/kibana'; +import { createPath, MlRoute, PageLoader, PageProps } from '../../router'; +import { useRouteResolver } from '../../use_resolver'; +import { + breadcrumbOnClickFactory, + DATA_COMPARISON_BREADCRUMB, + DATA_VISUALIZER_BREADCRUMB, + getBreadcrumbWithUrlForApp, +} from '../../breadcrumbs'; +import { basicResolvers } from '../../resolvers'; + +export const dataComparisonRouteFactory = ( + navigateToPath: NavigateToPath, + basePath: string +): MlRoute => ({ + id: 'data_comparison', + path: createPath(ML_PAGES.DATA_COMPARISON), + title: i18n.translate('xpack.ml.dataVisualizer.dataComparison.docTitle', { + defaultMessage: 'Data Comparison', + }), + render: (props, deps) => , + breadcrumbs: [ + getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath, basePath), + { + text: DATA_VISUALIZER_BREADCRUMB.text, + ...(navigateToPath + ? { + href: `${basePath}/app/ml${DATA_COMPARISON_BREADCRUMB.href}`, + onClick: breadcrumbOnClickFactory(DATA_COMPARISON_BREADCRUMB.href, navigateToPath), + } + : {}), + }, + { + text: i18n.translate('xpack.ml.trainedModelsBreadcrumbs.dataComparisonLabel', { + defaultMessage: 'Data Comparison', + }), + }, + ], + 'data-test-subj': 'mlPageDataComparison', +}); + +const PageWrapper: FC = () => { + const { context } = useRouteResolver('basic', [], basicResolvers()); + + return ( + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx index 3337257b8754c..087b1d7ccb35c 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/index_or_search.tsx @@ -183,6 +183,26 @@ export const changePointDetectionIndexOrSearchRouteFactory = ( breadcrumbs: getChangePointDetectionBreadcrumbs(navigateToPath, basePath), }); +export const dataComparisonIndexOrSearchRouteFactory = ( + navigateToPath: NavigateToPath, + basePath: string +): MlRoute => ({ + id: 'data_view_data_comparison', + path: createPath(ML_PAGES.DATA_COMPARISON_INDEX_SELECT), + title: i18n.translate('xpack.ml.selectDataViewLabel', { + defaultMessage: 'Select Data View', + }), + render: (props, deps) => ( + + ), + breadcrumbs: getDataVisBreadcrumbs(navigateToPath, basePath), +}); + const PageWrapper: FC = ({ nextStepPath, mode }) => { const { services: { diff --git a/x-pack/plugins/ml/public/application/routing/routes/trained_models/index.ts b/x-pack/plugins/ml/public/application/routing/routes/trained_models/index.ts index d69a2256335e3..0a1b1610e6f11 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/trained_models/index.ts +++ b/x-pack/plugins/ml/public/application/routing/routes/trained_models/index.ts @@ -6,3 +6,4 @@ */ export * from './models_list'; +export * from '../datavisualizer/data_comparison'; diff --git a/x-pack/plugins/ml/public/locator/ml_locator.ts b/x-pack/plugins/ml/public/locator/ml_locator.ts index 4b581e401b3eb..6008741860d6d 100644 --- a/x-pack/plugins/ml/public/locator/ml_locator.ts +++ b/x-pack/plugins/ml/public/locator/ml_locator.ts @@ -77,6 +77,8 @@ export class MlLocatorDefinition implements LocatorDefinition { case ML_PAGES.MEMORY_USAGE: path = formatMemoryUsageUrl('', params.pageState); break; + case ML_PAGES.DATA_COMPARISON_INDEX_SELECT: + case ML_PAGES.DATA_COMPARISON: case ML_PAGES.ANOMALY_DETECTION_CREATE_JOB: case ML_PAGES.ANOMALY_DETECTION_CREATE_JOB_RECOGNIZER: case ML_PAGES.ANOMALY_DETECTION_CREATE_JOB_ADVANCED: diff --git a/x-pack/plugins/ml/public/maps/util.ts b/x-pack/plugins/ml/public/maps/util.ts index 342938bdff698..f746ab5209963 100644 --- a/x-pack/plugins/ml/public/maps/util.ts +++ b/x-pack/plugins/ml/public/maps/util.ts @@ -20,8 +20,8 @@ import { VectorSourceRequestMeta } from '@kbn/maps-plugin/common'; import { LAYER_TYPE, SOURCE_TYPES, SCALING_TYPES } from '@kbn/maps-plugin/common'; import { type MLAnomalyDoc, ML_SEVERITY_COLOR_RAMP } from '@kbn/ml-anomaly-utils'; import { formatHumanReadableDateTimeSeconds } from '@kbn/ml-date-utils'; +import { SEARCH_QUERY_LANGUAGE } from '@kbn/ml-query-utils'; import type { MlApiServices } from '../application/services/ml_api_service'; -import { SEARCH_QUERY_LANGUAGE } from '../../common/constants/search'; import { tabColor } from '../../common/util/group_color_utils'; import { getIndexPattern } from '../application/explorer/reducers/explorer_reducer/get_index_pattern'; import { AnomalySource } from './anomaly_source'; diff --git a/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts b/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts index 470c025934649..dfe2b4bbaf200 100644 --- a/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts +++ b/x-pack/plugins/ml/public/register_helper/register_search_links/search_deep_links.ts @@ -210,6 +210,17 @@ function getIndexDataVisualizerDeepLink(mlCapabilities: MlCapabilities): AppDeep }; } +function getDataComparisonDeepLink(mlCapabilities: MlCapabilities): AppDeepLink { + return { + id: 'dataComparison', + title: i18n.translate('xpack.ml.deepLink.dataComparison', { + defaultMessage: 'Data Comparison', + }), + path: `/${ML_PAGES.DATA_COMPARISON_INDEX_SELECT}`, + navLinkStatus: getNavStatus(mlCapabilities, false), + }; +} + function getSettingsDeepLink(mlCapabilities: MlCapabilities): AppDeepLink { const navLinkStatus = getNavStatus(mlCapabilities, mlCapabilities.isADEnabled); return { @@ -256,6 +267,7 @@ export function getDeepLinks(isFullLicense: boolean, mlCapabilities: MlCapabilit getDataVisualizerDeepLink(mlCapabilities), getFileUploadDeepLink(mlCapabilities), getIndexDataVisualizerDeepLink(mlCapabilities), + getDataComparisonDeepLink(mlCapabilities), ]; if (isFullLicense === true) { diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index 441d3814496ba..13560d5e0b962 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -100,5 +100,6 @@ "@kbn/core-notifications-browser-mocks", "@kbn/unified-field-list", "@kbn/core-ui-settings-browser", + "@kbn/ml-in-memory-table", ], } diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 442a6ebedd0bd..f0367c0528648 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -11696,7 +11696,6 @@ "xpack.dataVisualizer.file.resultsLinks.fileBeatConfigTopText2": "Modifiez {filebeatYml} afin de définir les informations de connexion :", "xpack.dataVisualizer.file.welcomeContent.uploadedFilesAllowedSizeDescription": "Vous pouvez charger des fichiers d'une taille allant jusqu'à {maxFileSize}.", "xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage": "Erreur lors du chargement des données dans l'index {index}. {message}. La requête a peut-être expiré. Essayez d'utiliser un échantillon d'une taille inférieure ou de réduire la plage temporelle.", - "xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationTitle": "La vue de données {dataViewTitle} n'est pas basée sur une série temporelle", "xpack.dataVisualizer.index.errorLoadingDataMessage": "Erreur lors du chargement des données dans l'index {index}. {message}.", "xpack.dataVisualizer.index.lensChart.averageOfLabel": "Moyenne de {fieldName}", "xpack.dataVisualizer.index.lensChart.chartTitle": "Lens pour {fieldName}", @@ -11921,7 +11920,6 @@ "xpack.dataVisualizer.index.dataViewManagement.actionsPopoverLabel": "Paramètres Vue de données", "xpack.dataVisualizer.index.dataViewManagement.addFieldButton": "Ajouter un champ à la vue de données", "xpack.dataVisualizer.index.dataViewManagement.manageFieldButton": "Gérer les champs de la vue de données", - "xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationDescription": "La détection des anomalies ne s'exécute que sur des index temporels", "xpack.dataVisualizer.index.embeddableErrorDescription": "Une erreur s'est produite lors du chargement de l'incorporable. Veuillez vérifier si toutes les entrées requises sont valides.", "xpack.dataVisualizer.index.embeddableErrorTitle": "Erreur lors du chargement de l'incorporable", "xpack.dataVisualizer.index.embeddableNoResultsMessage": "Résultat introuvable", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ed681f32a13a3..d804e4a837a6b 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -11710,7 +11710,6 @@ "xpack.dataVisualizer.file.resultsLinks.fileBeatConfigTopText2": "{filebeatYml}を変更して接続情報を設定します:", "xpack.dataVisualizer.file.welcomeContent.uploadedFilesAllowedSizeDescription": "最大{maxFileSize}のファイルをアップロードできます。", "xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage": "インデックス{index}のデータの読み込み中にエラーが発生。{message}。リクエストがタイムアウトした可能性があります。小さなサンプルサイズを使うか、時間範囲を狭めてみてください。", - "xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationTitle": "データビュー{dataViewTitle}は時系列に基づいていません", "xpack.dataVisualizer.index.errorLoadingDataMessage": "インデックス{index}のデータの読み込み中にエラーが発生。{message}。", "xpack.dataVisualizer.index.lensChart.averageOfLabel": "{fieldName} の平均", "xpack.dataVisualizer.index.lensChart.chartTitle": "{fieldName}のLens", @@ -11935,7 +11934,6 @@ "xpack.dataVisualizer.index.dataViewManagement.actionsPopoverLabel": "データビュー設定", "xpack.dataVisualizer.index.dataViewManagement.addFieldButton": "フィールドをデータビューに追加", "xpack.dataVisualizer.index.dataViewManagement.manageFieldButton": "データビューフィールドを管理", - "xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationDescription": "異常検知は時間ベースのインデックスでのみ実行されます", "xpack.dataVisualizer.index.embeddableErrorDescription": "埋め込み可能オブジェクトの読み込みエラーが発生しました。すべての必須入力が有効であるかどうかを確認してください。", "xpack.dataVisualizer.index.embeddableErrorTitle": "埋め込み可能オブジェクトの読み込みエラー", "xpack.dataVisualizer.index.embeddableNoResultsMessage": "結果が見つかりませんでした", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 918538b3edfad..6a7287002e834 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -11710,7 +11710,6 @@ "xpack.dataVisualizer.file.resultsLinks.fileBeatConfigTopText2": "修改 {filebeatYml} 以设置连接信息：", "xpack.dataVisualizer.file.welcomeContent.uploadedFilesAllowedSizeDescription": "您可以上传不超过 {maxFileSize} 的文件。", "xpack.dataVisualizer.index.dataLoader.internalServerErrorMessage": "加载索引 {index} 中的数据时出错。{message}。请求可能已超时。请尝试使用较小的样例大小或缩小时间范围。", - "xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationTitle": "数据视图 {dataViewTitle} 并非基于时间序列", "xpack.dataVisualizer.index.errorLoadingDataMessage": "加载索引 {index} 中的数据时出错。{message}。", "xpack.dataVisualizer.index.lensChart.averageOfLabel": "{fieldName} 的平均值", "xpack.dataVisualizer.index.lensChart.chartTitle": "{fieldName} 的 Lens", @@ -11935,7 +11934,6 @@ "xpack.dataVisualizer.index.dataViewManagement.actionsPopoverLabel": "数据视图设置", "xpack.dataVisualizer.index.dataViewManagement.addFieldButton": "将字段添加到数据视图", "xpack.dataVisualizer.index.dataViewManagement.manageFieldButton": "管理数据视图字段", - "xpack.dataVisualizer.index.dataViewNotBasedOnTimeSeriesNotificationDescription": "仅针对基于时间的索引运行异常检测", "xpack.dataVisualizer.index.embeddableErrorDescription": "加载可嵌入对象时出现错误。请检查所有必需的输入是否有效。", "xpack.dataVisualizer.index.embeddableErrorTitle": "加载可嵌入对象时出错", "xpack.dataVisualizer.index.embeddableNoResultsMessage": "找不到结果", diff --git a/yarn.lock b/yarn.lock index c00213e71e0e6..b0b88d71ac615 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4789,6 +4789,10 @@ version "0.0.0" uid "" +"@kbn/ml-in-memory-table@link:x-pack/packages/ml/in_memory_table": + version "0.0.0" + uid "" + "@kbn/ml-is-defined@link:x-pack/packages/ml/is_defined": version "0.0.0" uid "" From 9900c0875cdb8b29a9f4681cbb831f1ecca49202 Mon Sep 17 00:00:00 2001 From: Adam Demjen Date: Mon, 31 Jul 2023 11:26:00 -0400 Subject: [PATCH 24/33] [Enterprise Search] Switch ML inference config to use the multi field selector (#162657) ## Summary Switching over to the multi field selector component for non-ELSER pipelines. We're also cleaning up obsolete code: - Remove `sourceField`, `destinationField`, `inferenceConfig` references - Remove generation of field mappings and full pipeline definition from `sourceField` and `destinationField` ![field_config_non_elser](https://github.com/elastic/kibana/assets/14224983/4f5910dc-1347-4293-8dbc-113d3c70b799) Use cases tested manually: - Create ELSER pipeline - Create non-ELSER pipeline - Create non-ELSER pipeline with custom target field name - Attach ELSER pipeline - Attach non-ELSER pipeline ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - ~[ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials~ - will do in a separate PR - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../ml_inference_pipeline/index.test.ts | 18 +- .../common/ml_inference_pipeline/index.ts | 14 +- .../common/types/pipelines.ts | 12 +- .../pipelines/create_ml_inference_pipeline.ts | 7 +- .../__mocks__/ml_inference_logic.mock.ts | 2 - .../add_inference_pipeline_flyout.test.tsx | 4 - .../ml_inference/configure_fields.test.tsx | 12 +- .../ml_inference/configure_fields.tsx | 23 +-- .../ml_inference/ml_inference_logic.test.ts | 78 ++++----- .../ml_inference/ml_inference_logic.ts | 23 +-- .../single_field_selector.test.tsx | 87 ---------- .../ml_inference/single_field_selector.tsx | 145 ---------------- .../pipelines/ml_inference/test_pipeline.tsx | 4 +- .../ml_inference/test_pipeline_logic.test.ts | 6 - .../pipelines/ml_inference/types.ts | 5 +- .../pipelines/ml_inference/utils.ts | 11 +- .../create_ml_inference_pipeline.test.ts | 86 +--------- .../create_ml_inference_pipeline.ts | 39 +---- .../create_pipeline_definitions.test.ts | 159 ------------------ .../pipelines/create_pipeline_definitions.ts | 38 ----- .../routes/enterprise_search/indices.test.ts | 80 +-------- .../routes/enterprise_search/indices.ts | 46 ----- .../translations/translations/fr-FR.json | 6 - .../translations/translations/ja-JP.json | 6 - .../translations/translations/zh-CN.json | 6 - 25 files changed, 67 insertions(+), 850 deletions(-) delete mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.test.tsx delete mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.tsx diff --git a/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.test.ts b/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.test.ts index 5cc0bdad3e121..7af18f59e53a9 100644 --- a/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.test.ts +++ b/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.test.ts @@ -65,34 +65,34 @@ describe('getMlModelTypesForModelConfig lib function', () => { }); describe('getRemoveProcessorForInferenceType lib function', () => { - const destinationField = 'dest'; + const targetField = 'ml.inference.target'; it('should return expected value for TEXT_CLASSIFICATION', () => { const inferenceType = SUPPORTED_PYTORCH_TASKS.TEXT_CLASSIFICATION; const expected: IngestRemoveProcessor = { - field: destinationField, + field: targetField, ignore_missing: true, }; - expect(getRemoveProcessorForInferenceType(destinationField, inferenceType)).toEqual(expected); + expect(getRemoveProcessorForInferenceType(targetField, inferenceType)).toEqual(expected); }); it('should return expected value for TEXT_EMBEDDING', () => { const inferenceType = SUPPORTED_PYTORCH_TASKS.TEXT_EMBEDDING; const expected: IngestRemoveProcessor = { - field: destinationField, + field: targetField, ignore_missing: true, }; - expect(getRemoveProcessorForInferenceType(destinationField, inferenceType)).toEqual(expected); + expect(getRemoveProcessorForInferenceType(targetField, inferenceType)).toEqual(expected); }); it('should return undefined for unknown inferenceType', () => { const inferenceType = 'wrongInferenceType'; - expect(getRemoveProcessorForInferenceType(destinationField, inferenceType)).toBeUndefined(); + expect(getRemoveProcessorForInferenceType(targetField, inferenceType)).toBeUndefined(); }); }); @@ -377,10 +377,9 @@ describe('parseMlInferenceParametersFromPipeline', () => { ], }) ).toEqual({ - destination_field: 'test', model_id: 'test-model', pipeline_name: 'unit-test', - source_field: 'body', + pipeline_definition: {}, field_mappings: [ { sourceField: 'body', @@ -414,10 +413,9 @@ describe('parseMlInferenceParametersFromPipeline', () => { ], }) ).toEqual({ - destination_field: 'body', model_id: 'test-model', pipeline_name: 'unit-test', - source_field: 'body', + pipeline_definition: {}, field_mappings: [ { sourceField: 'body', diff --git a/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.ts b/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.ts index 7870b22977a34..486e2fafbb081 100644 --- a/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.ts +++ b/x-pack/plugins/enterprise_search/common/ml_inference_pipeline/index.ts @@ -22,7 +22,7 @@ import { import { MlInferencePipeline, - CreateMlInferencePipelineParameters, + CreateMLInferencePipeline, TrainedModelState, InferencePipelineInferenceConfig, } from '../types/pipelines'; @@ -215,7 +215,7 @@ export const formatPipelineName = (rawName: string) => export const parseMlInferenceParametersFromPipeline = ( name: string, pipeline: IngestPipeline -): CreateMlInferencePipelineParameters | null => { +): CreateMLInferencePipeline | null => { const inferenceProcessors = pipeline?.processors ?.filter((p) => p.inference) .map((p) => p.inference) as IngestInferenceProcessor[]; @@ -239,12 +239,9 @@ export const parseMlInferenceParametersFromPipeline = ( return fieldMappings.length === 0 ? null : { - destination_field: fieldMappings[0].targetField // Backward compatibility - TODO: remove after multi-field selector is implemented for all inference types - ? stripMlInferencePrefix(fieldMappings[0].targetField) - : '', model_id: inferenceProcessors[0].model_id, pipeline_name: name, - source_field: fieldMappings[0].sourceField, // Backward compatibility - TODO: remove after multi-field selector is implemented for all inference types + pipeline_definition: {}, field_mappings: fieldMappings, }; }; @@ -278,8 +275,3 @@ export const parseModelStateReasonFromStats = (trainedModelStats?: Partial fieldName.startsWith(ML_INFERENCE_PREFIX) ? fieldName : `${ML_INFERENCE_PREFIX}${fieldName}`; - -const stripMlInferencePrefix = (fieldName: string) => - fieldName.startsWith(ML_INFERENCE_PREFIX) - ? fieldName.replace(ML_INFERENCE_PREFIX, '') - : fieldName; diff --git a/x-pack/plugins/enterprise_search/common/types/pipelines.ts b/x-pack/plugins/enterprise_search/common/types/pipelines.ts index b94249d33af2b..6ea8e6c46e3f8 100644 --- a/x-pack/plugins/enterprise_search/common/types/pipelines.ts +++ b/x-pack/plugins/enterprise_search/common/types/pipelines.ts @@ -74,18 +74,8 @@ export interface DeleteMlInferencePipelineResponse { updated?: string; } -export interface CreateMlInferencePipelineParameters { - destination_field?: string; - inference_config?: InferencePipelineInferenceConfig; - model_id: string; - pipeline_name: string; - source_field: string; - field_mappings: FieldMapping[]; -} - -export interface CreateMLInferencePipelineDefinition { +export interface CreateMLInferencePipeline { field_mappings: FieldMapping[]; - inference_config?: InferencePipelineInferenceConfig; model_id: string; pipeline_definition: MlInferencePipeline; pipeline_name: string; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/pipelines/create_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/pipelines/create_ml_inference_pipeline.ts index 8b7c990a14fc0..2a3780d343a2e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/pipelines/create_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/pipelines/create_ml_inference_pipeline.ts @@ -7,9 +7,8 @@ import { FieldMapping } from '../../../../../common/ml_inference_pipeline'; import { - CreateMLInferencePipelineDefinition, + CreateMLInferencePipeline, MlInferencePipeline, - InferencePipelineInferenceConfig, } from '../../../../../common/types/pipelines'; import { createApiLogic } from '../../../shared/api_logic/create_api_logic'; import { HttpLogic } from '../../../shared/http'; @@ -17,7 +16,6 @@ import { HttpLogic } from '../../../shared/http'; export interface CreateMlInferencePipelineApiLogicArgs { fieldMappings: FieldMapping[]; indexName: string; - inferenceConfig?: InferencePipelineInferenceConfig; modelId: string; pipelineDefinition: MlInferencePipeline; pipelineName: string; @@ -31,9 +29,8 @@ export const createMlInferencePipeline = async ( args: CreateMlInferencePipelineApiLogicArgs ): Promise => { const route = `/internal/enterprise_search/indices/${args.indexName}/ml_inference/pipeline_processors`; - const params: CreateMLInferencePipelineDefinition = { + const params: CreateMLInferencePipeline = { field_mappings: args.fieldMappings, - inference_config: args.inferenceConfig, model_id: args.modelId, pipeline_definition: args.pipelineDefinition, pipeline_name: args.pipelineName, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/__mocks__/ml_inference_logic.mock.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/__mocks__/ml_inference_logic.mock.ts index 65e3685ec9b42..6731dd0b96b78 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/__mocks__/ml_inference_logic.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/__mocks__/ml_inference_logic.mock.ts @@ -8,10 +8,8 @@ export const mockMlInferenceValues: any = { addInferencePipelineModal: { configuration: { - destinationField: '', modelID: '', pipelineName: '', - sourceField: '', }, indexName: '', step: 0, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx index 807e2f56d451f..340bd77160262 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx @@ -488,11 +488,9 @@ describe('AddInferencePipelineFlyout', () => { ...DEFAULT_VALUES.addInferencePipelineModal, step: AddInferencePipelineSteps.Review, configuration: { - destinationField: 'test', existingPipeline: false, modelID: 'test-model', pipelineName: 'my-test-pipeline', - sourceField: 'body', }, }, }); @@ -514,11 +512,9 @@ describe('AddInferencePipelineFlyout', () => { ...DEFAULT_VALUES.addInferencePipelineModal, step: AddInferencePipelineSteps.Review, configuration: { - destinationField: 'test', existingPipeline: true, modelID: 'test-model', pipelineName: 'my-test-pipeline', - sourceField: 'body', }, }, }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.test.tsx index ef2101c1de7da..58cc3c3c5c771 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.test.tsx @@ -13,7 +13,6 @@ import { shallow } from 'enzyme'; import { ConfigureFields } from './configure_fields'; import { MultiFieldMapping, SelectedFieldMappings } from './multi_field_selector'; -import { SingleFieldMapping } from './single_field_selector'; describe('ConfigureFields', () => { beforeEach(() => { @@ -26,20 +25,12 @@ describe('ConfigureFields', () => { addInferencePipelineModal: { configuration: { existingPipeline: false } }, }; - it('renders single field selector component if non-text expansion model is selected', () => { - setMockValues(mockValues); - const wrapper = shallow(); - expect(wrapper.find(SingleFieldMapping)).toHaveLength(1); - expect(wrapper.find(MultiFieldMapping)).toHaveLength(0); - expect(wrapper.find(SelectedFieldMappings)).toHaveLength(0); - }); - it('renders multi-field selector components if text expansion model is selected', () => { + it('renders multi-field selector components', () => { setMockValues({ ...mockValues, isTextExpansionModelSelected: true, }); const wrapper = shallow(); - expect(wrapper.find(SingleFieldMapping)).toHaveLength(0); expect(wrapper.find(MultiFieldMapping)).toHaveLength(1); expect(wrapper.find(SelectedFieldMappings)).toHaveLength(1); }); @@ -50,7 +41,6 @@ describe('ConfigureFields', () => { addInferencePipelineModal: { configuration: { existingPipeline: true } }, }); const wrapper = shallow(); - expect(wrapper.find(SingleFieldMapping)).toHaveLength(0); expect(wrapper.find(MultiFieldMapping)).toHaveLength(0); expect(wrapper.find(SelectedFieldMappings)).toHaveLength(1); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.tsx index c84266cc6cc03..18684b5b1637e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_fields.tsx @@ -16,11 +16,9 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { InferenceConfiguration } from './inference_config'; import { MLInferenceLogic } from './ml_inference_logic'; import { MultiFieldMapping, SelectedFieldMappings } from './multi_field_selector'; -import { SingleFieldMapping } from './single_field_selector'; export const ConfigureFields: React.FC = () => { const { - isTextExpansionModelSelected, addInferencePipelineModal: { configuration }, } = useValues(MLInferenceLogic); const areInputsDisabled = configuration.existingPipeline !== false; @@ -75,24 +73,9 @@ export const ConfigureFields: React.FC = () => { - {isTextExpansionModelSelected ? ( - <> - {areInputsDisabled ? ( - <> - ) : ( - <> - - - - )} - - - ) : ( - <> - - - - )} + {areInputsDisabled || } + + ); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts index 0d85cd3428c68..a725371de0242 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.test.ts @@ -43,9 +43,9 @@ const DEFAULT_VALUES: MLInferenceProcessorsValues = { existingPipeline: undefined, existingInferencePipelines: [], formErrors: { + fieldMappings: 'Field is required.', modelID: 'Field is required.', pipelineName: 'Field is required.', - sourceField: 'Field is required.', }, index: null, isConfigureStepValid: false, @@ -336,11 +336,9 @@ describe('MlInferenceLogic', () => { describe('mlInferencePipeline', () => { it('returns undefined when configuration is invalid', () => { MLInferenceLogic.actions.setInferencePipelineConfiguration({ - destinationField: '', modelID: '', - pipelineName: 'unit-test', - sourceField: '', - fieldMappings: [], + pipelineName: '', // Invalid + fieldMappings: [], // Invalid targetField: '', }); @@ -349,11 +347,14 @@ describe('MlInferenceLogic', () => { it('generates inference pipeline', () => { MLModelsApiLogic.actions.apiSuccess([nerModel]); MLInferenceLogic.actions.setInferencePipelineConfiguration({ - destinationField: '', modelID: nerModel.model_id, pipelineName: 'unit-test', - sourceField: 'body', - fieldMappings: [], + fieldMappings: [ + { + sourceField: 'body', + targetField: 'ml.inference.body', + }, + ], targetField: '', }); @@ -362,10 +363,8 @@ describe('MlInferenceLogic', () => { it('returns undefined when existing pipeline not yet selected', () => { MLInferenceLogic.actions.setInferencePipelineConfiguration({ existingPipeline: true, - destinationField: '', modelID: '', pipelineName: '', - sourceField: '', fieldMappings: [], targetField: '', }); @@ -382,11 +381,14 @@ describe('MlInferenceLogic', () => { }); MLInferenceLogic.actions.setInferencePipelineConfiguration({ existingPipeline: true, - destinationField: '', modelID: '', pipelineName: 'unit-test', - sourceField: '', - fieldMappings: [], + fieldMappings: [ + { + sourceField: 'body', + targetField: 'ml.inference.body', + }, + ], targetField: '', }); expect(MLInferenceLogic.values.mlInferencePipeline).not.toBeUndefined(); @@ -494,8 +496,8 @@ describe('MlInferenceLogic', () => { it('has errors when configuration is empty', () => { expect(MLInferenceLogic.values.formErrors).toEqual({ modelID: 'Field is required.', + fieldMappings: 'Field is required.', pipelineName: 'Field is required.', - sourceField: 'Field is required.', }); }); it('has error for invalid pipeline names', () => { @@ -503,7 +505,12 @@ describe('MlInferenceLogic', () => { ...MLInferenceLogic.values.addInferencePipelineModal.configuration, modelID: 'unit-test-model', existingPipeline: false, - sourceField: 'body', + fieldMappings: [ + { + sourceField: 'body', + targetField: 'ml.inference.body', + }, + ], pipelineName: 'Invalid Pipeline Name', }); const expectedErrors = { @@ -528,7 +535,12 @@ describe('MlInferenceLogic', () => { pipelineName: 'unit-test-pipeline', modelID: 'unit-test-model', existingPipeline: false, - sourceField: 'body', + fieldMappings: [ + { + sourceField: 'body', + targetField: 'ml.inference.body', + }, + ], }); MLInferenceLogic.actions.fetchPipelineSuccess({ 'mock-pipeline': {}, @@ -546,10 +558,8 @@ describe('MlInferenceLogic', () => { const mockModelConfiguration = { ...DEFAULT_VALUES.addInferencePipelineModal, configuration: { - destinationField: 'mock_destination_field', modelID: 'mock-model-id', pipelineName: 'mock-pipeline-name', - sourceField: 'mock_text_field', }, indexName: 'my-index-123', }; @@ -561,6 +571,7 @@ describe('MlInferenceLogic', () => { configuration: { ...mockModelConfiguration.configuration, modelID: textExpansionModel.model_id, + fieldMappings: [], }, }, }); @@ -589,37 +600,6 @@ describe('MlInferenceLogic', () => { pipelineName: mockModelConfiguration.configuration.pipelineName, }); }); - - it('calls makeCreatePipelineRequest with passed pipelineDefinition and default fieldMappings', () => { - mount({ - ...DEFAULT_VALUES, - addInferencePipelineModal: { - ...mockModelConfiguration, - configuration: { - ...mockModelConfiguration.configuration, - modelID: nerModel.model_id, - }, - }, - }); - jest.spyOn(MLInferenceLogic.actions, 'makeCreatePipelineRequest'); - - MLModelsApiLogic.actions.apiSuccess([nerModel]); - MLInferenceLogic.actions.createPipeline(); - - expect(MLInferenceLogic.actions.makeCreatePipelineRequest).toHaveBeenCalledWith({ - indexName: mockModelConfiguration.indexName, - inferenceConfig: undefined, - fieldMappings: [ - { - sourceField: mockModelConfiguration.configuration.sourceField, - targetField: `ml.inference.${mockModelConfiguration.configuration.destinationField}`, - }, - ], - modelId: nerModel.model_id, - pipelineDefinition: expect.any(Object), // Generation logic is tested elsewhere - pipelineName: mockModelConfiguration.configuration.pipelineName, - }); - }); }); describe('startTextExpansionModelSuccess', () => { it('fetches ml models', () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.ts index f67d764a97bb0..4d3747f79adc1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/ml_inference_logic.ts @@ -89,10 +89,8 @@ import { } from './utils'; export const EMPTY_PIPELINE_CONFIGURATION: InferencePipelineConfiguration = { - destinationField: '', modelID: '', pipelineName: '', - sourceField: '', targetField: '', }; @@ -308,14 +306,7 @@ export const MLInferenceLogic = kea< actions.makeCreatePipelineRequest({ indexName, - inferenceConfig: configuration.inferenceConfig, - fieldMappings: configuration.fieldMappings || [ - // Temporary while we're using single fields for non-ELSER pipelines - { - sourceField: configuration.sourceField, - targetField: getMlInferencePrefixedFieldName(configuration.destinationField), - }, - ], + fieldMappings: configuration.fieldMappings ?? [], modelId: configuration.modelID, pipelineDefinition: mlInferencePipeline!, pipelineName: configuration.pipelineName, @@ -327,13 +318,11 @@ export const MLInferenceLogic = kea< const params = parseMlInferenceParametersFromPipeline(pipelineName, pipeline); if (params === null) return; actions.setInferencePipelineConfiguration({ - destinationField: params.destination_field ?? '', existingPipeline: true, modelID: params.model_id, pipelineName, - sourceField: params.source_field, fieldMappings: params.field_mappings, - targetField: params.destination_field ?? '', + targetField: '', }); }, setIndexName: ({ indexName }) => { @@ -545,13 +534,7 @@ export const MLInferenceLogic = kea< return generateMlInferencePipelineBody({ model, pipelineName: configuration.pipelineName, - fieldMappings: configuration.fieldMappings || [ - { - sourceField: configuration.sourceField, - targetField: - configuration.destinationField || formatPipelineName(configuration.pipelineName), - }, - ], + fieldMappings: configuration.fieldMappings ?? [], inferenceConfig: configuration.inferenceConfig, }); }, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.test.tsx deleted file mode 100644 index ce4a9629c4542..0000000000000 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.test.tsx +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { setMockValues } from '../../../../../__mocks__/kea_logic'; - -import React from 'react'; - -import { shallow } from 'enzyme'; - -import { EuiFieldText, EuiSelect } from '@elastic/eui'; - -import { SingleFieldMapping } from './single_field_selector'; - -const DEFAULT_VALUES = { - addInferencePipelineModal: { - configuration: { - sourceField: 'my-source-field', - destinationField: 'my-target-field', - }, - }, - formErrors: {}, - sourceFields: ['my-source-field1', 'my-source-field2', 'my-source-field3'], - supportedMLModels: [], -}; - -describe('SingleFieldMapping', () => { - beforeEach(() => { - jest.clearAllMocks(); - setMockValues({}); - }); - it('renders source field selector and target field text field', () => { - setMockValues(DEFAULT_VALUES); - const wrapper = shallow(); - - expect(wrapper.find(EuiSelect)).toHaveLength(1); - const select = wrapper.find(EuiSelect); - expect(select.prop('options')).toEqual([ - { - disabled: true, - text: 'Select a schema field', - value: '', - }, - { - text: 'my-source-field1', - value: 'my-source-field1', - }, - { - text: 'my-source-field2', - value: 'my-source-field2', - }, - { - text: 'my-source-field3', - value: 'my-source-field3', - }, - ]); - expect(select.prop('value')).toEqual('my-source-field'); - - expect(wrapper.find(EuiFieldText)).toHaveLength(1); - const textField = wrapper.find(EuiFieldText); - expect(textField.prop('value')).toEqual('my-target-field'); - }); - it('disables inputs when selecting an existing pipeline', () => { - setMockValues({ - ...DEFAULT_VALUES, - addInferencePipelineModal: { - ...DEFAULT_VALUES.addInferencePipelineModal, - configuration: { - ...DEFAULT_VALUES.addInferencePipelineModal.configuration, - existingPipeline: true, - }, - }, - }); - const wrapper = shallow(); - - expect(wrapper.find(EuiSelect)).toHaveLength(1); - const select = wrapper.find(EuiSelect); - expect(select.prop('disabled')).toBe(true); - - expect(wrapper.find(EuiFieldText)).toHaveLength(1); - const textField = wrapper.find(EuiFieldText); - expect(textField.prop('disabled')).toBe(true); - }); -}); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.tsx deleted file mode 100644 index 354db83afbc34..0000000000000 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/single_field_selector.tsx +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; - -import { useValues, useActions } from 'kea'; - -import { - EuiFieldText, - EuiFlexGroup, - EuiFlexItem, - EuiFormRow, - EuiLink, - EuiSelect, -} from '@elastic/eui'; - -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; - -import { docLinks } from '../../../../../shared/doc_links'; - -import { IndexViewLogic } from '../../index_view_logic'; - -import { MLInferenceLogic } from './ml_inference_logic'; -import { TargetFieldHelpText } from './target_field_help_text'; - -const NoSourceFieldsError: React.FC = () => ( - - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error.docLink', - { defaultMessage: 'Learn more about field mapping' } - )} - - ), - }} - /> -); - -export const SingleFieldMapping: React.FC = () => { - const { - addInferencePipelineModal: { configuration }, - formErrors, - supportedMLModels, - sourceFields, - } = useValues(MLInferenceLogic); - const { setInferencePipelineConfiguration } = useActions(MLInferenceLogic); - const { ingestionMethod } = useValues(IndexViewLogic); - - const { destinationField, modelID, pipelineName, sourceField } = configuration; - const isEmptySourceFields = (sourceFields?.length ?? 0) === 0; - const areInputsDisabled = configuration.existingPipeline !== false; - const selectedModel = supportedMLModels.find((model) => model.model_id === modelID); - return ( - <> - - - } - isInvalid={isEmptySourceFields} - > - ({ - text: field, - value: field, - })) ?? []), - ]} - onChange={(e) => - setInferencePipelineConfiguration({ - ...configuration, - sourceField: e.target.value, - }) - } - /> - - - - - ) - } - error={formErrors.destinationField} - isInvalid={formErrors.destinationField !== undefined} - fullWidth - > - - setInferencePipelineConfiguration({ - ...configuration, - destinationField: e.target.value, - }) - } - fullWidth - /> - - - - - ); -}; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx index 5d6332b7c0475..d734d1bf9350b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx @@ -34,7 +34,7 @@ import './add_inference_pipeline_flyout.scss'; export const TestPipeline: React.FC = () => { const { addInferencePipelineModal: { - configuration: { sourceField, fieldMappings }, + configuration: { fieldMappings }, indexName, }, getDocumentsErr, @@ -168,7 +168,7 @@ export const TestPipeline: React.FC = () => { `"${fieldMapping.sourceField}": "${sampleFieldValue}"` ) .join(', ') - : `"${sourceField}":"${sampleFieldValue}"` + : `"my_field": "${sampleFieldValue}"` }}}]` ), null, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline_logic.test.ts index 83bf8f7eef545..61685bfeb0420 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline_logic.test.ts @@ -21,10 +21,8 @@ import { AddInferencePipelineSteps } from './types'; const DEFAULT_VALUES: TestPipelineValues = { addInferencePipelineModal: { configuration: { - destinationField: '', modelID: '', pipelineName: '', - sourceField: '', targetField: '', }, indexName: '', @@ -67,10 +65,8 @@ describe('TestPipelineLogic', () => { jest.clearAllMocks(); mockMlInferenceValues.addInferencePipelineModal = { configuration: { - destinationField: '', modelID: '', pipelineName: '', - sourceField: '', targetField: '', }, indexName: '', @@ -123,10 +119,8 @@ describe('TestPipelineLogic', () => { describe('simulatePipeline', () => { const mockModelConfiguration = { configuration: { - destinationField: '', modelID: nerModel.model_id, pipelineName: 'mock-pipeline-name', - sourceField: 'mock_text_field', }, indexName: 'my-index-123', }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts index 9754a378e7473..4e680d7dfba40 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/types.ts @@ -10,21 +10,18 @@ import { FieldMapping } from '../../../../../../../common/ml_inference_pipeline' import { InferencePipelineInferenceConfig } from '../../../../../../../common/types/pipelines'; export interface InferencePipelineConfiguration { - destinationField: string; existingPipeline?: boolean; inferenceConfig?: InferencePipelineInferenceConfig; modelID: string; pipelineName: string; - sourceField: string; fieldMappings?: FieldMapping[]; targetField: string; } export interface AddInferencePipelineFormErrors { - destinationField?: string; modelID?: string; + fieldMappings?: string; pipelineName?: string; - sourceField?: string; } export enum AddInferencePipelineSteps { diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts index 332f885a3543d..e70468d684279 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/utils.ts @@ -71,16 +71,9 @@ export const validateInferencePipelineFields = ( config: InferencePipelineConfiguration ): AddInferencePipelineFormErrors => { const errors: AddInferencePipelineFormErrors = {}; - - // If there are field mappings, we don't need to validate the single source field - if (config.fieldMappings && Object.keys(config.fieldMappings).length > 0) { - return errors; + if ((config.fieldMappings ?? []).length === 0) { + errors.fieldMappings = FIELD_REQUIRED_ERROR; } - - if (config.sourceField.trim().length === 0) { - errors.sourceField = FIELD_REQUIRED_ERROR; - } - return errors; }; diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.test.ts index 1603cd05c54a5..62fc7935b169a 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.test.ts @@ -23,9 +23,7 @@ const mockClient = { describe('createMlInferencePipeline lib function', () => { const pipelineName = 'my-pipeline'; - const modelId = 'my-model-id'; - const sourceField = 'my-source-field'; - const destinationField = 'my-dest-field'; + const pipelineDefinition = { processors: [] }; const inferencePipelineGeneratedName = getPrefixedInferencePipelineProcessorName(pipelineName); mockClient.ml.getTrainedModels.mockImplementation(() => @@ -58,11 +56,7 @@ describe('createMlInferencePipeline lib function', () => { const actualResult = await createMlInferencePipeline( pipelineName, - undefined, - modelId, - sourceField, - destinationField, - undefined, // Omitted inference config + pipelineDefinition, mockClient as unknown as ElasticsearchClient ); @@ -73,11 +67,7 @@ describe('createMlInferencePipeline lib function', () => { it('should convert spaces to underscores in the pipeline name', async () => { await createMlInferencePipeline( 'my pipeline with spaces ', - undefined, - modelId, - sourceField, - destinationField, - undefined, // Omitted inference config + pipelineDefinition, mockClient as unknown as ElasticsearchClient ); @@ -88,70 +78,6 @@ describe('createMlInferencePipeline lib function', () => { ); }); - it('should default the destination field to the pipeline name', async () => { - mockClient.ingest.getPipeline.mockImplementation(() => Promise.reject({ statusCode: 404 })); // Pipeline does not exist - mockClient.ingest.putPipeline.mockImplementation(() => Promise.resolve({ acknowledged: true })); - - await createMlInferencePipeline( - pipelineName, - undefined, - modelId, - sourceField, - undefined, // Omitted destination field - undefined, // Omitted inference config - mockClient as unknown as ElasticsearchClient - ); - - // Verify the object passed to pipeline creation contains the default target field name - expect(mockClient.ingest.putPipeline).toHaveBeenCalledWith( - expect.objectContaining({ - processors: expect.arrayContaining([ - expect.objectContaining({ - inference: expect.objectContaining({ - target_field: `ml.inference.${pipelineName}`, - }), - }), - ]), - }) - ); - }); - - it('should set inference config when provided', async () => { - mockClient.ingest.getPipeline.mockImplementation(() => Promise.reject({ statusCode: 404 })); // Pipeline does not exist - mockClient.ingest.putPipeline.mockImplementation(() => Promise.resolve({ acknowledged: true })); - - await createMlInferencePipeline( - pipelineName, - undefined, - modelId, - sourceField, - destinationField, - { - zero_shot_classification: { - labels: ['foo', 'bar'], - }, - }, - mockClient as unknown as ElasticsearchClient - ); - - // Verify the object passed to pipeline creation contains the default target field name - expect(mockClient.ingest.putPipeline).toHaveBeenCalledWith( - expect.objectContaining({ - processors: expect.arrayContaining([ - expect.objectContaining({ - inference: expect.objectContaining({ - inference_config: { - zero_shot_classification: { - labels: ['foo', 'bar'], - }, - }, - }), - }), - ]), - }) - ); - }); - it('should throw an error without creating the pipeline if it already exists', () => { mockClient.ingest.getPipeline.mockImplementation(() => Promise.resolve({ @@ -161,11 +87,7 @@ describe('createMlInferencePipeline lib function', () => { const actualResult = createMlInferencePipeline( pipelineName, - undefined, - modelId, - sourceField, - destinationField, - undefined, // Omitted inference config + pipelineDefinition, mockClient as unknown as ElasticsearchClient ); diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts index 21091fec67167..bdcc4eb2932d9 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/pipelines/ml_inference/pipeline_processors/create_ml_inference_pipeline.ts @@ -8,16 +8,14 @@ import { IngestGetPipelineResponse, IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { FieldMapping, formatPipelineName } from '../../../../../../common/ml_inference_pipeline'; +import { FieldMapping } from '../../../../../../common/ml_inference_pipeline'; import { ErrorCode } from '../../../../../../common/types/error_codes'; import type { PreparePipelineAndIndexForMlInferenceResult, - InferencePipelineInferenceConfig, CreatePipelineResult, } from '../../../../../../common/types/pipelines'; import { addSubPipelineToIndexSpecificMlPipeline } from '../../../../../utils/create_ml_inference_pipeline'; import { getPrefixedInferencePipelineProcessorName } from '../../../../../utils/ml_inference_pipeline_utils'; -import { formatMlPipelineBody } from '../../../../pipelines/create_pipeline_definitions'; import { updateMlInferenceMappings } from '../update_ml_inference_mappings'; /** @@ -28,10 +26,7 @@ import { updateMlInferenceMappings } from '../update_ml_inference_mappings'; * @param pipelineName pipeline name set by the user. * @param pipelineDefinition * @param modelId model ID selected by the user. - * @param sourceField The document field that model will read. - * @param destinationField The document field that the model will write to. * @param fieldMappings The array of objects representing the source field (text) names and target fields (ML output) names - * @param inferenceConfig The configuration for the model. * @param esClient the Elasticsearch Client to use when retrieving pipeline and model details. */ export const preparePipelineAndIndexForMlInference = async ( @@ -39,19 +34,12 @@ export const preparePipelineAndIndexForMlInference = async ( pipelineName: string, pipelineDefinition: IngestPipeline | undefined, modelId: string, - sourceField: string | undefined, - destinationField: string | null | undefined, fieldMappings: FieldMapping[] | undefined, - inferenceConfig: InferencePipelineInferenceConfig | undefined, esClient: ElasticsearchClient ): Promise => { const createPipelineResult = await createMlInferencePipeline( pipelineName, pipelineDefinition, - modelId, - sourceField, - destinationField, - inferenceConfig, esClient ); @@ -77,19 +65,11 @@ export const preparePipelineAndIndexForMlInference = async ( * Creates a Machine Learning Inference pipeline with the given settings, if it doesn't exist yet. * @param pipelineName pipeline name set by the user. * @param pipelineDefinition full definition of the pipeline - * @param modelId model ID selected by the user. - * @param sourceField The document field that model will read. - * @param destinationField The document field that the model will write to. - * @param inferenceConfig The configuration for the model. * @param esClient the Elasticsearch Client to use when retrieving pipeline and model details. */ export const createMlInferencePipeline = async ( pipelineName: string, pipelineDefinition: IngestPipeline | undefined, - modelId: string | undefined, - sourceField: string | undefined, - destinationField: string | null | undefined, - inferenceConfig: InferencePipelineInferenceConfig | undefined, esClient: ElasticsearchClient ): Promise => { const inferencePipelineGeneratedName = getPrefixedInferencePipelineProcessorName(pipelineName); @@ -107,24 +87,15 @@ export const createMlInferencePipeline = async ( throw new Error(ErrorCode.PIPELINE_ALREADY_EXISTS); } - if (!(modelId && sourceField) && !pipelineDefinition) { + // TODO: See if we can defer this error handling to putPipeline() + if (!pipelineDefinition) { throw new Error(ErrorCode.PARAMETER_CONFLICT); } - const mlInferencePipeline = - modelId && sourceField - ? await formatMlPipelineBody( - inferencePipelineGeneratedName, - modelId, - sourceField, - destinationField || formatPipelineName(pipelineName), - inferenceConfig, - esClient - ) - : { ...pipelineDefinition, version: 1 }; await esClient.ingest.putPipeline({ id: inferencePipelineGeneratedName, - ...mlInferencePipeline, + ...pipelineDefinition, + version: 1, }); return { diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts index 82dce66e0ed7d..863134080c0cd 100644 --- a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts @@ -4,12 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { merge } from 'lodash'; - import { ElasticsearchClient } from '@kbn/core/server'; import { createIndexPipelineDefinitions } from './create_pipeline_definitions'; -import { formatMlPipelineBody } from './create_pipeline_definitions'; describe('createIndexPipelineDefinitions util function', () => { const indexName = 'my-index'; @@ -30,159 +27,3 @@ describe('createIndexPipelineDefinitions util function', () => { expect(mockClient.ingest.putPipeline).toHaveBeenCalledTimes(3); }); }); - -describe('formatMlPipelineBody util function', () => { - const pipelineName = 'ml-inference-my-ml-proc'; - const modelId = 'my-model-id'; - const modelInputField = 'my-model-input-field'; - const modelType = 'pytorch'; - const inferenceConfigKey = 'my-model-type'; - const modelTypes = ['pytorch', 'my-model-type']; - const modelVersion = 3; - const sourceField = 'my-source-field'; - const destField = 'my-dest-field'; - - const expectedResult = { - description: '', - processors: [ - { - remove: { - field: `ml.inference.${destField}`, - ignore_missing: true, - }, - }, - { - inference: { - field_map: { - [sourceField]: modelInputField, - }, - model_id: modelId, - target_field: `ml.inference.${destField}`, - on_failure: [ - { - append: { - field: '_source._ingest.inference_errors', - allow_duplicates: false, - value: [ - { - pipeline: pipelineName, - message: `Processor 'inference' in pipeline '${pipelineName}' failed for field '${sourceField}' with message '{{ _ingest.on_failure_message }}'`, - timestamp: '{{{ _ingest.timestamp }}}', - }, - ], - }, - }, - ], - }, - }, - { - append: { - field: '_source._ingest.processors', - value: [ - { - model_version: modelVersion, - pipeline: pipelineName, - processed_timestamp: '{{{ _ingest.timestamp }}}', - types: modelTypes, - }, - ], - }, - }, - ], - version: 1, - }; - - const mockClient = { - ml: { - getTrainedModels: jest.fn(), - }, - }; - - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('should return the pipeline body', async () => { - const mockResponse = { - count: 1, - trained_model_configs: [ - { - inference_config: { - [inferenceConfigKey]: {}, - }, - input: { field_names: [modelInputField] }, - model_id: modelId, - model_type: modelType, - version: modelVersion, - }, - ], - }; - mockClient.ml.getTrainedModels.mockImplementation(() => Promise.resolve(mockResponse)); - const actualResult = await formatMlPipelineBody( - pipelineName, - modelId, - sourceField, - destField, - undefined, - mockClient as unknown as ElasticsearchClient - ); - expect(actualResult).toEqual(expectedResult); - expect(mockClient.ml.getTrainedModels).toHaveBeenCalledTimes(1); - }); - - it('should raise an error if no model found', async () => { - const mockError = new Error('No known trained model with model_id [my-model-id]'); - mockClient.ml.getTrainedModels.mockImplementation(() => Promise.reject(mockError)); - const asyncCall = formatMlPipelineBody( - pipelineName, - modelId, - sourceField, - destField, - undefined, - mockClient as unknown as ElasticsearchClient - ); - await expect(asyncCall).rejects.toThrow(Error); - expect(mockClient.ml.getTrainedModels).toHaveBeenCalledTimes(1); - }); - - it('should insert a placeholder if model has no input fields', async () => { - const expectedResultWithNoInputField = merge({}, expectedResult, { - processors: [ - {}, // append - we'll leave it untouched - { - inference: { - field_map: { - [sourceField]: 'MODEL_INPUT_FIELD', - }, - }, - }, - ], - }); - - const mockResponse = { - count: 1, - trained_model_configs: [ - { - inference_config: { - [inferenceConfigKey]: {}, - }, - input: { field_names: [] }, - model_id: modelId, - model_type: modelType, - version: modelVersion, - }, - ], - }; - mockClient.ml.getTrainedModels.mockImplementation(() => Promise.resolve(mockResponse)); - const actualResult = await formatMlPipelineBody( - pipelineName, - modelId, - sourceField, - destField, - undefined, - mockClient as unknown as ElasticsearchClient - ); - expect(actualResult).toEqual(expectedResultWithNoInputField); - expect(mockClient.ml.getTrainedModels).toHaveBeenCalledTimes(1); - }); -}); diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts index 8655ee6959f85..7900e196fe604 100644 --- a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts +++ b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts @@ -8,11 +8,6 @@ import { IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; -import { generateMlInferencePipelineBody } from '../../../common/ml_inference_pipeline'; -import { - InferencePipelineInferenceConfig, - MlInferencePipeline, -} from '../../../common/types/pipelines'; import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils'; /** @@ -227,36 +222,3 @@ export const createIndexPipelineDefinitions = async ( throw error; } }; - -/** - * Format the body of an ML inference pipeline for a specified model. - * Does not create the pipeline, only returns JSON for the user to preview. - * @param modelId modelId selected by user. - * @param sourceField The document field that model will read. - * @param destinationField The document field that the model will write to. - * @param inferenceConfig The configuration for the model. - * @param esClient the Elasticsearch Client to use when retrieving model details. - */ -export const formatMlPipelineBody = async ( - pipelineName: string, - modelId: string, - sourceField: string, - destinationField: string, - inferenceConfig: InferencePipelineInferenceConfig | undefined, - esClient: ElasticsearchClient -): Promise => { - // This will raise a 404 if model doesn't exist - const models = await esClient.ml.getTrainedModels({ model_id: modelId }); - const model = models.trained_model_configs[0]; - return generateMlInferencePipelineBody({ - inferenceConfig, - model, - pipelineName, - fieldMappings: [ - { - sourceField, - targetField: destinationField, - }, - ], - }); -}; diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts index 4b46eb8d5233a..1a8412ba8703c 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts @@ -293,78 +293,6 @@ describe('Enterprise Search Managed Indices', () => { mockRouter.shouldThrow(request); }); - it('responds with 400 BAD REQUEST with both source/target AND pipeline_definition', async () => { - await mockRouter.callRoute({ - body: { - model_id: 'my-model-id', - pipeline_name: 'my-pipeline-name', - source_field: 'my-source-field', - destination_field: 'my-dest-field', - pipeline_definition: { - processors: [], - }, - }, - params: { indexName: 'my-index-name' }, - }); - - expect(mockRouter.response.customError).toHaveBeenCalledWith( - expect.objectContaining({ - statusCode: 400, - }) - ); - }); - - it('responds with 400 BAD REQUEST with none of source/target/model OR pipeline_definition', async () => { - await mockRouter.callRoute({ - body: { - pipeline_name: 'my-pipeline-name', - }, - params: { indexName: 'my-index-name' }, - }); - - expect(mockRouter.response.customError).toHaveBeenCalledWith( - expect.objectContaining({ - statusCode: 400, - }) - ); - }); - - it('creates an ML inference pipeline from model and source_field', async () => { - (preparePipelineAndIndexForMlInference as jest.Mock).mockImplementationOnce(() => { - return Promise.resolve({ - added_to_parent_pipeline: true, - created_pipeline: true, - mapping_updated: false, - pipeline_id: 'ml-inference-my-pipeline-name', - }); - }); - - await mockRouter.callRoute({ - params: { indexName: 'my-index-name' }, - body: mockRequestBody, - }); - - expect(preparePipelineAndIndexForMlInference).toHaveBeenCalledWith( - 'my-index-name', - mockRequestBody.pipeline_name, - undefined, - mockRequestBody.model_id, - mockRequestBody.source_field, - mockRequestBody.destination_field, - undefined, - undefined, - mockClient.asCurrentUser - ); - - expect(mockRouter.response.ok).toHaveBeenCalledWith({ - body: { - created: 'ml-inference-my-pipeline-name', - mapping_updated: false, - }, - headers: { 'content-type': 'application/json' }, - }); - }); - it('creates an ML inference pipeline from pipeline definition', async () => { (preparePipelineAndIndexForMlInference as jest.Mock).mockImplementationOnce(() => { return Promise.resolve({ @@ -379,10 +307,11 @@ describe('Enterprise Search Managed Indices', () => { params: { indexName: 'my-index-name' }, body: { field_mappings: [], + model_id: mockRequestBody.model_id, pipeline_definition: { processors: [], }, - pipeline_name: 'my-pipeline-name', + pipeline_name: mockRequestBody.pipeline_name, }, }); @@ -392,11 +321,8 @@ describe('Enterprise Search Managed Indices', () => { { processors: [], }, - undefined, - undefined, - undefined, + mockRequestBody.model_id, [], - undefined, mockClient.asCurrentUser ); diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts index f2219b93919c9..b41f391fd66f6 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts @@ -394,21 +394,11 @@ export function registerIndexRoutes({ indexName: schema.string(), }), body: schema.object({ - destination_field: schema.maybe(schema.nullable(schema.string())), field_mappings: schema.maybe( schema.arrayOf( schema.object({ sourceField: schema.string(), targetField: schema.string() }) ) ), - inference_config: schema.maybe( - schema.object({ - zero_shot_classification: schema.maybe( - schema.object({ - labels: schema.arrayOf(schema.string()), - }) - ), - }) - ), model_id: schema.string(), pipeline_definition: schema.maybe( schema.object({ @@ -418,7 +408,6 @@ export function registerIndexRoutes({ }) ), pipeline_name: schema.string(), - source_field: schema.maybe(schema.string()), }), }, }, @@ -430,41 +419,9 @@ export function registerIndexRoutes({ model_id: modelId, pipeline_name: pipelineName, pipeline_definition: pipelineDefinition, - source_field: sourceField, - destination_field: destinationField, - inference_config: inferenceConfig, field_mappings: fieldMappings, } = request.body; - // additional validations - if ((pipelineDefinition || fieldMappings) && (sourceField || destinationField)) { - return createError({ - errorCode: ErrorCode.PARAMETER_CONFLICT, - message: i18n.translate( - 'xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterConflictError', - { - defaultMessage: - 'pipeline_definition and field_mappings should only be provided if source_field and destination_field are not provided', - } - ), - response, - statusCode: 400, - }); - } else if (!((pipelineDefinition && fieldMappings) || (sourceField && modelId))) { - return createError({ - errorCode: ErrorCode.PARAMETER_CONFLICT, - message: i18n.translate( - 'xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterMissingError', - { - defaultMessage: - 'either pipeline_definition AND fieldMappings or source_field AND model_id must be provided', - } - ), - response, - statusCode: 400, - }); - } - try { // Create the sub-pipeline for inference const createPipelineResult = await preparePipelineAndIndexForMlInference( @@ -472,10 +429,7 @@ export function registerIndexRoutes({ pipelineName, pipelineDefinition, modelId, - sourceField, - destinationField, fieldMappings, - inferenceConfig, client.asCurrentUser ); return response.ok({ diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index f0367c0528648..f6dc374bd4fe9 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -12148,7 +12148,6 @@ "xpack.enterpriseSearch.content.indices.deleteIndex.successToast.title": "Votre index {indexName} et toute configuration d'ingestion associée ont été supprimés avec succès", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.description": "Aucun de vos modèles entraînés de Machine Learning ne peut être utilisé par un pipeline d'inférence. {documentationLink}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText": "Les noms de pipeline sont uniques dans un déploiement, et ils peuvent uniquement contenir des lettres, des chiffres, des traits de soulignement et des traits d'union. Cela créera un pipeline nommé {pipelineName}.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error": "La sélection d'un champ source est requise pour la configuration du pipeline, mais cet index n'a pas de mapping de champ. {learnMore}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.default": "Cela attribue un nom au champ qui contient le résultat d'inférence. Votre nom de champ recevra le préfixe \"ml.inference.\". S'il n'est pas défini, le nom par défaut sera \"ml.inference.{pipelineName}\"", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textClassificationModel": "De plus, la valeur_prévue (predicted_value) sera copiée sur \"{fieldName}\" si la probabilité de prédiction (prediction_probability) est supérieure à {probabilityThreshold}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textEmbeddingModel": "De plus, la valeur_prévue (predicted_value) sera copiée sur \"{fieldName}\"", @@ -13586,12 +13585,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.fieldMappings.tableCaption": "Mappings de champs", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.fieldMappings.targetFieldHeader": "Champs cibles", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.selectedFields": "Champs sélectionnés", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error.docLink": "En savoir plus sur le mapping de champs", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.helpText": "Sélectionnez un champ existant ou tapez un nom de champ.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.placeholder": "Sélectionner un champ de schéma", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceFieldLabel": "Champ source", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText": "Ce nom est créé automatiquement en fonction de votre champ source.", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.label": "Champ cible (facultatif)", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetFieldLabel": "Champ cible", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.title": "Sélectionner les mappings de champs", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.description": "Ce pipeline sera créé et injecté en tant que processeur dans votre pipeline par défaut pour cet index. Vous pourrez également utiliser ce nouveau pipeline de façon indépendante.", @@ -14577,8 +14573,6 @@ "xpack.enterpriseSearch.server.routes.createApiIndex.connectorExistsError": "Un connecteur existe déjà pour cet index", "xpack.enterpriseSearch.server.routes.createApiIndex.crawlerExistsError": "Un robot d'indexation existe déjà pour cet index", "xpack.enterpriseSearch.server.routes.createApiIndex.indexExistsError": "L'index existe déjà.", - "xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterConflictError": "pipeline_definition et field_mappings doivent uniquement être fournis si source_field, destination_field et model_id ne sont pas fournis.", - "xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterMissingError": "pipeline_definition ET fieldMappings ou source_field ET model_id doivent être fournis", "xpack.enterpriseSearch.server.routes.createSearchApplication.searchApplciationExistsError": "Le nom de l’application de recherche est déjà pris. Choisissez un autre nom.", "xpack.enterpriseSearch.server.routes.indices.mlInference.pipelineProcessors.pipelineIsInUseError": "Le pipeline d'inférence est utilisé dans le pipeline géré \"{pipelineName}\" d'un autre index", "xpack.enterpriseSearch.server.routes.recreateConnector.connectorExistsError": "Un connecteur existe déjà pour cet index", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index d804e4a837a6b..ce4f83316e42a 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -12162,7 +12162,6 @@ "xpack.enterpriseSearch.content.indices.deleteIndex.successToast.title": "インデックス{indexName}と関連付けられたすべての統合構成が正常に削除されました", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.description": "推論パイプラインで使用できる学習済み機械学習モデルがありません。{documentationLink}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText": "パイプライン名はデプロイ内で一意であり、文字、数字、アンダースコア、ハイフンのみを使用できます。これにより、{pipelineName}という名前のパイプラインが作成されます。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error": "パイプラインのソースフィールドを選択する必要があります。ただし、このインデックスにはフィールドマッピングがありません。{learnMore}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.default": "これは、推論結果を保持するフィールドの名前を指定します。\"ml.inference.\"というプレフィックスが付きます。設定されていない場合は、デフォルトで\"ml.inference.{pipelineName}\"となります", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textClassificationModel": "さらに、predicted_probabilityが{probabilityThreshold}より大きい場合、predicted_valueは\"{fieldName}\"にコピーされます。", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textEmbeddingModel": "さらにpredicted_valueは\"{fieldName}\"にコピーされます。", @@ -13600,12 +13599,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.fieldMappings.tableCaption": "フィールドマッピング", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.fieldMappings.targetFieldHeader": "ターゲットフィールド", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.selectedFields": "選択したフィールド", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error.docLink": "フィールドマッピングの詳細", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.helpText": "既存のフィールドを選択するか、フィールド名を入力してください。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.placeholder": "スキーマフィールドを選択", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceFieldLabel": "ソースフィールド", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText": "この名前は、ソースフィールドに基づいて自動的に作成されます。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.label": "ターゲットフィールド（任意）", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetFieldLabel": "ターゲットフィールド", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.title": "フィールドマッピングを選択", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.description": "このパイプラインが作成され、プロセッサーとしてこのインデックスのデフォルトパイプラインに挿入されます。この新しいパイプラインは単独でも使用できます。", @@ -14591,8 +14587,6 @@ "xpack.enterpriseSearch.server.routes.createApiIndex.connectorExistsError": "このインデックスのコネクターはすでに存在します", "xpack.enterpriseSearch.server.routes.createApiIndex.crawlerExistsError": "このインデックスのクローラーはすでに存在します", "xpack.enterpriseSearch.server.routes.createApiIndex.indexExistsError": "このインデックスはすでに存在します", - "xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterConflictError": "pipeline_definitionおよびfield_mappingsは、source_field、destination_field、およびmodel_idが指定されていない場合にのみ指定してください。", - "xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterMissingError": "pipeline_definitionとfieldMappingsの両方、またはsource_fieldとmodel_idの両方を指定する必要があります。", "xpack.enterpriseSearch.server.routes.createSearchApplication.searchApplciationExistsError": "検索アプリケーション名はすでに取得されています。別の名前を選択してください。", "xpack.enterpriseSearch.server.routes.indices.mlInference.pipelineProcessors.pipelineIsInUseError": "推論パイプラインは、別のインデックスの管理されたパイプライン'{pipelineName}'で使用されています。", "xpack.enterpriseSearch.server.routes.recreateConnector.connectorExistsError": "このインデックスのコネクターはすでに存在します", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 6a7287002e834..402917505fc25 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -12162,7 +12162,6 @@ "xpack.enterpriseSearch.content.indices.deleteIndex.successToast.title": "您的索引 {indexName} 和任何关联的采集配置已成功删除", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.noModels.description": "您没有可供推理管道使用的已训练 Machine Learning 模型。{documentationLink}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText": "管道名称在部署内唯一，并且只能包含字母、数字、下划线和连字符。这会创建名为 {pipelineName} 的管道。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error": "配置管道需要设置源字段，但此索引没有字段映射。{learnMore}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.default": "这会命名存放推理结果的字段。它将加有“ml.inference”前缀，如果未设置，将默认前缀为“ml.inference.{pipelineName}”", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textClassificationModel": "此外，如果 prediction_probability 大于 {probabilityThreshold}，则会将 predicted_value 复制到“{fieldName}”", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textEmbeddingModel": "此外，还会将 predicted_value 复制到“{fieldName}”", @@ -13600,12 +13599,9 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.fieldMappings.tableCaption": "字段映射", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.fieldMappings.targetFieldHeader": "目标字段", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.selectedFields": "选定字段", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.error.docLink": "详细了解字段映射", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.helpText": "选择现有字段或键入字段名称。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceField.placeholder": "选择架构字段", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.sourceFieldLabel": "源字段", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText": "此名称基于您的源字段自动创建。", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.label": "目标字段（可选）", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetFieldLabel": "目标字段", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.title": "选择字段映射", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.description": "将创建此管道并将其作为处理器注入到该索引的默认管道。您还可以独立使用这个新管道。", @@ -14591,8 +14587,6 @@ "xpack.enterpriseSearch.server.routes.createApiIndex.connectorExistsError": "此索引的连接器已存在", "xpack.enterpriseSearch.server.routes.createApiIndex.crawlerExistsError": "此索引的网络爬虫已存在", "xpack.enterpriseSearch.server.routes.createApiIndex.indexExistsError": "此索引已存在", - "xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterConflictError": "如果未提供 source_field、destination_field 和 model_id，应仅提供 pipeline_definition 和 field_mappings", - "xpack.enterpriseSearch.server.routes.createMlInferencePipeline.ParameterMissingError": "必须提供 pipeline_definition 和 fieldMappings，或 source_field 和 model_id", "xpack.enterpriseSearch.server.routes.createSearchApplication.searchApplciationExistsError": "搜索应用程序名称已占用。请选择其他名称。", "xpack.enterpriseSearch.server.routes.indices.mlInference.pipelineProcessors.pipelineIsInUseError": "推理管道已用在不同索引的托管管道“{pipelineName}”中", "xpack.enterpriseSearch.server.routes.recreateConnector.connectorExistsError": "此索引的连接器已存在", From f0050dbc70ab1b4dd4a2a9b3c8b5f66b3b5ec884 Mon Sep 17 00:00:00 2001 From: Rickyanto Ang Date: Mon, 31 Jul 2023 08:38:59 -0700 Subject: [PATCH 25/33] [Cloud Security][Onboarding]GCP Onboarding - Manual (IMPROVEMENTS) (#162434) ## Summary Addressing PR Comments + Improvements from my previous PR (https://github.com/elastic/kibana/pull/161913) in this PR --- .../fleet_extensions/gcp_credential_form.tsx | 137 ++++++++++-------- .../components/fleet_extensions/mocks.ts | 33 ++++- .../policy_template_form.test.tsx | 133 ++++++++++++++++- 3 files changed, 241 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx index eeafb3cc40e00..ea192309310e3 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/gcp_credential_form.tsx @@ -4,7 +4,10 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { useEffect, useState } from 'react'; +import React, { useEffect } from 'react'; +import semverLt from 'semver/functions/lt'; +import semverCoerce from 'semver/functions/coerce'; +import semverValid from 'semver/functions/valid'; import { EuiFieldText, EuiFormRow, @@ -25,6 +28,12 @@ import { RadioGroup } from './csp_boxed_radio_group'; import { getPosturePolicy, NewPackagePolicyPostureInput } from './utils'; import { MIN_VERSION_GCP_CIS } from '../../common/constants'; +export const CIS_GCP_INPUT_FIELDS_TEST_SUBJECTS = { + PROJECT_ID: 'project_id_test_id', + CREDENTIALS_TYPE: 'credentials_type_test_id', + CREDENTIALS_FILE: 'credentials_file_test_id', + CREDENTIALS_JSON: 'credentials_json_test_id', +}; type SetupFormatGCP = 'google_cloud_shell' | 'manual'; const GCPSetupInfoContent = () => ( <> @@ -49,7 +58,7 @@ const GCPSetupInfoContent = () => ( ); -/* NEED TO FIND THE REAL URL HERE LATER*/ +/* NEED TO FIND THE REAL URL HERE LATER */ const DocsLink = ( ); -const CredentialFileText = i18n.translate( - 'xpack.csp.findings.gcpIntegration.gcpInputText.credentialFileText', - { defaultMessage: 'Path to JSON file containing the credentials and key used to subscribe' } -); -const CredentialJSONText = i18n.translate( - 'xpack.csp.findings.gcpIntegration.gcpInputText.credentialJSONText', - { defaultMessage: 'JSON blob containing the credentials and key used to subscribe' } -); - type GcpCredentialsType = 'credentials_file' | 'credentials_json'; type GcpFields = Record; interface GcpInputFields { @@ -90,32 +90,39 @@ const gcpField: GcpInputFields = { type: 'text', }, credentials_file: { - label: i18n.translate('xpack.csp.gcpIntegration.credentialsFileFieldLabel', { - defaultMessage: 'Credentials File', + label: i18n.translate('xpack.csp.findings.gcpIntegration.gcpInputText.credentialFileText', { + defaultMessage: 'Path to JSON file containing the credentials and key used to subscribe', }), type: 'text', }, credentials_json: { - label: i18n.translate('xpack.csp.gcpIntegration.credentialsJSONFieldLabel', { - defaultMessage: 'Credentials JSON', + label: i18n.translate('xpack.csp.findings.gcpIntegration.gcpInputText.credentialJSONText', { + defaultMessage: 'JSON blob containing the credentials and key used to subscribe', }), type: 'text', }, + credentials_type: { + label: i18n.translate( + 'xpack.csp.findings.gcpIntegration.gcpInputText.credentialSelectBoxTitle', + { defaultMessage: 'Credential' } + ), + type: 'text', + }, }, }; const credentialOptionsList = [ { - label: i18n.translate('xpack.csp.gcpIntegration.credentialsFileOption', { + text: i18n.translate('xpack.csp.gcpIntegration.credentialsFileOption', { defaultMessage: 'Credentials File', }), - text: 'Credentials File', + value: 'credentials-file', }, { - label: i18n.translate('xpack.csp.gcpIntegration.credentialsjsonOption', { + text: i18n.translate('xpack.csp.gcpIntegration.credentialsJsonOption', { defaultMessage: 'Credentials JSON', }), - text: 'Credentials JSON', + value: 'credentials-json', }, ]; @@ -140,7 +147,7 @@ const getSetupFormatOptions = (): Array<{ }, ]; -interface Props { +interface GcpFormProps { newPolicy: NewPackagePolicy; input: Extract< NewPackagePolicyPostureInput, @@ -175,12 +182,12 @@ export const GcpCredentialsForm = ({ packageInfo, setIsValid, onChange, -}: Props) => { +}: GcpFormProps) => { const fields = getInputVarsFields(input, gcpField.fields); - + const validSemantic = semverValid(packageInfo.version); + const integrationVersionNumberOnly = semverCoerce(validSemantic) || ''; + const isInvalid = semverLt(integrationVersionNumberOnly, MIN_VERSION_GCP_CIS); useEffect(() => { - const isInvalid = packageInfo.version < MIN_VERSION_GCP_CIS; - setIsValid(!isInvalid); onChange({ @@ -190,7 +197,7 @@ export const GcpCredentialsForm = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [input, packageInfo]); - if (packageInfo.version < MIN_VERSION_GCP_CIS) { + if (isInvalid) { return ( <> @@ -240,52 +247,64 @@ const GcpInputVarFields = ({ fields: Array; onChange: (key: string, value: string) => void; }) => { - const [credentialOption, setCredentialOption] = useState('Credentials File'); - const targetFieldName = (id: string) => { + const getFieldById = (id: keyof GcpInputFields['fields']) => { return fields.find((element) => element.id === id); }; + const projectIdFields = getFieldById('project_id'); + const credentialsTypeFields = getFieldById('credentials_type') || credentialOptionsList[0]; + const credentialFilesFields = getFieldById('credentials_file'); + const credentialJSONFields = getFieldById('credentials_json'); + + const credentialFieldValue = credentialOptionsList[0].value; + const credentialJSONValue = credentialOptionsList[1].value; + return (
- - onChange(targetFieldName('project_id')!.id, event.target.value)} - /> - - - { - setCredentialOption(optionElem.target.value); - }} - /> - - {credentialOption === 'Credentials File' && ( - + {projectIdFields && ( + + onChange(projectIdFields.id, event.target.value)} + /> + + )} + {credentialFilesFields && credentialJSONFields && ( + + { + onChange('credentials_type', optionElem.target.value); + }} + /> + + )} + + {credentialsTypeFields.value === credentialFieldValue && credentialFilesFields && ( + - onChange(targetFieldName('credentials_file')!.id, event.target.value) - } + value={credentialFilesFields.value || ''} + onChange={(event) => onChange(credentialFilesFields.id, event.target.value)} /> )} - {credentialOption === 'Credentials JSON' && ( - + {credentialsTypeFields?.value === credentialJSONValue && credentialJSONFields && ( + - onChange(targetFieldName('credentials_json')!.id, event.target.value) - } + value={credentialJSONFields.value || ''} + onChange={(event) => onChange(credentialJSONFields.id, event.target.value)} /> )} diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/mocks.ts b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/mocks.ts index ce14f80b3d965..6d8b8b648bf20 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/mocks.ts +++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/mocks.ts @@ -18,6 +18,7 @@ import { import type { PostureInput } from '../../../common/types'; export const getMockPolicyAWS = () => getPolicyMock(CLOUDBEAT_AWS, 'cspm', 'aws'); +export const getMockPolicyGCP = () => getPolicyMock(CLOUDBEAT_GCP, 'cspm', 'gcp'); export const getMockPolicyK8s = () => getPolicyMock(CLOUDBEAT_VANILLA, 'kspm', 'self_managed'); export const getMockPolicyEKS = () => getPolicyMock(CLOUDBEAT_EKS, 'kspm', 'eks'); export const getMockPolicyVulnMgmtAWS = () => @@ -79,6 +80,28 @@ export const getMockPackageInfoCspmAWS = (packageVersion = '1.5.0') => { } as PackageInfo; }; +export const getMockPackageInfoCspmGCP = (packageVersion = '1.5.0') => { + return { + version: packageVersion, + name: 'cspm', + policy_templates: [ + { + title: '', + description: '', + name: 'cspm', + inputs: [ + { + type: CLOUDBEAT_GCP, + title: 'GCP', + description: '', + vars: [{}], + }, + ], + }, + ], + } as PackageInfo; +}; + const getPolicyMock = ( type: PostureInput, posture: string, @@ -106,6 +129,12 @@ const getPolicyMock = ( 'aws.credentials.type': { value: 'assume_role', type: 'text' }, }; + const gcpVarsMock = { + project_id: { type: 'text' }, + credentials_file: { type: 'text' }, + credentials_json: { type: 'text' }, + }; + const dataStream = { type: 'logs', dataset: 'cloud_security_posture.findings' }; return { @@ -145,8 +174,8 @@ const getPolicyMock = ( { type: CLOUDBEAT_GCP, policy_template: 'cspm', - enabled: false, - streams: [{ enabled: false, data_stream: dataStream }], + enabled: type === CLOUDBEAT_GCP, + streams: [{ enabled: type === CLOUDBEAT_GCP, data_stream: dataStream, vars: gcpVarsMock }], }, { type: CLOUDBEAT_AZURE, diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx index aa1a26fbeea7f..808a3164fb41c 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.test.tsx @@ -14,9 +14,11 @@ import { import { TestProvider } from '../../test/test_provider'; import { getMockPackageInfoCspmAWS, + getMockPackageInfoCspmGCP, getMockPackageInfoVulnMgmtAWS, getMockPolicyAWS, getMockPolicyEKS, + getMockPolicyGCP, getMockPolicyK8s, getMockPolicyVulnMgmtAWS, } from './mocks'; @@ -28,11 +30,12 @@ import type { } from '@kbn/fleet-plugin/common'; import userEvent from '@testing-library/user-event'; import { getPosturePolicy } from './utils'; -import { CLOUDBEAT_AWS, CLOUDBEAT_EKS } from '../../../common/constants'; +import { CLOUDBEAT_AWS, CLOUDBEAT_EKS, CLOUDBEAT_GCP } from '../../../common/constants'; import { useParams } from 'react-router-dom'; import { createReactQueryResponse } from '../../test/fixtures/react_query'; import { useCspSetupStatusApi } from '../../common/api/use_setup_status_api'; import { usePackagePolicyList } from '../../common/api/use_package_policy_list'; +import { CIS_GCP_INPUT_FIELDS_TEST_SUBJECTS } from './gcp_credential_form'; // mock useParams jest.mock('react-router-dom', () => ({ @@ -974,4 +977,132 @@ describe('', () => { }); }); }); + + describe('GCP Credentials input fields', () => { + it(`renders ${CLOUDBEAT_GCP} Not supported when version is not at least version 1.5.0`, () => { + let policy = getMockPolicyGCP(); + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_type: { value: 'credentials-file' }, + }); + + const { getByText } = render( + + ); + + expect(onChange).toHaveBeenCalledWith({ + isValid: false, + updatedPolicy: policy, + }); + + expect( + getByText( + 'CIS GCP is not supported on the current Integration version, please upgrade your integration to the latest version to use CIS GCP' + ) + ).toBeInTheDocument(); + }); + + it(`renders ${CLOUDBEAT_GCP} Credentials File fields`, () => { + let policy = getMockPolicyGCP(); + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_type: { value: 'credentials-file' }, + }); + + const { getByLabelText, getByRole } = render( + + ); + + expect(getByRole('option', { name: 'Credentials File', selected: true })).toBeInTheDocument(); + + expect( + getByLabelText('Path to JSON file containing the credentials and key used to subscribe') + ).toBeInTheDocument(); + }); + + it(`updates ${CLOUDBEAT_GCP} Credentials File fields`, () => { + let policy = getMockPolicyGCP(); + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_type: { value: 'credentials-file' }, + }); + + const { rerender, getByTestId } = render( + + ); + + userEvent.type(getByTestId(CIS_GCP_INPUT_FIELDS_TEST_SUBJECTS.PROJECT_ID), 'a'); + + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + project_id: { value: 'a' }, + }); + + expect(onChange).toHaveBeenCalledWith({ + isValid: true, + updatedPolicy: policy, + }); + + rerender(); + + userEvent.type(getByTestId(CIS_GCP_INPUT_FIELDS_TEST_SUBJECTS.CREDENTIALS_FILE), 'b'); + + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_file: { value: 'b' }, + }); + + expect(onChange).toHaveBeenNthCalledWith(5, { + isValid: true, + updatedPolicy: policy, + }); + }); + + it(`renders ${CLOUDBEAT_GCP} Credentials JSON fields`, () => { + let policy = getMockPolicyGCP(); + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_type: { value: 'credentials-json' }, + }); + + const { getByLabelText, getByRole } = render( + + ); + + expect(getByRole('option', { name: 'Credentials JSON', selected: true })).toBeInTheDocument(); + + expect( + getByLabelText('JSON blob containing the credentials and key used to subscribe') + ).toBeInTheDocument(); + }); + + it(`updates ${CLOUDBEAT_GCP} Credentials JSON fields`, () => { + let policy = getMockPolicyGCP(); + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_type: { value: 'credentials-json' }, + }); + + const { rerender, getByTestId } = render( + + ); + + userEvent.type(getByTestId(CIS_GCP_INPUT_FIELDS_TEST_SUBJECTS.PROJECT_ID), 'a'); + + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + project_id: { value: 'a' }, + }); + + expect(onChange).toHaveBeenCalledWith({ + isValid: true, + updatedPolicy: policy, + }); + + rerender(); + + userEvent.type(getByTestId(CIS_GCP_INPUT_FIELDS_TEST_SUBJECTS.CREDENTIALS_JSON), 'b'); + + policy = getPosturePolicy(policy, CLOUDBEAT_GCP, { + credentials_json: { value: 'b' }, + }); + + expect(onChange).toHaveBeenNthCalledWith(5, { + isValid: true, + updatedPolicy: policy, + }); + }); + }); }); From 96de1482e09c82bd42dc73eb3fa21da0cd74b2a8 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 31 Jul 2023 17:39:25 +0200 Subject: [PATCH 26/33] [ML] `@kbn/ml-response-stream`: Fix race condition related to throttling. (#162803) Fixes a race condition in the case where a response stream finishes and sets `isRunning` to `false`, but `useThrottle` didn't trigger it's last update yet within the refresh rate. In the case of log rate analysis, `isRunning` could be set to `false` too early and the UI wouldn't consider later throttled updates (for example, setting `loaded=1` which would result in inconsistent UI state). The fix in this case is to return the unthrottled raw data instead of the throttled one as soon as the stream finished. --- .../packages/ml/response_stream/client/use_fetch_stream.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/packages/ml/response_stream/client/use_fetch_stream.ts b/x-pack/packages/ml/response_stream/client/use_fetch_stream.ts index bfd01b607ffac..d2d140f80ee41 100644 --- a/x-pack/packages/ml/response_stream/client/use_fetch_stream.ts +++ b/x-pack/packages/ml/response_stream/client/use_fetch_stream.ts @@ -129,7 +129,10 @@ export function useFetchStream>( return { cancel, - data: dataThrottled, + // To avoid a race condition where the stream already ended but `useThrottle` would + // yet have to trigger another update within the throttling interval, we'll return + // the unthrottled data once the stream is complete. + data: isRunning ? dataThrottled : data, dispatch, errors, isCancelled, From 7b2a00a712540921ceda91faadc9051c1f499e96 Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Mon, 31 Jul 2023 17:57:55 +0200 Subject: [PATCH 27/33] [Serverless] Update left nav, disable ML alerts (#162728) ## Summary This updates the left nav of Serverless Search to match the latest designs, and disables ML alerting rules. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../serverless_search/public/layout/nav.tsx | 11 ++-------- .../serverless_search/server/plugin.ts | 2 +- .../test_suites/search/navigation.ts | 22 +++++-------------- 3 files changed, 8 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/serverless_search/public/layout/nav.tsx b/x-pack/plugins/serverless_search/public/layout/nav.tsx index e1f4fe0ef04d7..fd460f8c95066 100644 --- a/x-pack/plugins/serverless_search/public/layout/nav.tsx +++ b/x-pack/plugins/serverless_search/public/layout/nav.tsx @@ -10,7 +10,6 @@ import { DefaultNavigation, NavigationKibanaProvider, type NavigationTreeDefinition, - getPresets, } from '@kbn/shared-ux-chrome-navigation'; import React from 'react'; import { i18n } from '@kbn/i18n'; @@ -40,7 +39,7 @@ const navigationTree: NavigationTreeDefinition = { title: i18n.translate('xpack.serverlessSearch.nav.devTools', { defaultMessage: 'Dev Tools', }), - children: getPresets('devtools').children[0].children, + children: [{ link: 'dev_tools:console' }, { link: 'dev_tools:searchprofiler' }], }, { id: 'explore', @@ -83,16 +82,14 @@ const navigationTree: NavigationTreeDefinition = { children: [ { title: i18n.translate('xpack.serverlessSearch.nav.content.indices', { - defaultMessage: 'Indices', + defaultMessage: 'Index Management', }), - // TODO: this will be updated to a new Indices page link: 'management:index_management', }, { title: i18n.translate('xpack.serverlessSearch.nav.content.pipelines', { defaultMessage: 'Pipelines', }), - // TODO: this will be updated to a new Pipelines page link: 'management:ingest_pipelines', }, { @@ -117,10 +114,6 @@ const navigationTree: NavigationTreeDefinition = { }, ], }, - { - type: 'navGroup', - ...getPresets('ml'), - }, ], footer: [ { diff --git a/x-pack/plugins/serverless_search/server/plugin.ts b/x-pack/plugins/serverless_search/server/plugin.ts index 767d51ff0c439..d57752e40068c 100644 --- a/x-pack/plugins/serverless_search/server/plugin.ts +++ b/x-pack/plugins/serverless_search/server/plugin.ts @@ -62,7 +62,7 @@ export class ServerlessSearchPlugin registerIndicesRoutes(dependencies); }); - pluginsSetup.ml.setFeaturesEnabled({ ad: false, dfa: false, nlp: true }); + pluginsSetup.ml.setFeaturesEnabled({ ad: false, dfa: false, nlp: false }); return {}; } diff --git a/x-pack/test_serverless/functional/test_suites/search/navigation.ts b/x-pack/test_serverless/functional/test_suites/search/navigation.ts index 82dac047ac2c3..edf4c41274403 100644 --- a/x-pack/test_serverless/functional/test_suites/search/navigation.ts +++ b/x-pack/test_serverless/functional/test_suites/search/navigation.ts @@ -37,7 +37,6 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await svlCommonNavigation.breadcrumbs.expectBreadcrumbExists({ deepLinkId: 'serverlessElasticsearch', }); - await svlCommonNavigation.sidenav.expectSectionClosed('rootNav:ml'); // TODO: test something search project specific instead of generic discover // navigate to discover @@ -48,14 +47,12 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await expect(await browser.getCurrentUrl()).contain('/app/discover'); // navigate to a different section - await svlCommonNavigation.sidenav.openSection('rootNav:ml'); - await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'ml:notifications' }); - await svlCommonNavigation.sidenav.expectLinkActive({ deepLinkId: 'ml:notifications' }); - await svlCommonNavigation.breadcrumbs.expectBreadcrumbExists({ text: `Machine Learning` }); - await svlCommonNavigation.breadcrumbs.expectBreadcrumbExists({ - deepLinkId: 'ml:notifications', + await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'management:index_management' }); + await svlCommonNavigation.sidenav.expectLinkActive({ + deepLinkId: 'management:index_management', }); - await testSubjects.existOrFail(`mlPageNotifications`); + await svlCommonNavigation.breadcrumbs.expectBreadcrumbExists({ text: `Index Management` }); + await testSubjects.existOrFail(`indicesTab`); // navigate back to serverless search overview await svlCommonNavigation.breadcrumbs.clickHome(); @@ -64,19 +61,10 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { }); await svlCommonNavigation.breadcrumbs.expectBreadcrumbExists({ text: `Getting started` }); await testSubjects.existOrFail(`svlSearchOverviewPage`); - await svlCommonNavigation.sidenav.expectSectionOpen(`rootNav:ml`); // remains open await expectNoPageReload(); }); - it('active sidenav section is auto opened on load', async () => { - await svlCommonNavigation.sidenav.openSection('rootNav:ml'); - await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'ml:notifications' }); - await browser.refresh(); - await testSubjects.existOrFail(`mlPageNotifications`); - await svlCommonNavigation.sidenav.expectSectionOpen('rootNav:ml'); - }); - it('navigate using search', async () => { await svlCommonNavigation.search.showSearch(); // TODO: test something search project specific instead of generic discover From 2d17d753ef242ba1c50b6c250e2f7e922b329056 Mon Sep 17 00:00:00 2001 From: ruhshan Date: Mon, 31 Jul 2023 22:02:20 +0600 Subject: [PATCH 28/33] Update role mappings empty UI for consistency (#162357) ## Summary Fixes #161137 Update style of empty role mappings prompt to meet consistency. ![image](https://github.com/elastic/kibana/assets/5312918/f419de0e-177a-4fac-aa36-1c6ec5f0d6c6) Update icon of create api key button of empty api key prompt. ![image](https://github.com/elastic/kibana/assets/5312918/9b69e91a-9267-42d7-85f0-aca1507879af) ### Checklist Delete any items that are not applicable to this PR. - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Jeramy Soucy --- .../api_keys_grid/api_keys_empty_prompt.tsx | 2 +- .../api_keys_grid/api_keys_grid_page.tsx | 2 +- .../create_role_mapping_button.tsx | 1 + .../empty_prompt/empty_prompt.tsx | 20 +++++++++---------- .../role_mappings_grid_page.tsx | 6 +----- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx b/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx index c76b385cfccc5..b6300832551e8 100644 --- a/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx +++ b/x-pack/plugins/security/public/management/api_keys/api_keys_grid/api_keys_empty_prompt.tsx @@ -147,7 +147,7 @@ export const ApiKeysEmptyPrompt: FunctionComponent = ({ return ( { diff --git a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/create_role_mapping_button/create_role_mapping_button.tsx b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/create_role_mapping_button/create_role_mapping_button.tsx index 4aa55840cc7f8..ab9ce044f7b4c 100644 --- a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/create_role_mapping_button/create_role_mapping_button.tsx +++ b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/create_role_mapping_button/create_role_mapping_button.tsx @@ -21,6 +21,7 @@ interface CreateRoleMappingButtonProps { export const CreateRoleMappingButton = ({ history }: CreateRoleMappingButtonProps) => { return ( = ({ history, readOnly = false, }) => ( - @@ -40,14 +40,12 @@ export const EmptyPrompt: React.FunctionComponent = ({ } body={ - -

- -

- +

+ +

} actions={readOnly ? null : } data-test-subj="roleMappingsEmptyPrompt" diff --git a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx index e6feadd199b02..ad8f204fa75cd 100644 --- a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx +++ b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx @@ -136,11 +136,7 @@ export class RoleMappingsGridPage extends Component { } if (loadState === 'finished' && roleMappings && roleMappings.length === 0) { - return ( - - - - ); + return ; } return ( From a1bc1994880eab09d768abc3a57d9c8651284d60 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 31 Jul 2023 17:10:42 +0100 Subject: [PATCH 29/33] skip flaky suites (#162813) --- x-pack/performance/journeys/apm_service_inventory.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/performance/journeys/apm_service_inventory.ts b/x-pack/performance/journeys/apm_service_inventory.ts index 03a5fce636dcd..d98b151c000d6 100644 --- a/x-pack/performance/journeys/apm_service_inventory.ts +++ b/x-pack/performance/journeys/apm_service_inventory.ts @@ -9,6 +9,7 @@ import { Journey } from '@kbn/journeys'; import { SynthtraceClient } from '../services/synthtrace'; import { generateData } from '../synthtrace_data/apm_data'; +// FLAKY: https://github.com/elastic/kibana/issues/162813 export const journey = new Journey({ beforeSteps: async ({ kbnUrl, log, auth, es }) => { // Install APM Package @@ -34,6 +35,7 @@ export const journey = new Journey({ ); }, ftrConfigPath: 'x-pack/performance/configs/apm_config.ts', + skipped: true }) .step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => { await page.goto(kbnUrl.get(`app/apm/services`)); From 235c6ec8cb2717c9a56f285fc3751ba4b14c73af Mon Sep 17 00:00:00 2001 From: Justin Kambic Date: Mon, 31 Jul 2023 13:02:24 -0400 Subject: [PATCH 30/33] [Synthetics] Fix TS error for TS upgrade (#162822) ## Summary Fixes the TS error encountered in https://github.com/elastic/kibana/pull/162738 in the synthetics plugin. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../server/legacy_uptime/lib/alerts/duration_anomaly.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/duration_anomaly.test.ts b/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/duration_anomaly.test.ts index 21c20503f31ab..7a48860265432 100644 --- a/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/duration_anomaly.test.ts +++ b/x-pack/plugins/uptime/server/legacy_uptime/lib/alerts/duration_anomaly.test.ts @@ -121,6 +121,8 @@ describe('duration anomaly alert', () => { jest.spyOn(Intl, 'DateTimeFormat').mockImplementation(() => ({ format: jest.fn(), formatToParts: jest.fn(), + formatRange: jest.fn(), + formatRangeToParts: jest.fn(), resolvedOptions: () => ({ locale: '', calendar: '', From db42331c0d7b862dc7f054dbeaeb1e237ba5458d Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Mon, 31 Jul 2023 13:05:59 -0400 Subject: [PATCH 31/33] [ILM] Unskip API integration test (#162525) --- .../index_lifecycle_management/fixtures.js | 30 +++++++ .../index_lifecycle_management/policies.js | 78 +++++++++++++------ .../snapshot_policies.ts | 2 +- .../snapshot_repositories.ts | 2 +- 4 files changed, 85 insertions(+), 27 deletions(-) diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/fixtures.js b/x-pack/test/api_integration/apis/management/index_lifecycle_management/fixtures.js index e61470cc2cc84..6a037f7f76315 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/fixtures.js +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/fixtures.js @@ -7,7 +7,37 @@ import { INDEX_TEMPLATE_PATTERN_PREFIX } from './constants'; +// ILM policy with minimal configuration export const getPolicyPayload = (name) => ({ + name, + phases: { + hot: { + min_age: '1d', + actions: { + set_priority: { + priority: 100, + }, + }, + }, + warm: { + min_age: '10d', + actions: { + set_priority: { + priority: 50, + }, + }, + }, + delete: { + min_age: '30d', + actions: { + delete: {}, + }, + }, + }, +}); + +// ILM policy with full configuration; includes searchable_snapshot configuration, which requires a valid repository and SLM policy +export const getPolicyPayloadWithSearchableSnapshots = (name) => ({ name, phases: { hot: { diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js b/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js index 25ce7d4b677a3..a4d0bc385b65b 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js @@ -9,9 +9,11 @@ import expect from '@kbn/expect'; import { registerHelpers as registerPoliciesHelpers } from './policies.helpers'; import { registerHelpers as registerIndexHelpers } from './indices.helpers'; -import { getPolicyPayload } from './fixtures'; +import { registerSnapshotPoliciesHelpers } from './snapshot_policies.helpers'; +import { registerSnapshotRepositoriesHelpers } from './snapshot_repositories.helpers'; + +import { getPolicyPayload, getPolicyPayloadWithSearchableSnapshots } from './fixtures'; import { initElasticsearchHelpers, getPolicyNames } from './lib'; -import { DEFAULT_POLICY_NAME } from './constants'; export default function ({ getService }) { const supertest = getService('supertest'); @@ -32,36 +34,22 @@ export default function ({ getService }) { const { addPolicyToIndex } = registerIndexHelpers({ supertest }); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/114030 - describe.skip('policies', () => { + const { createSnapshotPolicy, cleanupPolicies: cleanupSnapshotPolicies } = + registerSnapshotPoliciesHelpers(getService); + const { createSnapshotRepository, cleanupRepositories } = + registerSnapshotRepositoriesHelpers(getService); + + describe('policies', () => { after(() => Promise.all([cleanUpEsResources(), cleanUpPolicies()])); describe('list', () => { - it('should have a default policy to manage the Watcher history indices', async () => { - const { body } = await loadPolicies().expect(200); - const { version, name, policy } = body.find( - (policy) => policy.name === DEFAULT_POLICY_NAME - ); - - expect(version).to.eql(1); - expect(name).to.eql(DEFAULT_POLICY_NAME); - expect(policy.phases).to.eql({ - delete: { - min_age: '7d', - actions: { - delete: { - delete_searchable_snapshot: true, - }, - }, - }, - }); - }); - it('should add the indices linked to the policies', async () => { // Create a policy const policy = getPolicyPayload('link-test-policy'); const { name: policyName } = policy; - await createPolicy(policy); + + const { statusCode } = await createPolicy(policy); + expect(statusCode).to.eql(200); // Create a new index const indexName = await createIndex(); @@ -120,6 +108,46 @@ export default function ({ getService }) { }); }); + describe('searchable snapshots', function () { + this.tags(['skipCloud']); // file system repositories are not supported in cloud + + before(async () => { + try { + await createSnapshotRepository('backing_repo'); // This corresponds to the name set in the ILM policy + } catch (err) { + // eslint-disable-next-line no-console + console.log('[Setup error] Error creating repository'); + throw err; + } + + try { + await createSnapshotPolicy('policy', 'backing_repo'); // Policy name corresponds to the policy name specified in the ILM policy + } catch (err) { + // eslint-disable-next-line no-console + console.log('[Setup error] Error creating SLM policy'); + throw err; + } + }); + + after(async () => Promise.all([cleanupRepositories(), cleanupSnapshotPolicies()])); + + it('should create a lifecycle policy with searchable snapshot action', async () => { + const policy = getPolicyPayloadWithSearchableSnapshots('create-searchable-snapshot-policy'); + const { name } = policy; + + // Load current policies + const { body: bodyFirstLoad } = await loadPolicies(); + expect(getPolicyNames(bodyFirstLoad)).not.to.contain(name); + + // Create new policy + await createPolicy(policy).expect(200); + + // Make sure the new policy is returned + const { body: bodySecondLoad } = await loadPolicies(); + expect(getPolicyNames(bodySecondLoad)).to.contain(name); + }); + }); + describe('edit', () => { it('keeps _meta field intact', async () => { const policyName = 'edit-meta-test-policy'; diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_policies.ts b/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_policies.ts index f33115da9c3b3..af573fab10c8a 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_policies.ts +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_policies.ts @@ -22,7 +22,7 @@ export default function ({ getService }: FtrProviderContext) { registerSnapshotRepositoriesHelpers(getService); describe('snapshot policies', function describeSnapshotPoliciesTest() { - // skip Cloud failing test https://github.com/elastic/kibana/issues/136881 + // File-system repositories are not applicable on cloud this.tags(['skipCloud']); before(async () => Promise.all([cleanupPolicies(), cleanupRepositories()])); diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_repositories.ts b/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_repositories.ts index e758ab294b12b..c04fab48797d8 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_repositories.ts +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/snapshot_repositories.ts @@ -20,7 +20,7 @@ export default function ({ getService }: FtrProviderContext) { registerSnapshotRepositoriesHelpers(getService); describe('snapshot repositories', function describeSnapshotRepositoriesTest() { - // skip Cloud failing test https://github.com/elastic/kibana/issues/136882 + // file system repositories are not supported in cloud this.tags(['skipCloud']); before(async () => { From 92db0d39360eaca077bb872f5cc5b0569e410951 Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Tue, 1 Aug 2023 02:14:21 +0900 Subject: [PATCH 32/33] Allow variable names with underscores (#162208) Closes #162205 ## Summary With this PR, users will be able to define variables with names containing underscores (`_`) like the below picture. ![image](https://github.com/elastic/kibana/assets/721858/d6aa0b4c-0c12-4fe7-ab3f-da0aa5cc8862) ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../components/variables/utils.test.ts | 60 +++++++++++++++++++ .../application/components/variables/utils.ts | 9 +++ .../components/variables/variables_flyout.tsx | 5 +- 3 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 src/plugins/console/public/application/components/variables/utils.test.ts diff --git a/src/plugins/console/public/application/components/variables/utils.test.ts b/src/plugins/console/public/application/components/variables/utils.test.ts new file mode 100644 index 0000000000000..0b990ca998114 --- /dev/null +++ b/src/plugins/console/public/application/components/variables/utils.test.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { isValidVariableName } from './utils'; + +describe('utils', () => { + describe('isValidVariableName', () => { + it('returns `false` to `null`', () => { + // @ts-ignore passing a wrong type intentionally + expect(isValidVariableName(null)).toBe(false); + }); + + it('returns `false` to empty string', () => { + expect(isValidVariableName('')).toBe(false); + }); + + it('returns `false` to space string', () => { + expect(isValidVariableName(' ')).toBe(false); + }); + + it('returns `false` to integer zero', () => { + // @ts-ignore passing a wrong type intentionally + expect(isValidVariableName(0)).toBe(false); + }); + + it('returns `false` to float zero', () => { + // @ts-ignore passing a wrong type intentionally + expect(isValidVariableName(0.0)).toBe(false); + }); + + it('returns `true` to string zero', () => { + expect(isValidVariableName('0')).toBe(true); + }); + + it('returns `true` to allowed styles', () => { + for (const name of ['camelCase', 'snake_case', 'PascalCase', 'MACRO_CASE']) { + expect(isValidVariableName(name)).toBe(true); + } + }); + + it('returns `false` to disallowed styles', () => { + for (const name of ['kebab-case', 'COBOL-CASE', 'dot.notation', 'bracket[notation]']) { + expect(isValidVariableName(name)).toBe(false); + } + }); + + it('returns `true` to underscores prefix & suffix', () => { + expect(isValidVariableName('__name__')).toBe(true); + }); + + it('returns `true` to numbers prefix & suffix', () => { + expect(isValidVariableName('00name00')).toBe(true); + }); + }); +}); diff --git a/src/plugins/console/public/application/components/variables/utils.ts b/src/plugins/console/public/application/components/variables/utils.ts index 852498b92da98..3e0ac32f266c4 100644 --- a/src/plugins/console/public/application/components/variables/utils.ts +++ b/src/plugins/console/public/application/components/variables/utils.ts @@ -37,3 +37,12 @@ export const generateEmptyVariableField = (): DevToolsVariable => ({ name: '', value: '', }); + +export const isValidVariableName = (name: string) => { + /* + * MUST avoid characters that get URL-encoded, because they'll result in unusable variable names. + * Common variable names consist of letters, digits, and underscores and do not begin with a digit. + * However, the ones beginning with a digit are still allowed here for backward compatibility. + */ + return typeof name === 'string' && name.match(/^[a-zA-Z0-9_]+$/g) !== null; +}; diff --git a/src/plugins/console/public/application/components/variables/variables_flyout.tsx b/src/plugins/console/public/application/components/variables/variables_flyout.tsx index 46b67bdf9075f..5ca172b2e045b 100644 --- a/src/plugins/console/public/application/components/variables/variables_flyout.tsx +++ b/src/plugins/console/public/application/components/variables/variables_flyout.tsx @@ -85,15 +85,14 @@ export const DevToolsVariablesFlyout = (props: DevToolsVariablesFlyoutProps) => defaultMessage: 'Variable name', }), render: (name, { id }) => { - // Avoid characters that get URL-encoded, because they'll result in unusable variable names. - const isInvalid = name && !name.match(/^[a-zA-Z0-9]+$/g); + const isInvalid = !utils.isValidVariableName(name); return ( , ]} fullWidth={true} From 3efb83851fd2b9b1ffd6d10c9728478e9deffa51 Mon Sep 17 00:00:00 2001 From: Kevin Logan <56395104+kevinlog@users.noreply.github.com> Date: Mon, 31 Jul 2023 13:18:36 -0400 Subject: [PATCH 33/33] [Security Solution] Skip file operations tests until ES permissions is fixed (#162787) ## Summary Skip the broken File Operations tests until a permissions issue in ES is fixed: https://github.com/elastic/kibana/issues/162760 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../cypress/e2e/detection_alerts/cti_enrichments.cy.ts | 3 ++- .../management/cypress/e2e/endpoint/response_console.cy.ts | 3 ++- .../management/cypress/e2e/mocked_data/response_console.cy.ts | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/cti_enrichments.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/cti_enrichments.cy.ts index c14ae488d865d..fc424eb192e05 100644 --- a/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/cti_enrichments.cy.ts +++ b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/cti_enrichments.cy.ts @@ -47,7 +47,8 @@ describe('CTI Enrichment', () => { goToRuleDetails(); }); - it('Displays enrichment matched.* fields on the timeline', () => { + // Skipped: https://github.com/elastic/kibana/issues/162818 + it.skip('Displays enrichment matched.* fields on the timeline', () => { const expectedFields = { 'threat.enrichments.matched.atomic': indicatorRuleMatchingDoc.atomic, 'threat.enrichments.matched.type': indicatorRuleMatchingDoc.matchedType, diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint/response_console.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint/response_console.cy.ts index 245de3ce5512c..d2ea69dca1618 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint/response_console.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/endpoint/response_console.cy.ts @@ -188,7 +188,8 @@ describe('Response console', () => { }); }); - describe('File operations: get-file and execute', () => { + // Broken until this is fixed: https://github.com/elastic/kibana/issues/162760 + describe.skip('File operations: get-file and execute', () => { const homeFilePath = process.env.CI || true ? '/home/vagrant' : `/home/ubuntu`; const fileContent = 'This is a test file for the get-file command.'; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/mocked_data/response_console.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/mocked_data/response_console.cy.ts index dad53207b56e3..fa347d77581ae 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/mocked_data/response_console.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/mocked_data/response_console.cy.ts @@ -228,7 +228,8 @@ describe('Response console', () => { }); }); - describe('`get-file` command', () => { + // Broken until this is fixed: https://github.com/elastic/kibana/issues/162760 + describe.skip('`get-file` command', () => { let endpointData: ReturnTypeFromChainable; let endpointHostname: string; let getFileRequestResponse: ActionDetails;