From ce4c24570198605412fc970d89005d8e5035177f Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Mon, 8 Apr 2024 11:21:28 +0200 Subject: [PATCH 01/11] [BK] Add template for pipeline defs (#180189) ## Summary This is to aid new pipeline creation by example. --- .../_template/template.yml | 72 +++++++++++++++++++ .../fix-location-collection.ts | 2 +- catalog-info.yaml | 3 + 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 .buildkite/pipeline-resource-definitions/_template/template.yml diff --git a/.buildkite/pipeline-resource-definitions/_template/template.yml b/.buildkite/pipeline-resource-definitions/_template/template.yml new file mode 100644 index 0000000000000..f33e738882693 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/_template/template.yml @@ -0,0 +1,72 @@ +### +# For more information on authoring pipeline definitions, +# follow the guides at https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic +### +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + # This will be the URL slug in Backstage UI as: + # https://backstage.elastic.dev/catalog/default/resource/bk-kibana-your-pipeline-name + # bk-pipeline- + name: bk-kibana-your-pipeline-name + # This will be displayed in the Backstage UI + description: '' + links: + # These are relevant links to your pipeline that will be listed in the Backstage UI + # The URL slug here is the .spec.implementation.metadata.name field slugified + - url: 'https://buildkite.com/elastic/kibana-your-pipeline-name' + title: Pipeline link +spec: + type: buildkite-pipeline + system: buildkite + # The owner team's github group name in the format 'group:' + owner: 'group:github-group-name' + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + # - this will be displayed in the Buildkite UI as title + # and this will be slugified to form the URL in the Backstage UI + name: kibana / your pipeline name + # This will appear as description on the Buildkite UI + description: '' + spec: + # Environment variables that will be set for the pipeline + env: + # Slack channel to send notifications to, if ELASTIC_SLACK_NOTIFICATIONS_ENABLED = 'true' + SLACK_NOTIFICATIONS_CHANNEL: '#team-slack-channel-name' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + + allow_rebuilds: false + branch_configuration: main + default_branch: main + repository: elastic/kibana + # Point to a pipeline implementation, detailing the pipeline steps to run + pipeline_file: .buildkite/pipelines/your-pipeline-name.yml + skip_intermediate_builds: false + provider_settings: + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: true + trigger_mode: none + # Teams and their access levels to the pipeline, + # please keep [kibana-operations, appex-qa, kibana-tech-leads] as MANAGE_BUILD_AND_READ + # and [everyone] as BUILD_AND_READ + teams: + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ + # Scheduled runs for the pipeline + schedules: + Daily 6 am UTC: + cronline: 0 5 * * * + message: Daily 6 am UTC + branch: main + # Optionally, set schedule-specific env-vars here + env: + SCHEDULED: 'true' diff --git a/.buildkite/pipeline-resource-definitions/fix-location-collection.ts b/.buildkite/pipeline-resource-definitions/fix-location-collection.ts index 8ca4bf50f2cd9..d4e36f2559a89 100755 --- a/.buildkite/pipeline-resource-definitions/fix-location-collection.ts +++ b/.buildkite/pipeline-resource-definitions/fix-location-collection.ts @@ -11,7 +11,7 @@ import jsYaml from 'js-yaml'; import path from 'path'; import { execSync } from 'child_process'; -const EXCLUDE_LIST = ['locations.yml']; +const EXCLUDE_LIST = ['locations.yml', '_template/template.yml']; const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main'; type BackstageLocationResource = object & { diff --git a/catalog-info.yaml b/catalog-info.yaml index 05ef4c8482bfb..95d625355d4db 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -220,3 +220,6 @@ spec: access_level: BUILD_AND_READ everyone: access_level: READ_ONLY + +# Please avoid creating new kibana pipelines in this file to avoid bloating. +# Instead, create a new file in the pipeline-resource-definitions directory, and wire it in through the locations.yml file. From 0527f48ee70bbfecfb16cd5538795b8e676589fb Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Mon, 8 Apr 2024 13:28:45 +0300 Subject: [PATCH 02/11] fix: [Integrations > Install integration only][SCREEN READER]: Change default button accessible labels must contain the visual label (#180166) Closes: https://github.com/elastic/security-team/issues/9008 ## Description The `Change defaults` buttons have accessible labels that are different than the visible label. The accessible label needs to include the visible label and should also have an `aria-expanded="true | false"` attribute for screen reader usability. Screenshot below. ### Steps to recreate 1. Create a new Security Serverless project if none exist 2. When the project is ready, open it and go to Integrations, under the Project Settings in the lower left navigation 3. Search for Kubernetes in the Integrations, and click on the card 4. Click "Add Kubernetes" to load the prompt page 5. Click "Add integration only (skip agent installation)" 6. Turn on the screen reader of your choice 7. Press `Tab` until the first "Change defaults" button has keyboard focus 8. Verify this button text is not announced. Text is "Hide kubernetes/metrics inputs" or something similar. ### What was done? 1. Required `a11y` attributes were added ### Screen #### a11y tree ![image](https://github.com/elastic/kibana/assets/20072247/47af9636-2ddb-4988-bab6-fe20bb14b76c) --- .../components/package_policy_input_panel.tsx | 29 ++++--------------- .../single_page_layout/index.test.tsx | 4 +-- .../edit_package_policy_page/index.test.tsx | 2 +- .../translations/translations/fr-FR.json | 2 -- .../translations/translations/ja-JP.json | 2 -- .../translations/translations/zh-CN.json | 2 -- 6 files changed, 9 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx index 68f26a3cf6121..89df8d0074f71 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx @@ -7,7 +7,6 @@ import React, { useState, Fragment, memo, useMemo } from 'react'; import styled from 'styled-components'; -import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiFlexGroup, @@ -17,6 +16,7 @@ import { EuiHorizontalRule, EuiSpacer, EuiButtonEmpty, + htmlIdGenerator, } from '@elastic/eui'; import type { @@ -128,6 +128,8 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ [packageInputStreams, packagePolicyInput.streams] ); + const titleElementId = useMemo(() => htmlIdGenerator()(), []); + return ( <> {/* Header / input-level toggle */} @@ -138,7 +140,7 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ -

{packageInput.title || packageInput.type}

+

{packageInput.title || packageInput.type}

 @@ -179,27 +181,8 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{ onClick={() => setIsShowingStreams(!isShowingStreams)} iconType={isShowingStreams ? 'arrowUp' : 'arrowDown'} iconSide="right" - aria-label={ - isShowingStreams - ? i18n.translate( - 'xpack.fleet.createPackagePolicy.stepConfigure.hideStreamsAriaLabel', - { - defaultMessage: 'Hide {type} inputs', - values: { - type: packageInput.type, - }, - } - ) - : i18n.translate( - 'xpack.fleet.createPackagePolicy.stepConfigure.showStreamsAriaLabel', - { - defaultMessage: 'Show {type} inputs', - values: { - type: packageInput.type, - }, - } - ) - } + aria-expanded={isShowingStreams} + aria-labelledby={titleElementId} > { { test('should disable submit button on invalid form with empty package var', async () => { await act(async () => { - fireEvent.click(renderResult.getByLabelText('Show logfile inputs')); + fireEvent.click(renderResult.getByText('Change defaults')); }); await act(async () => { @@ -553,7 +553,7 @@ describe('when on the package policy create page', () => { test('should submit form with changed package var', async () => { await act(async () => { - fireEvent.click(renderResult.getByLabelText('Show logfile inputs')); + fireEvent.click(renderResult.getByText('Change defaults')); }); await act(async () => { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx index 901426e10f12b..450692f2201c3 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.test.tsx @@ -233,7 +233,7 @@ describe('edit package policy page', () => { }); await act(async () => { - fireEvent.click(renderResult.getByLabelText('Show logfile inputs')); + fireEvent.click(renderResult.getByText('Change defaults')); }); await act(async () => { diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index a9bd53e383dda..581489bbf2e06 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -17605,10 +17605,8 @@ "xpack.fleet.createPackagePolicy.multiPageTitle": "Configurer l'intégration de {title}", "xpack.fleet.createPackagePolicy.pageTitleWithPackageName": "Ajouter l'intégration {packageName}", "xpack.fleet.createPackagePolicy.stepConfigure.errorCountText": "{count, plural, one {# erreur} other {# erreurs}}", - "xpack.fleet.createPackagePolicy.stepConfigure.hideStreamsAriaLabel": "Masquer les entrées {type}", "xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyDataRetentionText": "Par défaut, tous les logs et toutes les données d'indicateurs sont stockés au niveau \"hot\". {learnMore} sur la modification de la politique de conservation des données pour cette intégration.", "xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyNamespaceHelpLabel": "Modifiez l'espace de nom par défaut hérité de la politique d'agent sélectionnée. Ce paramètre modifie le nom du flux de données de l'intégration. {learnMore}.", - "xpack.fleet.createPackagePolicy.stepConfigure.showStreamsAriaLabel": "Afficher les entrées {type}", "xpack.fleet.createPackagePolicy.StepSelectPolicy.agentPolicyAgentsDescriptionText": "{count, plural, one {# agent est enregistré} other {# agents sont enregistrés}} avec la politique d'agent sélectionnée.", "xpack.fleet.createPackagePolicy.transformInstallWithCurrentUserPermissionCallout": "Ce package a {count, plural, one {une ressource de transformation qui sera créée et démarrée} other {# ressources de transformation qui seront créées et démarrées}} avec les mêmes rôles que l'utilisateur responsable de l'installation du package.", "xpack.fleet.currentUpgrade.confirmDescription": "Cette action provoquera l'annulation de la mise à niveau de {nbAgents, plural, one {# agent} other {# agents}}", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 73cb47451f5b1..6e6d65a9668a6 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -17582,10 +17582,8 @@ "xpack.fleet.createPackagePolicy.multiPageTitle": "{title}統合を設定", "xpack.fleet.createPackagePolicy.pageTitleWithPackageName": "{packageName}統合の追加", "xpack.fleet.createPackagePolicy.stepConfigure.errorCountText": "{count, plural, other {# 件のエラー}}", - "xpack.fleet.createPackagePolicy.stepConfigure.hideStreamsAriaLabel": "{type}入力を非表示", "xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyDataRetentionText": "デフォルトでは、すべてのログとメトリックがホットティアに格納されます。この統合のデータ保持ポリシーの変更については、{learnMore}してください。", "xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyNamespaceHelpLabel": "選択したエージェントポリシーから継承されたデフォルト名前空間を変更します。この設定により、統合のデータストリームの名前が変更されます。{learnMore}。", - "xpack.fleet.createPackagePolicy.stepConfigure.showStreamsAriaLabel": "{type}入力を表示", "xpack.fleet.createPackagePolicy.StepSelectPolicy.agentPolicyAgentsDescriptionText": "{count, plural, other {# 個のエージェント}}が選択したエージェントポリシーで登録されています。", "xpack.fleet.createPackagePolicy.transformInstallWithCurrentUserPermissionCallout": "このパッケージには、パッケージをインストールするユーザーと同じロールで作成、開始される{count, plural, one {1個の変換アセット} other {# 変換アセット}}があります。", "xpack.fleet.currentUpgrade.confirmDescription": "{nbAgents, plural, other {# 個のエージェント}}のアップグレードがキャンセルされます", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index eb9d382b30eb5..246d461cb7efc 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -17609,10 +17609,8 @@ "xpack.fleet.createPackagePolicy.multiPageTitle": "设置 {title} 集成", "xpack.fleet.createPackagePolicy.pageTitleWithPackageName": "添加 {packageName} 集成", "xpack.fleet.createPackagePolicy.stepConfigure.errorCountText": "{count, plural, other {# 个错误}}", - "xpack.fleet.createPackagePolicy.stepConfigure.hideStreamsAriaLabel": "隐藏 {type} 输入", "xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyDataRetentionText": "默认情况下，所有日志和指标数据存储在热层中。{learnMore}如何更改此集成的数据保留策略。", "xpack.fleet.createPackagePolicy.stepConfigure.packagePolicyNamespaceHelpLabel": "更改从选定代理策略继承的默认命名空间。此设置将更改集成的数据流的名称。{learnMore}。", - "xpack.fleet.createPackagePolicy.stepConfigure.showStreamsAriaLabel": "显示 {type} 输入", "xpack.fleet.createPackagePolicy.StepSelectPolicy.agentPolicyAgentsDescriptionText": "{count, plural, other {# 个代理}}已注册到选定代理策略。", "xpack.fleet.createPackagePolicy.transformInstallWithCurrentUserPermissionCallout": "此软件包具有 {count, plural, one {一个转换资产} other {# 个转换资产}}，它们将使用与安装软件包的用户相同的角色创建并启动。", "xpack.fleet.currentUpgrade.confirmDescription": "此操作会取消升级 {nbAgents, plural, other {# 个代理}}", From 07a83d6a2849cc166fe00ca356ced7d857fd3c80 Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 8 Apr 2024 12:31:20 +0200 Subject: [PATCH 03/11] [Security Solution] Adds `serverlessQA` tag to the Cypress tests (#179737) ## Summary We want to start integrating our Cypress tests with the serverless Kibana quality gate. However, not all the teams feel comfortable enabling all the tests, to facilitate the effort of enabling tests in the quality gate we are adding the `@serverlessQA` tag. Once this PR is merged, the behavior will be the following: **Regular PR flow** All the tests tagged as `@serverless` will be executed as part of the PR validation process using the serverless FTR environment (not a real one). If you want to skip a test to be executed in this flow use `@brokenInServerless` or `@skipInServerless`. **Periodic pipeline** All the tests tagged as `@serverless` will be executed as part of the periodic pipeline using a real serverless project. QA environment is used to do so using the latest available commit in main at the time of the execution. If you want to skip a test to be executed in this flow use `@brokenInServerlessQA`. **Kibana second quality gate** All the tests tagged as `@serverlessQA` will be executed as part of the kibana release process using a real serverless project with the latest image available in the QA environment. If you want to skip a test to be executed in this flow use `@brokenInServerlessQA`. --- .../run_cypress/parallel_serverless.ts | 10 +- .../cypress/README.md | 8 +- .../cypress/cypress.config.ts | 2 +- .../cypress/cypress_ci.config.ts | 2 +- .../cypress/cypress_ci_serverless.config.ts | 2 +- .../cypress_ci_serverless_qa.config.ts | 2 +- .../cypress/cypress_serverless.config.ts | 2 +- .../callouts/missing_privileges_callout.cy.ts | 2 +- .../alert_threat_enrichments.cy.ts | 318 +++++++++--------- .../status/alert_status.cy.ts | 2 +- .../endpoint_exceptions.cy.ts | 2 +- .../auto_populate_with_alert_data.cy.ts | 2 +- .../add_edit_exception.cy.ts | 2 +- .../overview/cti_link_panel.cy.ts | 2 +- .../rule_actions/rule_actions.cy.ts | 2 +- .../sourcerer/sourcerer.cy.ts | 2 +- .../sourcerer/sourcerer_permissions.cy.ts | 2 +- .../sourcerer/sourcerer_timeline.cy.ts | 2 +- .../coverage_overview/coverage_overview.cy.ts | 2 +- .../install_update_authorization.cy.ts | 2 +- .../install_update_error_handling.cy.ts | 2 +- .../prebuilt_rules/install_workflow.cy.ts | 2 +- .../prebuilt_rules/management.cy.ts | 2 +- .../prebuilt_rules/notifications.cy.ts | 4 +- .../related_integrations.cy.ts | 2 +- .../bulk_actions/bulk_edit_rules.cy.ts | 6 +- .../bulk_edit_rules_actions.cy.ts | 2 +- .../import_export/export_rule.cy.ts | 6 +- .../rules_table/rules_table_selection.cy.ts | 94 +++--- .../entity_analytics/legacy_risk_score.cy.ts | 2 +- .../entity_analytics/new_risk_score.cy.ts | 2 +- .../e2e/entity_analytics/entity_flyout.cy.ts | 2 +- .../e2e/explore/inspect/inspect_button.cy.ts | 2 +- .../e2e/explore/ml/ml_conditional_links.cy.ts | 2 +- .../e2e/explore/overview/overview.cy.ts | 4 +- .../e2e/explore/urls/compatibility.cy.ts | 2 +- .../cypress/e2e/explore/urls/state.cy.ts | 2 +- .../alerts/changing_alert_status.cy.ts | 2 +- ...etails_left_panel_analyzer_graph_tab.cy.ts | 2 +- .../discover_timeline_state_integration.cy.ts | 2 +- .../investigations/timelines/overview.cy.ts | 2 +- .../investigations/timelines/query_tab.cy.ts | 2 +- .../unified_components/query_tab.cy.ts | 2 +- 43 files changed, 259 insertions(+), 261 deletions(-) diff --git a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts index 12b0957fe99f8..d4fa3866402e6 100644 --- a/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts +++ b/x-pack/plugins/security_solution/scripts/run_cypress/parallel_serverless.ts @@ -459,10 +459,16 @@ ${JSON.stringify(argv, null, 2)} `); const isOpen = argv._.includes('open'); - const cypressConfigFilePath = require.resolve(`../../${argv.configFile}`) as string; const cypressConfigFile = await import(cypressConfigFilePath); - + // KIBANA_MKI_USE_LATEST_COMMIT === 1 means that we are overriding the image for the periodic pipeline execution. + // We don't override the image when executing the tests on the second quality gate. + if ( + !process.env.KIBANA_MKI_USE_LATEST_COMMIT || + process.env.KIBANA_MKI_USE_LATEST_COMMIT !== '1' + ) { + cypressConfigFile.env.grepTags = '@serverlessQA --@skipInServerless'; + } const tier: string = argv.tier; const endpointAddon: boolean = argv.endpointAddon; const cloudAddon: boolean = argv.cloudAddon; diff --git a/x-pack/test/security_solution_cypress/cypress/README.md b/x-pack/test/security_solution_cypress/cypress/README.md index ed8fe3793d60b..dea0f72240d1a 100644 --- a/x-pack/test/security_solution_cypress/cypress/README.md +++ b/x-pack/test/security_solution_cypress/cypress/README.md @@ -44,11 +44,11 @@ Please, before opening a PR with the new test, please make sure that the test fa Note that we use tags in order to select which tests we want to execute: -- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate) and QA environemnt (the so-called second quality gate). You need to explicitly add this tag to any test you want to run in CI for serverless. +- `@serverless` includes a test in the Serverless test suite for PRs (the so-called first quality gate) and QA environment for the periodic pipeline. You need to explicitly add this tag to any test you want to run in CI for serverless. +- `@serverlessQA` includes a test in the Serverless test suite for the Kibana release process of serverless. You need to explicitly add this tag to any test you want yo run in CI for the second quality gate. These tests should be stable, otherviswe they will be blocking the release pipeline. They should be alsy critical enough, so that when they fail, there's a high chance of an SDH or blocker issue to be reported. - `@ess` includes a test in the normal, non-Serverless test suite. You need to explicitly add this tag to any test you want to run against a non-Serverless environment. -- `@brokenInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Indicates that a test should run in Serverless, but currently is broken. -- `@brokenInServerlessQA` excludes a test form the Serverless QA enviornment (second quality gate). Indicates that a test should run on it, but currently is broken. -- `@skipInServerless` excludes a test from the Serverless test suite (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environemnt", "the test has been temporarily excluded, see the comment above why". +- `@skipInEss` excludes a test from the non-Serverless test suite. The test will not be executed as part for the PR process. All the skipped tests should have a link to a ticket describing the reason why the test got skipped. +- `@skipInServerless` excludes a test from the Serverless test suite and Serverless QA environment for both, periodic pipeline and second quality gate (even if it's tagged as `@serverless`). Could indicate many things, e.g. "the test is flaky in Serverless", "the test is Flaky in any type of environment", "the test has been temporarily excluded, see the comment above why". All the skipped tests should have a link to a ticket describing the reason why the test got skipped. Please, before opening a PR with a new test, make sure that the test fails. If you never see your test fail you don’t know if your test is actually testing the right thing, or testing anything at all. diff --git a/x-pack/test/security_solution_cypress/cypress/cypress.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress.config.ts index d7f0bbc7a0254..387f3b7aec118 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress.config.ts @@ -12,7 +12,7 @@ export default defineCypressConfig({ defaultCommandTimeout: 60000, env: { grepFilterSpecs: true, - grepTags: '@ess', + grepTags: '@ess --@skipInEss', }, execTimeout: 60000, pageLoadTimeout: 60000, diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci.config.ts index efb3b64d36f4d..bb632fb237c9d 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci.config.ts @@ -18,7 +18,7 @@ export default defineCypressConfig({ env: { grepFilterSpecs: true, grepOmitFiltered: true, - grepTags: '@ess', + grepTags: '@ess --@skipInEss', }, execTimeout: 150000, pageLoadTimeout: 150000, diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless.config.ts index 62aec8c49e787..80afa64fbb7a5 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless.config.ts @@ -19,7 +19,7 @@ export default defineCypressConfig({ env: { grepFilterSpecs: true, grepOmitFiltered: true, - grepTags: '@serverless --@brokenInServerless --@skipInServerless', + grepTags: '@serverless --@skipInServerless', }, execTimeout: 150000, pageLoadTimeout: 150000, diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts index c88faf0d9cfe3..c2c3c9abccda1 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_ci_serverless_qa.config.ts @@ -19,7 +19,7 @@ export default defineCypressConfig({ env: { grepFilterSpecs: true, grepOmitFiltered: true, - grepTags: '@serverless --@brokenInServerless --@skipInServerless --@brokenInServerlessQA', + grepTags: '@serverless --@skipInServerless', }, execTimeout: 300000, pageLoadTimeout: 300000, diff --git a/x-pack/test/security_solution_cypress/cypress/cypress_serverless.config.ts b/x-pack/test/security_solution_cypress/cypress/cypress_serverless.config.ts index b76c7ff22bcbc..8f66e7f1a4173 100644 --- a/x-pack/test/security_solution_cypress/cypress/cypress_serverless.config.ts +++ b/x-pack/test/security_solution_cypress/cypress/cypress_serverless.config.ts @@ -24,7 +24,7 @@ export default defineCypressConfig({ numTestsKeptInMemory: 10, env: { grepFilterSpecs: true, - grepTags: '@serverless --@brokenInServerless --@skipInServerless', + grepTags: '@serverless --@skipInServerless', }, e2e: { experimentalCspAllowList: ['default-src', 'script-src', 'script-src-elem'], diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/callouts/missing_privileges_callout.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/callouts/missing_privileges_callout.cy.ts index aaf861dfd7558..bcb29a456bdd2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/callouts/missing_privileges_callout.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/callouts/missing_privileges_callout.cy.ts @@ -47,7 +47,7 @@ const waitForPageTitleToBeShown = () => { // FLAKY: https://github.com/elastic/kibana/issues/178176 describe.skip( 'Detections > Callouts', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { before(() => { // First, we have to open the app on behalf of a privileged user in order to initialize it. diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/enrichments/alert_threat_enrichments.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/enrichments/alert_threat_enrichments.cy.ts index b8a70ce7b13e8..383abd2e12d42 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/enrichments/alert_threat_enrichments.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/enrichments/alert_threat_enrichments.cy.ts @@ -29,175 +29,171 @@ import { openJsonView, openThreatIndicatorDetails } from '../../../../../tasks/a import { addsFieldsToTimeline, visitRuleDetailsPage } from '../../../../../tasks/rule_details'; // TODO: https://github.com/elastic/kibana/issues/161539 -describe( - 'Threat Match Enrichment', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, - () => { - before(() => { - // illegal_argument_exception: unknown setting [index.lifecycle.rollover_alias] - cy.task('esArchiverLoad', { archiveName: 'threat_indicator' }); - cy.task('esArchiverLoad', { archiveName: 'suspicious_source_event' }); - login(); - - disableExpandableFlyout(); +describe('Threat Match Enrichment', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { + before(() => { + // illegal_argument_exception: unknown setting [index.lifecycle.rollover_alias] + cy.task('esArchiverLoad', { archiveName: 'threat_indicator' }); + cy.task('esArchiverLoad', { archiveName: 'suspicious_source_event' }); + login(); + + disableExpandableFlyout(); + }); + + after(() => { + cy.task('esArchiverUnload', { archiveName: 'threat_indicator' }); + cy.task('esArchiverUnload', { archiveName: 'suspicious_source_event' }); + }); + + beforeEach(() => { + login(); + createRule({ ...getNewThreatIndicatorRule(), rule_id: 'rule_testing', enabled: true }).then( + (rule) => visitRuleDetailsPage(rule.body.id) + ); + }); + + // TODO: https://github.com/elastic/kibana/issues/161539 + // Skipped: https://github.com/elastic/kibana/issues/162818 + it.skip('Displays enrichment matched.* fields on the timeline', () => { + const expectedFields = { + 'threat.enrichments.matched.atomic': indicatorRuleMatchingDoc.atomic, + 'threat.enrichments.matched.type': indicatorRuleMatchingDoc.matchedType, + 'threat.enrichments.matched.field': + getNewThreatIndicatorRule().threat_mapping[0].entries[0].field, + 'threat.enrichments.matched.id': indicatorRuleMatchingDoc.matchedId, + 'threat.enrichments.matched.index': indicatorRuleMatchingDoc.matchedIndex, + }; + const fields = Object.keys(expectedFields) as Array; + + addsFieldsToTimeline('threat.enrichments.matched', fields); + + fields.forEach((field) => { + cy.get(TIMELINE_FIELD(field)).should('have.text', expectedFields[field]); }); - - after(() => { - cy.task('esArchiverUnload', { archiveName: 'threat_indicator' }); - cy.task('esArchiverUnload', { archiveName: 'suspicious_source_event' }); + }); + + it('Displays persisted enrichments on the JSON view', () => { + const expectedEnrichment = [ + { + 'indicator.file.hash.md5': ['9b6c3518a91d23ed77504b5416bfb5b3'], + 'matched.index': ['logs-ti_abusech.malware'], + 'indicator.file.type': ['elf'], + 'indicator.file.hash.tlsh': [ + '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE', + ], + 'feed.name': ['AbuseCH malware'], + 'indicator.file.hash.ssdeep': [ + '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL', + ], + 'indicator.file.hash.sha256': [ + 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + ], + 'indicator.first_seen': ['2021-03-10T08:02:14.000Z'], + 'matched.field': ['myhash.mysha256'], + 'indicator.type': ['file'], + 'matched.type': ['indicator_match_rule'], + 'matched.id': ['84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f'], + 'matched.atomic': ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'], + 'indicator.file.size': [80280], + }, + ]; + + expandFirstAlert(); + openJsonView(); + + cy.get(JSON_TEXT).then((x) => { + const parsed = JSON.parse(x.text()); + expect(parsed.fields['threat.enrichments']).to.deep.equal(expectedEnrichment); }); - - beforeEach(() => { - login(); - createRule({ ...getNewThreatIndicatorRule(), rule_id: 'rule_testing', enabled: true }).then( - (rule) => visitRuleDetailsPage(rule.body.id) - ); - }); - - // TODO: https://github.com/elastic/kibana/issues/161539 - // Skipped: https://github.com/elastic/kibana/issues/162818 - it.skip('Displays enrichment matched.* fields on the timeline', () => { - const expectedFields = { - 'threat.enrichments.matched.atomic': indicatorRuleMatchingDoc.atomic, - 'threat.enrichments.matched.type': indicatorRuleMatchingDoc.matchedType, - 'threat.enrichments.matched.field': - getNewThreatIndicatorRule().threat_mapping[0].entries[0].field, - 'threat.enrichments.matched.id': indicatorRuleMatchingDoc.matchedId, - 'threat.enrichments.matched.index': indicatorRuleMatchingDoc.matchedIndex, - }; - const fields = Object.keys(expectedFields) as Array; - - addsFieldsToTimeline('threat.enrichments.matched', fields); - - fields.forEach((field) => { - cy.get(TIMELINE_FIELD(field)).should('have.text', expectedFields[field]); + }); + + it('Displays threat indicator details on the threat intel tab', () => { + const expectedThreatIndicatorData = [ + { field: 'feed.name', value: 'AbuseCH malware' }, + { field: 'indicator.file.hash.md5', value: '9b6c3518a91d23ed77504b5416bfb5b3' }, + { + field: 'indicator.file.hash.sha256', + value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + }, + { + field: 'indicator.file.hash.ssdeep', + value: '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL', + }, + { + field: 'indicator.file.hash.tlsh', + value: '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE', + }, + { field: 'indicator.file.size', value: '80280' }, + { field: 'indicator.file.type', value: 'elf' }, + { field: 'indicator.first_seen', value: '2021-03-10T08:02:14.000Z' }, + { field: 'indicator.type', value: 'file' }, + { + field: 'matched.atomic', + value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + }, + { field: 'matched.field', value: 'myhash.mysha256' }, + { + field: 'matched.id', + value: '84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f', + }, + { field: 'matched.index', value: 'logs-ti_abusech.malware' }, + { field: 'matched.type', value: 'indicator_match_rule' }, + ]; + + expandFirstAlert(); + openThreatIndicatorDetails(); + + cy.get(ENRICHMENT_COUNT_NOTIFICATION).should('have.text', '1'); + cy.get(THREAT_DETAILS_VIEW).within(() => { + cy.get(TABLE_ROWS).should('have.length', expectedThreatIndicatorData.length); + expectedThreatIndicatorData.forEach((row, index) => { + cy.get(TABLE_ROWS) + .eq(index) + .within(() => { + cy.get(TABLE_CELL).eq(0).should('have.text', row.field); + cy.get(TABLE_CELL).eq(1).should('have.text', row.value); + }); }); }); + }); - it('Displays persisted enrichments on the JSON view', () => { - const expectedEnrichment = [ - { - 'indicator.file.hash.md5': ['9b6c3518a91d23ed77504b5416bfb5b3'], - 'matched.index': ['logs-ti_abusech.malware'], - 'indicator.file.type': ['elf'], - 'indicator.file.hash.tlsh': [ - '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE', - ], - 'feed.name': ['AbuseCH malware'], - 'indicator.file.hash.ssdeep': [ - '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL', - ], - 'indicator.file.hash.sha256': [ - 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - ], - 'indicator.first_seen': ['2021-03-10T08:02:14.000Z'], - 'matched.field': ['myhash.mysha256'], - 'indicator.type': ['file'], - 'matched.type': ['indicator_match_rule'], - 'matched.id': ['84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f'], - 'matched.atomic': ['a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3'], - 'indicator.file.size': [80280], - }, - ]; - - expandFirstAlert(); - openJsonView(); - - cy.get(JSON_TEXT).then((x) => { - const parsed = JSON.parse(x.text()); - expect(parsed.fields['threat.enrichments']).to.deep.equal(expectedEnrichment); - }); + describe('with additional indicators', () => { + before(() => { + cy.task('esArchiverLoad', { archiveName: 'threat_indicator2' }); }); - it('Displays threat indicator details on the threat intel tab', () => { - const expectedThreatIndicatorData = [ - { field: 'feed.name', value: 'AbuseCH malware' }, - { field: 'indicator.file.hash.md5', value: '9b6c3518a91d23ed77504b5416bfb5b3' }, - { - field: 'indicator.file.hash.sha256', - value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - }, - { - field: 'indicator.file.hash.ssdeep', - value: '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL', - }, - { - field: 'indicator.file.hash.tlsh', - value: '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE', - }, - { field: 'indicator.file.size', value: '80280' }, - { field: 'indicator.file.type', value: 'elf' }, - { field: 'indicator.first_seen', value: '2021-03-10T08:02:14.000Z' }, - { field: 'indicator.type', value: 'file' }, - { - field: 'matched.atomic', - value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - }, - { field: 'matched.field', value: 'myhash.mysha256' }, - { - field: 'matched.id', - value: '84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f', - }, - { field: 'matched.index', value: 'logs-ti_abusech.malware' }, - { field: 'matched.type', value: 'indicator_match_rule' }, - ]; - - expandFirstAlert(); - openThreatIndicatorDetails(); - - cy.get(ENRICHMENT_COUNT_NOTIFICATION).should('have.text', '1'); - cy.get(THREAT_DETAILS_VIEW).within(() => { - cy.get(TABLE_ROWS).should('have.length', expectedThreatIndicatorData.length); - expectedThreatIndicatorData.forEach((row, index) => { - cy.get(TABLE_ROWS) - .eq(index) - .within(() => { - cy.get(TABLE_CELL).eq(0).should('have.text', row.field); - cy.get(TABLE_CELL).eq(1).should('have.text', row.value); - }); - }); - }); + after(() => { + cy.task('esArchiverUnload', { archiveName: 'threat_indicator2' }); }); - describe('with additional indicators', () => { - before(() => { - cy.task('esArchiverLoad', { archiveName: 'threat_indicator2' }); - }); - - after(() => { - cy.task('esArchiverUnload', { archiveName: 'threat_indicator2' }); - }); + it('Displays matched fields from both indicator match rules and investigation time enrichments on Threat Intel tab', () => { + const indicatorMatchRuleEnrichment = { + field: 'myhash.mysha256', + value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + feedName: 'AbuseCH malware', + }; + const investigationTimeEnrichment = { + field: 'source.ip', + value: '192.168.1.1', + feedName: 'feed_name', + }; - it('Displays matched fields from both indicator match rules and investigation time enrichments on Threat Intel tab', () => { - const indicatorMatchRuleEnrichment = { - field: 'myhash.mysha256', - value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', - feedName: 'AbuseCH malware', - }; - const investigationTimeEnrichment = { - field: 'source.ip', - value: '192.168.1.1', - feedName: 'feed_name', - }; - - expandFirstAlert(); - viewThreatIntelTab(); - setEnrichmentDates('08/05/2018 10:00 AM'); - - cy.get(`${INDICATOR_MATCH_ENRICHMENT_SECTION} ${THREAT_DETAILS_ACCORDION}`) - .should('exist') - .should( - 'have.text', - `${indicatorMatchRuleEnrichment.field} ${indicatorMatchRuleEnrichment.value} from ${indicatorMatchRuleEnrichment.feedName}` - ); - - cy.get(`${INVESTIGATION_TIME_ENRICHMENT_SECTION} ${THREAT_DETAILS_ACCORDION}`) - .should('exist') - .should( - 'have.text', - `${investigationTimeEnrichment.field} ${investigationTimeEnrichment.value} from ${investigationTimeEnrichment.feedName}` - ); - }); + expandFirstAlert(); + viewThreatIntelTab(); + setEnrichmentDates('08/05/2018 10:00 AM'); + + cy.get(`${INDICATOR_MATCH_ENRICHMENT_SECTION} ${THREAT_DETAILS_ACCORDION}`) + .should('exist') + .should( + 'have.text', + `${indicatorMatchRuleEnrichment.field} ${indicatorMatchRuleEnrichment.value} from ${indicatorMatchRuleEnrichment.feedName}` + ); + + cy.get(`${INVESTIGATION_TIME_ENRICHMENT_SECTION} ${THREAT_DETAILS_ACCORDION}`) + .should('exist') + .should( + 'have.text', + `${investigationTimeEnrichment.field} ${investigationTimeEnrichment.value} from ${investigationTimeEnrichment.feedName}` + ); }); - } -); + }); +}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/status/alert_status.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/status/alert_status.cy.ts index 911168362c946..f4a11f17aa59f 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/status/alert_status.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/detection_alerts/status/alert_status.cy.ts @@ -240,7 +240,7 @@ describe.skip('Changing alert status', { tags: ['@ess', '@serverless'] }, () => // This test is unable to be run in serverless as `reader` is not available and viewer is currently reserved // https://github.com/elastic/kibana/pull/169723#issuecomment-1793191007 // https://github.com/elastic/kibana/issues/170583 - context('User is readonly', { tags: ['@ess', '@brokenInServerless'] }, () => { + context('User is readonly', { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { login(); visit(ALERTS_URL); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/endpoint_exceptions.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/endpoint_exceptions.cy.ts index 34fa454e7a641..5ba114e9de58c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/endpoint_exceptions.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/endpoint_exceptions.cy.ts @@ -43,7 +43,7 @@ import { // See https://github.com/elastic/kibana/issues/163967 describe.skip( 'Endpoint Exceptions workflows from Alert', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { const ITEM_NAME = 'Sample Exception List Item'; const ITEM_NAME_EDIT = 'Sample Exception List Item'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/rule_exceptions/auto_populate_with_alert_data.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/rule_exceptions/auto_populate_with_alert_data.cy.ts index 788907e461526..b230c1a0a3ceb 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/rule_exceptions/auto_populate_with_alert_data.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/alerts_table_flow/rule_exceptions/auto_populate_with_alert_data.cy.ts @@ -40,7 +40,7 @@ import { waitForAlertsToPopulate } from '../../../../../../tasks/create_new_rule // See https://github.com/elastic/kibana/issues/163967 describe.skip( 'Auto populate exception with Alert data', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { const ITEM_NAME = 'Sample Exception Item'; const ITEM_NAME_EDIT = 'Sample Exception Item Edit'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/rule_details_flow/add_edit_exception.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/rule_details_flow/add_edit_exception.cy.ts index 12a44ab6b40bd..25a01389c7eaa 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/rule_details_flow/add_edit_exception.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/exceptions/rule_details_flow/add_edit_exception.cy.ts @@ -61,7 +61,7 @@ import { waitForAlertsToPopulate } from '../../../../../tasks/create_new_rule'; // TODO: https://github.com/elastic/kibana/issues/161539 describe( 'Add/edit exception from rule details', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { const NUMBER_OF_AUDITBEAT_EXCEPTIONS_ALERTS = '3 alerts'; const FIELD_DIFFERENT_FROM_EXISTING_ITEM_FIELD = 'agent.name'; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/overview/cti_link_panel.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/overview/cti_link_panel.cy.ts index 65cd15dbce8fe..5beb203443b88 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/overview/cti_link_panel.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/overview/cti_link_panel.cy.ts @@ -36,7 +36,7 @@ describe.skip('CTI Link Panel', { tags: ['@ess', '@serverless', '@skipInServerle // TODO: https://github.com/elastic/kibana/issues/161539 describe( 'enabled threat intel module', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { before(() => { // illegal_argument_exception: unknown setting [index.lifecycle.name] diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_actions/rule_actions.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_actions/rule_actions.cy.ts index 11de935630876..050b10e1c85a9 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_actions/rule_actions.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_actions/rule_actions.cy.ts @@ -35,7 +35,7 @@ import { CREATE_RULE_URL } from '../../../../urls/navigation'; // TODO: https://github.com/elastic/kibana/issues/161539 describe( 'Rule actions during detection rule creation', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { const indexConnector = getIndexConnector(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer.cy.ts index 0de3f32d9cc6a..50cc4c3995684 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer.cy.ts @@ -100,7 +100,7 @@ describe.skip('Sourcerer', { tags: ['@ess', '@serverless'] }, () => { it( 'adds a pattern to the default index and correctly filters out auditbeat-*', - { tags: '@brokenInServerless' }, + { tags: '@skipInServerless' }, () => { openSourcerer(); isSourcererSelection(`auditbeat-*`); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_permissions.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_permissions.cy.ts index a1692fca5b859..c6d03fe8ec151 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_permissions.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_permissions.cy.ts @@ -15,7 +15,7 @@ import { login } from '../../../../tasks/login'; const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kibana*,fakebeat-*']; -describe('Sourcerer permissions', { tags: ['@ess', '@brokenInServerless'] }, () => { +describe('Sourcerer permissions', { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { dataViews.forEach((dataView: string) => postDataView(dataView)); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_timeline.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_timeline.cy.ts index 252bb222c2f3e..b99b93dd956ee 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_timeline.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/sourcerer/sourcerer_timeline.cy.ts @@ -39,7 +39,7 @@ import { closeTimeline, openTimelineById } from '../../../../tasks/timeline'; const siemDataViewTitle = 'Security Default Data View'; const dataViews = ['logs-*', 'metrics-*', '.kibana-event-log-*']; -describe('Timeline scope', { tags: ['@ess', '@serverless', '@brokenInServerless'] }, () => { +describe('Timeline scope', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { beforeEach(() => { cy.clearLocalStorage(); login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/coverage_overview/coverage_overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/coverage_overview/coverage_overview.cy.ts index adfab13fe619f..ce7a1c8d0b535 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/coverage_overview/coverage_overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/coverage_overview/coverage_overview.cy.ts @@ -190,7 +190,7 @@ const prebuiltRules = [ ]; // https://github.com/elastic/kibana/issues/179052 -describe('Coverage overview', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { +describe('Coverage overview', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { describe('base cases', () => { beforeEach(() => { login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts index b165eb1fc6845..4e2b57e7a5e26 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_authorization.cy.ts @@ -68,7 +68,7 @@ const loginPageAsWriteAuthorizedUser = (url: string) => { // https://github.com/elastic/kibana/issues/179965 describe( 'Detection rules, Prebuilt Rules Installation and Update - Authorization/RBAC', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { beforeEach(() => { preventPrebuiltRulesPackageInstallation(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts index 524ce86e6e73c..0102cfb349edf 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_update_error_handling.cy.ts @@ -39,7 +39,7 @@ import { visitRulesManagementTable } from '../../../../tasks/rules_management'; // https://github.com/elastic/kibana/issues/179970 describe( 'Detection rules, Prebuilt Rules Installation and Update - Error handling', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { beforeEach(() => { preventPrebuiltRulesPackageInstallation(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts index 523d0ec0ad4e0..259440f1c2abd 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/install_workflow.cy.ts @@ -32,7 +32,7 @@ import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; describe( 'Detection rules, Prebuilt Rules Installation and Update workflow', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { describe('Installation of prebuilt rules', () => { const RULE_1 = createRuleAssetSavedObject({ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts index 5c353729c571a..63290d850729c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/management.cy.ts @@ -51,7 +51,7 @@ const rules = Array.from(Array(5)).map((_, i) => { }); // https://github.com/elastic/kibana/issues/179973 -describe('Prebuilt rules', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { +describe('Prebuilt rules', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { beforeEach(() => { login(); deleteAlertsAndRules(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts index c2467bf1707e2..02a59e1b17f2e 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/notifications.cy.ts @@ -57,7 +57,7 @@ describe( // https://github.com/elastic/kibana/issues/179967 it( 'should NOT display install or update notifications when latest rules are installed', - { tags: ['@brokenInServerlessQA'] }, + { tags: ['@skipInServerless'] }, () => { visitRulesManagementTable(); createAndInstallMockedPrebuiltRules([RULE_1]); @@ -72,7 +72,7 @@ describe( }); // https://github.com/elastic/kibana/issues/179968 - describe('Notifications', { tags: ['@brokenInServerlessQA'] }, () => { + describe('Notifications', { tags: ['@skipInServerless'] }, () => { beforeEach(() => { installPrebuiltRuleAssets([RULE_1]); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts index a2242331bf211..ad0cc76f623f5 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/related_integrations/related_integrations.cy.ts @@ -47,7 +47,7 @@ import { // https://github.com/elastic/kibana/issues/179943 -describe('Related integrations', { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, () => { +describe('Related integrations', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { const DATA_STREAM_NAME = 'logs-related-integrations-test'; const PREBUILT_RULE_NAME = 'Prebuilt rule with related integrations'; const RULE_RELATED_INTEGRATIONS: IntegrationDefinition[] = [ diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts index f997ff1524b45..c98f28fef2da0 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules.cy.ts @@ -183,7 +183,7 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () => }); // github.com/elastic/kibana/issues/179954 - it('Only prebuilt rules selected', { tags: ['@brokenInServerlessQA'] }, () => { + it('Only prebuilt rules selected', { tags: ['@skipInServerless'] }, () => { createAndInstallMockedPrebuiltRules(PREBUILT_RULES); // select Elastic(prebuilt) rules, check if we can't proceed further, as Elastic rules are not editable @@ -204,7 +204,7 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () => // https://github.com/elastic/kibana/issues/179955 it( 'Prebuilt and custom rules selected: user proceeds with custom rules editing', - { tags: ['@brokenInServerlessQA'] }, + { tags: ['@skipInServerless'] }, () => { getRulesManagementTableRows().then((existedRulesRows) => { createAndInstallMockedPrebuiltRules(PREBUILT_RULES); @@ -235,7 +235,7 @@ describe('Detection rules, bulk edit', { tags: ['@ess', '@serverless'] }, () => // https://github.com/elastic/kibana/issues/179956 it( 'Prebuilt and custom rules selected: user cancels action', - { tags: ['@brokenInServerlessQA'] }, + { tags: ['@skipInServerless'] }, () => { createAndInstallMockedPrebuiltRules(PREBUILT_RULES); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts index e2acc801a6652..5762e273e9686 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/bulk_actions/bulk_edit_rules_actions.cy.ts @@ -75,7 +75,7 @@ const expectedSlackMessage = 'Slack action test message'; // https://github.com/elastic/kibana/issues/179958 describe( 'Detection rules, bulk edit of rule actions', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { beforeEach(() => { login(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts index d0a113f7253fe..36bdcd7c23379 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rule_actions/import_export/export_rule.cy.ts @@ -96,7 +96,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { // https://github.com/elastic/kibana/issues/179959 it( 'shows a modal saying that no rules can be exported if all the selected rules are prebuilt', - { tags: ['@brokenInServerlessQA'] }, + { tags: ['@skipInServerless'] }, function () { createAndInstallMockedPrebuiltRules(prebuiltRules); @@ -111,7 +111,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { ); // https://github.com/elastic/kibana/issues/179960 - it('exports only custom rules', { tags: ['@brokenInServerlessQA'] }, function () { + it('exports only custom rules', { tags: ['@skipInServerless'] }, function () { const expectedNumberCustomRulesToBeExported = 1; createAndInstallMockedPrebuiltRules(prebuiltRules); @@ -164,7 +164,7 @@ describe('Export rules', { tags: ['@ess', '@serverless'] }, () => { }); // https://github.com/elastic/kibana/issues/180029 - it('exports custom rules with exceptions', { tags: ['@brokenInServerlessQA'] }, function () { + it('exports custom rules with exceptions', { tags: ['@skipInServerless'] }, function () { // one rule with exception, one without it const expectedNumberCustomRulesToBeExported = 2; diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts index 275c24fca866c..b07069b65a4ef 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/rules_table/rules_table_selection.cy.ts @@ -35,69 +35,65 @@ const RULE_2 = createRuleAssetSavedObject({ }); // https://github.com/elastic/kibana/issues/179961 -describe( - 'Rules table: selection', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, - () => { - beforeEach(() => { - login(); - /* Create and install two mock rules */ - createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]); - visit(RULES_MANAGEMENT_URL); - waitForPrebuiltDetectionRulesToBeLoaded(); - disableAutoRefresh(); - }); +describe('Rules table: selection', { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { + beforeEach(() => { + login(); + /* Create and install two mock rules */ + createAndInstallMockedPrebuiltRules([RULE_1, RULE_2]); + visit(RULES_MANAGEMENT_URL); + waitForPrebuiltDetectionRulesToBeLoaded(); + disableAutoRefresh(); + }); - it('should correctly update the selection label when rules are individually selected and unselected', () => { - waitForPrebuiltDetectionRulesToBeLoaded(); + it('should correctly update the selection label when rules are individually selected and unselected', () => { + waitForPrebuiltDetectionRulesToBeLoaded(); - selectRulesByName(['Test rule 1', 'Test rule 2']); + selectRulesByName(['Test rule 1', 'Test rule 2']); - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '2'); + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '2'); - unselectRulesByName(['Test rule 1', 'Test rule 2']); + unselectRulesByName(['Test rule 1', 'Test rule 2']); - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); - }); + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); + }); - it('should correctly update the selection label when rules are bulk selected and then bulk un-selected', () => { - waitForPrebuiltDetectionRulesToBeLoaded(); + it('should correctly update the selection label when rules are bulk selected and then bulk un-selected', () => { + waitForPrebuiltDetectionRulesToBeLoaded(); - cy.get(SELECT_ALL_RULES_BTN).click(); + cy.get(SELECT_ALL_RULES_BTN).click(); - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); - }); + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); + }); - // Un-select all rules via the Bulk Selection button from the Utility bar - cy.get(SELECT_ALL_RULES_BTN).click(); + // Un-select all rules via the Bulk Selection button from the Utility bar + cy.get(SELECT_ALL_RULES_BTN).click(); - // Current selection should be 0 rules - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); - // Bulk selection button should be back to displaying all rules - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); - }); + // Current selection should be 0 rules + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); + // Bulk selection button should be back to displaying all rules + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); }); + }); - it('should correctly update the selection label when rules are bulk selected and then unselected via the table select all checkbox', () => { - waitForPrebuiltDetectionRulesToBeLoaded(); + it('should correctly update the selection label when rules are bulk selected and then unselected via the table select all checkbox', () => { + waitForPrebuiltDetectionRulesToBeLoaded(); - cy.get(SELECT_ALL_RULES_BTN).click(); + cy.get(SELECT_ALL_RULES_BTN).click(); - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); - }); + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', availablePrebuiltRulesCount); + }); - // Un-select all rules via the Un-select All checkbox from the table - cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click(); + // Un-select all rules via the Un-select All checkbox from the table + cy.get(SELECT_ALL_RULES_ON_PAGE_CHECKBOX).click(); - // Current selection should be 0 rules - cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); - // Bulk selection button should be back to displaying all rules - getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { - cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); - }); + // Current selection should be 0 rules + cy.get(SELECTED_RULES_NUMBER_LABEL).should('contain.text', '0'); + // Bulk selection button should be back to displaying all rules + getAvailablePrebuiltRulesCount().then((availablePrebuiltRulesCount) => { + cy.get(SELECT_ALL_RULES_BTN).should('contain.text', availablePrebuiltRulesCount); }); - } -); + }); +}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/legacy_risk_score.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/legacy_risk_score.cy.ts index 32b27077ed9d3..5f697f706c40c 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/legacy_risk_score.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/legacy_risk_score.cy.ts @@ -49,7 +49,7 @@ const DATE_BEFORE_ALERT_CREATION = moment().format(DATE_FORMAT); // https://github.com/elastic/kibana/issues/179686 describe( 'Entity Analytics Dashboard', - { tags: ['@ess', '@serverless', '@brokenInServerlessQA'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { before(() => { cy.task('esArchiverLoad', { archiveName: 'auditbeat_multiple' }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/new_risk_score.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/new_risk_score.cy.ts index 6bacd88defad7..2f16d5c77a833 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/new_risk_score.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/dashboards/entity_analytics/new_risk_score.cy.ts @@ -76,7 +76,7 @@ describe('Entity Analytics Dashboard', { tags: ['@ess', '@serverless'] }, () => }); // https://github.com/elastic/kibana/issues/179687 - describe('When risk engine is enabled', { tags: ['@brokenInServerlessQA'] }, () => { + describe('When risk engine is enabled', { tags: ['@skipInServerless'] }, () => { beforeEach(() => { login(); mockRiskEngineEnabled(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/entity_flyout.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/entity_flyout.cy.ts index 398813da901ee..75f3ea23515ba 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/entity_flyout.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics/entity_flyout.cy.ts @@ -132,7 +132,7 @@ describe( }); // https://github.com/elastic/kibana/issues/179248 - describe('Managed data section', { tags: ['@brokenInServerlessQA'] }, () => { + describe('Managed data section', { tags: ['@skipInServerless'] }, () => { beforeEach(() => { mockFleetInstalledIntegrations([ { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/inspect/inspect_button.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/inspect/inspect_button.cy.ts index fbe1f5fe8c33a..e015d26887148 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/inspect/inspect_button.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/inspect/inspect_button.cy.ts @@ -27,7 +27,7 @@ const DATA_VIEW = 'auditbeat-*'; // FLAKY: https://github.com/elastic/kibana/issues/178367 describe.skip( 'Inspect Explore pages', - { tags: ['@ess', '@serverless', '@brokenInServerless'] }, + { tags: ['@ess', '@serverless', '@skipInServerless'] }, () => { before(() => { // illegal_argument_exception: unknown setting [index.lifecycle.name] diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/ml/ml_conditional_links.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/ml/ml_conditional_links.cy.ts index c90f8bea96fd7..f2fc45db975e2 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/ml/ml_conditional_links.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/ml/ml_conditional_links.cy.ts @@ -26,7 +26,7 @@ import { mlNetworkSingleIpNullKqlQuery, } from '../../../urls/ml_conditional_links'; -describe('ml conditional links', { tags: ['@ess', '@brokenInServerless'] }, () => { +describe('ml conditional links', { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { login(); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts index e1159a811a2ae..8ad3315bf36b1 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/overview/overview.cy.ts @@ -49,7 +49,7 @@ describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { }); // https://github.com/elastic/kibana/issues/173168 - describe('Favorite Timelines', { tags: ['@brokenInServerless'] }, () => { + describe('Favorite Timelines', { tags: ['@skipInServerless'] }, () => { it('should appear on overview page', () => { createTimeline() .then((response) => response.body.data.persistTimeline.timeline.savedObjectId) @@ -66,7 +66,7 @@ describe('Overview Page', { tags: ['@ess', '@serverless'] }, () => { }); }); -describe('Overview page with no data', { tags: '@brokenInServerless' }, () => { +describe('Overview page with no data', { tags: '@skipInServerless' }, () => { it('Splash screen should be here', () => { login(); visitWithTimeRange(OVERVIEW_URL); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/compatibility.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/compatibility.cy.ts index fa11642abc172..1a13708e0d779 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/compatibility.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/compatibility.cy.ts @@ -32,7 +32,7 @@ const ABSOLUTE_DATE = { const RULE_ID = '5a4a0460-d822-11eb-8962-bfd4aff0a9b3'; -describe('URL compatibility', { tags: ['@ess', '@brokenInServerless'] }, () => { +describe('URL compatibility', { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { login(); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/state.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/state.cy.ts index dbc92ebf3fb67..4e34fedb1dd43 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/state.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/explore/urls/state.cy.ts @@ -70,7 +70,7 @@ const ABSOLUTE_DATE = { const mockTimeline = getTimeline(); -describe('url state', { tags: ['@ess', '@brokenInServerless'] }, () => { +describe('url state', { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { login(); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/changing_alert_status.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/changing_alert_status.cy.ts index f173e78988638..bb5d45654e287 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/changing_alert_status.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/changing_alert_status.cy.ts @@ -40,7 +40,7 @@ import { visit } from '../../../tasks/navigation'; import { ALERTS_URL } from '../../../urls/navigation'; // Iusse tracked in: https://github.com/elastic/kibana/issues/167809 -describe('Changing alert status', { tags: ['@ess', '@brokenInServerless'] }, () => { +describe('Changing alert status', { tags: ['@ess', '@skipInServerless'] }, () => { before(() => { cy.task('esArchiverLoad', { archiveName: 'auditbeat_multiple' }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_analyzer_graph_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_analyzer_graph_tab.cy.ts index 3458eac37fe80..bb273093c474d 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_analyzer_graph_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/alerts/expandable_flyout/alert_details_left_panel_analyzer_graph_tab.cy.ts @@ -25,7 +25,7 @@ import { waitForAlertsToPopulate } from '../../../../tasks/create_new_rule'; // TODO enable once the visualize tabs are back describe.skip( 'Alert details expandable flyout left panel analyzer graph', - { tags: ['@ess', '@brokenInServerless'] }, + { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { deleteAlertsAndRules(); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/esql/discover_timeline_state_integration.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/esql/discover_timeline_state_integration.cy.ts index d644b0fa2a790..ff932a44b0da4 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/esql/discover_timeline_state_integration.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/esql/discover_timeline_state_integration.cy.ts @@ -72,7 +72,7 @@ const handleIntercepts = () => { describe( 'Discover Timeline State Integration', { - tags: ['@ess', '@brokenInServerless'], + tags: ['@ess', '@skipInServerless'], }, () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/overview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/overview.cy.ts index 412e0bab6a948..ef5d9c75df406 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/overview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/overview.cy.ts @@ -30,7 +30,7 @@ import { TIMELINES_URL } from '../../../urls/navigation'; const mockTimeline = getTimeline(); const mockFavoritedTimeline = getFavoritedTimeline(); -describe('timeline overview search', { tags: ['@ess', 'serverless'] }, () => { +describe('timeline overview search', { tags: ['@ess', '@skipInServerless'] }, () => { beforeEach(() => { deleteTimelines(); createTimeline(mockFavoritedTimeline) diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/query_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/query_tab.cy.ts index 3e7687bd7050b..961be88c42ff3 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/query_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/query_tab.cy.ts @@ -64,7 +64,7 @@ describe('Timeline query tab', { tags: ['@ess', '@serverless'] }, () => { .and('match', /Unpin the event in row 2/); }); - it('should have an unlock icon', { tags: '@brokenInServerless' }, () => { + it('should have an unlock icon', { tags: '@skipInServerless' }, () => { cy.get(UNLOCKED_ICON).should('be.visible'); }); }); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/unified_components/query_tab.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/unified_components/query_tab.cy.ts index 058025b596956..c54b593472dcd 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/unified_components/query_tab.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/unified_components/query_tab.cy.ts @@ -28,7 +28,7 @@ import { ALERTS_URL } from '../../../../urls/navigation'; describe( 'Unsaved Timeline query tab', { - tags: ['@ess', '@serverless', '@brokenInServerlessQA'], + tags: ['@ess', '@serverless', '@skipInServerless'], env: { ftrConfig: { kbnServerArgs: [ From 4047cc862b9f2a7c4a799805871eb001922e317b Mon Sep 17 00:00:00 2001 From: Jan Monschke Date: Mon, 8 Apr 2024 12:43:23 +0200 Subject: [PATCH 04/11] [Threat Hunting][ADR] 0001 - Saving of timeline-associated saved objects (#179830) ## Summary Moving the ADR discussion into Kibana for easier association of code to an ADR. Pinging @michaelolo24 and @kqualters-elastic since they previously commented on the ADR in a private repo. --- ...ng_of_timeline_associated_saved_objects.md | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 x-pack/plugins/security_solution/docs/adrs/threat_hunting/0001_saving_of_timeline_associated_saved_objects.md diff --git a/x-pack/plugins/security_solution/docs/adrs/threat_hunting/0001_saving_of_timeline_associated_saved_objects.md b/x-pack/plugins/security_solution/docs/adrs/threat_hunting/0001_saving_of_timeline_associated_saved_objects.md new file mode 100644 index 0000000000000..a2d98fcb750e1 --- /dev/null +++ b/x-pack/plugins/security_solution/docs/adrs/threat_hunting/0001_saving_of_timeline_associated_saved_objects.md @@ -0,0 +1,85 @@ +--- +status: accepted +date: 2024-04-05 +--- + +# 0001 - Saving of timeline-associated saved objects + +## Context and Problem Statement + +As described in [#178182](https://github.com/elastic/kibana/issues/178182), the removal of autosave on timeline resulted in a regression in which pinned events and comments on unsaved timelines are lost. + +When commenting on an unsaved timeline or an event in a timeline or when pinning an event in a timeline, the pins/comments are lost when the timeline has not been saved before. This used to work in 8.11 and is broken from version 8.12 onwards. + +What's causing this bug is that the associated saved objects have a field `timelineId` that connects them to the timeline. When a (pin/note) save request comes in, the server checks for that field and if it doesn't exist, it will create a new timeline on the fly and return its `timelineId` and `timelineVersion` as part of the saved object's response. + +https://github.com/elastic/kibana/blob/2df44b9f7f76f3d03f6e32be7f2a39034f97c22e/x-pack/plugins/security_solution/server/lib/timeline/saved_object/notes/saved_object.ts#L138-L148 + +These two fields are currently not used in the timeline middleware: + +https://github.com/elastic/kibana/blob/2df44b9f7f76f3d03f6e32be7f2a39034f97c22e/x-pack/plugins/security_solution/public/timelines/store/middlewares/timeline_note.ts#L60 + +This gives the false impression that the associated saved object has been stored but it actually has been associated to a different timeline. Subsequent saves of the active timeline (which is not the associated timeline) will then create a new `timelineId`. When the page is reloaded, the associated saved objects will be gone from that timeline. + +## Considered Options + +- Auto-saving timeline as draft when an a note or pinned event is created +- Disabling notes and pinned events until the timeline is saved +- Opening the save modal when clicking the note/pin buttons on an unsaved timeline +- Caching notes and pinned events locally, saving when timeline is saved + +## Decision Outcome + +Chosen option: **Auto-saving timeline as draft when an a note or pinned event is created** + +### Confirmation + +https://github.com/elastic/kibana/pull/178212 + +## Discussion of options + +### Auto-saving timeline as draft when an a note or pinned event is created + +In this proposed solution we're bringing back parts of the "auto-save" behaviour. A request to save the associated saved object will precede request to create a draft timeline (given that timeline has not been saved previously). Draft timelines are ephemeral and tied to a specific user so "auto-saving" will not create version conflicts since they cannot be concurrently edited. +This approach has been implemented in https://github.com/elastic/kibana/pull/178212 . + +| Pros | Cons | +| -------------------------------------------- | --------------------------------------------------------------- | +| Mimics the previous auto-save behaviour | Draft timelines are not surfaced anywhere in the UI (by design) | +| Easy to test in unit and in acceptance tests | | +| Simple, very few things can go wrong | | +| No public API changes necessary | | + +### Disabling notes and pinned events until the timeline is saved + +Disabling all note/pin buttons and lists until the timeline is saved makes sure that no associated saved objects can be created with a missing `timelineId`. Subsequently, the code that creates timelines when `timelineId` is missing should be removed as well. +This approach has been implemented in https://github.com/elastic/kibana/pull/178525 . + +| Pros | Cons | +| -------------------------------------------- | ------------------------------------------------------------------------- | +| Easy to test in unit and in acceptance tests | Requires public API changes | +| Simple, very few things can go wrong | Users might use note/pin less often, since they are disabled "by default" | +| | Possibility to create orphaned notes when duplicating timelines or creating one from an alert that has an investigation guide | + +### Opening the save modal when clicking the note/pin buttons on an unsaved timeline + +Instead of disabling the note/pin buttons on an unsaved timeline, they are enabled by default but instead of performing the note/pin action right away, they're opening the timeline's save modal. The modal contains a callout, explaining that they need to save the timeline first in order to perform their original action. After saving the timeline, their original action is performed. +This approach has been proposed here: https://github.com/elastic/kibana/pull/178525#issuecomment-1992596905 + +| Pros | Cons | +| -------------------------------------------- | --------------------------------------------------------- | +| Easy to test in unit and in acceptance tests | Users might be confused as to why the save modal opens up | +| Simple, very few things can go wrong | | +| Requires no public API changes | | + +### Caching notes and pinned events locally, saving when timeline is saved + +Notes and pinned events are stored locally, until the timeline is persisted. When the timeline is saved, the locally stored saved objects are either sent alongside the save request or they are saved on the client once the save request finishes. +This approach comes out of an internal Slack thread. + +| Pros | Cons +| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| | (potentially) requires public API changes | +| | We need to make sure, this local cache is emptied properly when switching timelines | +| | Unclear what should happen if a save of an associated saved object fails, while the timeline has been created and the other objects could be created. There are no transactions. | +| | Likely requires changes to the "Unsaved changes" modal | From 9f8433e56413f5961df9793a6424ffa68a5318c3 Mon Sep 17 00:00:00 2001 From: Davis Plumlee <56367316+dplumlee@users.noreply.github.com> Date: Mon, 8 Apr 2024 07:55:29 -0400 Subject: [PATCH 05/11] [Security Solution] Setup field form component (#178131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Addresses https://github.com/elastic/kibana/issues/173626 Adds a markdown component in the create and edit rule forms so that users are able to add their own setup guides to custom rules. Also updates the `create` and `update` rule schemas and route logic to handle these new cases through the API. [Flaky test run (internal)](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5603) ### Screenshots ![Screenshot 2024-03-08 at 11 12 25 AM](https://github.com/elastic/kibana/assets/56367316/5a00b007-d02d-4f1e-b1ba-ca7ba7f68bbd) ![Screenshot 2024-03-06 at 10 25 47 AM](https://github.com/elastic/kibana/assets/56367316/a3973e10-1c82-4981-b38d-69faf06a5993) ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [x] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../model/rule_schema/rule_schemas.gen.ts | 4 +- .../rule_schema/rule_schemas.schema.yaml | 9 ++-- .../components/markdown_editor/editor.tsx | 6 ++- .../components/markdown_editor/eui_form.tsx | 4 +- .../step_about_rule_details/index.test.tsx | 48 ++++++++++--------- .../components/description_step/helpers.tsx | 23 +++++++++ .../description_step/index.test.tsx | 17 ++++++- .../components/description_step/index.tsx | 4 ++ .../step_about_rule/default_value.ts | 1 + .../components/step_about_rule/index.test.tsx | 2 + .../components/step_about_rule/index.tsx | 12 +++++ .../components/step_about_rule/schema.tsx | 17 +++++++ .../step_about_rule/translations.ts | 7 +++ .../pages/rule_creation/helpers.test.ts | 9 ++++ .../components/rules_table/__mocks__/mock.ts | 5 +- .../detection_engine/rules/helpers.test.tsx | 23 +++++---- .../pages/detection_engine/rules/helpers.tsx | 11 +++-- .../pages/detection_engine/rules/types.ts | 2 + .../pages/detection_engine/rules/utils.ts | 1 + .../logic/actions/duplicate_rule.test.ts | 16 ------- .../logic/actions/duplicate_rule.ts | 2 - .../logic/crud/update_rules.ts | 2 +- .../normalization/rule_converters.ts | 3 -- .../create_rules.ts | 22 +++++++++ .../patch_rules.ts | 32 +++++++++++++ .../update_rules.ts | 34 +++++++++++++ 26 files changed, 245 insertions(+), 71 deletions(-) diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index 9d27297d11bbe..d7a8b83ec28f4 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -52,12 +52,12 @@ import { RuleReferenceArray, MaxSignals, ThreatArray, + SetupGuide, RuleObjectId, RuleSignatureId, IsRuleImmutable, RelatedIntegrationArray, RequiredFieldArray, - SetupGuide, RuleQuery, IndexPatternArray, DataViewId, @@ -134,6 +134,7 @@ export const BaseDefaultableFields = z.object({ references: RuleReferenceArray.optional(), max_signals: MaxSignals.optional(), threat: ThreatArray.optional(), + setup: SetupGuide.optional(), }); export type BaseCreateProps = z.infer; @@ -162,7 +163,6 @@ export const ResponseFields = z.object({ revision: z.number().int().min(0), related_integrations: RelatedIntegrationArray, required_fields: RequiredFieldArray, - setup: SetupGuide, execution_summary: RuleExecutionSummary.optional(), }); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index 22308557c3aaa..d3a09d8355727 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -128,6 +128,8 @@ components: $ref: './common_attributes.schema.yaml#/components/schemas/MaxSignals' threat: $ref: './common_attributes.schema.yaml#/components/schemas/ThreatArray' + setup: + $ref: './common_attributes.schema.yaml#/components/schemas/SetupGuide' BaseCreateProps: x-inline: true @@ -174,7 +176,7 @@ components: revision: type: integer minimum: 0 - # NOTE: For now, Related Integrations, Required Fields and Setup Guide are + # NOTE: For now, Related Integrations and Required Fields are # supported for prebuilt rules only. We don't want to allow users to edit these 3 # fields via the API. If we added them to baseParams.defaultable, they would # become a part of the request schema as optional fields. This is why we add them @@ -183,8 +185,6 @@ components: $ref: './common_attributes.schema.yaml#/components/schemas/RelatedIntegrationArray' required_fields: $ref: './common_attributes.schema.yaml#/components/schemas/RequiredFieldArray' - setup: - $ref: './common_attributes.schema.yaml#/components/schemas/SetupGuide' execution_summary: $ref: '../../rule_monitoring/model/execution_summary.schema.yaml#/components/schemas/RuleExecutionSummary' required: @@ -198,7 +198,6 @@ components: - revision - related_integrations - required_fields - - setup SharedCreateProps: x-inline: true @@ -279,7 +278,7 @@ components: $ref: './specific_attributes/eql_attributes.schema.yaml#/components/schemas/TiebreakerField' timestamp_field: $ref: './specific_attributes/eql_attributes.schema.yaml#/components/schemas/TimestampField' - + EqlRuleCreateFields: allOf: - $ref: '#/components/schemas/EqlRequiredFields' diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/editor.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/editor.tsx index 64d289cd65f3e..2f439c55a7d1c 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/editor.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/editor.tsx @@ -31,6 +31,7 @@ interface MarkdownEditorProps { height?: number; autoFocusDisabled?: boolean; setIsMarkdownInvalid: (value: boolean) => void; + includePlugins?: boolean; } type EuiMarkdownEditorRef = ElementRef; @@ -52,6 +53,7 @@ const MarkdownEditorComponent = forwardRef { @@ -73,8 +75,8 @@ const MarkdownEditorComponent = forwardRef { - return uiPlugins({ insightsUpsellingMessage }); - }, [insightsUpsellingMessage]); + return includePlugins ? uiPlugins({ insightsUpsellingMessage }) : undefined; + }, [insightsUpsellingMessage, includePlugins]); // @ts-expect-error update types useImperativeHandle(ref, () => { diff --git a/x-pack/plugins/security_solution/public/common/components/markdown_editor/eui_form.tsx b/x-pack/plugins/security_solution/public/common/components/markdown_editor/eui_form.tsx index dc157a85afa2b..8fdbc3559bbc4 100644 --- a/x-pack/plugins/security_solution/public/common/components/markdown_editor/eui_form.tsx +++ b/x-pack/plugins/security_solution/public/common/components/markdown_editor/eui_form.tsx @@ -23,6 +23,7 @@ type MarkdownEditorFormProps = EuiMarkdownEditorProps & { idAria: string; isDisabled?: boolean; bottomRightContent?: React.ReactNode; + includePlugins?: boolean; }; /* eslint-enable react/no-unused-prop-types */ @@ -34,7 +35,7 @@ const BottomContentWrapper = styled(EuiFlexGroup)` export const MarkdownEditorForm = React.memo( forwardRef( - ({ id, field, dataTestSubj, idAria, bottomRightContent }, ref) => { + ({ id, field, dataTestSubj, idAria, bottomRightContent, includePlugins }, ref) => { const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field); const [isMarkdownInvalid, setIsMarkdownInvalid] = useState(false); @@ -58,6 +59,7 @@ export const MarkdownEditorForm = React.memo( value={field.value as string} data-test-subj={`${dataTestSubj}-markdown-editor`} setIsMarkdownInvalid={setIsMarkdownInvalid} + includePlugins={includePlugins} /> {bottomRightContent && ( diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/step_about_rule_details/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/step_about_rule_details/index.test.tsx index 953e75ab5ceda..ec39abb61465a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/step_about_rule_details/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/step_about_rule_details/index.test.tsx @@ -40,7 +40,7 @@ describe('StepAboutRuleToggleDetails', () => { stepDataDetails={{ note: stepDataMock.note, description: stepDataMock.description, - setup: '', + setup: stepDataMock.setup, }} stepData={stepDataMock} rule={mockRule('mocked-rule-id')} @@ -82,28 +82,30 @@ describe('StepAboutRuleToggleDetails', () => { }); describe('note value is empty string', () => { - test('it does not render toggle buttons', () => { + test('it does render toggle buttons if setup is not empty', () => { const mockAboutStepWithoutNote = { ...stepDataMock, note: '', }; - const wrapper = shallow( - + const wrapper = mount( + + + ); - expect(wrapper.find('[data-test-subj="stepAboutDetailsToggle"]').exists()).toBeFalsy(); + expect(wrapper.find(EuiButtonGroup).exists()).toBeTruthy(); + expect(wrapper.find('#details').at(0).prop('isSelected')).toBeTruthy(); + expect(wrapper.find('#setup').at(0).prop('isSelected')).toBeFalsy(); expect(wrapper.find('[data-test-subj="stepAboutDetailsNoteContent"]').exists()).toBeFalsy(); - expect(wrapper.find('[data-test-subj="stepAboutDetailsSetupContent"]').exists()).toBeFalsy(); - expect(wrapper.find('[data-test-subj="stepAboutDetailsContent"]').exists()).toBeTruthy(); }); }); @@ -116,7 +118,7 @@ describe('StepAboutRuleToggleDetails', () => { stepDataDetails={{ note: stepDataMock.note, description: stepDataMock.description, - setup: '', + setup: stepDataMock.setup, }} stepData={stepDataMock} rule={mockRule('mocked-rule-id')} @@ -137,7 +139,7 @@ describe('StepAboutRuleToggleDetails', () => { stepDataDetails={{ note: stepDataMock.note, description: stepDataMock.description, - setup: '', + setup: stepDataMock.setup, }} stepData={stepDataMock} rule={mockRule('mocked-rule-id')} @@ -212,7 +214,7 @@ describe('StepAboutRuleToggleDetails', () => { stepDataDetails={{ note: stepDataMock.note, description: stepDataMock.description, - setup: stepDataMock.note, // TODO: Update to mockRule.setup once supported in UI (and mock can be updated) + setup: stepDataMock.setup, }} stepData={stepDataMock} rule={mockRule('mocked-rule-id')} @@ -234,7 +236,7 @@ describe('StepAboutRuleToggleDetails', () => { stepDataDetails={{ note: stepDataMock.note, description: stepDataMock.description, - setup: stepDataMock.note, // TODO: Update to mockRule.setup once supported in UI (and mock can be updated) + setup: stepDataMock.setup, }} stepData={stepDataMock} rule={mockRule('mocked-rule-id')} @@ -253,7 +255,7 @@ describe('StepAboutRuleToggleDetails', () => { expect(wrapper.find('[idSelected="setup"]').exists()).toBeTruthy(); }); - test('it displays notes markdown when user toggles to "setup"', () => { + test('it displays setup markdown when user toggles to "setup"', () => { const wrapper = mount( { stepDataDetails={{ note: stepDataMock.note, description: stepDataMock.description, - setup: stepDataMock.note, // TODO: Update to mockRule.setup once supported in UI (and mock can be updated) + setup: stepDataMock.setup, }} stepData={stepDataMock} rule={mockRule('mocked-rule-id')} @@ -273,7 +275,7 @@ describe('StepAboutRuleToggleDetails', () => { expect(wrapper.find('EuiButtonGroup[idSelected="setup"]').exists()).toBeTruthy(); expect(wrapper.find('div.euiMarkdownFormat').text()).toEqual( - 'this is some markdown documentation' + 'this is some setup documentation' ); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/helpers.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/helpers.tsx index 446ff21d9414d..222920c536917 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/helpers.tsx @@ -56,6 +56,11 @@ const NoteDescriptionContainer = styled(EuiFlexItem)` overflow-y: hidden; `; +const SetupDescriptionContainer = styled(EuiFlexItem)` + height: 105px; + overflow-y: hidden; +`; + export const isNotEmptyArray = (values: string[]) => !isEmpty(values.join('')); const EuiBadgeWrap = styled(EuiBadge)` @@ -647,3 +652,21 @@ export const buildAlertSuppressionMissingFieldsDescription = ( }, ]; }; + +export const buildSetupDescription = (label: string, setup: string): ListItems[] => { + if (setup.trim() !== '') { + return [ + { + title: label, + description: ( + +
+ {setup} +
+ + ), + }, + ]; + } + return []; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx index c4770a1640704..f341476c4d8f9 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx @@ -263,7 +263,7 @@ describe('description_step', () => { mockLicenseService ); - expect(result.length).toEqual(12); + expect(result.length).toEqual(13); }); }); @@ -559,6 +559,21 @@ describe('description_step', () => { }); }); + describe('setup', () => { + test('returns default "setup" description', () => { + const result: ListItems[] = getDescriptionItem( + 'setup', + 'Setup guide', + mockAboutStep, + mockFilterManager, + mockLicenseService + ); + + expect(result[0].title).toEqual('Setup guide'); + expect(React.isValidElement(result[0].description)).toBeTruthy(); + }); + }); + describe('alert suppression', () => { const ruleTypesWithoutSuppression: Type[] = ['eql', 'esql', 'machine_learning', 'new_terms']; const suppressionFields = { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx index 9f377755c769e..78bcd60e5c0d6 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.tsx @@ -47,6 +47,7 @@ import { buildAlertSuppressionWindowDescription, buildAlertSuppressionMissingFieldsDescription, buildHighlightedFieldsOverrideDescription, + buildSetupDescription, getQueryLabel, } from './helpers'; import * as i18n from './translations'; @@ -305,6 +306,9 @@ export const getDescriptionItem = ( } else if (field === 'note') { const val: string = get(field, data); return buildNoteDescription(label, val); + } else if (field === 'setup') { + const val: string = get(field, data); + return buildSetupDescription(label, val); } else if (field === 'ruleType') { const ruleType: Type = get(field, data); return buildRuleTypeDescription(label, ruleType); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/default_value.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/default_value.ts index 91057eb3ff5f8..26f842384ef25 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/default_value.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/default_value.ts @@ -33,4 +33,5 @@ export const stepAboutDefaultValue: AboutStepRule = { timestampOverride: '', threat: threatDefault, note: '', + setup: '', }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx index d654eaef9cca7..dc3fc5645b138 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.test.tsx @@ -269,6 +269,7 @@ describe('StepAboutRuleComponent', () => { falsePositives: [''], name: 'Test name text', note: '', + setup: '', references: [''], riskScore: { value: 21, mapping: [], isMappingChecked: false }, severity: { value: 'low', mapping: fillEmptySeverityMappings([]), isMappingChecked: false }, @@ -329,6 +330,7 @@ describe('StepAboutRuleComponent', () => { falsePositives: [''], name: 'Test name text', note: '', + setup: '', references: [''], riskScore: { value: 80, mapping: [], isMappingChecked: false }, severity: { value: 'low', mapping: fillEmptySeverityMappings([]), isMappingChecked: false }, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx index 839618669dc06..99e65f33e486a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx @@ -253,6 +253,18 @@ const StepAboutRuleComponent: FC = ({ }} /> + + = { ), labelAppend: OptionalFieldLabel, }, + setup: { + type: FIELD_TYPES.TEXTAREA, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupLabel', + { + defaultMessage: 'Setup guide', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupHelpText', + { + defaultMessage: + 'Provide instructions on rule prerequisites such as required integrations, configuration steps, and anything else needed for the rule to work correctly.', + } + ), + labelAppend: OptionalFieldLabel, + }, }; const threatIndicatorPathRequiredSchemaValue = { diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/translations.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/translations.ts index 007cf4d9dd4c6..d07fe22a8ed7b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/translations.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/translations.ts @@ -90,3 +90,10 @@ export const ADD_RULE_NOTE_HELP_TEXT = i18n.translate( defaultMessage: 'Add rule investigation guide...', } ); + +export const ADD_RULE_SETUP_HELP_TEXT = i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText', + { + defaultMessage: 'Add rule setup guide...', + } +); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.test.ts index 86bcebb72ded5..71fe20ba3e6fb 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.test.ts @@ -556,6 +556,7 @@ describe('helpers', () => { tags: ['tag1', 'tag2'], threat: getThreatMock(), investigation_fields: { field_names: ['foo', 'bar'] }, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -637,6 +638,7 @@ describe('helpers', () => { tags: ['tag1', 'tag2'], threat: getThreatMock(), investigation_fields: { field_names: ['foo', 'bar'] }, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -662,6 +664,7 @@ describe('helpers', () => { tags: ['tag1', 'tag2'], threat: getThreatMock(), investigation_fields: { field_names: ['foo', 'bar'] }, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -706,6 +709,7 @@ describe('helpers', () => { tags: ['tag1', 'tag2'], threat: getThreatMock(), investigation_fields: { field_names: ['foo', 'bar'] }, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -759,6 +763,7 @@ describe('helpers', () => { }, ], investigation_fields: { field_names: ['foo', 'bar'] }, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -788,6 +793,7 @@ describe('helpers', () => { timestamp_override: 'event.ingest', timestamp_override_fallback_disabled: true, investigation_fields: { field_names: ['foo', 'bar'] }, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -818,6 +824,7 @@ describe('helpers', () => { timestamp_override_fallback_disabled: undefined, threat: getThreatMock(), investigation_fields: undefined, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -847,6 +854,7 @@ describe('helpers', () => { threat_indicator_path: undefined, timestamp_override: undefined, timestamp_override_fallback_disabled: undefined, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); @@ -876,6 +884,7 @@ describe('helpers', () => { threat_indicator_path: undefined, timestamp_override: undefined, timestamp_override_fallback_disabled: undefined, + setup: '# this is some setup documentation', }; expect(result).toEqual(expected); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts index 4c23c14871067..49bd1649c3471 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/__mocks__/mock.ts @@ -81,7 +81,7 @@ export const mockRule = (id: string): SavedQueryRule => ({ meta: { from: '0m' }, related_integrations: [], required_fields: [], - setup: '', + setup: '# this is some setup documentation', severity: 'low', severity_mapping: [], updated_by: 'elastic', @@ -149,7 +149,7 @@ export const mockRuleWithEverything = (id: string): RuleResponse => ({ meta: { from: '0m' }, related_integrations: [], required_fields: [], - setup: '', + setup: '# this is some setup documentation', severity: 'low', severity_mapping: [], updated_by: 'elastic', @@ -197,6 +197,7 @@ export const mockAboutStepRule = (): AboutStepRule => ({ tags: ['tag1', 'tag2'], threat: getThreatMock(), note: '# this is some markdown documentation', + setup: '# this is some setup documentation', investigationFields: ['foo', 'bar'], }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 2ffedcdc55568..bcb73b1f9edc2 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -146,6 +146,7 @@ describe('rule helpers', () => { timestampOverride: 'event.ingested', timestampOverrideFallbackDisabled: false, investigationFields: [], + setup: '# this is some setup documentation', }; const scheduleRuleStepData = { from: '0s', interval: '5m' }; const ruleActionsStepData = { @@ -156,7 +157,7 @@ describe('rule helpers', () => { const aboutRuleDataDetailsData = { note: '# this is some markdown documentation', description: '24/7', - setup: '', + setup: '# this is some setup documentation', }; expect(defineRuleData).toEqual(defineRuleStepData); @@ -195,18 +196,18 @@ describe('rule helpers', () => { describe('determineDetailsValue', () => { test('returns name, description, and note as empty string if detailsView is true', () => { - const result: Pick = determineDetailsValue( + const result: Pick = determineDetailsValue( mockRuleWithEverything('test-id'), true ); - const expected = { name: '', description: '', note: '' }; + const expected = { name: '', description: '', note: '', setup: '' }; expect(result).toEqual(expected); }); test('returns name, description, and note values if detailsView is false', () => { const mockedRule = mockRuleWithEverything('test-id'); - const result: Pick = determineDetailsValue( + const result: Pick = determineDetailsValue( mockedRule, false ); @@ -214,6 +215,7 @@ describe('rule helpers', () => { name: mockedRule.name, description: mockedRule.description, note: mockedRule.note, + setup: mockedRule.setup, }; expect(result).toEqual(expected); @@ -222,11 +224,16 @@ describe('rule helpers', () => { test('returns note as empty string if property does not exist on rule', () => { const mockedRule = mockRuleWithEverything('test-id'); delete mockedRule.note; - const result: Pick = determineDetailsValue( + const result: Pick = determineDetailsValue( mockedRule, false ); - const expected = { name: mockedRule.name, description: mockedRule.description, note: '' }; + const expected = { + name: mockedRule.name, + description: mockedRule.description, + note: '', + setup: mockedRule.setup, + }; expect(result).toEqual(expected); }); @@ -418,7 +425,7 @@ describe('rule helpers', () => { const aboutRuleDataDetailsData = { note: '# this is some markdown documentation', description: '24/7', - setup: '', + setup: '# this is some setup documentation', }; expect(result).toEqual(aboutRuleDataDetailsData); @@ -431,7 +438,7 @@ describe('rule helpers', () => { const aboutRuleDetailsData = { note: '', description: mockRuleWithoutNote.description, - setup: '', + setup: '# this is some setup documentation', }; expect(result).toEqual(aboutRuleDetailsData); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 96a3b17a77871..574397c80e767 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -222,7 +222,7 @@ export const getHumanizedDuration = (from: string, interval: string): string => }; export const getAboutStepsData = (rule: RuleResponse, detailsView: boolean): AboutStepRule => { - const { name, description, note } = determineDetailsValue(rule, detailsView); + const { name, description, note, setup } = determineDetailsValue(rule, detailsView); const { author, building_block_type: buildingBlockType, @@ -272,6 +272,7 @@ export const getAboutStepsData = (rule: RuleResponse, detailsView: boolean): Abo investigationFields: investigationFields?.field_names ?? [], threat: threat as Threats, threatIndicatorPath, + setup, }; }; @@ -296,13 +297,13 @@ export const fillEmptySeverityMappings = (mappings: SeverityMapping): SeverityMa export const determineDetailsValue = ( rule: RuleResponse, detailsView: boolean -): Pick => { - const { name, description, note } = rule; +): Pick => { + const { name, description, note, setup } = rule; if (detailsView) { - return { name: '', description: '', note: '' }; + return { name: '', description: '', note: '', setup: '' }; } - return { name, description, note: note ?? '' }; + return { name, description, setup, note: note ?? '' }; }; export const getModifiedAboutDetailsData = (rule: RuleResponse): AboutStepRuleDetails => ({ diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index f57184a3a490b..fa0168c7d2e98 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -101,6 +101,7 @@ export interface AboutStepRule { threatIndicatorPath?: string; threat: Threats; note: string; + setup: SetupGuide; } export interface AboutStepRuleDetails { @@ -240,6 +241,7 @@ export interface AboutStepRuleJson { rule_name_override?: RuleNameOverride; tags: string[]; threat: Threats; + setup: string; threat_indicator_path?: string; timestamp_override?: TimestampOverride; timestamp_override_fallback_disabled?: boolean; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts index 9e54856b7b28c..565180217f842 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/utils.ts @@ -93,6 +93,7 @@ export const stepAboutDefaultValue: AboutStepRule = { timestampOverride: '', threat: threatDefault, note: '', + setup: '', threatIndicatorPath: undefined, timestampOverrideFallbackDisabled: undefined, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts index 7223b920c7bdc..c0cb5f903c3ea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts @@ -224,22 +224,6 @@ describe('duplicateRule', () => { }) ); }); - - it('resets setup guide to an empty string', async () => { - const rule = createPrebuiltRule(); - rule.params.setup = `## Config\n\nThe 'Audit Detailed File Share' audit policy must be configured...`; - const result = await duplicateRule({ - rule, - }); - - expect(result).toEqual( - expect.objectContaining({ - params: expect.objectContaining({ - setup: '', - }), - }) - ); - }); }); describe('when duplicating a custom (mutable) rule', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.ts index 315517504def4..57931dca00c1e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.ts @@ -33,7 +33,6 @@ export const duplicateRule = async ({ rule }: DuplicateRuleParams): Promise ): InternalRuleUpdate => { @@ -487,7 +485,6 @@ export const convertCreateAPIToInternalSchema = ( input: RuleCreateProps & { related_integrations?: RelatedIntegrationArray; required_fields?: RequiredFieldArray; - setup?: SetupGuide; }, immutable = false, defaultEnabled = true diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/create_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/create_rules.ts index 19b3188a4b8ad..e7967df45b5f9 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/create_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/create_rules.ts @@ -691,5 +691,27 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('setup guide', async () => { + beforeEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + it('creates a rule with a setup guide when setup parameter is present', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send( + getCustomQueryRuleParams({ + setup: 'A setup guide', + }) + ) + .expect(200); + + expect(body.setup).toEqual('A setup guide'); + }); + }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/patch_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/patch_rules.ts index edd84f6c86650..24919448b8522 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/patch_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/patch_rules.ts @@ -656,5 +656,37 @@ export default ({ getService }: FtrProviderContext) => { }); }); }); + + describe('setup guide', () => { + beforeEach(async () => { + await createAlertsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + it('should overwrite setup field on patch', async () => { + await createRule(supertest, log, { + ...getSimpleRule('rule-1'), + setup: 'A setup guide', + }); + + const rulePatch = { + rule_id: 'rule-1', + setup: 'A different setup guide', + }; + + const { body } = await supertest + .patch(DETECTION_ENGINE_RULES_URL) + .set('kbn-xsrf', 'true') + .set('elastic-api-version', '2023-10-31') + .send(rulePatch) + .expect(200); + + expect(body.setup).to.eql('A different setup guide'); + }); + }); }); }; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/update_rules.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/update_rules.ts index fe59a0127bb82..500eedb5bc2fd 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/update_rules.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/update_rules.ts @@ -757,6 +757,40 @@ export default ({ getService }: FtrProviderContext) => { expect(body.investigation_fields).to.eql(undefined); }); }); + + describe('setup guide', () => { + it('should overwrite setup value on update', async () => { + await createRule(supertest, log, { + ...getSimpleRule('rule-1'), + setup: 'A setup guide', + }); + + const ruleUpdate = { + ...getSimpleRuleUpdate('rule-1'), + setup: 'A different setup guide', + }; + + const { body } = await securitySolutionApi.updateRule({ body: ruleUpdate }).expect(200); + + expect(body.setup).to.eql('A different setup guide'); + }); + + it('should reset setup field to empty string on unset', async () => { + await createRule(supertest, log, { + ...getSimpleRule('rule-1'), + setup: 'A setup guide', + }); + + const ruleUpdate = { + ...getSimpleRuleUpdate('rule-1'), + setup: undefined, + }; + + const { body } = await securitySolutionApi.updateRule({ body: ruleUpdate }).expect(200); + + expect(body.setup).to.eql(''); + }); + }); }); }); }; From 3a31ee0872541a622ac66ec2501e51312422180c Mon Sep 17 00:00:00 2001 From: Cristina Amico Date: Mon, 8 Apr 2024 14:31:00 +0200 Subject: [PATCH 06/11] [Fleet] Implement state machine behavior for package install (#178657) Closes https://github.com/elastic/kibana/issues/175592 ## Summary Implement state machine behavior for package install. It keeps track of the current step and save it in the SO , then exposes it in the`installationInfo` property. - Implemented a generic state machine function that can automatically handle state transitions based on a simple data structure: https://github.com/elastic/kibana/pull/178657/files#diff-f350d9630cd1f22cd1b3e70c9e95388d72dc877190bbeb33c739cb0433949e95R1-R88. In theory, this state machine could be reused for something else, since is generic enough and it's decoupled from the transition functions that we pass to it. - The state transitions passed to the state machine are defined in [services/epm/packages/install_steps.ts](https://github.com/elastic/kibana/blob/5f09e58ae7a300f459c3d1157fb747cfbb0c11aa/x-pack/plugins/fleet/server/services/epm/packages/install_steps.ts) and are based off the existing steps in https://github.com/elastic/kibana/blob/10d5167fa78c1a4c65f8607dad1e6a681e39f4b0/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts#L61 I simply divided that long function in smaller steps and wrapped them to accept a common parameter, based off [InstallContext](https://github.com/elastic/kibana/pull/178657/files#diff-39d1f59e77a329eb06241c220156e5cf2d350649bb548707b0b0f54365ea91bfR49-R72) - Defined a **feature flag** `enablePackagesStateMachine` and called the new [installPackageWitStateMachine](https://github.com/elastic/kibana/pull/178657/files#diff-cf9cec44de2ad0a6a3b74cca05e5308231d57d5c3e180ae4bc5114c2bf9af4ebR466-R483) only when it's enabled. - For now this new function is only applied to `InstallPackageFromRegistry`, so `upload` and `bundled` case don't use it yet. ### Testing - Enable `enablePackagesStateMachine` in kibana.dev.yml - Try to install an integration from registry, either from API or UI. For instance ``` POST kbn:api/fleet/epm/packages/nginx/1.20.0 ``` The installation process should succeed and the installationInfo property will expose the `latest_executed_state` along with the error.
Screenshots ### Logging With `logger.debug` enabled: ![Screenshot 2024-03-27 at 16 12 33](https://github.com/elastic/kibana/assets/16084106/75fb4af8-675e-483e-a51f-eb4adbf9d2aa) ![Screenshot 2024-03-27 at 16 12 48](https://github.com/elastic/kibana/assets/16084106/74092f6d-528c-4e8f-85ee-85e2852487b8) ### InstallationInfo object Content of `installationInfo` property when install process was successful: ![Screenshot 2024-03-27 at 16 13 54](https://github.com/elastic/kibana/assets/16084106/c2535c8f-24f7-4b6c-8f58-dadf4c9b4b28) ### Errors during install process I manually triggered an error inside `stepInstallIndexTemplatePipelines` and it's reported in the `installationInfo` property along with the latest executed step (latest successful state) and error message: ![Screenshot 2024-03-27 at 17 26 29](https://github.com/elastic/kibana/assets/16084106/47d77330-bcbb-4608-9e42-c9f46e8831a1)
### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../src/constants.ts | 2 +- .../current_fields.json | 1 + .../current_mappings.json | 4 + .../check_registered_types.test.ts | 2 +- .../fleet/common/experimental_features.ts | 1 + .../plugins/fleet/common/openapi/bundled.json | 28 + .../plugins/fleet/common/openapi/bundled.yaml | 22 + .../components/schemas/installation_info.yaml | 22 + .../plugins/fleet/common/types/models/epm.ts | 32 + .../fleet/common/types/rest_spec/epm.ts | 3 +- .../fleet/server/routes/epm/handlers.ts | 1 + .../fleet/server/saved_objects/index.ts | 11 + .../services/epm/kibana/assets/install.ts | 2 - .../services/epm/packages/_install_package.ts | 1 - .../services/epm/packages/install.test.ts | 644 +++++++++++++----- .../server/services/epm/packages/install.ts | 242 ++++++- .../_state_machine_package_install.test.ts | 438 ++++++++++++ .../_state_machine_package_install.ts | 179 +++++ .../state_machine.test.ts | 542 +++++++++++++++ .../install_state_machine/state_machine.ts | 139 ++++ .../install_state_machine/steps/index.ts | 20 + .../step_create_restart_installation.test.ts | 198 ++++++ .../steps/step_create_restart_installation.ts | 84 +++ .../step_delete_previous_pipelines.test.ts | 481 +++++++++++++ .../steps/step_delete_previous_pipelines.ts | 65 ++ .../steps/step_install_ilm_policies.test.ts | 374 ++++++++++ .../steps/step_install_ilm_policies.ts | 45 ++ ...p_install_index_template_pipelines.test.ts | 592 ++++++++++++++++ .../step_install_index_template_pipelines.ts | 66 ++ .../steps/step_install_kibana_assets.test.ts | 111 +++ .../steps/step_install_kibana_assets.ts | 48 ++ .../steps/step_install_mlmodel.test.ts | 155 +++++ .../steps/step_install_mlmodel.ts | 22 + .../steps/step_install_transforms.test.ts | 161 +++++ .../steps/step_install_transforms.ts | 37 + .../step_remove_legacy_templates.test.ts | 155 +++++ .../steps/step_remove_legacy_templates.ts | 20 + .../steps/step_resolve_kibana_promise.ts | 15 + .../steps/step_save_archive_entries.test.ts | 184 +++++ .../steps/step_save_archive_entries.ts | 39 ++ .../steps/step_save_system_object.test.ts | 180 +++++ .../steps/step_save_system_object.ts | 82 +++ .../step_update_current_write_indices.test.ts | 163 +++++ .../step_update_current_write_indices.ts | 25 + .../update_latest_executed_state.test.ts | 195 ++++++ .../steps/update_latest_executed_state.ts | 39 ++ x-pack/plugins/fleet/server/types/index.tsx | 2 + .../apis/epm/install_error_rollback.ts | 5 +- .../apis/epm/install_remove_assets.ts | 4 + .../apis/epm/update_assets.ts | 4 + .../test/fleet_api_integration/config.base.ts | 1 + 51 files changed, 5672 insertions(+), 216 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts index cdc47c69caef3..124515299efc7 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/constants.ts @@ -159,7 +159,7 @@ export const HASH_TO_VERSION_MAP = { 'endpoint:user-artifact-manifest|7502b5c5bc923abe8aa5ccfd636e8c3d': '10.0.0', 'enterprise_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0', 'epm-packages-assets|44621b2f6052ef966da47b7c3a00f33b': '10.0.0', - 'epm-packages|c1e2020399dbebba2448096ca007c668': '10.1.0', + 'epm-packages|8ce219acd0f6f3529237d52193866afb': '10.2.0', 'event_loop_delays_daily|5df7e292ddd5028e07c1482e130e6654': '10.0.0', 'event-annotation-group|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0', 'exception-list-agnostic|8a1defe5981db16792cb9a772e84bb9a': '10.0.0', diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 5a26aca6bacc1..a66ab72d98aa3 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -290,6 +290,7 @@ "installed_kibana_space_id", "internal", "keep_policies_up_to_date", + "latest_executed_state", "latest_install_failed_attempts", "name", "package_assets", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 246c02b7dfec3..00d9a5f82fd1c 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -1003,6 +1003,10 @@ "index": false, "type": "boolean" }, + "latest_executed_state": { + "enabled": false, + "type": "object" + }, "latest_install_failed_attempts": { "enabled": false, "type": "object" diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index a466bda78df4f..7ec28e55fad21 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -85,7 +85,7 @@ describe('checking migration metadata changes on all registered SO types', () => "dashboard": "211e9ca30f5a95d5f3c27b1bf2b58e6cfa0c9ae9", "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b", "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d", - "epm-packages": "c23d3d00c051a08817335dba26f542b64b18a56a", + "epm-packages": "f8ee125b57df31fd035dc04ad81aef475fd2f5bd", "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1", "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582", "event_loop_delays_daily": "01b967e8e043801357503de09199dfa3853bab88", diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 8271f0403beda..48e77dbe1988d 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -28,6 +28,7 @@ export const allowedExperimentalValues = Object.freeze>( agentless: false, enableStrictKQLValidation: false, subfeaturePrivileges: false, + enablePackagesStateMachine: true, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index a526294d963fa..1ed749fe31dea 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -6302,6 +6302,34 @@ } } }, + "latest_executed_state": { + "description": "Latest successfully executed state in package install state machine", + "type": "object", + "properties": { + "name": { + "type": "string", + "enum": [ + "create_restart_installation", + "install_kibana_assets", + "install_ilm_policies", + "install_ml_model", + "install_index_template_pipelines", + "remove_legacy_templates", + "update_current_write_indices", + "install_transforms", + "delete_previous_pipelines", + "save_archive_entries_from_assets_map", + "update_so" + ] + }, + "started_at": { + "type": "string" + }, + "error": { + "type": "string" + } + } + }, "verification_status": { "type": "string", "enum": [ diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index f76e50c095aed..03bb90fd84d73 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -3970,6 +3970,28 @@ components: type: string stack: type: string + latest_executed_state: + description: Latest successfully executed state in package install state machine + type: object + properties: + name: + type: string + enum: + - create_restart_installation + - install_kibana_assets + - install_ilm_policies + - install_ml_model + - install_index_template_pipelines + - remove_legacy_templates + - update_current_write_indices + - install_transforms + - delete_previous_pipelines + - save_archive_entries_from_assets_map + - update_so + started_at: + type: string + error: + type: string verification_status: type: string enum: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml index c5db5f12d4cc3..b8d82bc669d04 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/installation_info.yaml @@ -66,6 +66,28 @@ properties: type: string stack: type: string + latest_executed_state: + description: Latest successfully executed state in package install state machine + type: object + properties: + name: + type: string + enum: + - create_restart_installation + - install_kibana_assets + - install_ilm_policies + - install_ml_model + - install_index_template_pipelines + - remove_legacy_templates + - update_current_write_indices + - install_transforms + - delete_previous_pipelines + - save_archive_entries_from_assets_map + - update_so + started_at: + type: string + error: + type: string verification_status: type: string enum: diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index b1bea249ee9de..a62833dfdcfb5 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -37,6 +37,7 @@ export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'in export type InstallSource = 'registry' | 'upload' | 'bundled' | 'custom'; export type EpmPackageInstallStatus = 'installed' | 'installing' | 'install_failed'; +export type InstallResultStatus = 'installed' | 'already_installed'; export type ServiceName = 'kibana' | 'elasticsearch'; export type AgentAssetType = typeof agentAssetTypes; @@ -548,6 +549,36 @@ export interface InstallFailedAttempt { }; } +export enum INSTALL_STATES { + CREATE_RESTART_INSTALLATION = 'create_restart_installation', + INSTALL_KIBANA_ASSETS = 'install_kibana_assets', + INSTALL_ILM_POLICIES = 'install_ilm_policies', + INSTALL_ML_MODEL = 'install_ml_model', + INSTALL_INDEX_TEMPLATE_PIPELINES = 'install_index_template_pipelines', + REMOVE_LEGACY_TEMPLATES = 'remove_legacy_templates', + UPDATE_CURRENT_WRITE_INDICES = 'update_current_write_indices', + INSTALL_TRANSFORMS = 'install_transforms', + DELETE_PREVIOUS_PIPELINES = 'delete_previous_pipelines', + SAVE_ARCHIVE_ENTRIES = 'save_archive_entries_from_assets_map', + RESOLVE_KIBANA_PROMISE = 'resolve_kibana_promise', + UPDATE_SO = 'update_so', +} +type StatesKeys = keyof typeof INSTALL_STATES; +export type StateNames = typeof INSTALL_STATES[StatesKeys]; + +export interface LatestExecutedState { + name: T; + started_at: string; + error?: string; +} + +export type InstallLatestExecutedState = LatestExecutedState; + +export interface StateContext { + [key: string]: any; + latestExecutedState?: LatestExecutedState; +} + export interface Installation { installed_kibana: KibanaAssetReference[]; installed_es: EsAssetReference[]; @@ -568,6 +599,7 @@ export interface Installation { internal?: boolean; removable?: boolean; latest_install_failed_attempts?: InstallFailedAttempt[]; + latest_executed_state?: InstallLatestExecutedState; } export interface PackageUsageStats { diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index 7300bd5449333..4882c1c0652e6 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -18,6 +18,7 @@ import type { EpmPackageInstallStatus, SimpleSOAssetType, AssetSOObject, + InstallResultStatus, } from '../models/epm'; export interface GetCategoriesRequest { @@ -154,7 +155,7 @@ export interface IBulkInstallPackageHTTPError { export interface InstallResult { assets?: AssetReference[]; - status?: 'installed' | 'already_installed'; + status?: InstallResultStatus; error?: Error; installType: InstallType; installSource: InstallSource; diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 5c14cf3d3bac8..47324cfc493f1 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -612,6 +612,7 @@ const soToInstallationInfo = (pkg: PackageListItem | PackageInfo) => { verification_key_id: attributes.verification_key_id, experimental_data_stream_features: attributes.experimental_data_stream_features, latest_install_failed_attempts: attributes.latest_install_failed_attempts, + latest_executed_state: attributes.latest_executed_state, }; return { diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 80665f381e871..4aef23990ffec 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -530,6 +530,7 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, }, latest_install_failed_attempts: { type: 'object', enabled: false }, + latest_executed_state: { type: 'object', enabled: false }, installed_kibana: { dynamic: false, properties: {}, @@ -571,6 +572,16 @@ export const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ }, ], }, + '2': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + latest_executed_state: { type: 'object', enabled: false }, + }, + }, + ], + }, }, migrations: { '7.14.0': migrateInstallationToV7140, diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts index 47c4da20b9d05..2956cb5fe20c2 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts @@ -172,7 +172,6 @@ export async function installKibanaAssetsAndReferences({ pkgName, pkgTitle, packageInstallContext, - paths, installedPkg, spaceId, assetTags, @@ -185,7 +184,6 @@ export async function installKibanaAssetsAndReferences({ pkgName: string; pkgTitle: string; packageInstallContext: PackageInstallContext; - paths: string[]; installedPkg?: SavedObject; spaceId: string; assetTags?: PackageSpecTags[]; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 8b8a44b55e222..4a6cb0306a9cb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -161,7 +161,6 @@ export async function _installPackage({ pkgName, pkgTitle, packageInstallContext, - paths, installedPkg, logger, spaceId, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index 97817b063b730..bbaa10728754b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -29,6 +29,7 @@ import { isPackageVersionOrLaterInstalled, } from './install'; import * as install from './_install_package'; +import * as installStateMachine from './install_state_machine/_state_machine_package_install'; import { getBundledPackageByPkgKey } from './bundled_packages'; import { getInstalledPackageWithAssets, getInstallationObject } from './get'; @@ -57,6 +58,7 @@ jest.mock('../../app_context', () => { getConfig: jest.fn(() => ({})), getSavedObjectsTagging: jest.fn(() => mockedSavedObjectTagging), getInternalUserSOClientForSpaceId: jest.fn(), + getExperimentalFeatures: jest.fn(), }, }; }); @@ -79,6 +81,11 @@ jest.mock('./_install_package', () => { _installPackage: jest.fn(() => Promise.resolve()), }; }); +jest.mock('./install_state_machine/_state_machine_package_install', () => { + return { + _stateMachineInstallPackage: jest.fn(() => Promise.resolve()), + }; +}); jest.mock('../kibana/index_pattern/install', () => { return { installIndexPatterns: jest.fn(() => Promise.resolve()), @@ -161,246 +168,504 @@ describe('install', () => { jest.mocked(Registry.getPackage).mockImplementation(() => Promise.resolve({ packageInfo: { license: 'basic', conditions: { elastic: { subscription: 'basic' } } }, + paths: [], } as any) ); mockGetBundledPackageByPkgKey.mockReset(); (install._installPackage as jest.Mock).mockClear(); + (installStateMachine._stateMachineInstallPackage as jest.Mock).mockClear(); jest.mocked(appContextService.getInternalUserSOClientForSpaceId).mockReset(); }); describe('registry', () => { - beforeEach(() => { - mockGetBundledPackageByPkgKey.mockResolvedValue(undefined); - }); - - it('should send telemetry on install failure, out of date', async () => { - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.1.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + describe('with enablePackagesStateMachine = false', () => { + beforeEach(() => { + mockGetBundledPackageByPkgKey.mockResolvedValue(undefined); + jest.mocked(appContextService.getExperimentalFeatures).mockReturnValue({ + enablePackagesStateMachine: false, + } as any); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - errorMessage: 'apache-1.1.0 is out-of-date and cannot be installed or updated', - eventType: 'package-install', - installType: 'install', - newVersion: '1.1.0', - packageName: 'apache', - status: 'failure', + it('should send telemetry on install failure, out of date', async () => { + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.1.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'apache-1.1.0 is out-of-date and cannot be installed or updated', + eventType: 'package-install', + installType: 'install', + newVersion: '1.1.0', + packageName: 'apache', + status: 'failure', + }); }); - }); - it('should send telemetry on install failure, license error', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on install failure, license error', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'Installation requires basic license', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - errorMessage: 'Installation requires basic license', - eventType: 'package-install', - installType: 'install', - newVersion: '1.3.0', - packageName: 'apache', - status: 'failure', + it('should send telemetry on install success', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'success', + }); }); - }); - it('should send telemetry on install success', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on update success', async () => { + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: '1.2.0', + dryRun: false, + eventType: 'package-install', + installType: 'update', + newVersion: '1.3.0', + packageName: 'apache', + status: 'success', + }); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - eventType: 'package-install', - installType: 'install', - newVersion: '1.3.0', - packageName: 'apache', - status: 'success', + it('should send telemetry on install failure, async error', async () => { + jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'error', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - }); - it('should send telemetry on update success', async () => { - jest - .mocked(getInstallationObject) - .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + it('should install from bundled package if one exists', async () => { + (install._installPackage as jest.Mock).mockResolvedValue({}); + mockGetBundledPackageByPkgKey.mockResolvedValue({ + name: 'test_package', + version: '1.0.0', + getBuffer: async () => Buffer.from('test_package'), + }); + + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package-1.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.error).toBeUndefined(); + + expect(install._installPackage).toHaveBeenCalledWith( + expect.objectContaining({ installSource: 'bundled' }) + ); + }); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should fetch latest version if version not provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('installed'); + + expect(sendTelemetryEvents).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.objectContaining({ + newVersion: '1.3.0', + }) + ); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: '1.2.0', - dryRun: false, - eventType: 'package-install', - installType: 'update', - newVersion: '1.3.0', - packageName: 'apache', - status: 'success', + it('should do nothing if same version is installed', async () => { + jest.mocked(getInstallationObject).mockResolvedValueOnce({ + attributes: { + version: '1.2.0', + install_status: 'installed', + installed_es: [], + installed_kibana: [], + }, + } as any); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.2.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('already_installed'); }); - }); - it('should send telemetry on install failure, async error', async () => { - jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => { + jest.mocked(appContextService.getConfig).mockReturnValueOnce({ + internal: { + fleetServerStandalone: true, + }, + } as any); - await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'fleet_server-2.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('installed'); }); - expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { - currentVersion: 'not_installed', - dryRun: false, - errorMessage: 'error', - eventType: 'package-install', - installType: 'install', - newVersion: '1.3.0', - packageName: 'apache', - status: 'failure', + it('should use a scoped to package space soClient for tagging', async () => { + const mockedTaggingSo = savedObjectsClientMock.create(); + jest + .mocked(appContextService.getInternalUserSOClientForSpaceId) + .mockReturnValue(mockedTaggingSo); + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: 'test', + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(appContextService.getInternalUserSOClientForSpaceId).toBeCalledWith('test'); + expect(appContextService.getSavedObjectsTagging().createTagClient).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); + expect( + appContextService.getSavedObjectsTagging().createInternalAssignmentService + ).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); }); }); - it('should install from bundled package if one exists', async () => { - (install._installPackage as jest.Mock).mockResolvedValue({}); - mockGetBundledPackageByPkgKey.mockResolvedValue({ - name: 'test_package', - version: '1.0.0', - getBuffer: async () => Buffer.from('test_package'), + describe('with enablePackagesStateMachine = true', () => { + beforeEach(() => { + mockGetBundledPackageByPkgKey.mockResolvedValue(undefined); + jest.mocked(appContextService.getExperimentalFeatures).mockReturnValue({ + enablePackagesStateMachine: true, + } as any); }); - - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'test_package-1.0.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + afterEach(() => { + (install._installPackage as jest.Mock).mockClear(); + // jest.resetAllMocks(); + }); + afterAll(() => { + jest.mocked(appContextService.getExperimentalFeatures).mockReturnValue({ + enablePackagesStateMachine: false, + } as any); }); - expect(response.error).toBeUndefined(); - - expect(install._installPackage).toHaveBeenCalledWith( - expect.objectContaining({ installSource: 'bundled' }) - ); - }); + it('should send telemetry on install failure, out of date', async () => { + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.1.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'apache-1.1.0 is out-of-date and cannot be installed or updated', + eventType: 'package-install', + installType: 'install', + newVersion: '1.1.0', + packageName: 'apache', + status: 'failure', + }); + }); - it('should fetch latest version if version not provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'test_package', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on install failure, license error', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'Installation requires basic license', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - expect(response.status).toEqual('installed'); + it('should send telemetry on install success', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'success', + }); + }); - expect(sendTelemetryEvents).toHaveBeenCalledWith( - expect.anything(), - undefined, - expect.objectContaining({ + it('should send telemetry on update success', async () => { + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0', installed_kibana: [] } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: '1.2.0', + dryRun: false, + eventType: 'package-install', + installType: 'update', newVersion: '1.3.0', - }) - ); - }); + packageName: 'apache', + status: 'success', + }); + }); - it('should do nothing if same version is installed', async () => { - jest.mocked(getInstallationObject).mockResolvedValueOnce({ - attributes: { - version: '1.2.0', - install_status: 'installed', - installed_es: [], - installed_kibana: [], - }, - } as any); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'apache-1.2.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should send telemetry on install failure, async error', async () => { + jest + .mocked(installStateMachine._stateMachineInstallPackage) + .mockRejectedValue(new Error('error')); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + + await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(sendTelemetryEvents).toHaveBeenCalledWith(expect.anything(), undefined, { + currentVersion: 'not_installed', + dryRun: false, + errorMessage: 'error', + eventType: 'package-install', + installType: 'install', + newVersion: '1.3.0', + packageName: 'apache', + status: 'failure', + }); }); - expect(response.status).toEqual('already_installed'); - }); + it('should install from bundled package if one exists', async () => { + (installStateMachine._stateMachineInstallPackage as jest.Mock).mockResolvedValue({}); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + mockGetBundledPackageByPkgKey.mockResolvedValue({ + name: 'test_package', + version: '1.0.0', + getBuffer: async () => Buffer.from('test_package'), + }); + + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package-1.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.error).toBeUndefined(); + + expect(install._installPackage).toHaveBeenCalledWith( + expect.objectContaining({ installSource: 'bundled' }) + ); + }); - it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => { - jest.mocked(appContextService.getConfig).mockReturnValueOnce({ - internal: { - fleetServerStandalone: true, - }, - } as any); + it('should fetch latest version if version not provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'test_package', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('installed'); + + expect(sendTelemetryEvents).toHaveBeenCalledWith( + expect.anything(), + undefined, + expect.objectContaining({ + newVersion: '1.3.0', + }) + ); + }); - const response = await installPackage({ - spaceId: DEFAULT_SPACE_ID, - installSource: 'registry', - pkgkey: 'fleet_server-2.0.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + it('should do nothing if same version is installed', async () => { + jest.mocked(getInstallationObject).mockResolvedValueOnce({ + attributes: { + version: '1.2.0', + install_status: 'installed', + installed_es: [], + installed_kibana: [], + }, + } as any); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'apache-1.2.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(response.status).toEqual('already_installed'); }); - expect(response.status).toEqual('installed'); - }); + // failing + it('should allow to install fleet_server if internal.fleetServerStandalone is configured', async () => { + jest.mocked(appContextService.getConfig).mockReturnValueOnce({ + internal: { + fleetServerStandalone: true, + }, + } as any); - it('should use a scopped to package space soClient for tagging', async () => { - const mockedTaggingSo = savedObjectsClientMock.create(); - jest - .mocked(appContextService.getInternalUserSOClientForSpaceId) - .mockReturnValue(mockedTaggingSo); - jest - .mocked(getInstallationObject) - .mockResolvedValueOnce({ attributes: { version: '1.2.0' } } as any); + const response = await installPackage({ + spaceId: DEFAULT_SPACE_ID, + installSource: 'registry', + pkgkey: 'fleet_server-2.0.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - await installPackage({ - spaceId: 'test', - installSource: 'registry', - pkgkey: 'apache-1.3.0', - savedObjectsClient: savedObjectsClientMock.create(), - esClient: {} as ElasticsearchClient, + expect(response.status).toEqual('installed'); }); - expect(appContextService.getInternalUserSOClientForSpaceId).toBeCalledWith('test'); - expect(appContextService.getSavedObjectsTagging().createTagClient).toBeCalledWith( - expect.objectContaining({ - client: mockedTaggingSo, - }) - ); - expect( - appContextService.getSavedObjectsTagging().createInternalAssignmentService - ).toBeCalledWith( - expect.objectContaining({ - client: mockedTaggingSo, - }) - ); + it('should use a scoped to package space soClient for tagging', async () => { + const mockedTaggingSo = savedObjectsClientMock.create(); + jest + .mocked(appContextService.getInternalUserSOClientForSpaceId) + .mockReturnValue(mockedTaggingSo); + jest + .mocked(getInstallationObject) + .mockResolvedValueOnce({ attributes: { version: '1.2.0', installed_kibana: [] } } as any); + + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + await installPackage({ + spaceId: 'test', + installSource: 'registry', + pkgkey: 'apache-1.3.0', + savedObjectsClient: savedObjectsClientMock.create(), + esClient: {} as ElasticsearchClient, + }); + + expect(appContextService.getInternalUserSOClientForSpaceId).toBeCalledWith('test'); + expect(appContextService.getSavedObjectsTagging().createTagClient).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); + expect( + appContextService.getSavedObjectsTagging().createInternalAssignmentService + ).toBeCalledWith( + expect.objectContaining({ + client: mockedTaggingSo, + }) + ); + }); }); }); @@ -453,6 +718,7 @@ describe('install', () => { it('should send telemetry on install failure, async error', async () => { jest.mocked(install._installPackage).mockRejectedValue(new Error('error')); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); await installPackage({ spaceId: DEFAULT_SPACE_ID, installSource: 'upload', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 43b0c9d68a04c..c8c2e542bfe8b 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -38,6 +38,7 @@ import type { NewPackagePolicy, PackageInfo, PackageVerificationResult, + InstallResultStatus, } from '../../../types'; import { AUTO_UPGRADE_POLICIES_PACKAGES, @@ -70,6 +71,8 @@ import { sendTelemetryEvents, UpdateEventType } from '../../upgrade_sender'; import { auditLoggingService } from '../../audit_logging'; import { getFilteredInstallPackages } from '../filtered_packages'; +import { _stateMachineInstallPackage } from './install_state_machine/_state_machine_package_install'; + import { formatVerificationResultForSO } from './package_verification'; import { getInstallation, getInstallationObject } from './get'; import { removeInstallation } from './remove'; @@ -458,24 +461,44 @@ async function installPackageFromRegistry({ }` ); } - - return await installPackageCommon({ - pkgName, - pkgVersion, - installSource, - installedPkg, - installType, - savedObjectsClient, - esClient, - spaceId, - force, - packageInstallContext, - paths, - verificationResult, - authorizationHeader, - ignoreMappingUpdateErrors, - skipDataStreamRollover, - }); + const { enablePackagesStateMachine } = appContextService.getExperimentalFeatures(); + if (enablePackagesStateMachine) { + return await installPackageWitStateMachine({ + pkgName, + pkgVersion, + installSource, + installedPkg, + installType, + savedObjectsClient, + esClient, + spaceId, + force, + packageInstallContext, + paths, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }); + } else { + return await installPackageCommon({ + pkgName, + pkgVersion, + installSource, + installedPkg, + installType, + savedObjectsClient, + esClient, + spaceId, + force, + packageInstallContext, + paths, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }); + } } catch (e) { sendEvent({ ...telemetryEvent, @@ -607,7 +630,6 @@ async function installPackageCommon(options: { .createTagClient({ client: savedObjectClientWithSpace }); // try installing the package, if there was an error, call error handler and rethrow - // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return await _installPackage({ savedObjectsClient, savedObjectsImporter, @@ -637,7 +659,187 @@ async function installPackageCommon(options: { ...telemetryEvent!, status: 'success', }); - return { assets, status: 'installed', installType, installSource }; + return { assets, status: 'installed' as InstallResultStatus, installType, installSource }; + }) + .catch(async (err: Error) => { + logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`, { + error: { stack_trace: err.stack }, + }); + await handleInstallPackageFailure({ + savedObjectsClient, + error: err, + pkgName, + pkgVersion, + installedPkg, + spaceId, + esClient, + authorizationHeader, + }); + sendEvent({ + ...telemetryEvent!, + errorMessage: err.message, + }); + return { error: err, installType, installSource }; + }); + } catch (e) { + sendEvent({ + ...telemetryEvent, + errorMessage: e.message, + }); + return { + error: e, + installType, + installSource, + }; + } finally { + span?.end(); + } +} + +async function installPackageWitStateMachine(options: { + pkgName: string; + pkgVersion: string; + installSource: InstallSource; + installedPkg?: SavedObject; + installType: InstallType; + savedObjectsClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; + spaceId: string; + force?: boolean; + packageInstallContext: PackageInstallContext; + paths: string[]; + verificationResult?: PackageVerificationResult; + telemetryEvent?: PackageUpdateEvent; + authorizationHeader?: HTTPAuthorizationHeader | null; + ignoreMappingUpdateErrors?: boolean; + skipDataStreamRollover?: boolean; +}): Promise { + const packageInfo = options.packageInstallContext.packageInfo; + + const { + pkgName, + pkgVersion, + installSource, + installedPkg, + installType, + savedObjectsClient, + force, + esClient, + spaceId, + verificationResult, + authorizationHeader, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + packageInstallContext, + } = options; + let { telemetryEvent } = options; + const logger = appContextService.getLogger(); + logger.info( + `Install with enablePackagesStateMachine - Starting installation of ${pkgName}@${pkgVersion} from ${installSource} ` + ); + + // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611 + await Promise.resolve(); + const span = apm.startSpan( + `Install package from ${installSource} ${pkgName}@${pkgVersion}`, + 'package' + ); + + if (!telemetryEvent) { + telemetryEvent = getTelemetryEvent(pkgName, pkgVersion); + telemetryEvent.installType = installType; + telemetryEvent.currentVersion = installedPkg?.attributes.version || 'not_installed'; + } + + try { + span?.addLabels({ + packageName: pkgName, + packageVersion: pkgVersion, + installType, + }); + + const filteredPackages = getFilteredInstallPackages(); + if (filteredPackages.includes(pkgName)) { + throw new FleetUnauthorizedError(`${pkgName} installation is not authorized`); + } + + // if the requested version is the same as installed version, check if we allow it based on + // current installed package status and force flag, if we don't allow it, + // just return the asset references from the existing installation + if ( + installedPkg?.attributes.version === pkgVersion && + installedPkg?.attributes.install_status === 'installed' + ) { + if (!force) { + logger.debug(`${pkgName}-${pkgVersion} is already installed, skipping installation`); + return { + assets: [ + ...installedPkg.attributes.installed_es, + ...installedPkg.attributes.installed_kibana, + ], + status: 'already_installed', + installType, + installSource, + }; + } + } + const elasticSubscription = getElasticSubscription(packageInfo); + if (!licenseService.hasAtLeast(elasticSubscription)) { + logger.error(`Installation requires ${elasticSubscription} license`); + const err = new FleetError(`Installation requires ${elasticSubscription} license`); + sendEvent({ + ...telemetryEvent, + errorMessage: err.message, + }); + return { error: err, installType, installSource }; + } + + // Saved object client need to be scopped with the package space for saved object tagging + const savedObjectClientWithSpace = appContextService.getInternalUserSOClientForSpaceId(spaceId); + + const savedObjectsImporter = appContextService + .getSavedObjects() + .createImporter(savedObjectClientWithSpace, { importSizeLimit: 15_000 }); + + const savedObjectTagAssignmentService = appContextService + .getSavedObjectsTagging() + .createInternalAssignmentService({ client: savedObjectClientWithSpace }); + + const savedObjectTagClient = appContextService + .getSavedObjectsTagging() + .createTagClient({ client: savedObjectClientWithSpace }); + + // try installing the package, if there was an error, call error handler and rethrow + return await _stateMachineInstallPackage({ + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + esClient, + logger, + installedPkg, + packageInstallContext, + installType, + spaceId, + verificationResult, + installSource, + authorizationHeader, + force, + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }) + .then(async (assets) => { + logger.debug(`Removing old assets from previous versions of ${pkgName}`); + await removeOldAssets({ + soClient: savedObjectsClient, + pkgName: packageInfo.name, + currentVersion: packageInfo.version, + }); + sendEvent({ + ...telemetryEvent!, + status: 'success', + }); + return { assets, status: 'installed' as InstallResultStatus, installType, installSource }; }) .catch(async (err: Error) => { logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`, { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts new file mode 100644 index 0000000000000..c77433774a5cf --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.test.ts @@ -0,0 +1,438 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { + PackageSavedObjectConflictError, + ConcurrentInstallOperationError, +} from '../../../../errors'; + +import type { Installation } from '../../../../../common'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; + +import { appContextService } from '../../../app_context'; +import { createAppContextStartContractMock } from '../../../../mocks'; +import { saveArchiveEntriesFromAssetsMap } from '../../archive/storage'; +import { installILMPolicy } from '../../elasticsearch/ilm/install'; +import { installIlmForDataStream } from '../../elasticsearch/datastream_ilm/install'; + +jest.mock('../../elasticsearch/template/template'); +jest.mock('../../kibana/assets/install'); +jest.mock('../../kibana/index_pattern/install'); +jest.mock('../install'); +jest.mock('../get'); +jest.mock('../install_index_template_pipeline'); + +jest.mock('../../archive/storage'); +jest.mock('../../elasticsearch/ilm/install'); +jest.mock('../../elasticsearch/datastream_ilm/install'); + +import { updateCurrentWriteIndices } from '../../elasticsearch/template/template'; +import { installKibanaAssetsAndReferences } from '../../kibana/assets/install'; + +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../common/constants'; + +import { restartInstallation } from '../install'; +import { installIndexTemplatesAndPipelines } from '../install_index_template_pipeline'; + +import { _stateMachineInstallPackage } from './_state_machine_package_install'; + +const mockedInstallIndexTemplatesAndPipelines = + installIndexTemplatesAndPipelines as jest.MockedFunction< + typeof installIndexTemplatesAndPipelines + >; +const mockedUpdateCurrentWriteIndices = updateCurrentWriteIndices as jest.MockedFunction< + typeof updateCurrentWriteIndices +>; +const mockedInstallKibanaAssetsAndReferences = + installKibanaAssetsAndReferences as jest.MockedFunction; + +function sleep(millis: number) { + return new Promise((resolve) => setTimeout(resolve, millis)); +} + +describe('_stateMachineInstallPackage', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + + soClient.update.mockImplementation(async (type, id, attributes) => { + return { id, attributes } as any; + }); + soClient.get.mockImplementation(async (type, id) => { + return { id, attributes: {} } as any; + }); + + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + jest.mocked(installILMPolicy).mockReset(); + jest.mocked(installIlmForDataStream).mockReset(); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [], + installedIlms: [], + }); + jest.mocked(saveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [], + }); + jest.mocked(restartInstallation).mockReset(); + }); + + it('Handles errors from installKibanaAssets', async () => { + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { + throw new Error('mocked async error A: should be caught'); + }); + + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + + const installationPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + // if we have a .catch this will fail nicely (test pass) + // otherwise the test will fail with either of the mocked errors + await expect(installationPromise).rejects.toThrow('mocked'); + await expect(installationPromise).rejects.toThrow('should be caught'); + }); + + it('Do not install ILM policies if disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]); + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + }); + + it('Installs ILM policies if not disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]); + // pick any function between when those are called and when await Promise.all is defined later + // and force it to take long enough for the errors to occur + // @ts-expect-error about call signature + mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000)); + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [], + esReferences: [], + }); + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toBeCalled(); + expect(installIlmForDataStream).toBeCalled(); + }); + + describe('When package is stuck in `installing`', () => { + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(() => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + }); + + describe('When timeout is reached', () => { + it('restarts installation', async () => { + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date( + Date.now() - MAX_TIME_COMPLETE_INSTALL * 2 + ).toISOString(), + }, + }, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + + describe('When timeout is not reached', () => { + describe('With no force flag', () => { + it('throws concurrent installation error', async () => { + const installPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + }); + + await expect(installPromise).rejects.toThrowError(ConcurrentInstallOperationError); + }); + }); + + describe('With force flag provided', () => { + it('restarts installation', async () => { + await _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + paths: [], + assetsMap: new Map(), + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + }); + }); + + it('Surfaces saved object conflicts error', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + + mockedInstallKibanaAssetsAndReferences.mockRejectedValueOnce( + new PackageSavedObjectConflictError('test') + ); + + const installPromise = _stateMachineInstallPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + await expect(installPromise).rejects.toThrowError(PackageSavedObjectConflictError); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts new file mode 100644 index 0000000000000..d66334b315a42 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/_state_machine_package_install.ts @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + ElasticsearchClient, + Logger, + SavedObject, + SavedObjectsClientContract, + ISavedObjectsImporter, +} from '@kbn/core/server'; +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; + +import type { IAssignmentService, ITagsClient } from '@kbn/saved-objects-tagging-plugin/server'; + +import { PackageSavedObjectConflictError } from '../../../../errors'; + +import type { HTTPAuthorizationHeader } from '../../../../../common/http_authorization_header'; +import { INSTALL_STATES } from '../../../../../common/types'; +import type { PackageInstallContext, StateNames, StateContext } from '../../../../../common/types'; +import type { PackageAssetReference } from '../../../../types'; + +import type { + Installation, + InstallType, + InstallSource, + PackageVerificationResult, + EsAssetReference, + KibanaAssetReference, + IndexTemplateEntry, + AssetReference, +} from '../../../../types'; + +import { + stepCreateRestartInstallation, + stepInstallKibanaAssets, + stepInstallILMPolicies, + stepInstallMlModel, + stepInstallIndexTemplatePipelines, + stepRemoveLegacyTemplates, + stepUpdateCurrentWriteIndices, + stepInstallTransforms, + stepDeletePreviousPipelines, + stepSaveArchiveEntries, + stepResolveKibanaPromise, + stepSaveSystemObject, + updateLatestExecutedState, +} from './steps'; +import type { StateMachineDefinition } from './state_machine'; +import { handleState } from './state_machine'; + +export interface InstallContext extends StateContext { + savedObjectsClient: SavedObjectsClientContract; + savedObjectsImporter: Pick; + savedObjectTagAssignmentService: IAssignmentService; + savedObjectTagClient: ITagsClient; + esClient: ElasticsearchClient; + logger: Logger; + installedPkg?: SavedObject; + packageInstallContext: PackageInstallContext; + installType: InstallType; + installSource: InstallSource; + spaceId: string; + force?: boolean; + verificationResult?: PackageVerificationResult; + authorizationHeader?: HTTPAuthorizationHeader | null; + ignoreMappingUpdateErrors?: boolean; + skipDataStreamRollover?: boolean; + + indexTemplates?: IndexTemplateEntry[]; + packageAssetRefs?: PackageAssetReference[]; + // output values + esReferences?: EsAssetReference[]; + kibanaAssetPromise?: Promise; +} +/* + * _stateMachineInstallPackage installs packages using the generic state machine in ./state_machine + * installStates is the data structure providing the state machine definition + * Usually the install process starts with `create_restart_installation` and continues based on nextState parameter in the definition + * The `onTransition` functions are the steps executed to go from one state to another, and all accept an `InstallContext` object as input parameter + * After each transition `updateLatestExecutedState` is executed, it updates the executed state in the SO + */ +export async function _stateMachineInstallPackage( + context: InstallContext +): Promise { + const installStates: StateMachineDefinition = { + context, + states: { + create_restart_installation: { + nextState: 'install_kibana_assets', + onTransition: stepCreateRestartInstallation, + onPostTransition: updateLatestExecutedState, + }, + install_kibana_assets: { + onTransition: stepInstallKibanaAssets, + nextState: 'install_ilm_policies', + onPostTransition: updateLatestExecutedState, + }, + install_ilm_policies: { + onTransition: stepInstallILMPolicies, + nextState: 'install_ml_model', + onPostTransition: updateLatestExecutedState, + }, + install_ml_model: { + onTransition: stepInstallMlModel, + nextState: 'install_index_template_pipelines', + onPostTransition: updateLatestExecutedState, + }, + install_index_template_pipelines: { + onTransition: stepInstallIndexTemplatePipelines, + nextState: 'remove_legacy_templates', + onPostTransition: updateLatestExecutedState, + }, + remove_legacy_templates: { + onTransition: stepRemoveLegacyTemplates, + nextState: 'update_current_write_indices', + onPostTransition: updateLatestExecutedState, + }, + update_current_write_indices: { + onTransition: stepUpdateCurrentWriteIndices, + nextState: 'install_transforms', + onPostTransition: updateLatestExecutedState, + }, + install_transforms: { + onTransition: stepInstallTransforms, + nextState: 'delete_previous_pipelines', + onPostTransition: updateLatestExecutedState, + }, + delete_previous_pipelines: { + onTransition: stepDeletePreviousPipelines, + nextState: 'save_archive_entries_from_assets_map', + onPostTransition: updateLatestExecutedState, + }, + save_archive_entries_from_assets_map: { + onTransition: stepSaveArchiveEntries, + nextState: 'resolve_kibana_promise', + onPostTransition: updateLatestExecutedState, + }, + resolve_kibana_promise: { + onTransition: stepResolveKibanaPromise, + nextState: 'update_so', + onPostTransition: updateLatestExecutedState, + }, + update_so: { + onTransition: stepSaveSystemObject, + nextState: 'end', + onPostTransition: updateLatestExecutedState, + }, + }, + }; + try { + const { installedKibanaAssetsRefs, esReferences } = await handleState( + INSTALL_STATES.CREATE_RESTART_INSTALLATION, + installStates, + installStates.context + ); + return [ + ...(installedKibanaAssetsRefs as KibanaAssetReference[]), + ...(esReferences as EsAssetReference[]), + ]; + } catch (err) { + const { packageInfo } = installStates.context.packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + + if (SavedObjectsErrorHelpers.isConflictError(err)) { + throw new PackageSavedObjectConflictError( + `Saved Object conflict encountered while installing ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + }. There may be a conflicting Saved Object saved to another Space. Original error: ${ + err.message + }` + ); + } else { + throw err; + } + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts new file mode 100644 index 0000000000000..f6e1f8fba5a20 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.test.ts @@ -0,0 +1,542 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createAppContextStartContractMock } from '../../../../mocks'; +import { appContextService } from '../../..'; + +import { handleState } from './state_machine'; + +const getTestDefinition = ( + mockOnTransition1: any, + mockOnTransition2: any, + mockOnTransition3: any, + context?: any, + onPostTransition?: any +) => { + return { + context, + states: { + state1: { + onTransition: mockOnTransition1, + onPostTransition, + nextState: 'state2', + }, + state2: { + onTransition: mockOnTransition2, + onPostTransition, + nextState: 'state3', + }, + state3: { + onTransition: mockOnTransition3, + onPostTransition, + nextState: 'end', + }, + }, + }; +}; + +describe('handleState', () => { + let mockContract: ReturnType; + beforeEach(async () => { + // prevents `Logger not set.` and other appContext errors + mockContract = createAppContextStartContractMock(); + appContextService.start(mockContract); + }); + afterEach(() => { + jest.resetAllMocks(); + appContextService.stop(); + }); + + it('should execute all the state machine transitions based on the provided data structure', async () => { + const mockOnTransitionState1 = jest.fn(); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3 + ); + + await handleState('state1', testDefinition, testDefinition.context); + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state1 with status: success - nextState: state2' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state2 with status: success - nextState: state3' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state3 with status: success - nextState: end' + ); + }); + + it('should call the onTransition function with context data and the return value is saved for the next iteration', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ promiseData: {} })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + await handleState('state1', testDefinition, testDefinition.context); + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + promiseData: {}, + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state1 with status: success - nextState: state2' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state2 with status: success - nextState: state3' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state3 with status: success - nextState: end' + ); + }); + + it('should save the return data from transitions also when return type is function', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ arrayData: ['test1', 'test2'] }); + const state2Result = () => { + return { + result: 'test', + }; + }; + const mockOnTransitionState2 = jest.fn().mockImplementation(() => { + return state2Result; + }); + const mockOnTransitionState3 = jest.fn(); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + await handleState('state1', testDefinition, testDefinition.context); + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + state2Result, + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + arrayData: ['test1', 'test2'], + state2Result, + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state1 with status: success - nextState: state2' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state2 with status: success - nextState: state3' + ); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executed state: state3 with status: success - nextState: end' + ); + }); + + it('should return updated context data', async () => { + const mockOnTransitionState1 = jest + .fn() + .mockImplementation(() => Promise.resolve({ promiseData: {} })); + const state2Result = () => { + return { + result: 'test', + }; + }; + const mockOnTransitionState2 = jest.fn().mockImplementation(() => { + return state2Result; + }); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ lastData: ['test3'] }); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + expect(mockOnTransitionState1).toHaveBeenCalledWith({ testData: 'test' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + promiseData: {}, + state2Result, + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + testData: 'test', + promiseData: {}, + state2Result, + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + promiseData: {}, + state2Result, + lastData: ['test3'], + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + + it('should update a variable in the context at every call and return the updated value', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ runningVal: 'test1' }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ runningVal: 'test2' })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ runningVal: 'test3' }); + const contextData = { runningVal: [], fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + expect(mockOnTransitionState1).toHaveBeenCalledWith({ runningVal: [], fixedVal: 'something' }); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: 'test1', + fixedVal: 'something', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: 'test2', + fixedVal: 'something', + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + runningVal: 'test3', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + + it('should execute the transition starting from the provided state', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ runningVal: 'test1' }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ runningVal: 'test2' })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ runningVal: 'test3' }); + const contextData = { runningVal: [], fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + + const updatedContext = await handleState('state2', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState2).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: [], + fixedVal: 'something', + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledWith( + expect.objectContaining({ + runningVal: 'test2', + fixedVal: 'something', + latestExecutedState: { + name: 'state2', + started_at: expect.anything(), + }, + }) + ); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + runningVal: 'test3', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + + it('should throw and return updated context with latest error when a state returns error', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockRejectedValue(error); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const contextData = { fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + const promise = handleState('state1', testDefinition, testDefinition.context); + await expect(promise).rejects.toThrowError('Installation failed'); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(0); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Error during execution of state "state1" with status "failed": Installation failed' + ); + }); + + it('should execute postTransition function after the transition is complete', async () => { + const mockOnTransitionState1 = jest.fn(); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const mockPostTransition = jest.fn(); + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + undefined, + mockPostTransition + ); + await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockOnTransitionState2).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockOnTransitionState3).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executing post transition function: mockConstructor' + ); + }); + + it('should execute postTransition function after the transition passing the updated context', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ runningVal: 'test1' }); + const mockOnTransitionState2 = jest + .fn() + .mockImplementation(() => Promise.resolve({ runningVal: 'test2' })); + const mockOnTransitionState3 = jest.fn().mockReturnValue({ runningVal: 'test3' }); + const mockPostTransition = jest.fn(); + const contextData = { fixedVal: 'something' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData, + mockPostTransition + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockOnTransitionState2).toHaveBeenCalled(); + expect(mockPostTransition).toHaveBeenCalled(); + expect(mockOnTransitionState3).toHaveBeenCalled(); + expect(updatedContext).toEqual( + expect.objectContaining({ + fixedVal: 'something', + runningVal: 'test3', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + expect(mockPostTransition).toHaveBeenCalledWith(updatedContext); + expect(mockContract.logger?.debug).toHaveBeenCalledWith( + 'Executing post transition function: mockConstructor' + ); + }); + + it('should execute postTransition correctly also when a transition throws', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = jest.fn().mockRejectedValue(error); + const mockOnTransitionState3 = jest.fn(); + const mockPostTransition = jest.fn(); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData, + mockPostTransition + ); + const promise = handleState('state1', testDefinition, testDefinition.context); + await expect(promise).rejects.toThrowError('Installation failed'); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockPostTransition).toHaveBeenCalledWith( + expect.objectContaining({ + result1: 'test', + testData: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + error: + 'Error during execution of state "state2" with status "failed": Installation failed', + }, + }) + ); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockPostTransition).toHaveBeenCalledWith( + expect.objectContaining({ + result1: 'test', + testData: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + }); + + it('should log a warning when postTransition exits with errors and continue executing the states', async () => { + const error = new Error('Installation failed'); + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = jest.fn(); + const mockOnTransitionState3 = jest.fn(); + const mockPostTransition = jest.fn().mockRejectedValue(error); + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData, + mockPostTransition + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockPostTransition).toHaveBeenCalledWith( + expect.objectContaining({ + result1: 'test', + testData: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + expect(mockOnTransitionState2).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(1); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Error during execution of post transition function: Installation failed' + ); + + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state3', + started_at: expect.anything(), + }, + }) + ); + }); + + it('should exit and log a warning when the provided OnTransition is not a function', async () => { + const mockOnTransitionState1 = jest.fn().mockReturnValue({ result1: 'test' }); + const mockOnTransitionState2 = undefined; + const mockOnTransitionState3 = jest.fn(); + + const contextData = { testData: 'test' }; + const testDefinition = getTestDefinition( + mockOnTransitionState1, + mockOnTransitionState2, + mockOnTransitionState3, + contextData + ); + const updatedContext = await handleState('state1', testDefinition, testDefinition.context); + + expect(mockOnTransitionState1).toHaveBeenCalledTimes(1); + expect(mockOnTransitionState3).toHaveBeenCalledTimes(0); + expect(mockContract.logger?.warn).toHaveBeenCalledWith( + 'Execution of state "state2" with status "failed": provided onTransition is not a valid function' + ); + + expect(updatedContext).toEqual( + expect.objectContaining({ + testData: 'test', + result1: 'test', + latestExecutedState: { + name: 'state1', + started_at: expect.anything(), + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts new file mode 100644 index 0000000000000..c70a99e272361 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/state_machine.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; + +import { appContextService } from '../../../app_context'; +import type { StateContext, LatestExecutedState } from '../../../../../common/types'; +export interface State { + onTransition: any; + nextState?: string; + currentStatus?: string; + onPostTransition?: any; +} + +export type StatusName = 'success' | 'failed' | 'pending'; +export type StateMachineStates = Record; +/* + * Data structure defining the state machine + * { + * context: {}, + * states: { + * state1: { + * onTransition: onState1Transition, + * onPostTransition: onPostTransition, + * nextState: 'state2', + * }, + * state2: { + * onTransition: onState2Transition, + * onPostTransition: onPostTransition,, + * nextState: 'state3', + * }, + * state3: { + * onTransition: onState3Transition, + * onPostTransition: onPostTransition, + * nextState: 'end', + * } + * } + */ +export interface StateMachineDefinition { + context: StateContext; + states: StateMachineStates; +} +/* + * Generic state machine implemented to handle state transitions, based on a provided data structure + * currentStateName: iniital state + * definition: data structure defined as a StateMachineDefinition + * context: object keeping the state between transitions. All the transition functions accept it as input parameter and write to it + * + * It recursively traverses all the states until it finds the last state. + */ +export async function handleState( + currentStateName: string, + definition: StateMachineDefinition, + context: StateContext +): Promise> { + const logger = appContextService.getLogger(); + const { states } = definition; + const currentState = states[currentStateName]; + let currentStatus = 'pending'; + let stateResult; + let updatedContext = { ...context }; + if (typeof currentState.onTransition === 'function') { + logger.debug( + `Current state ${currentStateName}: running transition ${currentState.onTransition.name}` + ); + try { + // inject information about the state into context + const startedAt = new Date(Date.now()).toISOString(); + const latestExecutedState: LatestExecutedState = { + name: currentStateName, + started_at: startedAt, + }; + stateResult = await currentState.onTransition.call(undefined, updatedContext); + // check if is a function/promise + if (typeof stateResult === 'function') { + const promiseName = `${currentStateName}Result`; + updatedContext[promiseName] = stateResult; + updatedContext = { ...updatedContext, latestExecutedState }; + } else { + updatedContext = { + ...updatedContext, + ...stateResult, + latestExecutedState, + }; + } + currentStatus = 'success'; + logger.debug( + `Executed state: ${currentStateName} with status: ${currentStatus} - nextState: ${currentState.nextState}` + ); + } catch (error) { + currentStatus = 'failed'; + const errorMessage = `Error during execution of state "${currentStateName}" with status "${currentStatus}": ${error.message}`; + const latestStateWithError = { + ...updatedContext.latestExecutedState, + error: errorMessage, + } as LatestExecutedState; + updatedContext = { ...updatedContext, latestExecutedState: latestStateWithError }; + logger.warn(errorMessage); + + // execute post transition function when transition failed too + await executePostTransition(logger, updatedContext, currentState); + + // bubble up the error + throw error; + } + } else { + currentStatus = 'failed'; + logger.warn( + `Execution of state "${currentStateName}" with status "${currentStatus}": provided onTransition is not a valid function` + ); + } + // execute post transition function + await executePostTransition(logger, updatedContext, currentState); + + if (currentStatus === 'success' && currentState?.nextState && currentState?.nextState !== 'end') { + return await handleState(currentState.nextState, definition, updatedContext); + } else { + return updatedContext; + } +} + +async function executePostTransition( + logger: Logger, + updatedContext: StateContext, + currentState: State +) { + if (typeof currentState.onPostTransition === 'function') { + try { + await currentState.onPostTransition.call(undefined, updatedContext); + logger.debug(`Executing post transition function: ${currentState.onPostTransition.name}`); + } catch (error) { + logger.warn(`Error during execution of post transition function: ${error.message}`); + } + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts new file mode 100644 index 0000000000000..c34c4f566715b --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './step_create_restart_installation'; +export * from './step_install_kibana_assets'; +export * from './step_install_mlmodel'; +export * from './step_install_ilm_policies'; +export * from './step_install_index_template_pipelines'; +export * from './step_remove_legacy_templates'; +export * from './step_update_current_write_indices'; +export * from './step_install_transforms'; +export * from './step_delete_previous_pipelines'; +export * from './step_save_archive_entries'; +export * from './step_save_system_object'; +export * from './step_resolve_kibana_promise'; +export * from './update_latest_executed_state'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts new file mode 100644 index 0000000000000..9323841daba00 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { + MAX_TIME_COMPLETE_INSTALL, + PACKAGES_SAVED_OBJECT_TYPE, +} from '../../../../../../common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; + +import { INSTALL_STATES } from '../../../../../../common/types'; + +import { auditLoggingService } from '../../../../audit_logging'; +import { restartInstallation, createInstallation } from '../../install'; +import type { Installation } from '../../../../../../common'; + +import { stepCreateRestartInstallation } from './step_create_restart_installation'; + +jest.mock('../../../../audit_logging'); +jest.mock('../../install'); + +const mockedRestartInstallation = jest.mocked(restartInstallation); +const mockedCreateInstallation = createInstallation as jest.Mocked; + +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +describe('stepCreateRestartInstallation', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(() => { + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + soClient.update.mockReset(); + // mockedCreateInstallation.mockReset(); + }); + + it('Should call createInstallation if no installedPkg is available', async () => { + await stepCreateRestartInstallation({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + expect(logger.debug).toHaveBeenCalledWith(`Package install - Create installation`); + expect(mockedCreateInstallation).toHaveBeenCalledTimes(1); + }); + + it('Should call restartInstallation if installedPkg is available and force = true', async () => { + await stepCreateRestartInstallation({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + expect(mockedRestartInstallation).toHaveBeenCalledTimes(1); + }); + + it('Should call restartInstallation and throw if installedPkg is available and force is not provided', async () => { + const promise = stepCreateRestartInstallation({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + await expect(promise).rejects.toThrowError( + 'Concurrent installation or upgrade of xyz-4.5.6 detected, aborting.' + ); + }); + expect(mockedRestartInstallation).toHaveBeenCalledTimes(0); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts new file mode 100644 index 0000000000000..58daa6c379134 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_create_restart_installation.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ConcurrentInstallOperationError } from '../../../../../errors'; +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../../constants'; + +import { restartInstallation, createInstallation } from '../../install'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepCreateRestartInstallation(context: InstallContext) { + const { + savedObjectsClient, + logger, + installSource, + packageInstallContext, + spaceId, + force, + verificationResult, + installedPkg, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + + // if some installation already exists + if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + logger.debug(`Package install - Install status ${installedPkg.attributes.install_status}`); + + // if the installation is currently running, don't try to install + // instead, only return already installed assets + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + logger.debug(`Package install - Installation is running and has exceeded timeout`); + + if (force) { + logger.debug(`Package install - Forced installation, restarting`); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } + } else { + // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL + // (it might be stuck) update the saved object and proceed + logger.debug( + `Package install - no installation running or the installation has been running longer than ${MAX_TIME_COMPLETE_INSTALL}, restarting` + ); + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } + } else { + logger.debug(`Package install - Create installation`); + + await createInstallation({ + savedObjectsClient, + packageInfo, + installSource, + spaceId, + verificationResult, + }); + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts new file mode 100644 index 0000000000000..7d8a251433bb5 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.test.ts @@ -0,0 +1,481 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { + isTopLevelPipeline, + deletePreviousPipelines, +} from '../../../elasticsearch/ingest_pipeline'; + +import { stepDeletePreviousPipelines } from './step_delete_previous_pipelines'; + +jest.mock('../../../elasticsearch/ingest_pipeline'); + +const mockedDeletePreviousPipelines = deletePreviousPipelines as jest.MockedFunction< + typeof deletePreviousPipelines +>; +const mockedIsTopLevelPipeline = isTopLevelPipeline as jest.MockedFunction< + typeof isTopLevelPipeline +>; + +describe('stepDeletePreviousPipelines', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedDeletePreviousPipelines).mockReset(); + jest.mocked(mockedIsTopLevelPipeline).mockReset(); + }); + + describe('Should call deletePreviousPipelines', () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + beforeEach(async () => { + jest.mocked(mockedDeletePreviousPipelines).mockResolvedValue([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); + + it('if installType is update', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + installedPkg.attributes.name, + installedPkg.attributes.version, + [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ] + ); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is reupdate', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'reupdate', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + installedPkg.attributes.name, + installedPkg.attributes.version, + [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ] + ); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is rollback', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'rollback', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + installedPkg.attributes.name, + installedPkg.attributes.version, + [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ] + ); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + }); + + describe('Should not call deletePreviousPipelines', () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + beforeEach(async () => { + jest.mocked(mockedDeletePreviousPipelines).mockResolvedValue([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); + + it('if installType is update and installedPkg is not present', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is reupdate and installedPkg is not present', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'reupdate', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType is rollback and installedPkg is not present', async () => { + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'rollback', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installType type is of different type', async () => { + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + jest.mocked(mockedIsTopLevelPipeline).mockImplementation(() => true); + + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { ...packageInstallContext, paths: ['some/path/1', 'some/path/2'] }, + installType: 'install', + installedPkg, + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('if installedPkg is present and there is a top level pipeline', async () => { + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + jest.mocked(mockedIsTopLevelPipeline).mockImplementation(() => true); + + const res = await stepDeletePreviousPipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { ...packageInstallContext, paths: ['some/path/1', 'some/path/2'] }, + installType: 'update', + installedPkg, + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedDeletePreviousPipelines).not.toBeCalled(); + expect(res).toEqual({ + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts new file mode 100644 index 0000000000000..eb80ef16dbcb0 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_delete_previous_pipelines.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + isTopLevelPipeline, + deletePreviousPipelines, +} from '../../../elasticsearch/ingest_pipeline'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepDeletePreviousPipelines(context: InstallContext) { + const { + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + installType, + installedPkg, + } = context; + const { packageInfo, paths } = packageInstallContext; + const { name: pkgName } = packageInfo; + let updatedESReferences; + // If this is an update or retrying an update, delete the previous version's pipelines + // Top-level pipeline assets will not be removed on upgrade as of ml model package addition which requires previous + // assets to remain installed. This is a temporary solution - more robust solution tracked here https://github.com/elastic/kibana/issues/115035 + if ( + paths.filter((path) => isTopLevelPipeline(path)).length === 0 && + (installType === 'update' || installType === 'reupdate') && + installedPkg + ) { + logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); + updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => + deletePreviousPipelines( + esClient, + savedObjectsClient, + pkgName, + installedPkg!.attributes.version, + esReferences || [] + ) + ); + } else if (installType === 'rollback' && installedPkg) { + // pipelines from a different version may have been installed during a failed update + logger.debug(`Package install - installType ${installType} Deleting previous ingest pipelines`); + updatedESReferences = await withPackageSpan('Delete previous ingest pipelines', () => + deletePreviousPipelines( + esClient, + savedObjectsClient, + pkgName, + installedPkg!.attributes.install_version, + esReferences || [] + ) + ); + } else { + // if none of the previous cases, return the original esReferences + updatedESReferences = esReferences; + } + return { esReferences: updatedESReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts new file mode 100644 index 0000000000000..210a6b882ceed --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.test.ts @@ -0,0 +1,374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; + +import type { Installation } from '../../../../../../common'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installILMPolicy } from '../../../elasticsearch/ilm/install'; +import { installIlmForDataStream } from '../../../elasticsearch/datastream_ilm/install'; +import { ElasticsearchAssetType } from '../../../../../types'; + +jest.mock('../../../archive/storage'); +jest.mock('../../../elasticsearch/ilm/install'); +jest.mock('../../../elasticsearch/datastream_ilm/install'); + +import { stepInstallILMPolicies } from './step_install_ilm_policies'; + +describe('stepInstallILMPolicies', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(installILMPolicy).mockReset(); + jest.mocked(installIlmForDataStream).mockReset(); + }); + + it('Should not install ILM policies if disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + }); + + it('Should not install ILM policies if disabled in config and should return esReferences form installedPkg', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const res = await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + installed_es: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).not.toBeCalled(); + expect(installIlmForDataStream).not.toBeCalled(); + expect(res?.esReferences).toEqual([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ]); + }); + + it('Should installs ILM policies if not disabled in config', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + jest.mocked(installILMPolicy).mockResolvedValue([]); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ], + installedIlms: [], + }); + const res = await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toHaveBeenCalled(); + expect(installIlmForDataStream).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + expect.any(Object), + expect.any(Object), + [] + ); + expect(res?.esReferences).toEqual([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ]); + }); + + it('should return updated esReferences', async () => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + jest.mocked(installILMPolicy).mockResolvedValue([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ] as any); + jest.mocked(installIlmForDataStream).mockResolvedValue({ + esReferences: [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ], + installedIlms: [], + }); + + const res = await stepInstallILMPolicies({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(installILMPolicy).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + expect.any(Object), + expect.any(Object), + [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + ] + ); + expect(installIlmForDataStream).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + expect.any(Object), + expect.any(Object), + [ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ] + ); + expect(res?.esReferences).toEqual([ + { + id: 'metrics-endpoint.policy-0.1.0-dev.0', + type: ElasticsearchAssetType.ingestPipeline, + }, + { + id: 'endpoint.metadata_current-default-0.1.0', + type: ElasticsearchAssetType.transform, + }, + { + id: 'endpoint.metadata_current-default-0.2.0', + type: ElasticsearchAssetType.transform, + }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts new file mode 100644 index 0000000000000..0e0d4ca2779f2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_ilm_policies.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { appContextService } from '../../../..'; + +import { installIlmForDataStream } from '../../../elasticsearch/datastream_ilm/install'; +import { installILMPolicy } from '../../../elasticsearch/ilm/install'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallILMPolicies(context: InstallContext) { + const { logger, packageInstallContext, esClient, savedObjectsClient, installedPkg } = context; + + // Array that gets updated by each operation. This allows each operation to accurately update the + // installation object with its references without requiring a refresh of the SO index on each update (faster). + let esReferences = installedPkg?.attributes.installed_es ?? []; + + // currently only the base package has an ILM policy + // at some point ILM policies can be installed/modified + // per data stream and we should then save them + const isILMPoliciesDisabled = + appContextService.getConfig()?.internal?.disableILMPolicies ?? false; + if (!isILMPoliciesDisabled) { + esReferences = await withPackageSpan('Install ILM policies', () => + installILMPolicy(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) + ); + ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () => + installIlmForDataStream( + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences + ) + )); + } + // always return esReferences even when isILMPoliciesDisabled is false as it's the first time we are writing to it + return { esReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts new file mode 100644 index 0000000000000..92a76eada06ec --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.test.ts @@ -0,0 +1,592 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installIndexTemplatesAndPipelines } from '../../install_index_template_pipeline'; + +jest.mock('../../install_index_template_pipeline'); + +import { stepInstallIndexTemplatePipelines } from './step_install_index_template_pipelines'; +const mockedInstallIndexTemplatesAndPipelines = + installIndexTemplatesAndPipelines as jest.MockedFunction< + typeof installIndexTemplatesAndPipelines + >; + +describe('stepInstallIndexTemplatePipelines', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedInstallIndexTemplatesAndPipelines).mockReset(); + }); + + it('Should call installIndexTemplatesAndPipelines if packageInfo type is integration', async () => { + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + const res = await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(mockedInstallIndexTemplatesAndPipelines).toHaveBeenCalledWith({ + installedPkg: installedPkg.attributes, + packageInstallContext: expect.any(Object), + esClient: expect.any(Object), + savedObjectsClient: expect.any(Object), + logger: expect.any(Object), + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(res).toEqual({ + indexTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('Should call installIndexTemplatesAndPipelines if packageInfo type is input and installedPkg exists', async () => { + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'input', + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_0001', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + { + name: 'path_2', + type: 'text', + }, + ], + }, + ], + }, + ], + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'type-template_0001', + type: ElasticsearchAssetType.indexTemplate, + }, + ]); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + const res = await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(res).toEqual({ + indexTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + }); + + it('Should not call installIndexTemplatesAndPipelines if packageInfo type is input and no data streams are found', async () => { + mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({ + installedTemplates: [ + { + templateName: 'template-01', + indexTemplate: { + priority: 1, + index_patterns: [], + template: { + settings: {}, + mappings: {}, + }, + data_stream: { hidden: false }, + composed_of: [], + _meta: {}, + }, + }, + ], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + { + id: 'something-01', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'input', + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_0001', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + { + name: 'path_2', + type: 'text', + }, + ], + }, + ], + }, + ], + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallIndexTemplatesAndPipelines).not.toBeCalled(); + }); + + it('Should not call installIndexTemplatesAndPipelines if packageInfo type is input and installedPkg does not exist', async () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'input', + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_0001', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + { + name: 'path_2', + type: 'text', + }, + ], + }, + ], + }, + ], + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + + await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallIndexTemplatesAndPipelines).not.toBeCalled(); + }); + + it('Should not call installIndexTemplatesAndPipelines if packageInfo type is undefined', async () => { + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: undefined, + categories: ['cloud'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + assetsMap: new Map(), + paths: [], + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + + await stepInstallIndexTemplatePipelines({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + expect(mockedInstallIndexTemplatesAndPipelines).not.toBeCalled(); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts new file mode 100644 index 0000000000000..e2b6918b722cf --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_index_template_pipelines.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getNormalizedDataStreams } from '../../../../../../common/services'; + +import { installIndexTemplatesAndPipelines } from '../../install_index_template_pipeline'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallIndexTemplatePipelines(context: InstallContext) { + const { esClient, savedObjectsClient, packageInstallContext, logger, installedPkg } = context; + const { packageInfo } = packageInstallContext; + const esReferences = context.esReferences ?? []; + + if (packageInfo.type === 'integration') { + logger.debug( + `Package install - Installing index templates and pipelines, packageInfo.type: ${packageInfo.type}` + ); + const { installedTemplates, esReferences: templateEsReferences } = + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkg ? installedPkg.attributes : undefined, + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + }); + return { + esReferences: templateEsReferences, + indexTemplates: installedTemplates, + }; + } + + if (packageInfo.type === 'input' && installedPkg) { + // input packages create their data streams during package policy creation + // we must use installed_es to infer which streams exist first then + // we can install the new index templates + logger.debug(`Package install - packageInfo.type: ${packageInfo.type}`); + const dataStreamNames = installedPkg.attributes.installed_es + .filter((ref) => ref.type === 'index_template') + // index templates are named {type}-{dataset}, remove everything before first hyphen + .map((ref) => ref.id.replace(/^[^-]+-/, '')); + + const dataStreams = dataStreamNames.flatMap((dataStreamName) => + getNormalizedDataStreams(packageInfo, dataStreamName) + ); + + if (dataStreams.length) { + const { installedTemplates, esReferences: templateEsReferences } = + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkg ? installedPkg.attributes : undefined, + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + onlyForDataStreams: dataStreams, + }); + return { esReferences: templateEsReferences, indexTemplates: installedTemplates }; + } + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts new file mode 100644 index 0000000000000..e13e3c9b095b2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.test.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installKibanaAssetsAndReferences } from '../../../kibana/assets/install'; + +jest.mock('../../../kibana/assets/install'); + +import { stepInstallKibanaAssets } from './step_install_kibana_assets'; + +const mockedInstallKibanaAssetsAndReferences = + installKibanaAssetsAndReferences as jest.MockedFunction; + +describe('stepInstallKibanaAssets', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + + soClient.update.mockImplementation(async (type, id, attributes) => { + return { id, attributes } as any; + }); + soClient.get.mockImplementation(async (type, id) => { + return { id, attributes: {} } as any; + }); + }); + + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + + it('Should call installKibanaAssetsAndReferences', async () => { + const installationPromise = stepInstallKibanaAssets({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + await expect(installationPromise).resolves.not.toThrowError(); + expect(mockedInstallKibanaAssetsAndReferences).toBeCalledTimes(1); + }); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + + it('Should correctly handle errors', async () => { + // force errors from this function + mockedInstallKibanaAssetsAndReferences.mockImplementation(async () => { + throw new Error('mocked async error A: should be caught'); + }); + + const installationPromise = stepInstallKibanaAssets({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + await expect(installationPromise).resolves.not.toThrowError(); + await expect(installationPromise).resolves.not.toThrowError(); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts new file mode 100644 index 0000000000000..56649c04428ac --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_kibana_assets.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installKibanaAssetsAndReferences } from '../../../kibana/assets/install'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallKibanaAssets(context: InstallContext) { + const { + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + logger, + installedPkg, + packageInstallContext, + spaceId, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, title: pkgTitle } = packageInfo; + + const kibanaAssetPromise = withPackageSpan('Install Kibana assets', () => + installKibanaAssetsAndReferences({ + savedObjectsClient, + savedObjectsImporter, + savedObjectTagAssignmentService, + savedObjectTagClient, + pkgName, + pkgTitle, + packageInstallContext, + installedPkg, + logger, + spaceId, + assetTags: packageInfo?.asset_tags, + }) + ); + // Necessary to avoid async promise rejection warning + // See https://stackoverflow.com/questions/40920179/should-i-refrain-from-handling-promise-rejection-asynchronously + kibanaAssetPromise.catch(() => {}); + + return { kibanaAssetPromise }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts new file mode 100644 index 0000000000000..ac67f8abfaccb --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installMlModel } from '../../../elasticsearch/ml_model'; + +import { stepInstallMlModel } from './step_install_mlmodel'; + +jest.mock('../../../elasticsearch/ml_model'); + +const mockedInstallMlModel = installMlModel as jest.MockedFunction; + +describe('stepInstallMlModel', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedInstallMlModel).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should update esReferences', async () => { + jest.mocked(mockedInstallMlModel).mockResolvedValue([]); + const res = await stepInstallMlModel({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallMlModel).toHaveBeenCalled(); + expect(res.esReferences).toEqual([]); + }); + + it('Should call installTransforms and return updated esReferences', async () => { + jest.mocked(mockedInstallMlModel).mockResolvedValue([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + const res = await stepInstallMlModel({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + expect(mockedInstallMlModel).toHaveBeenCalled(); + expect(res.esReferences).toEqual([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts new file mode 100644 index 0000000000000..31d571fee4505 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_mlmodel.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installMlModel } from '../../../elasticsearch/ml_model'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallMlModel(context: InstallContext) { + const { logger, packageInstallContext, esClient, savedObjectsClient } = context; + let esReferences = context.esReferences ?? []; + + esReferences = await withPackageSpan('Install ML models', () => + installMlModel(packageInstallContext, esClient, savedObjectsClient, logger, esReferences) + ); + return { esReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts new file mode 100644 index 0000000000000..63ea9c203bf43 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.test.ts @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { installTransforms } from '../../../elasticsearch/transform/install'; + +import { stepInstallTransforms } from './step_install_transforms'; + +jest.mock('../../../elasticsearch/transform/install'); + +const mockedInstallTransforms = installTransforms as jest.MockedFunction; + +describe('stepInstallTransforms', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedInstallTransforms).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should update esReferences', async () => { + jest.mocked(mockedInstallTransforms).mockResolvedValue({ + installedTransforms: [], + esReferences: [], + }); + const res = await stepInstallTransforms({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedInstallTransforms).toHaveBeenCalled(); + expect(res.esReferences).toEqual([]); + }); + + it('Should call installTransforms and return updated esReferences', async () => { + jest.mocked(mockedInstallTransforms).mockResolvedValue({ + installedTransforms: [], + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + const res = await stepInstallTransforms({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + expect(mockedInstallTransforms).toHaveBeenCalled(); + expect(res.esReferences).toEqual([ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ]); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts new file mode 100644 index 0000000000000..cd7d7404db5ad --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_install_transforms.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { installTransforms } from '../../../elasticsearch/transform/install'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepInstallTransforms(context: InstallContext) { + const { + packageInstallContext, + esClient, + savedObjectsClient, + logger, + force, + authorizationHeader, + } = context; + let esReferences = context.esReferences ?? []; + + ({ esReferences } = await withPackageSpan('Install transforms', () => + installTransforms({ + packageInstallContext, + esClient, + savedObjectsClient, + logger, + esReferences, + force, + authorizationHeader, + }) + )); + return { esReferences }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts new file mode 100644 index 0000000000000..39e7159596ba8 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.test.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { removeLegacyTemplates } from '../../../elasticsearch/template/remove_legacy'; + +import { stepRemoveLegacyTemplates } from './step_remove_legacy_templates'; + +jest.mock('../../../elasticsearch/template/remove_legacy'); + +const mockedRemoveLegacyTemplates = removeLegacyTemplates as jest.MockedFunction< + typeof removeLegacyTemplates +>; + +describe('stepRemoveLegacyTemplates', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedRemoveLegacyTemplates).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should call removeLegacyTemplates', async () => { + await stepRemoveLegacyTemplates({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedRemoveLegacyTemplates).toHaveBeenCalled(); + }); + + it('Should catch the error when removeLegacyTemplates fails', async () => { + jest.mocked(mockedRemoveLegacyTemplates).mockRejectedValue(Error('Error!')); + await stepRemoveLegacyTemplates({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + expect(mockedRemoveLegacyTemplates).toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith('Error removing legacy templates: Error!'); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts new file mode 100644 index 0000000000000..0c70989a67096 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_remove_legacy_templates.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { removeLegacyTemplates } from '../../../elasticsearch/template/remove_legacy'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepRemoveLegacyTemplates(context: InstallContext) { + const { esClient, packageInstallContext, logger } = context; + const { packageInfo } = packageInstallContext; + try { + await removeLegacyTemplates({ packageInfo, esClient, logger }); + } catch (e) { + logger.warn(`Error removing legacy templates: ${e.message}`); + } +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts new file mode 100644 index 0000000000000..72782438c20b6 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_resolve_kibana_promise.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepResolveKibanaPromise(context: InstallContext) { + const { kibanaAssetPromise } = context; + const installedKibanaAssetsRefs = await kibanaAssetPromise; + + return { installedKibanaAssetsRefs }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts new file mode 100644 index 0000000000000..3515fd304b356 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.test.ts @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; +import { ElasticsearchAssetType } from '../../../../../types'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; + +import { stepSaveArchiveEntries } from './step_save_archive_entries'; + +jest.mock('../../../archive/storage'); + +const mockedSaveArchiveEntriesFromAssetsMap = + saveArchiveEntriesFromAssetsMap as jest.MockedFunction; + +describe('stepSaveArchiveEntries', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map([ + [ + 'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json', + Buffer.from('{"content": "data"}'), + ], + ]), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should return empty packageAssetRefs if saved_objects were not found', async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [], + }); + const res = await stepSaveArchiveEntries({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(res).toEqual({ + packageAssetRefs: [], + }); + }); + + it('Should return packageAssetRefs', async () => { + jest.mocked(mockedSaveArchiveEntriesFromAssetsMap).mockResolvedValue({ + saved_objects: [ + { + id: 'test', + attributes: { + package_name: 'test-package', + package_version: '1.0.0', + install_source: 'registry', + asset_path: 'some/path', + media_type: '', + data_utf8: '', + data_base64: '', + }, + type: '', + references: [], + }, + ], + }); + const res = await stepSaveArchiveEntries({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'update', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [ + { + id: 'something', + type: ElasticsearchAssetType.ilmPolicy, + }, + ], + }); + + expect(res).toEqual({ + packageAssetRefs: [ + { + id: 'test', + type: 'epm-packages-assets', + }, + ], + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts new file mode 100644 index 0000000000000..ca65b04e55303 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_archive_entries.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ASSETS_SAVED_OBJECT_TYPE } from '../../../../../constants'; +import type { PackageAssetReference } from '../../../../../types'; + +import { saveArchiveEntriesFromAssetsMap } from '../../../archive/storage'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepSaveArchiveEntries(context: InstallContext) { + const { packageInstallContext, savedObjectsClient, installSource } = context; + + const { packageInfo } = packageInstallContext; + + const packageAssetResults = await withPackageSpan('Update archive entries', () => + saveArchiveEntriesFromAssetsMap({ + savedObjectsClient, + assetsMap: packageInstallContext?.assetsMap, + paths: packageInstallContext?.paths, + packageInfo, + installSource, + }) + ); + const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map( + (result) => ({ + id: result.id, + type: ASSETS_SAVED_OBJECT_TYPE, + }) + ); + + return { packageAssetRefs }; +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts new file mode 100644 index 0000000000000..e91826c99793c --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.test.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; + +import { auditLoggingService } from '../../../../audit_logging'; +import { packagePolicyService } from '../../../../package_policy'; + +import { stepSaveSystemObject } from './step_save_system_object'; + +jest.mock('../../../../audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +jest.mock('../../../../package_policy'); +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; + +describe('updateLatestExecutedState', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + + afterEach(() => { + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + soClient.get.mockReset(); + soClient.update.mockReset(); + }); + + it('Should save the SO and should not call packagePolicy upgrade if keep_policies_up_to_date = false', async () => { + soClient.get.mockResolvedValue({ + id: 'test-integration', + attributes: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + install_source: 'registry', + install_status: 'installed', + package_assets: [], + }, + } as any); + + await stepSaveSystemObject({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'test-integration', + { + install_format_schema_version: '1.2.0', + install_status: 'installed', + install_version: '1.0.0', + latest_install_failed_attempts: [], + package_assets: undefined, + version: '1.0.0', + }, + ], + ]); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-integration', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + expect(mockedPackagePolicyService.upgrade).not.toBeCalled(); + }); + + it('Should call packagePolicy upgrade if keep_policies_up_to_date = true', async () => { + soClient.get.mockResolvedValue({ + id: 'test-integration', + attributes: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + install_source: 'registry', + install_status: 'installed', + package_assets: [], + keep_policies_up_to_date: true, + }, + } as any); + mockedPackagePolicyService.listIds.mockReturnValue({ + items: ['packagePolicy1', 'packagePolicy2'], + } as any); + + await stepSaveSystemObject({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'test-integration', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual([ + [ + 'epm-packages', + 'test-integration', + { + install_format_schema_version: '1.2.0', + install_status: 'installed', + install_version: '1.0.0', + latest_install_failed_attempts: [], + package_assets: undefined, + version: '1.0.0', + }, + ], + ]); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-integration', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + expect(packagePolicyService.upgrade).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + ['packagePolicy1', 'packagePolicy2'] + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts new file mode 100644 index 0000000000000..f7bca891da6f7 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_save_system_object.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + PACKAGES_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, + FLEET_INSTALL_FORMAT_VERSION, +} from '../../../../../constants'; +import type { Installation } from '../../../../../types'; + +import { packagePolicyService } from '../../../..'; + +import { auditLoggingService } from '../../../../audit_logging'; + +import { withPackageSpan } from '../../utils'; + +import { clearLatestFailedAttempts } from '../../install_errors_helpers'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepSaveSystemObject(context: InstallContext) { + const { + packageInstallContext, + savedObjectsClient, + logger, + esClient, + installedPkg, + packageAssetRefs, + } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName, version: pkgVersion } = packageInfo; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + + await withPackageSpan('Update install status', () => + savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + version: pkgVersion, + install_version: pkgVersion, + install_status: 'installed', + package_assets: packageAssetRefs, + install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, + latest_install_failed_attempts: clearLatestFailedAttempts( + pkgVersion, + installedPkg?.attributes.latest_install_failed_attempts ?? [] + ), + }) + ); + + // Need to refetch the installation again to retrieve all the attributes + const updatedPackage = await savedObjectsClient.get( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName + ); + logger.debug(`Package install - Install status ${updatedPackage?.attributes?.install_status}`); + // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its + // associated package policies after installation + if (updatedPackage.attributes.keep_policies_up_to_date) { + await withPackageSpan('Upgrade package policies', async () => { + const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { + page: 1, + perPage: SO_SEARCH_LIMIT, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, + }); + logger.debug( + `Package install - Package is flagged with keep_policies_up_to_date, upgrading its associated package policies ${policyIdsToUpgrade}` + ); + await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); + }); + } + logger.debug( + `Install status ${updatedPackage?.attributes?.install_status} - Installation complete!` + ); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts new file mode 100644 index 0000000000000..c7f3c040b7966 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; +import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import type { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/types'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../../common/constants'; + +import type { EsAssetReference, Installation } from '../../../../../../common'; +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; +import { updateCurrentWriteIndices } from '../../../elasticsearch/template/template'; + +import { stepUpdateCurrentWriteIndices } from './step_update_current_write_indices'; + +jest.mock('../../../elasticsearch/template/template'); + +const mockedUpdateCurrentWriteIndices = updateCurrentWriteIndices as jest.MockedFunction< + typeof updateCurrentWriteIndices +>; + +const createMockTemplate = ({ name, composedOf = [] }: { name: string; composedOf?: string[] }) => + ({ + name, + index_template: { + composed_of: composedOf, + }, + } as IndicesGetIndexTemplateIndexTemplateItem); + +describe('stepUpdateCurrentWriteIndices', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const getMockInstalledPackageSo = ( + installedEs: EsAssetReference[] = [] + ): SavedObject => { + return { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: installedEs, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + }; + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(async () => { + jest.mocked(mockedUpdateCurrentWriteIndices).mockReset(); + }); + + const packageInstallContext = { + packageInfo: { + title: 'title', + name: 'test-package', + version: '1.0.0', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + } as any, + paths: ['some/path/1', 'some/path/2'], + assetsMap: new Map(), + }; + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + retrySetupOnBoot: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + excludePackages: [], + }, + }, + }) + ); + const mockInstalledPackageSo = getMockInstalledPackageSo(); + const installedPkg = { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }; + + it('Should call updateCurrentWriteIndices', async () => { + await stepUpdateCurrentWriteIndices({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + }); + + expect(mockedUpdateCurrentWriteIndices).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + [], + { ignoreMappingUpdateErrors: undefined, skipDataStreamRollover: undefined } + ); + }); + + it('Should call updateCurrentWriteIndices with passed parameters', async () => { + const indexTemplates = [createMockTemplate({ name: 'tmpl1' })] as any; + await stepUpdateCurrentWriteIndices({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + packageInstallContext, + installedPkg, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + esReferences: [], + indexTemplates, + ignoreMappingUpdateErrors: true, + skipDataStreamRollover: true, + }); + + expect(mockedUpdateCurrentWriteIndices).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + indexTemplates, + { ignoreMappingUpdateErrors: true, skipDataStreamRollover: true } + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts new file mode 100644 index 0000000000000..094f1110d9021 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/step_update_current_write_indices.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { updateCurrentWriteIndices } from '../../../elasticsearch/template/template'; + +import { withPackageSpan } from '../../utils'; + +import type { InstallContext } from '../_state_machine_package_install'; + +export async function stepUpdateCurrentWriteIndices(context: InstallContext) { + const { esClient, logger, ignoreMappingUpdateErrors, skipDataStreamRollover, indexTemplates } = + context; + + // update current backing indices of each data stream + await withPackageSpan('Update write indices', () => + updateCurrentWriteIndices(esClient, logger, indexTemplates || [], { + ignoreMappingUpdateErrors, + skipDataStreamRollover, + }) + ); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts new file mode 100644 index 0000000000000..afce673348d7e --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.test.ts @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObjectsUpdateResponse, +} from '@kbn/core/server'; +import { + savedObjectsClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; + +import { + MAX_TIME_COMPLETE_INSTALL, + PACKAGES_SAVED_OBJECT_TYPE, +} from '../../../../../../common/constants'; + +import { appContextService } from '../../../../app_context'; +import { createAppContextStartContractMock } from '../../../../../mocks'; + +import { INSTALL_STATES } from '../../../../../../common/types'; + +import { auditLoggingService } from '../../../../audit_logging'; + +import type { PackagePolicySOAttributes } from '../../../../../types'; + +import { updateLatestExecutedState } from './update_latest_executed_state'; + +jest.mock('../../../../audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +describe('updateLatestExecutedState', () => { + let soClient: jest.Mocked; + let esClient: jest.Mocked; + const logger = loggingSystemMock.createLogger(); + + beforeEach(async () => { + soClient = savedObjectsClientMock.create(); + esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(() => { + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + soClient.update.mockReset(); + }); + + it('Updates the SO after each transition', async () => { + await updateLatestExecutedState({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual( + expect.objectContaining([ + [ + 'epm-packages', + 'xyz', + { + latest_executed_state: { + name: 'save_archive_entries_from_assets_map', + started_at: expect.anything(), + }, + }, + ], + ]) + ); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'xyz', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); + + it('Should not update the SO if the context contains concurrent installation error', async () => { + await updateLatestExecutedState({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + error: `Concurrent installation or upgrade of xyz-4.5.6 detected.`, + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(soClient.update.mock.calls).toEqual([]); + expect(mockedAuditLoggingService.writeCustomSoAuditLog).not.toHaveBeenCalled(); + }); + + it('Should log error if the update failed', async () => { + soClient.update.mockImplementation( + async ( + _type: string, + _id: string + ): Promise> => { + throw SavedObjectsErrorHelpers.createConflictError('abc', '123'); + } + ); + + await updateLatestExecutedState({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger, + packageInstallContext: { + assetsMap: new Map(), + paths: [], + packageInfo: { + title: 'title', + name: 'xyz', + version: '4.5.6', + description: 'test', + type: 'integration', + categories: ['cloud', 'custom'], + format_version: 'string', + release: 'experimental', + conditions: { kibana: { version: 'x.y.z' } }, + owner: { github: 'elastic/fleet' }, + }, + }, + latestExecutedState: { + name: INSTALL_STATES.SAVE_ARCHIVE_ENTRIES, + started_at: new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL * 2).toISOString(), + }, + installType: 'install', + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'xyz', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + expect(logger.error).toHaveBeenCalledWith( + 'Failed to update SO with latest executed state: Error: Saved object [abc/123] conflict' + ); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts new file mode 100644 index 0000000000000..55d7997ad58f7 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/install_state_machine/steps/update_latest_executed_state.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsErrorHelpers } from '@kbn/core/server'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../constants'; + +import { auditLoggingService } from '../../../../audit_logging'; + +import type { InstallContext } from '../_state_machine_package_install'; + +// Function invoked after each transition +export const updateLatestExecutedState = async (context: InstallContext) => { + const { logger, savedObjectsClient, packageInstallContext, latestExecutedState } = context; + const { packageInfo } = packageInstallContext; + const { name: pkgName } = packageInfo; + + try { + // If the error is of type ConcurrentInstallationError, don't save it in the SO + if (latestExecutedState?.error?.includes('Concurrent installation or upgrade')) return; + + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + return await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { + latest_executed_state: latestExecutedState, + }); + } catch (err) { + if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { + logger.error(`Failed to update SO with latest executed state: ${err}`); + } + } +}; diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index bec381d311937..da4d793989e8b 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -97,6 +97,8 @@ export type { ActionStatusOptions, PackageSpecTags, AssetsMap, + InstallResultStatus, + InstallLatestExecutedState, } from '../../common/types'; export { ElasticsearchAssetType, KibanaAssetType, KibanaSavedObjectType } from '../../common/types'; export { dataTypes } from '../../common/constants'; diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts index 5192b8a4e914b..5f4c5b784a280 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_error_rollback.ts @@ -27,7 +27,10 @@ export default function (providerContext: FtrProviderContext) { }; const uninstallPackage = async (pkg: string, version: string) => { - await supertest.delete(`/api/fleet/epm/packages/${pkg}/${version}`).set('kbn-xsrf', 'xxxx'); + await supertest + .delete(`/api/fleet/epm/packages/${pkg}/${version}`) + .set('kbn-xsrf', 'xxxx') + .send({ force: true }); }; const getPackageInfo = async (pkg: string, version: string) => { diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 9c0b5cd8a426e..96e5e95e720ad 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -778,6 +778,10 @@ const expectAssetsInstalled = ({ install_started_at: res.attributes.install_started_at, install_source: 'registry', latest_install_failed_attempts: [], + latest_executed_state: { + name: 'update_so', + started_at: res.attributes.latest_executed_state.started_at, + }, install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, verification_status: 'unknown', verification_key_id: null, diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index cd3898a58c6a7..fe584f9cd04f7 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -486,6 +486,10 @@ export default function (providerContext: FtrProviderContext) { install_source: 'registry', install_format_schema_version: FLEET_INSTALL_FORMAT_VERSION, latest_install_failed_attempts: [], + latest_executed_state: { + name: 'update_so', + started_at: res.attributes.latest_executed_state.started_at, + }, verification_status: 'unknown', verification_key_id: null, }); diff --git a/x-pack/test/fleet_api_integration/config.base.ts b/x-pack/test/fleet_api_integration/config.base.ts index 5626ee4d85d6e..fd9d8e08779c0 100644 --- a/x-pack/test/fleet_api_integration/config.base.ts +++ b/x-pack/test/fleet_api_integration/config.base.ts @@ -76,6 +76,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { 'agentTamperProtectionEnabled', 'enableStrictKQLValidation', 'subfeaturePrivileges', + 'enablePackagesStateMachine', ])}`, `--logging.loggers=${JSON.stringify([ ...getKibanaCliLoggers(xPackAPITestsConfig.get('kbnTestServer.serverArgs')), From 8f88fc8343ffb4f63cec5f1f25721af147f4994c Mon Sep 17 00:00:00 2001 From: amyjtechwriter <61687663+amyjtechwriter@users.noreply.github.com> Date: Mon, 8 Apr 2024 14:15:04 +0100 Subject: [PATCH 07/11] [DOCS] 8.13.2 release notes (#180264) Adds the 8.13.2 release notes. ![Screenshot 2024-04-08 at 11 54 17](https://github.com/elastic/kibana/assets/61687663/98ab9525-5645-4bc6-bea2-3e7e439705d2) --------- Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/CHANGELOG.asciidoc | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 0ff00622c7cc7..6188e010be28f 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -10,6 +10,7 @@ Review important information about the {kib} 8.x releases. +* <> * <> * <> * <> @@ -62,6 +63,35 @@ Review important information about the {kib} 8.x releases. * <> -- +[[release-notes-8.13.2]] +== {kib} 8.13.2 + +The 8.13.2 release includes the following bug fixes. + +[float] +[[fixes-v8.13.2]] +=== Bug Fixes +Canvas:: +* Fixes text settings to be honored in Canvas markdown elements ({kibana-pull}179948[#179948]). +Discover:: +* Fixes keyboard navigation for search input on the document viewer flyout ({kibana-pull}180022[#180022]). +Elastic Security:: +For the Elastic Security 8.13.2 release information, refer to {security-guide}/release-notes.html[_Elastic Security Solution Release Notes_]. +Fleet:: +* Fixes having to wait ten minutes after agent upgrade if agent cleared watching state ({kibana-pull}179917[#179917]). +Fixes using the latest available version in K8's manifest instead of the latest compatible version ({kibana-pull}179662[#179662]). +* Fixes a step in add agent instructions where a query to get all agents was unnecessary ({kibana-pull}179603[#179603]). +Machine Learning:: +* Single Metric Viewer embeddable: Ensures the detector index is passed to chart correctly ({kibana-pull}179761[#179761]). +* AIOps: Fixes text field candidate selection for log rate analysis ({kibana-pull}179699[#179699]). +Management:: +* Fixes the Response tab loading time to be faster ({kibana-pull}180035[#180035]). +Maps:: +* Fixes APM data view ID ({kibana-pull}179257[#179257]). +Monitoring:: +* Fixes a runtime error by adding a default value for source and target ({kibana-pull}180043[#180043]). +Operations:: +* Fixes an issue with {kib} looking for a configuration file outside of the {kib} home directory, potentially preventing startup due to insufficient permissions ({kibana-pull}179847[#179847]). [[release-notes-8.13.1]] == {kib} 8.13.1 From c708c499481fcc2394b301db019e22b2b5a8831a Mon Sep 17 00:00:00 2001 From: Julia Rechkunova Date: Mon, 8 Apr 2024 16:04:52 +0200 Subject: [PATCH 08/11] [Discover] Allow to store the configured ES|QL visualization v3 (#175227) - Resolves https://github.com/elastic/kibana/issues/167887 ## Summary On Discover page user can see a visualization for data view and ES|QL modes. For ES|QL mode it's also allowed to customize the visualization. This PR allows to save such customization together with a saved search. In more details, various types of Lens visualization can be shown on Discover page: - If in the default (data view) mode, Unified Histogram shows a "formBased" histogram (`type: UnifiedHistogramSuggestionType.histogramForDataView` in this PR) - If in the ES|QL mode, 2 scenarios are possible (so far only these are customizable): - If Lens has suggestions for the current query, Unified Histogram shows one of them (`type: UnifiedHistogramSuggestionType.lensSuggestion` in this PR) Example query: `from kibana_sample_data_logs | stats avg(bytes) by message.keyword` - If Lens suggestion list is empty, Unified Histogram shows a "textBased" histogram (`type: UnifiedHistogramSuggestionType.histogramForESQL` in this PR). Example query: `from kibana_sample_data_logs | limit 10` The main flow is that Unified Histogram first picks a suggestion (one of `UnifiedHistogramSuggestionType` type), then calculates lens attributes which are necessary to build Lens embeddable. With a saved search we are saving those calculated lens attributes under `savedSearch.visContext`. For handling this logic, I refactored `useLensSuggestion`, `getLensAttributes` into `LensVisService`. Restoring a saved customization adds complexity to the flow as it should pick now not just any available suggestion but the suggestion which matches to the previously saved lens attributes. Changes to the current query, time range, time field etc can make the current vis context incompatible and we have to drop the vis customization. This PR already includes this logic of invalidating the stored lens attributes if they are not compatible any more. New vis context will override the previous one when user presses Save for the current search. Until then, we try to restore the customization from the previously saved vis context (for example when the query changes back to the compatible one). What can invalidate the saved vis context and drop the user's customization: - data view id - data view time field name - query/filters - time range if it has a different time interval - text based columns affect what lens suggestions are available Flow of creating a new search: ![1](https://github.com/elastic/kibana/assets/1415710/9274d895-cedb-454a-9a9d-3b0cf600d801) Flow of editing a saved search: ![2](https://github.com/elastic/kibana/assets/1415710/086ce4a0-f679-4d96-892b-631bcfee7ee3)
Previous details - Previous approach https://github.com/elastic/kibana/pull/174373 (saving current suggestion instead of lens attributes) - Previous approach https://github.com/elastic/kibana/pull/174783 (saving lens attributes but it's based on existing hooks) But I was stuck with how to make "Unsaved changes" badge work well when user tries to revert changes. For testing in ES|QL mode I use `from kibana_sample_data_logs | limit 10` as query, customize color of a lens histogram, and save it with a saved search. Next I check 2 cases: 1. edit query limit `from kibana_sample_data_logs | limit 100`, see that vis customization gets reset which is expected, press "Revert changes" in the "Unsaved changes" badge => notice that reset did not work 2. edit only histogram color, press "Revert changes" in the "Unsaved changes" badge => notice that reset did not work Here are some nuances with the state management I am seeing which together do not allow to successfully revert unsaved changes: - For ES|QL histogram lens attributes include a modified query `from kibana_sample_data_logs | limit 10 | EVAL timestamp=DATE_TRUNC(30 second, @timestamp) | stats results = count(*) by timestamp | rename timestamp as "@timestamp every 30 second"` which means that not only changes to the original query but also a different time interval invalidates the saved lens attributes. - In ES|QL mode, `query` prop update is delayed for `UnifiedHistogramContainer` component until Discover finishes the documents fetch https://github.com/elastic/kibana/blob/fc2ec957fe78900967da26c80817aea8a0bd2c65/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts#L346 which means that Discover should make a request on revert changes. And It's not happening for (2) as it does not make sense for Discover to trigger refetch if only `visContext` changes so we should find another way. With (1) there is another problem that Discover `visContext` state gets hijacked by lens attributes invalidation logic (as query is not sync yet to UnifiedHistogram) before fetch is completed or get [a chance to be fired](https://github.com/elastic/kibana/blob/6038f92b1fcaeedf635a0eab68fd9cdadd1103d3/src/plugins/discover/public/application/main/hooks/utils/build_state_subscribe.ts#L51-L54). I tried delaying `externalVisContext` prop update too (to keep in sync with `query` update) but it does not help https://github.com/elastic/kibana/blob/fc2ec957fe78900967da26c80817aea8a0bd2c65/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts#L437 - Unified Histogram should signal to Discover to start a refetch when current suggestion changes https://github.com/elastic/kibana/blob/fc2ec957fe78900967da26c80817aea8a0bd2c65/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts#L289 - for some reason this logic is required for "Revert changes" to work as it triggers the refetch. I would expect Discover on its own to notice the change in query and refetch data but it does not seem to be the case.
Other challenges - [ ] Since we are starting to save lens attributes inside a saved search object (similar to how Dashboard saves lens vis by value), we should integrate Lens migrations into saved search migrations logic. I made a quick PoC how it could look like here https://github.com/jughosta/kibana/commit/4529711d0ddfd1e559be099f5c3263e099847b46 This showed that adding Lens plugin as a dependency to saved search plugin causes lots of circular deps in Kibana. To resolve that I am suggesting to spit saved search plugin into 2 plugins https://github.com/elastic/kibana/pull/174939 - not the best solution but it seems impossible to split lens plugins instead. Updates here: - [x] revert the code regarding migrations and saved search plugin split - [x] create a github issue to handle client side migrations once their API is available https://github.com/elastic/kibana/issues/179151 - [x] Discover syncs app state with URL which means that the new `visContext` (large lens attributes object) ends up in the URL too. We should exclude `visContext` from URL sync as it can make the URL too long. Updates here: we are not using appState for this any more - [x] Changes from https://github.com/elastic/kibana/pull/171081 would need to be refactored and integrated into the new `LensVisService`. - [x] Refactor after https://github.com/elastic/kibana/pull/177790 - [x] Handle a case when no chart is available for current ES|QL query - [ ] For ES|QL histogram lens attributes include a modified query `from kibana_sample_data_logs | limit 10 | EVAL timestamp=DATE_TRUNC(30 second, @timestamp) | stats results = count(*) by timestamp | rename timestamp as "@timestamp every 30 second"` which means that not only changes to the original query but also a different time range can reset the customization of lens vis as it gets a different time interval based on current time range - New update from Stratoula: - [ ] would it help to persist response of `onApplyCb` instead of lens attributes? <= the shape does not seem to be different and it works as it is so I'm keeping lens attributes - [x] use new `getLensAttributes` from https://github.com/elastic/kibana/pull/174677
10x flaky test https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5578 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [x] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [x] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Matthias Wilhelm Co-authored-by: Stratoula Kalafateli --- .../src/get_lens_attributes.test.ts | 1 + .../src/get_lens_attributes.ts | 24 +- .../check_registered_types.test.ts | 2 +- .../layout/discover_histogram_layout.tsx | 2 +- .../layout/use_discover_histogram.ts | 120 ++- .../components/top_nav/get_top_nav_badges.tsx | 8 +- .../components/top_nav/on_save_search.tsx | 11 + .../discover_internal_state_container.ts | 23 +- .../discover_saved_search_container.ts | 23 + .../main/services/discover_state.ts | 1 + .../main/services/load_saved_search.ts | 3 + .../application/main/utils/fetch_all.test.ts | 3 + .../content_management/v1/cm_services.ts | 19 + .../common/saved_searches_utils.ts | 1 + .../common/service/get_saved_searches.test.ts | 2 + .../service/saved_searches_utils.test.ts | 2 + .../common/service/saved_searches_utils.ts | 1 + src/plugins/saved_search/common/types.ts | 17 + .../saved_search_attribute_service.test.ts | 1 + .../saved_search_storage.ts | 1 + .../server/saved_objects/schema.ts | 22 + .../server/saved_objects/search.ts | 8 + .../public/__mocks__/data_view.ts | 6 +- .../__mocks__/data_view_with_timefield.ts | 6 + .../public/__mocks__/lens_vis.ts | 97 +++ .../public/__mocks__/services.tsx | 19 +- .../public/__mocks__/suggestions.ts | 85 ++ .../public/__mocks__/table.ts | 49 ++ .../public/chart/breakdown_field_selector.tsx | 2 +- .../public/chart/chart.test.tsx | 64 +- .../unified_histogram/public/chart/chart.tsx | 170 ++-- .../public/chart/chart_config_panel.test.tsx | 22 +- .../public/chart/chart_config_panel.tsx | 42 +- .../public/chart/histogram.test.tsx | 57 +- .../public/chart/histogram.tsx | 12 +- .../chart/hooks/use_edit_visualization.ts | 4 +- .../public/chart/hooks/use_lens_props.test.ts | 99 +-- .../public/chart/hooks/use_lens_props.ts | 17 +- .../chart/hooks/use_time_range.test.tsx | 2 +- .../public/chart/hooks/use_time_range.tsx | 2 +- .../public/chart/hooks/use_total_hits.ts | 2 +- .../public/chart/suggestion_selector.tsx | 78 +- .../public/chart/utils/get_lens_attributes.ts | 233 ------ .../public/container/container.tsx | 38 +- .../container/hooks/use_state_props.test.ts | 23 +- .../public/container/hooks/use_state_props.ts | 8 +- .../container/services/state_service.test.ts | 5 +- .../container/services/state_service.ts | 18 +- .../public/container/utils/state_selectors.ts | 1 - .../hooks/use_request_params.test.ts | 2 +- .../{chart => }/hooks/use_request_params.tsx | 12 +- .../hooks/use_stable_callback.test.ts | 0 .../{chart => }/hooks/use_stable_callback.ts | 0 src/plugins/unified_histogram/public/index.ts | 4 +- .../layout/hooks/use_lens_suggestions.test.ts | 224 ------ .../layout/hooks/use_lens_suggestions.ts | 152 ---- .../public/layout/layout.test.tsx | 1 + .../public/layout/layout.tsx | 171 ++-- .../lens_vis_service.attributes.test.ts} | 206 +++-- .../lens_vis_service.suggestions.test.ts | 194 +++++ .../public/services/lens_vis_service.ts | 754 ++++++++++++++++++ src/plugins/unified_histogram/public/types.ts | 47 +- .../external_vis_context.test.ts.snap | 342 ++++++++ .../hooks => utils}/compute_interval.test.ts | 0 .../hooks => utils}/compute_interval.ts | 0 .../public/utils/external_vis_context.test.ts | 164 ++++ .../public/utils/external_vis_context.ts | 89 +++ .../utils/field_supports_breakdown.test.ts | 0 .../utils/field_supports_breakdown.ts | 0 .../public/utils/lens_vis_from_table.ts | 57 ++ src/plugins/unified_histogram/tsconfig.json | 3 +- .../apps/discover/group3/_lens_vis.ts | 675 ++++++++++++++++ test/functional/apps/discover/group3/index.ts | 1 + test/functional/page_objects/discover_page.ts | 6 + .../shared/edit_on_the_fly/flyout_wrapper.tsx | 1 + .../get_edit_lens_configuration.tsx | 1 + .../public/functions/visualize_esql.test.tsx | 1 + 77 files changed, 3452 insertions(+), 1111 deletions(-) create mode 100644 src/plugins/unified_histogram/public/__mocks__/lens_vis.ts create mode 100644 src/plugins/unified_histogram/public/__mocks__/table.ts delete mode 100644 src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.ts rename src/plugins/unified_histogram/public/{chart => }/hooks/use_request_params.test.ts (95%) rename src/plugins/unified_histogram/public/{chart => }/hooks/use_request_params.tsx (85%) rename src/plugins/unified_histogram/public/{chart => }/hooks/use_stable_callback.test.ts (100%) rename src/plugins/unified_histogram/public/{chart => }/hooks/use_stable_callback.ts (100%) delete mode 100644 src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts delete mode 100644 src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts rename src/plugins/unified_histogram/public/{chart/utils/get_lens_attributes.test.ts => services/lens_vis_service.attributes.test.ts} (87%) create mode 100644 src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts create mode 100644 src/plugins/unified_histogram/public/services/lens_vis_service.ts create mode 100644 src/plugins/unified_histogram/public/utils/__snapshots__/external_vis_context.test.ts.snap rename src/plugins/unified_histogram/public/{layout/hooks => utils}/compute_interval.test.ts (100%) rename src/plugins/unified_histogram/public/{layout/hooks => utils}/compute_interval.ts (100%) create mode 100644 src/plugins/unified_histogram/public/utils/external_vis_context.test.ts create mode 100644 src/plugins/unified_histogram/public/utils/external_vis_context.ts rename src/plugins/unified_histogram/public/{chart => }/utils/field_supports_breakdown.test.ts (100%) rename src/plugins/unified_histogram/public/{chart => }/utils/field_supports_breakdown.ts (100%) create mode 100644 src/plugins/unified_histogram/public/utils/lens_vis_from_table.ts create mode 100644 test/functional/apps/discover/group3/_lens_vis.ts diff --git a/packages/kbn-visualization-utils/src/get_lens_attributes.test.ts b/packages/kbn-visualization-utils/src/get_lens_attributes.test.ts index 94e0b8e752926..8b0a22c63d005 100644 --- a/packages/kbn-visualization-utils/src/get_lens_attributes.test.ts +++ b/packages/kbn-visualization-utils/src/get_lens_attributes.test.ts @@ -20,6 +20,7 @@ describe('getLensAttributesFromSuggestion', () => { timeFieldName: '@timestamp', isPersisted: () => false, toSpec: () => ({}), + toMinimalSpec: () => ({}), } as unknown as DataView; const query: AggregateQuery = { esql: 'from foo | limit 10' }; diff --git a/packages/kbn-visualization-utils/src/get_lens_attributes.ts b/packages/kbn-visualization-utils/src/get_lens_attributes.ts index 38a2dd29b841e..3a62c7bee736f 100644 --- a/packages/kbn-visualization-utils/src/get_lens_attributes.ts +++ b/packages/kbn-visualization-utils/src/get_lens_attributes.ts @@ -20,7 +20,17 @@ export const getLensAttributesFromSuggestion = ({ query: Query | AggregateQuery; suggestion: Suggestion | undefined; dataView?: DataView; -}) => { +}): { + references: Array<{ name: string; id: string; type: string }>; + visualizationType: string; + state: { + visualization: {}; + datasourceStates: Record; + query: Query | AggregateQuery; + filters: Filter[]; + }; + title: string; +} => { const suggestionDatasourceState = Object.assign({}, suggestion?.datasourceState); const suggestionVisualizationState = Object.assign({}, suggestion?.visualizationState); const datasourceStates = @@ -35,11 +45,11 @@ export const getLensAttributesFromSuggestion = ({ }; const visualization = suggestionVisualizationState; const attributes = { - title: suggestion - ? suggestion.title - : i18n.translate('visualizationUtils.config.suggestion.title', { - defaultMessage: 'New suggestion', - }), + title: + suggestion?.title ?? + i18n.translate('visualizationUtils.config.suggestion.title', { + defaultMessage: 'New suggestion', + }), references: [ { id: dataView?.id ?? '', @@ -55,7 +65,7 @@ export const getLensAttributesFromSuggestion = ({ ...(dataView && dataView.id && !dataView.isPersisted() && { - adHocDataViews: { [dataView.id]: dataView.toSpec(false) }, + adHocDataViews: { [dataView.id]: dataView.toMinimalSpec() }, }), }, visualizationType: suggestion ? suggestion.visualizationId : 'lnsXY', diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 7ec28e55fad21..9a1299b6f1fe4 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -135,7 +135,7 @@ describe('checking migration metadata changes on all registered SO types', () => "risk-engine-configuration": "aea0c371a462e6d07c3ceb3aff11891b47feb09d", "rules-settings": "892a2918ebaeba809a612b8d97cec0b07c800b5f", "sample-data-telemetry": "37441b12f5b0159c2d6d5138a494c9f440e950b5", - "search": "cf69e2bf8ae25c10af21887cd6effc4a9ea73064", + "search": "7598e4a701ddcaa5e3f44f22e797618a48595e6f", "search-session": "b2fcd840e12a45039ada50b1355faeafa39876d1", "search-telemetry": "b568601618744720b5662946d3103e3fb75fe8ee", "security-rule": "07abb4d7e707d91675ec0495c73816394c7b521f", diff --git a/src/plugins/discover/public/application/main/components/layout/discover_histogram_layout.tsx b/src/plugins/discover/public/application/main/components/layout/discover_histogram_layout.tsx index 79f4e9e74cc96..6b53ea8769017 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_histogram_layout.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_histogram_layout.tsx @@ -10,7 +10,7 @@ import React, { useCallback, useMemo } from 'react'; import { UnifiedHistogramContainer } from '@kbn/unified-histogram-plugin/public'; import { css } from '@emotion/react'; import useObservable from 'react-use/lib/useObservable'; -import { Datatable } from '@kbn/expressions-plugin/common'; +import type { Datatable } from '@kbn/expressions-plugin/common'; import { useDiscoverHistogram } from './use_discover_histogram'; import { type DiscoverMainContentProps, DiscoverMainContent } from './discover_main_content'; import { useAppStateSelector } from '../../services/discover_app_state_container'; diff --git a/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts b/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts index b8bfed44563a5..5617b724df490 100644 --- a/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts +++ b/src/plugins/discover/public/application/main/components/layout/use_discover_histogram.ts @@ -8,12 +8,15 @@ import { useQuerySubscriber } from '@kbn/unified-field-list/src/hooks/use_query_subscriber'; import { + canImportVisContext, UnifiedHistogramApi, + UnifiedHistogramExternalVisContextStatus, UnifiedHistogramFetchStatus, UnifiedHistogramState, + UnifiedHistogramVisContext, } from '@kbn/unified-histogram-plugin/public'; import { isEqual } from 'lodash'; -import { useCallback, useEffect, useRef, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { debounceTime, distinctUntilChanged, @@ -26,6 +29,9 @@ import { } from 'rxjs'; import useObservable from 'react-use/lib/useObservable'; import type { RequestAdapter } from '@kbn/inspector-plugin/common'; +import type { DatatableColumn } from '@kbn/expressions-plugin/common'; +import type { SavedSearch } from '@kbn/saved-search-plugin/common'; +import type { Filter } from '@kbn/es-query'; import { useDiscoverCustomization } from '../../../../customizations'; import { useDiscoverServices } from '../../../../hooks/use_discover_services'; import { FetchStatus } from '../../../types'; @@ -35,7 +41,11 @@ import type { DiscoverStateContainer } from '../../services/discover_state'; import { addLog } from '../../../../utils/add_log'; import { useInternalStateSelector } from '../../services/discover_internal_state_container'; import type { DiscoverAppState } from '../../services/discover_app_state_container'; -import { RecordRawType } from '../../services/discover_data_state_container'; +import { DataDocumentsMsg, RecordRawType } from '../../services/discover_data_state_container'; +import { useSavedSearch } from '../../services/discover_state_provider'; + +const EMPTY_TEXT_BASED_COLUMNS: DatatableColumn[] = []; +const EMPTY_FILTERS: Filter[] = []; export interface UseDiscoverHistogramProps { stateContainer: DiscoverStateContainer; @@ -52,6 +62,7 @@ export const useDiscoverHistogram = ({ }: UseDiscoverHistogramProps) => { const services = useDiscoverServices(); const savedSearchData$ = stateContainer.dataState.data$; + const savedSearchState = useSavedSearch(); /** * API initialization @@ -219,15 +230,18 @@ export const useDiscoverHistogram = ({ [stateContainer] ); + const [initialTextBasedProps] = useState(() => + getUnifiedHistogramPropsForTextBased({ + documentsValue: savedSearchData$.documents$.getValue(), + savedSearch: stateContainer.savedSearchState.getState(), + }) + ); + const { dataView: textBasedDataView, query: textBasedQuery, - columns, - } = useObservable(textBasedFetchComplete$, { - dataView: stateContainer.internalState.getState().dataView!, - query: stateContainer.appState.getState().query, - columns: savedSearchData$.documents$.getValue().textBasedQueryColumns ?? [], - }); + columns: textBasedColumns, + } = useObservable(textBasedFetchComplete$, initialTextBasedProps); useEffect(() => { if (!isPlainRecord) { @@ -316,14 +330,53 @@ export const useDiscoverHistogram = ({ const histogramCustomization = useDiscoverCustomization('unified_histogram'); - const filtersMemoized = useMemo( - () => [...(filters ?? []), ...customFilters], - [filters, customFilters] - ); + const filtersMemoized = useMemo(() => { + const allFilters = [...(filters ?? []), ...customFilters]; + return allFilters.length ? allFilters : EMPTY_FILTERS; + }, [filters, customFilters]); // eslint-disable-next-line react-hooks/exhaustive-deps const timeRangeMemoized = useMemo(() => timeRange, [timeRange?.from, timeRange?.to]); + const onVisContextChanged = useCallback( + ( + nextVisContext: UnifiedHistogramVisContext | undefined, + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus + ) => { + switch (externalVisContextStatus) { + case UnifiedHistogramExternalVisContextStatus.manuallyCustomized: + // if user customized the visualization manually + // (only this action should trigger Unsaved changes badge) + stateContainer.savedSearchState.updateVisContext({ + nextVisContext, + }); + stateContainer.internalState.transitions.setOverriddenVisContextAfterInvalidation( + undefined + ); + break; + case UnifiedHistogramExternalVisContextStatus.automaticallyOverridden: + // if the visualization was invalidated as incompatible and rebuilt + // (it will be used later for saving the visualization via Save button) + stateContainer.internalState.transitions.setOverriddenVisContextAfterInvalidation( + nextVisContext + ); + break; + case UnifiedHistogramExternalVisContextStatus.automaticallyCreated: + case UnifiedHistogramExternalVisContextStatus.applied: + // clearing the value in the internal state so we don't use it during saved search saving + stateContainer.internalState.transitions.setOverriddenVisContextAfterInvalidation( + undefined + ); + break; + case UnifiedHistogramExternalVisContextStatus.unknown: + // using `{}` to overwrite the value inside the saved search SO during saving + stateContainer.internalState.transitions.setOverriddenVisContextAfterInvalidation({}); + break; + } + }, + [stateContainer] + ); + return { ref, getCreationOptions, @@ -333,12 +386,18 @@ export const useDiscoverHistogram = ({ filters: filtersMemoized, timeRange: timeRangeMemoized, relativeTimeRange, - columns, + columns: isPlainRecord ? textBasedColumns : undefined, onFilter: histogramCustomization?.onFilter, onBrushEnd: histogramCustomization?.onBrushEnd, withDefaultActions: histogramCustomization?.withDefaultActions, disabledActions: histogramCustomization?.disabledActions, isChartLoading: isSuggestionLoading, + // visContext should be in sync with current query + externalVisContext: + isPlainRecord && canImportVisContext(savedSearchState?.visContext) + ? savedSearchState?.visContext + : undefined, + onVisContextChanged: isPlainRecord ? onVisContextChanged : undefined, }; }; @@ -412,12 +471,13 @@ const createAppStateObservable = (state$: Observable) => { const createFetchCompleteObservable = (stateContainer: DiscoverStateContainer) => { return stateContainer.dataState.data$.documents$.pipe( distinctUntilChanged((prev, curr) => prev.fetchStatus === curr.fetchStatus), - filter(({ fetchStatus }) => fetchStatus === FetchStatus.COMPLETE), - map(({ textBasedQueryColumns }) => ({ - dataView: stateContainer.internalState.getState().dataView!, - query: stateContainer.appState.getState().query!, - columns: textBasedQueryColumns ?? [], - })) + filter(({ fetchStatus }) => [FetchStatus.COMPLETE, FetchStatus.ERROR].includes(fetchStatus)), + map((documentsValue) => { + return getUnifiedHistogramPropsForTextBased({ + documentsValue, + savedSearch: stateContainer.savedSearchState.getState(), + }); + }) ); }; @@ -430,7 +490,27 @@ const createTotalHitsObservable = (state$?: Observable) = const createCurrentSuggestionObservable = (state$: Observable) => { return state$.pipe( - map((state) => state.currentSuggestion), + map((state) => state.currentSuggestionContext), distinctUntilChanged(isEqual) ); }; + +function getUnifiedHistogramPropsForTextBased({ + documentsValue, + savedSearch, +}: { + documentsValue: DataDocumentsMsg | undefined; + savedSearch: SavedSearch; +}) { + const columns = documentsValue?.textBasedQueryColumns || EMPTY_TEXT_BASED_COLUMNS; + + const nextProps = { + dataView: savedSearch.searchSource.getField('index')!, + query: savedSearch.searchSource.getField('query'), + columns, + }; + + addLog('[UnifiedHistogram] delayed next props for text-based', nextProps); + + return nextProps; +} diff --git a/src/plugins/discover/public/application/main/components/top_nav/get_top_nav_badges.tsx b/src/plugins/discover/public/application/main/components/top_nav/get_top_nav_badges.tsx index 37daf11478bfc..30d58a58e1882 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/get_top_nav_badges.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/get_top_nav_badges.tsx @@ -45,7 +45,13 @@ export const getTopNavBadges = ({ if (hasUnsavedChanges && !defaultBadges?.unsavedChangesBadge?.disabled) { entries.push({ data: getTopNavUnsavedChangesBadge({ - onRevert: stateContainer.actions.undoSavedSearchChanges, + onRevert: async () => { + const lensEditFlyoutCancelButton = document.getElementById('lnsCancelEditOnFlyFlyout'); + if (lensEditFlyoutCancelButton) { + lensEditFlyoutCancelButton.click?.(); + } + await stateContainer.actions.undoSavedSearchChanges(); + }, onSave: services.capabilities.discover.save && !isManaged ? async () => { diff --git a/src/plugins/discover/public/application/main/components/top_nav/on_save_search.tsx b/src/plugins/discover/public/application/main/components/top_nav/on_save_search.tsx index 84c056f60ad01..f22d07b4d4d89 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/on_save_search.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/on_save_search.tsx @@ -93,6 +93,9 @@ export async function onSaveSearch({ }) { const { uiSettings, savedObjectsTagging } = services; const dataView = state.internalState.getState().dataView; + const overriddenVisContextAfterInvalidation = + state.internalState.getState().overriddenVisContextAfterInvalidation; + const onSave = async ({ newTitle, newCopyOnSave, @@ -116,6 +119,7 @@ export async function onSaveSearch({ const currentSampleSize = savedSearch.sampleSize; const currentDescription = savedSearch.description; const currentTags = savedSearch.tags; + const currentVisContext = savedSearch.visContext; savedSearch.title = newTitle; savedSearch.description = newDescription; savedSearch.timeRestore = newTimeRestore; @@ -134,6 +138,11 @@ export async function onSaveSearch({ if (savedObjectsTagging) { savedSearch.tags = newTags; } + + if (overriddenVisContextAfterInvalidation) { + savedSearch.visContext = overriddenVisContextAfterInvalidation; + } + const saveOptions: SaveSavedSearchOptions = { onTitleDuplicate, copyOnSave: newCopyOnSave, @@ -159,10 +168,12 @@ export async function onSaveSearch({ savedSearch.rowsPerPage = currentRowsPerPage; savedSearch.sampleSize = currentSampleSize; savedSearch.description = currentDescription; + savedSearch.visContext = currentVisContext; if (savedObjectsTagging) { savedSearch.tags = currentTags; } } else { + state.internalState.transitions.resetOnSavedSearchChange(); state.appState.resetInitialState(); } onSaveCb?.(); diff --git a/src/plugins/discover/public/application/main/services/discover_internal_state_container.ts b/src/plugins/discover/public/application/main/services/discover_internal_state_container.ts index 5825e4d2cef6c..4b26822bf04a5 100644 --- a/src/plugins/discover/public/application/main/services/discover_internal_state_container.ts +++ b/src/plugins/discover/public/application/main/services/discover_internal_state_container.ts @@ -11,9 +11,10 @@ import { createStateContainerReactHelpers, ReduxLikeStateContainer, } from '@kbn/kibana-utils-plugin/common'; -import { DataView, DataViewListItem } from '@kbn/data-views-plugin/common'; -import { Filter } from '@kbn/es-query'; +import type { DataView, DataViewListItem } from '@kbn/data-views-plugin/common'; +import type { Filter } from '@kbn/es-query'; import type { DataTableRecord } from '@kbn/discover-utils/types'; +import type { UnifiedHistogramVisContext } from '@kbn/unified-histogram-plugin/public'; export interface InternalState { dataView: DataView | undefined; @@ -22,6 +23,7 @@ export interface InternalState { adHocDataViews: DataView[]; expandedDoc: DataTableRecord | undefined; customFilters: Filter[]; + overriddenVisContextAfterInvalidation: UnifiedHistogramVisContext | {} | undefined; // it will be used during saved search saving } export interface InternalStateTransitions { @@ -40,6 +42,12 @@ export interface InternalStateTransitions { state: InternalState ) => (dataView: DataTableRecord | undefined) => InternalState; setCustomFilters: (state: InternalState) => (customFilters: Filter[]) => InternalState; + setOverriddenVisContextAfterInvalidation: ( + state: InternalState + ) => ( + overriddenVisContextAfterInvalidation: UnifiedHistogramVisContext | {} | undefined + ) => InternalState; + resetOnSavedSearchChange: (state: InternalState) => () => InternalState; } export type DiscoverInternalStateContainer = ReduxLikeStateContainer< @@ -59,6 +67,7 @@ export function getInternalStateContainer() { savedDataViews: [], expandedDoc: undefined, customFilters: [], + overriddenVisContextAfterInvalidation: undefined, }, { setDataView: (prevState: InternalState) => (nextDataView: DataView) => ({ @@ -112,6 +121,16 @@ export function getInternalStateContainer() { ...prevState, customFilters, }), + setOverriddenVisContextAfterInvalidation: + (prevState: InternalState) => + (overriddenVisContextAfterInvalidation: UnifiedHistogramVisContext | {} | undefined) => ({ + ...prevState, + overriddenVisContextAfterInvalidation, + }), + resetOnSavedSearchChange: (prevState: InternalState) => () => ({ + ...prevState, + overriddenVisContextAfterInvalidation: undefined, + }), }, {}, { freeze: (state) => state } diff --git a/src/plugins/discover/public/application/main/services/discover_saved_search_container.ts b/src/plugins/discover/public/application/main/services/discover_saved_search_container.ts index e1544ffffbe4c..76ffca5443017 100644 --- a/src/plugins/discover/public/application/main/services/discover_saved_search_container.ts +++ b/src/plugins/discover/public/application/main/services/discover_saved_search_container.ts @@ -12,6 +12,7 @@ import { cloneDeep } from 'lodash'; import { COMPARE_ALL_OPTIONS, FilterCompareOptions } from '@kbn/es-query'; import type { SearchSourceFields } from '@kbn/data-plugin/common'; import type { DataView } from '@kbn/data-views-plugin/common'; +import type { UnifiedHistogramVisContext } from '@kbn/unified-histogram-plugin/public'; import { SavedObjectSaveOpts } from '@kbn/saved-objects-plugin/public'; import { isEqual, isFunction } from 'lodash'; import { restoreStateFromSavedSearch } from '../../../services/saved_searches/restore_from_saved_search'; @@ -120,6 +121,11 @@ export interface DiscoverSavedSearchContainer { * @param params */ updateWithFilterManagerFilters: () => SavedSearch; + /** + * Updates the current value of visContext in saved search + * @param params + */ + updateVisContext: (params: { nextVisContext: UnifiedHistogramVisContext | undefined }) => void; } export function getSavedSearchContainer({ @@ -239,6 +245,22 @@ export function getSavedSearchContainer({ addLog('[savedSearch] updateWithTimeRange done', nextSavedSearch); }; + const updateVisContext = ({ + nextVisContext, + }: { + nextVisContext: UnifiedHistogramVisContext | undefined; + }) => { + const previousSavedSearch = getState(); + const nextSavedSearch: SavedSearch = { + ...previousSavedSearch, + visContext: nextVisContext, + }; + + assignNextSavedSearch({ nextSavedSearch }); + + addLog('[savedSearch] updateVisContext done', nextSavedSearch); + }; + const load = async (id: string, dataView: DataView | undefined): Promise => { addLog('[savedSearch] load', { id, dataView }); @@ -268,6 +290,7 @@ export function getSavedSearchContainer({ update, updateTimeRange, updateWithFilterManagerFilters, + updateVisContext, }; } diff --git a/src/plugins/discover/public/application/main/services/discover_state.ts b/src/plugins/discover/public/application/main/services/discover_state.ts index 1bab2e0328af8..94a0a80c54fd9 100644 --- a/src/plugins/discover/public/application/main/services/discover_state.ts +++ b/src/plugins/discover/public/application/main/services/discover_state.ts @@ -496,6 +496,7 @@ export function getDiscoverStateContainer({ }); } + internalStateContainer.transitions.resetOnSavedSearchChange(); await appStateContainer.replaceUrlState(newAppState); return nextSavedSearch; }; diff --git a/src/plugins/discover/public/application/main/services/load_saved_search.ts b/src/plugins/discover/public/application/main/services/load_saved_search.ts index d5a5be0935d8c..ac9e6f60526d1 100644 --- a/src/plugins/discover/public/application/main/services/load_saved_search.ts +++ b/src/plugins/discover/public/application/main/services/load_saved_search.ts @@ -53,6 +53,7 @@ export const loadSavedSearch = async ( globalStateContainer, services, } = deps; + const appStateExists = !appStateContainer.isEmptyURL(); const appState = appStateExists ? appStateContainer.getState() : initialAppState; @@ -124,6 +125,8 @@ export const loadSavedSearch = async ( nextSavedSearch = savedSearchContainer.updateWithFilterManagerFilters(); } + internalStateContainer.transitions.resetOnSavedSearchChange(); + return nextSavedSearch; }; diff --git a/src/plugins/discover/public/application/main/utils/fetch_all.test.ts b/src/plugins/discover/public/application/main/utils/fetch_all.test.ts index d9c96e9bce0e9..13aaedeeb6e9e 100644 --- a/src/plugins/discover/public/application/main/utils/fetch_all.test.ts +++ b/src/plugins/discover/public/application/main/utils/fetch_all.test.ts @@ -77,6 +77,7 @@ describe('test fetchAll', () => { adHocDataViews: [], expandedDoc: undefined, customFilters: [], + overriddenVisContextAfterInvalidation: undefined, }), searchSessionId: '123', initialFetchStatus: FetchStatus.UNINITIALIZED, @@ -275,6 +276,7 @@ describe('test fetchAll', () => { adHocDataViews: [], expandedDoc: undefined, customFilters: [], + overriddenVisContextAfterInvalidation: undefined, }), }; fetchAll(subjects, false, deps); @@ -401,6 +403,7 @@ describe('test fetchAll', () => { adHocDataViews: [], expandedDoc: undefined, customFilters: [], + overriddenVisContextAfterInvalidation: undefined, }), }; fetchAll(subjects, false, deps); diff --git a/src/plugins/saved_search/common/content_management/v1/cm_services.ts b/src/plugins/saved_search/common/content_management/v1/cm_services.ts index 24319df7d43ac..bc9d18b21e5b7 100644 --- a/src/plugins/saved_search/common/content_management/v1/cm_services.ts +++ b/src/plugins/saved_search/common/content_management/v1/cm_services.ts @@ -69,6 +69,25 @@ const savedSearchAttributesSchema = schema.object( }) ), breakdownField: schema.maybe(schema.string()), + visContext: schema.maybe( + schema.oneOf([ + // existing value + schema.object({ + // unified histogram state + suggestionType: schema.string(), + requestData: schema.object({ + dataViewId: schema.maybe(schema.string()), + timeField: schema.maybe(schema.string()), + timeInterval: schema.maybe(schema.string()), + breakdownField: schema.maybe(schema.string()), + }), + // lens attributes + attributes: schema.recordOf(schema.string(), schema.any()), + }), + // cleared previous value + schema.object({}), + ]) + ), version: schema.maybe(schema.number()), }, { unknowns: 'forbid' } diff --git a/src/plugins/saved_search/common/saved_searches_utils.ts b/src/plugins/saved_search/common/saved_searches_utils.ts index b71819e96e210..d8a1dbcd4cafa 100644 --- a/src/plugins/saved_search/common/saved_searches_utils.ts +++ b/src/plugins/saved_search/common/saved_searches_utils.ts @@ -36,5 +36,6 @@ export const fromSavedSearchAttributes = ( rowsPerPage: attributes.rowsPerPage, sampleSize: attributes.sampleSize, breakdownField: attributes.breakdownField, + visContext: attributes.visContext, managed, }); diff --git a/src/plugins/saved_search/common/service/get_saved_searches.test.ts b/src/plugins/saved_search/common/service/get_saved_searches.test.ts index be971f1469ade..ea9403fda6476 100644 --- a/src/plugins/saved_search/common/service/get_saved_searches.test.ts +++ b/src/plugins/saved_search/common/service/get_saved_searches.test.ts @@ -148,6 +148,7 @@ describe('getSavedSearch', () => { "title": "test1", "usesAdHocDataView": undefined, "viewMode": undefined, + "visContext": undefined, } `); }); @@ -256,6 +257,7 @@ describe('getSavedSearch', () => { "title": "test2", "usesAdHocDataView": undefined, "viewMode": undefined, + "visContext": undefined, } `); }); diff --git a/src/plugins/saved_search/common/service/saved_searches_utils.test.ts b/src/plugins/saved_search/common/service/saved_searches_utils.test.ts index 716d9db855a02..3972f38caa5b5 100644 --- a/src/plugins/saved_search/common/service/saved_searches_utils.test.ts +++ b/src/plugins/saved_search/common/service/saved_searches_utils.test.ts @@ -91,6 +91,7 @@ describe('saved_searches_utils', () => { "title": "saved search", "usesAdHocDataView": false, "viewMode": undefined, + "visContext": undefined, } `); }); @@ -143,6 +144,7 @@ describe('saved_searches_utils', () => { "title": "title", "usesAdHocDataView": false, "viewMode": undefined, + "visContext": undefined, } `); }); diff --git a/src/plugins/saved_search/common/service/saved_searches_utils.ts b/src/plugins/saved_search/common/service/saved_searches_utils.ts index ce3da85a2d3bd..11a848f8baaf8 100644 --- a/src/plugins/saved_search/common/service/saved_searches_utils.ts +++ b/src/plugins/saved_search/common/service/saved_searches_utils.ts @@ -50,4 +50,5 @@ export const toSavedSearchAttributes = ( rowsPerPage: savedSearch.rowsPerPage, sampleSize: savedSearch.sampleSize, breakdownField: savedSearch.breakdownField, + visContext: savedSearch.visContext, }); diff --git a/src/plugins/saved_search/common/types.ts b/src/plugins/saved_search/common/types.ts index d58f8c1cec7fc..34ada26b0c1a4 100644 --- a/src/plugins/saved_search/common/types.ts +++ b/src/plugins/saved_search/common/types.ts @@ -20,6 +20,20 @@ export interface DiscoverGridSettingsColumn extends SerializableRecord { width?: number; } +export type VisContextUnmapped = + | { + // UnifiedHistogramVisContext (can't be referenced here directly due to circular dependency) + attributes: unknown; + requestData: { + dataViewId?: string; + timeField?: string; + timeInterval?: string; + breakdownField?: string; + }; + suggestionType: string; + } + | {}; // cleared value + /** @internal **/ export interface SavedSearchAttributes { title: string; @@ -45,6 +59,7 @@ export interface SavedSearchAttributes { rowsPerPage?: number; sampleSize?: number; breakdownField?: string; + visContext?: VisContextUnmapped; } /** @internal **/ @@ -76,6 +91,8 @@ export interface SavedSearch { rowsPerPage?: number; sampleSize?: number; breakdownField?: string; + visContext?: VisContextUnmapped; + // Whether or not this saved search is managed by the system managed: boolean; references?: SavedObjectReference[]; diff --git a/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts b/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts index 81b7e68cae319..ae1e457fc7d4b 100644 --- a/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts +++ b/src/plugins/saved_search/public/services/saved_searches/saved_search_attribute_service.test.ts @@ -242,6 +242,7 @@ describe('getSavedSearchAttributeService', () => { "title": "saved-search-title", "usesAdHocDataView": undefined, "viewMode": undefined, + "visContext": undefined, } `); }); diff --git a/src/plugins/saved_search/server/content_management/saved_search_storage.ts b/src/plugins/saved_search/server/content_management/saved_search_storage.ts index 53fe82eb6e1e4..d3ff8633637d2 100644 --- a/src/plugins/saved_search/server/content_management/saved_search_storage.ts +++ b/src/plugins/saved_search/server/content_management/saved_search_storage.ts @@ -45,6 +45,7 @@ export class SavedSearchStorage extends SOContentStorage { 'rowsPerPage', 'breakdownField', 'sampleSize', + 'visContext', ], logger, throwOnResultValidationError, diff --git a/src/plugins/saved_search/server/saved_objects/schema.ts b/src/plugins/saved_search/server/saved_objects/schema.ts index 851a14417b400..fb0308915fe72 100644 --- a/src/plugins/saved_search/server/saved_objects/schema.ts +++ b/src/plugins/saved_search/server/saved_objects/schema.ts @@ -97,3 +97,25 @@ export const SCHEMA_SEARCH_MODEL_VERSION_1 = SCHEMA_SEARCH_BASE.extends({ export const SCHEMA_SEARCH_MODEL_VERSION_2 = SCHEMA_SEARCH_MODEL_VERSION_1.extends({ headerRowHeight: schema.maybe(schema.number()), }); + +export const SCHEMA_SEARCH_MODEL_VERSION_3 = SCHEMA_SEARCH_MODEL_VERSION_2.extends({ + visContext: schema.maybe( + schema.oneOf([ + // existing value + schema.object({ + // unified histogram state + suggestionType: schema.string(), + requestData: schema.object({ + dataViewId: schema.maybe(schema.string()), + timeField: schema.maybe(schema.string()), + timeInterval: schema.maybe(schema.string()), + breakdownField: schema.maybe(schema.string()), + }), + // lens attributes + attributes: schema.recordOf(schema.string(), schema.any()), + }), + // cleared previous value + schema.object({}), + ]) + ), +}); diff --git a/src/plugins/saved_search/server/saved_objects/search.ts b/src/plugins/saved_search/server/saved_objects/search.ts index a913e513e897f..6c6a9bb81c1ed 100644 --- a/src/plugins/saved_search/server/saved_objects/search.ts +++ b/src/plugins/saved_search/server/saved_objects/search.ts @@ -14,6 +14,7 @@ import { SCHEMA_SEARCH_V8_8_0, SCHEMA_SEARCH_MODEL_VERSION_1, SCHEMA_SEARCH_MODEL_VERSION_2, + SCHEMA_SEARCH_MODEL_VERSION_3, } from './schema'; export function getSavedSearchObjectType( @@ -54,6 +55,13 @@ export function getSavedSearchObjectType( create: SCHEMA_SEARCH_MODEL_VERSION_2, }, }, + 3: { + changes: [], + schemas: { + forwardCompatibility: SCHEMA_SEARCH_MODEL_VERSION_3.extends({}, { unknowns: 'ignore' }), + create: SCHEMA_SEARCH_MODEL_VERSION_3, + }, + }, }, mappings: { dynamic: false, diff --git a/src/plugins/unified_histogram/public/__mocks__/data_view.ts b/src/plugins/unified_histogram/public/__mocks__/data_view.ts index ffc429c1aa887..62184359c5abd 100644 --- a/src/plugins/unified_histogram/public/__mocks__/data_view.ts +++ b/src/plugins/unified_histogram/public/__mocks__/data_view.ts @@ -84,13 +84,16 @@ export const buildDataViewMock = ({ return dataViewFields; }; + const indexPattern = `${name}-title`; + const dataView = { id: `${name}-id`, - title: `${name}-title`, + title: indexPattern, name, metaFields: ['_index', '_score'], fields: dataViewFields, getName: () => name, + getIndexPattern: () => indexPattern, getComputedFields: () => ({ docvalueFields: [], scriptFields: {} }), getSourceFiltering: () => ({}), getFieldByName: jest.fn((fieldName: string) => dataViewFields.getByName(fieldName)), @@ -103,6 +106,7 @@ export const buildDataViewMock = ({ return dataViewFields.find((field) => field.name === timeFieldName); }, toSpec: () => ({}), + toMinimalSpec: () => ({}), } as unknown as DataView; dataView.isTimeBased = () => !!timeFieldName; diff --git a/src/plugins/unified_histogram/public/__mocks__/data_view_with_timefield.ts b/src/plugins/unified_histogram/public/__mocks__/data_view_with_timefield.ts index 3868ed2c70af5..2075b28c92226 100644 --- a/src/plugins/unified_histogram/public/__mocks__/data_view_with_timefield.ts +++ b/src/plugins/unified_histogram/public/__mocks__/data_view_with_timefield.ts @@ -64,3 +64,9 @@ export const dataViewWithTimefieldMock = buildDataViewMock({ fields, timeFieldName: 'timestamp', }); + +export const dataViewWithAtTimefieldMock = buildDataViewMock({ + name: 'index-pattern-with-@timefield', + fields, + timeFieldName: '@timestamp', +}); diff --git a/src/plugins/unified_histogram/public/__mocks__/lens_vis.ts b/src/plugins/unified_histogram/public/__mocks__/lens_vis.ts new file mode 100644 index 0000000000000..9ac64493806fe --- /dev/null +++ b/src/plugins/unified_histogram/public/__mocks__/lens_vis.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DataViewField } from '@kbn/data-views-plugin/common'; +import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/common'; +import type { Suggestion } from '@kbn/lens-plugin/public'; +import type { TimeRange } from '@kbn/data-plugin/common'; +import { LensVisService, type QueryParams } from '../services/lens_vis_service'; +import { unifiedHistogramServicesMock } from './services'; +import { histogramESQLSuggestionMock } from './suggestions'; +import { UnifiedHistogramSuggestionContext, UnifiedHistogramVisContext } from '../types'; + +const TIME_RANGE: TimeRange = { + from: '2022-11-17T00:00:00.000Z', + to: '2022-11-17T12:00:00.000Z', +}; + +export const getLensVisMock = async ({ + filters, + query, + columns, + isPlainRecord, + timeInterval, + timeRange, + breakdownField, + dataView, + allSuggestions, + hasHistogramSuggestionForESQL, + table, +}: { + filters: QueryParams['filters']; + query: QueryParams['query']; + dataView: QueryParams['dataView']; + columns: DatatableColumn[]; + isPlainRecord: boolean; + timeInterval: string; + timeRange?: TimeRange | null; + breakdownField: DataViewField | undefined; + allSuggestions?: Suggestion[]; + hasHistogramSuggestionForESQL?: boolean; + table?: Datatable; +}): Promise<{ + lensService: LensVisService; + visContext: UnifiedHistogramVisContext | undefined; + currentSuggestionContext: UnifiedHistogramSuggestionContext | undefined; +}> => { + const lensApi = await unifiedHistogramServicesMock.lens.stateHelperApi(); + const lensService = new LensVisService({ + services: unifiedHistogramServicesMock, + lensSuggestionsApi: allSuggestions + ? (...params) => { + const context = params[0]; + if ('query' in context && context.query === query) { + return allSuggestions; + } + return hasHistogramSuggestionForESQL ? [histogramESQLSuggestionMock] : []; + } + : lensApi.suggestions, + }); + + let visContext: UnifiedHistogramVisContext | undefined; + lensService.visContext$.subscribe((nextAttributesContext) => { + visContext = nextAttributesContext; + }); + + let currentSuggestionContext: UnifiedHistogramSuggestionContext | undefined; + lensService.currentSuggestionContext$.subscribe((nextSuggestionContext) => { + currentSuggestionContext = nextSuggestionContext; + }); + + lensService.update({ + queryParams: { + query, + filters, + dataView, + timeRange: timeRange ?? TIME_RANGE, + columns, + isPlainRecord, + }, + timeInterval, + breakdownField, + externalVisContext: undefined, + table, + onSuggestionContextChange: () => {}, + }); + + return { + lensService, + visContext, + currentSuggestionContext, + }; +}; diff --git a/src/plugins/unified_histogram/public/__mocks__/services.tsx b/src/plugins/unified_histogram/public/__mocks__/services.tsx index b7efb79941412..ddfbc9eecc405 100644 --- a/src/plugins/unified_histogram/public/__mocks__/services.tsx +++ b/src/plugins/unified_histogram/public/__mocks__/services.tsx @@ -6,6 +6,8 @@ * Side Public License, v 1. */ import React from 'react'; +import { of } from 'rxjs'; +import { calculateBounds } from '@kbn/data-plugin/common'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; import { expressionsPluginMock } from '@kbn/expressions-plugin/public/mocks'; import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks'; @@ -15,6 +17,11 @@ import { allSuggestionsMock } from './suggestions'; const dataPlugin = dataPluginMock.createStartContract(); dataPlugin.query.filterManager.getFilters = jest.fn(() => []); +dataPlugin.query.timefilter.timefilter = { + ...dataPlugin.query.timefilter.timefilter, + calculateBounds: jest.fn((timeRange) => calculateBounds(timeRange)), +}; + export const unifiedHistogramServicesMock = { data: dataPlugin, fieldFormats: fieldFormatsMock, @@ -43,7 +50,17 @@ export const unifiedHistogramServicesMock = { remove: jest.fn(), clear: jest.fn(), }, - expressions: expressionsPluginMock.createStartContract(), + expressions: { + ...expressionsPluginMock.createStartContract(), + run: jest.fn(() => + of({ + partial: false, + result: { + rows: [{}, {}, {}], + }, + }) + ), + }, capabilities: { dashboard: { showWriteControls: true, diff --git a/src/plugins/unified_histogram/public/__mocks__/suggestions.ts b/src/plugins/unified_histogram/public/__mocks__/suggestions.ts index 9e3a00d396047..9da1fb2fc4317 100644 --- a/src/plugins/unified_histogram/public/__mocks__/suggestions.ts +++ b/src/plugins/unified_histogram/public/__mocks__/suggestions.ts @@ -133,6 +133,91 @@ export const currentSuggestionMock = { changeType: 'initial', } as Suggestion; +export const histogramESQLSuggestionMock = { + title: 'Bar vertical stacked', + score: 0.16666666666666666, + hide: false, + incomplete: false, + visualizationId: 'lnsXY', + visualizationState: { + legend: { + isVisible: true, + position: 'right', + }, + valueLabels: 'hide', + fittingFunction: 'None', + axisTitlesVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + tickLabelsVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + labelsOrientation: { + x: 0, + yLeft: 0, + yRight: 0, + }, + gridlinesVisibilitySettings: { + x: true, + yLeft: true, + yRight: true, + }, + preferredSeriesType: 'bar_stacked', + layers: [ + { + layerId: '662552df-2cdc-4539-bf3b-73b9f827252c', + seriesType: 'bar_stacked', + xAccessor: '@timestamp every 30 second', + accessors: ['results'], + layerType: 'data', + }, + ], + }, + keptLayerIds: ['662552df-2cdc-4539-bf3b-73b9f827252c'], + datasourceState: { + layers: { + '662552df-2cdc-4539-bf3b-73b9f827252c': { + index: 'e3465e67bdeced2befff9f9dca7ecf9c48504cad68a10efd881f4c7dd5ade28a', + query: { + esql: 'from kibana_sample_data_logs | limit 10 | EVAL timestamp=DATE_TRUNC(30 second, @timestamp) | stats results = count(*) by timestamp | rename timestamp as `@timestamp every 30 second`', + }, + columns: [ + { + columnId: '@timestamp every 30 second', + fieldName: '@timestamp every 30 second', + meta: { + type: 'date', + }, + }, + { + columnId: 'results', + fieldName: 'results', + meta: { + type: 'number', + }, + inMetricDimension: true, + }, + ], + timeField: '@timestamp', + }, + }, + indexPatternRefs: [ + { + id: 'e3465e67bdeced2befff9f9dca7ecf9c48504cad68a10efd881f4c7dd5ade28a', + title: 'kibana_sample_data_logs', + timeField: '@timestamp', + }, + ], + }, + datasourceId: 'textBased', + columns: 2, + changeType: 'unchanged', +} as Suggestion; + export const allSuggestionsMock = [ currentSuggestionMock, { diff --git a/src/plugins/unified_histogram/public/__mocks__/table.ts b/src/plugins/unified_histogram/public/__mocks__/table.ts new file mode 100644 index 0000000000000..9aa28fdd5ed4c --- /dev/null +++ b/src/plugins/unified_histogram/public/__mocks__/table.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Datatable } from '@kbn/expressions-plugin/common'; + +export const tableQueryMock = { + esql: 'from logstash | stats avg(bytes) by extension.keyword', +}; + +export const tableMock = { + type: 'datatable', + rows: [ + { + 'avg(bytes)': 3850, + 'extension.keyword': '', + }, + { + 'avg(bytes)': 5393.5, + 'extension.keyword': 'css', + }, + { + 'avg(bytes)': 3252, + 'extension.keyword': 'deb', + }, + ], + columns: [ + { + id: 'avg(bytes)', + name: 'avg(bytes)', + meta: { + type: 'number', + }, + isNull: false, + }, + { + id: 'extension.keyword', + name: 'extension.keyword', + meta: { + type: 'string', + }, + isNull: false, + }, + ], +} as Datatable; diff --git a/src/plugins/unified_histogram/public/chart/breakdown_field_selector.tsx b/src/plugins/unified_histogram/public/chart/breakdown_field_selector.tsx index 78df66f50873e..5fbae47f63109 100644 --- a/src/plugins/unified_histogram/public/chart/breakdown_field_selector.tsx +++ b/src/plugins/unified_histogram/public/chart/breakdown_field_selector.tsx @@ -13,7 +13,7 @@ import { css } from '@emotion/react'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { i18n } from '@kbn/i18n'; import { UnifiedHistogramBreakdownContext } from '../types'; -import { fieldSupportsBreakdown } from './utils/field_supports_breakdown'; +import { fieldSupportsBreakdown } from '../utils/field_supports_breakdown'; import { ToolbarSelector, ToolbarSelectorProps, diff --git a/src/plugins/unified_histogram/public/chart/chart.test.tsx b/src/plugins/unified_histogram/public/chart/chart.test.tsx index 474da6bce5bf7..05f4e1a2b079a 100644 --- a/src/plugins/unified_histogram/public/chart/chart.test.tsx +++ b/src/plugins/unified_histogram/public/chart/chart.test.tsx @@ -13,9 +13,10 @@ import type { Capabilities } from '@kbn/core/public'; import type { DataView } from '@kbn/data-views-plugin/public'; import type { Suggestion } from '@kbn/lens-plugin/public'; import type { UnifiedHistogramFetchStatus } from '../types'; -import { Chart } from './chart'; +import { Chart, type ChartProps } from './chart'; import type { ReactWrapper } from 'enzyme'; import { unifiedHistogramServicesMock } from '../__mocks__/services'; +import { getLensVisMock } from '../__mocks__/lens_vis'; import { searchSourceInstanceMock } from '@kbn/data-plugin/common/search/search_source/mocks'; import { of } from 'rxjs'; import { dataViewWithTimefieldMock } from '../__mocks__/data_view_with_timefield'; @@ -23,8 +24,7 @@ import { dataViewMock } from '../__mocks__/data_view'; import { BreakdownFieldSelector } from './breakdown_field_selector'; import { SuggestionSelector } from './suggestion_selector'; import { checkChartAvailability } from './check_chart_availability'; - -import { currentSuggestionMock, allSuggestionsMock } from '../__mocks__/suggestions'; +import { allSuggestionsMock } from '../__mocks__/suggestions'; let mockUseEditVisualization: jest.Mock | undefined = jest.fn(); @@ -40,11 +40,11 @@ async function mountComponent({ chartHidden = false, appendHistogram, dataView = dataViewWithTimefieldMock, - currentSuggestion, allSuggestions, isPlainRecord, hasDashboardPermissions, isChartLoading, + hasHistogramSuggestionForESQL, }: { customToggle?: ReactElement; noChart?: boolean; @@ -53,11 +53,11 @@ async function mountComponent({ chartHidden?: boolean; appendHistogram?: ReactElement; dataView?: DataView; - currentSuggestion?: Suggestion; allSuggestions?: Suggestion[]; isPlainRecord?: boolean; hasDashboardPermissions?: boolean; isChartLoading?: boolean; + hasHistogramSuggestionForESQL?: boolean; } = {}) { (searchSourceInstanceMock.fetch$ as jest.Mock).mockImplementation( jest.fn().mockReturnValue(of({ rawResponse: { hits: { total: noHits ? 0 : 2 } } })) @@ -85,25 +85,46 @@ async function mountComponent({ }, }; - const props = { - dataView, - query: { - language: 'kuery', - query: '', - }, + const requestParams = { + query: isPlainRecord + ? { esql: 'from logs | limit 10' } + : { + language: 'kuery', + query: '', + }, filters: [], - timeRange: { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' }, + relativeTimeRange: { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' }, + getTimeRange: () => ({ from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' }), + updateTimeRange: () => {}, + }; + + const lensVisService = ( + await getLensVisMock({ + query: requestParams.query, + filters: requestParams.filters, + isPlainRecord: Boolean(isPlainRecord), + timeInterval: 'auto', + dataView, + breakdownField: undefined, + columns: [], + allSuggestions, + hasHistogramSuggestionForESQL, + }) + ).lensService; + + const props: ChartProps = { + lensVisService, + dataView, + requestParams, services, hits: noHits ? undefined : { status: 'complete' as UnifiedHistogramFetchStatus, - number: 2, + total: 2, }, chart, breakdown: noBreakdown ? undefined : { field: undefined }, - currentSuggestion, - allSuggestions, isChartLoading: Boolean(isChartLoading), isPlainRecord, appendHistogram, @@ -248,7 +269,7 @@ describe('Chart', () => { it('should render the Lens SuggestionsSelector when chart is visible and suggestions exist', async () => { const component = await mountComponent({ - currentSuggestion: currentSuggestionMock, + isPlainRecord: true, allSuggestions: allSuggestionsMock, }); expect(component.find(SuggestionSelector).exists()).toBeTruthy(); @@ -256,7 +277,6 @@ describe('Chart', () => { it('should render the edit on the fly button when chart is visible and suggestions exist', async () => { const component = await mountComponent({ - currentSuggestion: currentSuggestionMock, allSuggestions: allSuggestionsMock, isPlainRecord: true, }); @@ -267,8 +287,8 @@ describe('Chart', () => { it('should not render the edit on the fly button when chart is visible and suggestions dont exist', async () => { const component = await mountComponent({ - currentSuggestion: undefined, - allSuggestions: undefined, + allSuggestions: [], + hasHistogramSuggestionForESQL: false, isPlainRecord: true, }); expect( @@ -278,8 +298,8 @@ describe('Chart', () => { it('should render the save button when chart is visible and suggestions exist', async () => { const component = await mountComponent({ - currentSuggestion: currentSuggestionMock, allSuggestions: allSuggestionsMock, + isPlainRecord: true, }); expect( component.find('[data-test-subj="unifiedHistogramSaveVisualization"]').exists() @@ -288,7 +308,6 @@ describe('Chart', () => { it('should not render the save button when the dashboard save by value permissions are false', async () => { const component = await mountComponent({ - currentSuggestion: currentSuggestionMock, allSuggestions: allSuggestionsMock, hasDashboardPermissions: false, }); @@ -300,14 +319,13 @@ describe('Chart', () => { it('should not render the Lens SuggestionsSelector when chart is hidden', async () => { const component = await mountComponent({ chartHidden: true, - currentSuggestion: currentSuggestionMock, allSuggestions: allSuggestionsMock, }); expect(component.find(SuggestionSelector).exists()).toBeFalsy(); }); it('should not render the Lens SuggestionsSelector when chart is visible and suggestions are undefined', async () => { - const component = await mountComponent({ currentSuggestion: currentSuggestionMock }); + const component = await mountComponent({}); expect(component.find(SuggestionSelector).exists()).toBeFalsy(); }); }); diff --git a/src/plugins/unified_histogram/public/chart/chart.tsx b/src/plugins/unified_histogram/public/chart/chart.tsx index cfc096e8f197f..7c93e8bf5254d 100644 --- a/src/plugins/unified_histogram/public/chart/chart.tsx +++ b/src/plugins/unified_histogram/public/chart/chart.tsx @@ -6,45 +6,47 @@ * Side Public License, v 1. */ -import React, { ReactElement, useMemo, useState, useEffect, useCallback, memo } from 'react'; +import React, { memo, ReactElement, useCallback, useEffect, useMemo, useState } from 'react'; import type { Observable } from 'rxjs'; +import { Subject } from 'rxjs'; +import useObservable from 'react-use/lib/useObservable'; import { IconButtonGroup, type IconButtonGroupProps } from '@kbn/shared-ux-button-toolbar'; import { EuiFlexGroup, EuiFlexItem, EuiProgress } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import type { EmbeddableComponentProps, - Suggestion, + LensEmbeddableInput, LensEmbeddableOutput, + Suggestion, } from '@kbn/lens-plugin/public'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import type { LensEmbeddableInput } from '@kbn/lens-plugin/public'; -import type { AggregateQuery, Filter, Query, TimeRange } from '@kbn/es-query'; -import { Subject } from 'rxjs'; -import type { LensAttributes } from '@kbn/lens-embeddable-utils'; -import type { TextBasedPersistedState } from '@kbn/lens-plugin/public/datasources/text_based/types'; +import type { TimeRange } from '@kbn/es-query'; import { Histogram } from './histogram'; import type { + UnifiedHistogramSuggestionContext, UnifiedHistogramBreakdownContext, UnifiedHistogramChartContext, + UnifiedHistogramChartLoadEvent, UnifiedHistogramFetchStatus, UnifiedHistogramHitsContext, - UnifiedHistogramChartLoadEvent, - UnifiedHistogramRequestContext, - UnifiedHistogramServices, UnifiedHistogramInput$, UnifiedHistogramInputMessage, + UnifiedHistogramRequestContext, + UnifiedHistogramServices, } from '../types'; +import { UnifiedHistogramSuggestionType } from '../types'; import { BreakdownFieldSelector } from './breakdown_field_selector'; import { SuggestionSelector } from './suggestion_selector'; import { TimeIntervalSelector } from './time_interval_selector'; import { useTotalHits } from './hooks/use_total_hits'; -import { useRequestParams } from './hooks/use_request_params'; import { useChartStyles } from './hooks/use_chart_styles'; import { useChartActions } from './hooks/use_chart_actions'; import { ChartConfigPanel } from './chart_config_panel'; -import { getLensAttributes } from './utils/get_lens_attributes'; import { useRefetch } from './hooks/use_refetch'; import { useEditVisualization } from './hooks/use_edit_visualization'; +import { LensVisService } from '../services/lens_vis_service'; +import type { UseRequestParamsResult } from '../hooks/use_request_params'; +import { removeTablesFromLensAttributes } from '../utils/lens_vis_from_table'; export interface ChartProps { abortController?: AbortController; @@ -53,12 +55,9 @@ export interface ChartProps { className?: string; services: UnifiedHistogramServices; dataView: DataView; - query?: Query | AggregateQuery; - filters?: Filter[]; + requestParams: UseRequestParamsResult; isPlainRecord?: boolean; - currentSuggestion?: Suggestion; - allSuggestions?: Suggestion[]; - timeRange?: TimeRange; + lensVisService: LensVisService; relativeTimeRange?: TimeRange; request?: UnifiedHistogramRequestContext; hits?: UnifiedHistogramHitsContext; @@ -72,13 +71,10 @@ export interface ChartProps { input$?: UnifiedHistogramInput$; lensAdapters?: UnifiedHistogramChartLoadEvent['adapters']; lensEmbeddableOutput$?: Observable; - isOnHistogramMode?: boolean; - histogramQuery?: AggregateQuery; isChartLoading?: boolean; onChartHiddenChange?: (chartHidden: boolean) => void; onTimeIntervalChange?: (timeInterval: string) => void; onBreakdownFieldChange?: (breakdownField: DataViewField | undefined) => void; - onSuggestionChange?: (suggestion: Suggestion | undefined) => void; onTotalHitsChange?: (status: UnifiedHistogramFetchStatus, result?: number | Error) => void; onChartLoad?: (event: UnifiedHistogramChartLoadEvent) => void; onFilter?: LensEmbeddableInput['onFilter']; @@ -93,16 +89,13 @@ export function Chart({ className, services, dataView, - query: originalQuery, - filters: originalFilters, - timeRange: originalTimeRange, + requestParams, relativeTimeRange: originalRelativeTimeRange, request, hits, chart, breakdown, - currentSuggestion, - allSuggestions, + lensVisService, isPlainRecord, renderCustomChartToggleActions, appendHistogram, @@ -112,12 +105,9 @@ export function Chart({ input$: originalInput$, lensAdapters, lensEmbeddableOutput$, - isOnHistogramMode, - histogramQuery, isChartLoading, onChartHiddenChange, onTimeIntervalChange, - onSuggestionChange, onBreakdownFieldChange, onTotalHitsChange, onChartLoad, @@ -126,6 +116,13 @@ export function Chart({ withDefaultActions, abortController, }: ChartProps) { + const lensVisServiceCurrentSuggestionContext = useObservable( + lensVisService.currentSuggestionContext$ + ); + const visContext = useObservable(lensVisService.visContext$); + const allSuggestions = useObservable(lensVisService.allSuggestions$); + const currentSuggestion = lensVisServiceCurrentSuggestionContext?.suggestion; + const [isSaveModalVisible, setIsSaveModalVisible] = useState(false); const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); const { chartRef, toggleHideChart } = useChartActions({ @@ -133,19 +130,15 @@ export function Chart({ onChartHiddenChange, }); - const chartVisible = isChartAvailable && !!chart && !chart.hidden; + const chartVisible = + isChartAvailable && !!chart && !chart.hidden && !!visContext && !!visContext?.attributes; const input$ = useMemo( () => originalInput$ ?? new Subject(), [originalInput$] ); - const { filters, query, getTimeRange, updateTimeRange, relativeTimeRange } = useRequestParams({ - services, - query: originalQuery, - filters: originalFilters, - timeRange: originalTimeRange, - }); + const { filters, query, getTimeRange, updateTimeRange, relativeTimeRange } = requestParams; const refetch$ = useRefetch({ dataView, @@ -179,34 +172,24 @@ export function Chart({ const { chartToolbarCss, histogramCss } = useChartStyles(chartVisible); - const lensAttributesContext = useMemo( - () => - getLensAttributes({ - title: chart?.title, - filters, - query: histogramQuery ?? query, - dataView, - timeInterval: chart?.timeInterval, - breakdownField: breakdown?.field, - suggestion: currentSuggestion, - }), - [ - breakdown?.field, - chart?.timeInterval, - chart?.title, - currentSuggestion, - dataView, - filters, - query, - histogramQuery, - ] + const onSuggestionContextEdit = useCallback( + (editedSuggestionContext: UnifiedHistogramSuggestionContext | undefined) => { + lensVisService.onSuggestionEdited({ + editedSuggestionContext, + }); + }, + [lensVisService] ); const onSuggestionSelectorChange = useCallback( - (s: Suggestion | undefined) => { - onSuggestionChange?.(s); + (suggestion: Suggestion | undefined) => { + setIsFlyoutVisible(false); + onSuggestionContextEdit({ + suggestion, + type: UnifiedHistogramSuggestionType.lensSuggestion, + }); }, - [onSuggestionChange] + [onSuggestionContextEdit, setIsFlyoutVisible] ); useEffect(() => { @@ -221,7 +204,7 @@ export function Chart({ services, dataView, relativeTimeRange: originalRelativeTimeRange ?? relativeTimeRange, - lensAttributes: lensAttributesContext.attributes, + lensAttributes: visContext?.attributes, isPlainRecord, }); @@ -234,9 +217,22 @@ export function Chart({ } const LensSaveModalComponent = services.lens.SaveModalComponent; + const hasLensSuggestions = Boolean( + isPlainRecord && + lensVisServiceCurrentSuggestionContext?.type === UnifiedHistogramSuggestionType.lensSuggestion + ); + + const canCustomizeVisualization = + isPlainRecord && + currentSuggestion && + [ + UnifiedHistogramSuggestionType.lensSuggestion, + UnifiedHistogramSuggestionType.histogramForESQL, + ].includes(lensVisServiceCurrentSuggestionContext?.type); + + const canEditVisualizationOnTheFly = canCustomizeVisualization && chartVisible; const canSaveVisualization = - chartVisible && currentSuggestion && services.capabilities.dashboard?.showWriteControls; - const canEditVisualizationOnTheFly = currentSuggestion && chartVisible; + canEditVisualizationOnTheFly && services.capabilities.dashboard?.showWriteControls; const actions: IconButtonGroupProps['buttons'] = []; @@ -260,6 +256,7 @@ export function Chart({ onClick: onEditVisualization, }); } + if (canSaveVisualization) { actions.push({ label: i18n.translate('unifiedHistogram.saveVisualizationButton', { @@ -271,37 +268,6 @@ export function Chart({ }); } - const removeTables = (attributes: LensAttributes) => { - if (!attributes.state.datasourceStates.textBased) { - return attributes; - } - const layers = attributes.state.datasourceStates.textBased?.layers; - - const newState = { - ...attributes, - state: { - ...attributes.state, - datasourceStates: { - ...attributes.state.datasourceStates, - textBased: { - ...(attributes.state.datasourceStates.textBased || {}), - layers: {} as TextBasedPersistedState['layers'], - }, - }, - }, - }; - - if (layers) { - for (const key of Object.keys(layers)) { - const newLayer = { ...layers[key] }; - delete newLayer.table; - newState.state.datasourceStates.textBased!.layers[key] = newLayer; - } - } - - return newState; - }; - return ( )} - {canSaveVisualization && isSaveModalVisible && lensAttributesContext.attributes && ( + {canSaveVisualization && isSaveModalVisible && visContext.attributes && ( {}} onClose={() => setIsSaveModalVisible(false)} isSaveable={false} /> )} - {isFlyoutVisible && ( + {isFlyoutVisible && !!visContext && !!lensVisServiceCurrentSuggestionContext && ( )} diff --git a/src/plugins/unified_histogram/public/chart/chart_config_panel.test.tsx b/src/plugins/unified_histogram/public/chart/chart_config_panel.test.tsx index 5238fc0ac12bb..4f4eaa9faf6cc 100644 --- a/src/plugins/unified_histogram/public/chart/chart_config_panel.test.tsx +++ b/src/plugins/unified_histogram/public/chart/chart_config_panel.test.tsx @@ -13,9 +13,11 @@ import { act } from 'react-dom/test-utils'; import { setTimeout } from 'timers/promises'; import { dataViewWithTimefieldMock } from '../__mocks__/data_view_with_timefield'; import { unifiedHistogramServicesMock } from '../__mocks__/services'; +import { currentSuggestionMock } from '../__mocks__/suggestions'; import { lensAdaptersMock } from '../__mocks__/lens_adapters'; import { ChartConfigPanel } from './chart_config_panel'; -import type { LensAttributesContext } from './utils/get_lens_attributes'; +import type { UnifiedHistogramVisContext } from '../types'; +import { UnifiedHistogramSuggestionType } from '../types'; describe('ChartConfigPanel', () => { it('should return a jsx element to edit the visualization', async () => { @@ -28,16 +30,21 @@ describe('ChartConfigPanel', () => { {...{ services: unifiedHistogramServicesMock, dataView: dataViewWithTimefieldMock, - lensAttributesContext: { + visContext: { attributes: lensAttributes, - } as unknown as LensAttributesContext, + } as unknown as UnifiedHistogramVisContext, isFlyoutVisible: true, setIsFlyoutVisible: jest.fn(), + onSuggestionContextChange: jest.fn(), isPlainRecord: true, lensAdapters: lensAdaptersMock, query: { esql: 'from test', }, + currentSuggestionContext: { + suggestion: currentSuggestionMock, + type: UnifiedHistogramSuggestionType.lensSuggestion, + }, }} /> ); @@ -55,12 +62,17 @@ describe('ChartConfigPanel', () => { {...{ services: unifiedHistogramServicesMock, dataView: dataViewWithTimefieldMock, - lensAttributesContext: { + visContext: { attributes: lensAttributes, - } as unknown as LensAttributesContext, + } as unknown as UnifiedHistogramVisContext, isFlyoutVisible: true, setIsFlyoutVisible: jest.fn(), + onSuggestionContextChange: jest.fn(), isPlainRecord: false, + currentSuggestionContext: { + suggestion: currentSuggestionMock, + type: UnifiedHistogramSuggestionType.histogramForDataView, + }, }} /> ); diff --git a/src/plugins/unified_histogram/public/chart/chart_config_panel.tsx b/src/plugins/unified_histogram/public/chart/chart_config_panel.tsx index 314226525296e..654d4e9ab93ab 100644 --- a/src/plugins/unified_histogram/public/chart/chart_config_panel.tsx +++ b/src/plugins/unified_histogram/public/chart/chart_config_panel.tsx @@ -12,31 +12,35 @@ import { isEqual } from 'lodash'; import type { LensEmbeddableOutput, Suggestion } from '@kbn/lens-plugin/public'; import type { Datatable } from '@kbn/expressions-plugin/common'; -import type { UnifiedHistogramServices, UnifiedHistogramChartLoadEvent } from '../types'; -import type { LensAttributesContext } from './utils/get_lens_attributes'; +import type { + UnifiedHistogramServices, + UnifiedHistogramChartLoadEvent, + UnifiedHistogramVisContext, + UnifiedHistogramSuggestionContext, +} from '../types'; export function ChartConfigPanel({ services, - lensAttributesContext, + visContext, lensAdapters, lensEmbeddableOutput$, - currentSuggestion, + currentSuggestionContext, isFlyoutVisible, setIsFlyoutVisible, isPlainRecord, query, - onSuggestionChange, + onSuggestionContextChange, }: { services: UnifiedHistogramServices; - lensAttributesContext: LensAttributesContext; + visContext: UnifiedHistogramVisContext; isFlyoutVisible: boolean; setIsFlyoutVisible: (flag: boolean) => void; lensAdapters?: UnifiedHistogramChartLoadEvent['adapters']; lensEmbeddableOutput$?: Observable; - currentSuggestion?: Suggestion; + currentSuggestionContext: UnifiedHistogramSuggestionContext; isPlainRecord?: boolean; query?: Query | AggregateQuery; - onSuggestionChange?: (suggestion: Suggestion | undefined) => void; + onSuggestionContextChange: (suggestion: UnifiedHistogramSuggestionContext | undefined) => void; }) { const [editLensConfigPanel, setEditLensConfigPanel] = useState(null); const previousSuggestion = useRef(undefined); @@ -44,16 +48,21 @@ export function ChartConfigPanel({ const previousQuery = useRef(undefined); const updateSuggestion = useCallback( (datasourceState, visualizationState) => { - const updatedSuggestion = { - ...currentSuggestion, + const updatedSuggestion: Suggestion = { + ...currentSuggestionContext?.suggestion, ...(datasourceState && { datasourceState }), ...(visualizationState && { visualizationState }), - } as Suggestion; - onSuggestionChange?.(updatedSuggestion); + }; + onSuggestionContextChange({ + ...currentSuggestionContext, + suggestion: updatedSuggestion, + }); }, - [currentSuggestion, onSuggestionChange] + [currentSuggestionContext, onSuggestionContextChange] ); + const currentSuggestion = currentSuggestionContext.suggestion; + useEffect(() => { const tablesAdapters = lensAdapters?.tables?.tables; const dataHasChanged = @@ -64,7 +73,7 @@ export function ChartConfigPanel({ const Component = await services.lens.EditLensConfigPanelApi(); const panel = ( - getLensAttributes({ - title: 'test', - filters: [], - query: { - language: 'kuery', - query: '', - }, - dataView: dataViewWithTimefieldMock, - timeInterval: 'auto', - breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), - suggestion: undefined, - }); +const getMockLensAttributes = async () => { + const query = { + language: 'kuery', + query: '', + }; + return ( + await getLensVisMock({ + filters: [], + query, + columns: [], + isPlainRecord: false, + dataView: dataViewWithTimefieldMock, + timeInterval: 'auto', + breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), + }) + ).visContext; +}; -function mountComponent(isPlainRecord = false, hasLensSuggestions = false) { +async function mountComponent(isPlainRecord = false, hasLensSuggestions = false) { const services = unifiedHistogramServicesMock; services.data.query.timefilter.timefilter.getAbsoluteTime = () => { return { from: '2020-05-14T11:05:13.590', to: '2020-05-14T11:20:13.590' }; @@ -69,7 +72,7 @@ function mountComponent(isPlainRecord = false, hasLensSuggestions = false) { to: '2020-05-14T11:20:13.590', }), refetch$, - lensAttributesContext: getMockLensAttributes(), + visContext: (await getMockLensAttributes())!, onTotalHitsChange: jest.fn(), onChartLoad: jest.fn(), withDefaultActions: undefined, @@ -82,20 +85,20 @@ function mountComponent(isPlainRecord = false, hasLensSuggestions = false) { } describe('Histogram', () => { - it('renders correctly', () => { - const { component } = mountComponent(); + it('renders correctly', async () => { + const { component } = await mountComponent(); expect(component.find('[data-test-subj="unifiedHistogramChart"]').exists()).toBe(true); }); it('should only update lens.EmbeddableComponent props when refetch$ is triggered', async () => { - const { component, props } = mountComponent(); + const { component, props } = await mountComponent(); const embeddable = unifiedHistogramServicesMock.lens.EmbeddableComponent; expect(component.find(embeddable).exists()).toBe(true); let lensProps = component.find(embeddable).props(); const originalProps = getLensProps({ searchSessionId: props.request.searchSessionId, getTimeRange: props.getTimeRange, - attributes: getMockLensAttributes().attributes, + attributes: (await getMockLensAttributes())!.attributes, onLoad: lensProps.onLoad, }); expect(lensProps).toMatchObject(expect.objectContaining(originalProps)); @@ -113,7 +116,7 @@ describe('Histogram', () => { }); it('should execute onLoad correctly', async () => { - const { component, props } = mountComponent(); + const { component, props } = await mountComponent(); const embeddable = unifiedHistogramServicesMock.lens.EmbeddableComponent; const onLoad = component.find(embeddable).props().onLoad; const adapters = createDefaultInspectorAdapters(); @@ -193,7 +196,7 @@ describe('Histogram', () => { }); it('should execute onLoad correctly when the request has a failure status', async () => { - const { component, props } = mountComponent(); + const { component, props } = await mountComponent(); const embeddable = unifiedHistogramServicesMock.lens.EmbeddableComponent; const onLoad = component.find(embeddable).props().onLoad; const adapters = createDefaultInspectorAdapters(); @@ -209,7 +212,7 @@ describe('Histogram', () => { }); it('should execute onLoad correctly when the response has shard failures', async () => { - const { component, props } = mountComponent(); + const { component, props } = await mountComponent(); const embeddable = unifiedHistogramServicesMock.lens.EmbeddableComponent; const onLoad = component.find(embeddable).props().onLoad; const adapters = createDefaultInspectorAdapters(); @@ -242,7 +245,7 @@ describe('Histogram', () => { }); it('should execute onLoad correctly for textbased language and no Lens suggestions', async () => { - const { component, props } = mountComponent(true, false); + const { component, props } = await mountComponent(true, false); const embeddable = unifiedHistogramServicesMock.lens.EmbeddableComponent; const onLoad = component.find(embeddable).props().onLoad; const adapters = createDefaultInspectorAdapters(); @@ -278,7 +281,7 @@ describe('Histogram', () => { }); it('should execute onLoad correctly for textbased language and Lens suggestions', async () => { - const { component, props } = mountComponent(true, true); + const { component, props } = await mountComponent(true, true); const embeddable = unifiedHistogramServicesMock.lens.EmbeddableComponent; const onLoad = component.find(embeddable).props().onLoad; const adapters = createDefaultInspectorAdapters(); diff --git a/src/plugins/unified_histogram/public/chart/histogram.tsx b/src/plugins/unified_histogram/public/chart/histogram.tsx index 70d406b7f9be8..8a65426e4a9a2 100644 --- a/src/plugins/unified_histogram/public/chart/histogram.tsx +++ b/src/plugins/unified_histogram/public/chart/histogram.tsx @@ -30,12 +30,12 @@ import { UnifiedHistogramRequestContext, UnifiedHistogramServices, UnifiedHistogramInputMessage, + UnifiedHistogramVisContext, } from '../types'; import { buildBucketInterval } from './utils/build_bucket_interval'; import { useTimeRange } from './hooks/use_time_range'; -import { useStableCallback } from './hooks/use_stable_callback'; +import { useStableCallback } from '../hooks/use_stable_callback'; import { useLensProps } from './hooks/use_lens_props'; -import type { LensAttributesContext } from './utils/get_lens_attributes'; export interface HistogramProps { abortController?: AbortController; @@ -48,7 +48,7 @@ export interface HistogramProps { hasLensSuggestions: boolean; getTimeRange: () => TimeRange; refetch$: Observable; - lensAttributesContext: LensAttributesContext; + visContext: UnifiedHistogramVisContext; disableTriggers?: LensEmbeddableInput['disableTriggers']; disabledActions?: LensEmbeddableInput['disabledActions']; onTotalHitsChange?: (status: UnifiedHistogramFetchStatus, result?: number | Error) => void; @@ -95,7 +95,7 @@ export function Histogram({ hasLensSuggestions, getTimeRange, refetch$, - lensAttributesContext: attributesContext, + visContext, disableTriggers, disabledActions, onTotalHitsChange, @@ -117,7 +117,7 @@ export function Histogram({ }); const chartRef = useRef(null); const { height: containerHeight, width: containerWidth } = useResizeObserver(chartRef.current); - const { attributes } = attributesContext; + const { attributes } = visContext; useEffect(() => { if (attributes.visualizationType === 'lnsMetric') { @@ -178,7 +178,7 @@ export function Histogram({ request, getTimeRange, refetch$, - attributesContext, + visContext, onLoad, }); diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_edit_visualization.ts b/src/plugins/unified_histogram/public/chart/hooks/use_edit_visualization.ts index b02732bfcbfc9..8b70e08684971 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_edit_visualization.ts +++ b/src/plugins/unified_histogram/public/chart/hooks/use_edit_visualization.ts @@ -26,7 +26,7 @@ export const useEditVisualization = ({ services: UnifiedHistogramServices; dataView: DataView; relativeTimeRange?: TimeRange; - lensAttributes: TypedLensByValueInput['attributes']; + lensAttributes?: TypedLensByValueInput['attributes']; isPlainRecord?: boolean; }) => { const [canVisualize, setCanVisualize] = useState(false); @@ -51,7 +51,7 @@ export const useEditVisualization = ({ }, [dataView, isPlainRecord, services.uiActions]); const onEditVisualization = useMemo(() => { - if (!canVisualize) { + if (!canVisualize || !lensAttributes) { return undefined; } diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.test.ts b/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.test.ts index 36b4e5c8f4e4d..de483cbdb63ec 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.test.ts +++ b/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.test.ts @@ -11,27 +11,29 @@ import { act } from 'react-test-renderer'; import { Subject } from 'rxjs'; import type { UnifiedHistogramInputMessage } from '../../types'; import { dataViewWithTimefieldMock } from '../../__mocks__/data_view_with_timefield'; -import { currentSuggestionMock } from '../../__mocks__/suggestions'; -import { getLensAttributes } from '../utils/get_lens_attributes'; +import { getLensVisMock } from '../../__mocks__/lens_vis'; import { getLensProps, useLensProps } from './use_lens_props'; describe('useLensProps', () => { - it('should return lens props', () => { + it('should return lens props', async () => { const getTimeRange = jest.fn(); const refetch$ = new Subject(); const onLoad = jest.fn(); - const attributesContext = getLensAttributes({ - title: 'test', - filters: [], - query: { - language: 'kuery', - query: '', - }, - dataView: dataViewWithTimefieldMock, - timeInterval: 'auto', - breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), - suggestion: undefined, - }); + const query = { + language: 'kuery', + query: '', + }; + const attributesContext = ( + await getLensVisMock({ + filters: [], + query, + columns: [], + isPlainRecord: false, + dataView: dataViewWithTimefieldMock, + timeInterval: 'auto', + breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), + }) + ).visContext; const lensProps = renderHook(() => { return useLensProps({ request: { @@ -40,7 +42,7 @@ describe('useLensProps', () => { }, getTimeRange, refetch$, - attributesContext, + visContext: attributesContext!, onLoad, }); }); @@ -48,28 +50,31 @@ describe('useLensProps', () => { getLensProps({ searchSessionId: 'id', getTimeRange, - attributes: attributesContext.attributes, + attributes: attributesContext!.attributes, onLoad, }) ); }); - it('should return lens props for text based languages', () => { + it('should return lens props for text based languages', async () => { const getTimeRange = jest.fn(); const refetch$ = new Subject(); const onLoad = jest.fn(); - const attributesContext = getLensAttributes({ - title: 'test', - filters: [], - query: { - language: 'kuery', - query: '', - }, - dataView: dataViewWithTimefieldMock, - timeInterval: 'auto', - breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), - suggestion: currentSuggestionMock, - }); + const query = { + language: 'kuery', + query: '', + }; + const attributesContext = ( + await getLensVisMock({ + filters: [], + query, + columns: [], + isPlainRecord: false, + dataView: dataViewWithTimefieldMock, + timeInterval: 'auto', + breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), + }) + ).visContext; const lensProps = renderHook(() => { return useLensProps({ request: { @@ -78,7 +83,7 @@ describe('useLensProps', () => { }, getTimeRange, refetch$, - attributesContext, + visContext: attributesContext!, onLoad, }); }); @@ -86,16 +91,31 @@ describe('useLensProps', () => { getLensProps({ searchSessionId: 'id', getTimeRange, - attributes: attributesContext.attributes, + attributes: attributesContext!.attributes, onLoad, }) ); }); - it('should only update lens props when refetch$ is triggered', () => { + it('should only update lens props when refetch$ is triggered', async () => { const getTimeRange = jest.fn(); const refetch$ = new Subject(); const onLoad = jest.fn(); + const query = { + language: 'kuery', + query: '', + }; + const attributesContext = ( + await getLensVisMock({ + filters: [], + query, + columns: [], + isPlainRecord: false, + dataView: dataViewWithTimefieldMock, + timeInterval: 'auto', + breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), + }) + ).visContext; const lensProps = { request: { searchSessionId: '123', @@ -103,18 +123,7 @@ describe('useLensProps', () => { }, getTimeRange, refetch$, - attributesContext: getLensAttributes({ - title: 'test', - filters: [], - query: { - language: 'kuery', - query: '', - }, - dataView: dataViewWithTimefieldMock, - timeInterval: 'auto', - breakdownField: dataViewWithTimefieldMock.getFieldByName('extension'), - suggestion: undefined, - }), + visContext: attributesContext!, onLoad, }; const hook = renderHook( diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.ts b/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.ts index 29827a46dd705..8c4d1ec9b16a7 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.ts +++ b/src/plugins/unified_histogram/public/chart/hooks/use_lens_props.ts @@ -12,25 +12,28 @@ import type { DefaultInspectorAdapters } from '@kbn/expressions-plugin/common'; import type { TypedLensByValueInput } from '@kbn/lens-plugin/public'; import { useCallback, useEffect, useState } from 'react'; import type { Observable } from 'rxjs'; -import type { UnifiedHistogramInputMessage, UnifiedHistogramRequestContext } from '../../types'; -import type { LensAttributesContext } from '../utils/get_lens_attributes'; -import { useStableCallback } from './use_stable_callback'; +import type { + UnifiedHistogramInputMessage, + UnifiedHistogramRequestContext, + UnifiedHistogramVisContext, +} from '../../types'; +import { useStableCallback } from '../../hooks/use_stable_callback'; export const useLensProps = ({ request, getTimeRange, refetch$, - attributesContext, + visContext, onLoad, }: { request?: UnifiedHistogramRequestContext; getTimeRange: () => TimeRange; refetch$: Observable; - attributesContext: LensAttributesContext; + visContext: UnifiedHistogramVisContext; onLoad: (isLoading: boolean, adapters: Partial | undefined) => void; }) => { const buildLensProps = useCallback(() => { - const { attributes, requestData } = attributesContext; + const { attributes, requestData } = visContext; return { requestData: JSON.stringify(requestData), lensProps: getLensProps({ @@ -40,7 +43,7 @@ export const useLensProps = ({ onLoad, }), }; - }, [attributesContext, getTimeRange, onLoad, request?.searchSessionId]); + }, [visContext, getTimeRange, onLoad, request?.searchSessionId]); const [lensPropsContext, setLensPropsContext] = useState(buildLensProps()); const updateLensPropsContext = useStableCallback(() => setLensPropsContext(buildLensProps())); diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_time_range.test.tsx b/src/plugins/unified_histogram/public/chart/hooks/use_time_range.test.tsx index 1e86bf5d9614e..e37e8fdf44c8f 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_time_range.test.tsx +++ b/src/plugins/unified_histogram/public/chart/hooks/use_time_range.test.tsx @@ -264,7 +264,7 @@ describe('useTimeRange', () => { size="xs" textAlign="center" > - 2022-11-17T00:00:00.000Z - 2022-11-17T12:00:00.000Z + 2022-11-17T00:00:00.000Z - 2022-11-17T12:00:00.000Z `); }); diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_time_range.tsx b/src/plugins/unified_histogram/public/chart/hooks/use_time_range.tsx index 791d332a3a89f..f04b18de28f61 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_time_range.tsx +++ b/src/plugins/unified_histogram/public/chart/hooks/use_time_range.tsx @@ -75,7 +75,7 @@ export const useTimeRange = ({ }, }); - return `${toMoment(timeRange.from)} - ${toMoment(timeRange.to)} ${intervalText}`; + return `${toMoment(timeRange.from)} - ${toMoment(timeRange.to)} ${intervalText}`.trim(); }, [bucketInterval?.description, from, isPlainRecord, timeField, timeInterval, to, toMoment]); const { euiTheme } = useEuiTheme(); diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_total_hits.ts b/src/plugins/unified_histogram/public/chart/hooks/use_total_hits.ts index dfd14df6f452b..038847db56150 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_total_hits.ts +++ b/src/plugins/unified_histogram/public/chart/hooks/use_total_hits.ts @@ -19,7 +19,7 @@ import { UnifiedHistogramRequestContext, UnifiedHistogramServices, } from '../../types'; -import { useStableCallback } from './use_stable_callback'; +import { useStableCallback } from '../../hooks/use_stable_callback'; export const useTotalHits = ({ services, diff --git a/src/plugins/unified_histogram/public/chart/suggestion_selector.tsx b/src/plugins/unified_histogram/public/chart/suggestion_selector.tsx index cad20279bfdf0..82a7cc4d814c2 100644 --- a/src/plugins/unified_histogram/public/chart/suggestion_selector.tsx +++ b/src/plugins/unified_histogram/public/chart/suggestion_selector.tsx @@ -19,6 +19,10 @@ import type { Suggestion } from '@kbn/lens-plugin/public'; import { i18n } from '@kbn/i18n'; import React, { useCallback, useState } from 'react'; +const unfamiliarSuggestionTitle = i18n.translate('unifiedHistogram.lensUnfamiliarVisSubtypeTitle', { + defaultMessage: 'Customized', +}); + export interface SuggestionSelectorProps { suggestions: Suggestion[]; activeSuggestion?: Suggestion; @@ -30,21 +34,37 @@ export const SuggestionSelector = ({ activeSuggestion, onSuggestionChange, }: SuggestionSelectorProps) => { - const suggestionOptions = suggestions.map((sug) => { + const isUnfamiliarSuggestion = activeSuggestion && !activeSuggestion.previewIcon; + const activeSuggestionTitle = isUnfamiliarSuggestion + ? unfamiliarSuggestionTitle + : activeSuggestion?.title; + + let suggestionOptions = suggestions.map((sug) => { return { label: sug.title, value: sug.title, }; }); - const selectedSuggestion = activeSuggestion - ? [ - { - label: activeSuggestion.title, - value: activeSuggestion.title, - }, - ] - : []; + const selectedSuggestion = + activeSuggestion && activeSuggestionTitle + ? [ + { + label: activeSuggestionTitle, + value: activeSuggestionTitle, + }, + ] + : []; + + if (isUnfamiliarSuggestion && activeSuggestionTitle) { + suggestionOptions = [ + ...suggestionOptions, + { + label: activeSuggestionTitle, + value: activeSuggestionTitle, + }, + ]; + } const onSelectionChange = useCallback( (newOptions) => { @@ -80,7 +100,15 @@ export const SuggestionSelector = ({ > } + prepend={ + + } placeholder={i18n.translate('unifiedHistogram.suggestionSelectorPlaceholder', { defaultMessage: 'Select visualization', })} @@ -100,7 +128,13 @@ export const SuggestionSelector = ({ return ( - + {option.label} @@ -110,3 +144,25 @@ export const SuggestionSelector = ({ ); }; + +function getSuggestionIconWithFallback({ + suggestion, + suggestions, + activeSuggestion, +}: { + suggestion: Suggestion | undefined; + suggestions: Suggestion[]; + activeSuggestion?: Suggestion; +}) { + if (!suggestion) { + const similarKnownSuggestionWithIcon = suggestions.find( + (s) => s.title === activeSuggestion?.title && s.previewIcon + ); + + if (similarKnownSuggestionWithIcon?.previewIcon) { + return similarKnownSuggestionWithIcon.previewIcon; + } + } + + return suggestion?.previewIcon ?? 'lensApp'; +} diff --git a/src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.ts b/src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.ts deleted file mode 100644 index b5c9bca754ac5..0000000000000 --- a/src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.ts +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import type { AggregateQuery, Filter, Query } from '@kbn/es-query'; -import { i18n } from '@kbn/i18n'; -import type { - CountIndexPatternColumn, - DateHistogramIndexPatternColumn, - GenericIndexPatternColumn, - TermsIndexPatternColumn, - TypedLensByValueInput, - Suggestion, -} from '@kbn/lens-plugin/public'; -import { LegendSize } from '@kbn/visualizations-plugin/public'; -import { XYConfiguration } from '@kbn/visualizations-plugin/common'; -import { fieldSupportsBreakdown } from './field_supports_breakdown'; - -export interface LensRequestData { - dataViewId?: string; - timeField?: string; - timeInterval?: string; - breakdownField?: string; -} - -export interface LensAttributesContext { - attributes: TypedLensByValueInput['attributes']; - requestData: LensRequestData; -} - -export const getLensAttributes = ({ - title, - filters, - query, - dataView, - timeInterval, - breakdownField, - suggestion, -}: { - title?: string; - filters: Filter[]; - query: Query | AggregateQuery; - dataView: DataView; - timeInterval: string | undefined; - breakdownField: DataViewField | undefined; - suggestion: Suggestion | undefined; -}): LensAttributesContext => { - const showBreakdown = breakdownField && fieldSupportsBreakdown(breakdownField); - - let columnOrder = ['date_column', 'count_column']; - - if (showBreakdown) { - columnOrder = ['breakdown_column', ...columnOrder]; - } - - let columns: Record = { - date_column: { - dataType: 'date', - isBucketed: true, - label: dataView.timeFieldName ?? '', - operationType: 'date_histogram', - scale: 'interval', - sourceField: dataView.timeFieldName, - params: { - interval: timeInterval ?? 'auto', - }, - } as DateHistogramIndexPatternColumn, - count_column: { - dataType: 'number', - isBucketed: false, - label: i18n.translate('unifiedHistogram.countColumnLabel', { - defaultMessage: 'Count of records', - }), - operationType: 'count', - scale: 'ratio', - sourceField: '___records___', - params: { - format: { - id: 'number', - params: { - decimals: 0, - }, - }, - }, - } as CountIndexPatternColumn, - }; - - if (showBreakdown) { - columns = { - ...columns, - breakdown_column: { - dataType: 'string', - isBucketed: true, - label: i18n.translate('unifiedHistogram.breakdownColumnLabel', { - defaultMessage: 'Top 3 values of {fieldName}', - values: { fieldName: breakdownField?.displayName }, - }), - operationType: 'terms', - scale: 'ordinal', - sourceField: breakdownField.name, - params: { - size: 3, - orderBy: { - type: 'column', - columnId: 'count_column', - }, - orderDirection: 'desc', - otherBucket: true, - missingBucket: false, - parentFormat: { - id: 'terms', - }, - }, - } as TermsIndexPatternColumn, - }; - } - - const suggestionDatasourceState = Object.assign({}, suggestion?.datasourceState); - const suggestionVisualizationState = Object.assign({}, suggestion?.visualizationState); - const datasourceStates = - suggestion && suggestion.datasourceState - ? { - [suggestion.datasourceId!]: { - ...suggestionDatasourceState, - }, - } - : { - formBased: { - layers: { - unifiedHistogram: { columnOrder, columns }, - }, - }, - }; - const visualization = suggestion - ? { - ...suggestionVisualizationState, - } - : ({ - layers: [ - { - accessors: ['count_column'], - layerId: 'unifiedHistogram', - layerType: 'data', - seriesType: 'bar_stacked', - xAccessor: 'date_column', - ...(showBreakdown - ? { splitAccessor: 'breakdown_column' } - : { - yConfig: [ - { - forAccessor: 'count_column', - }, - ], - }), - }, - ], - legend: { - isVisible: true, - position: 'right', - legendSize: LegendSize.EXTRA_LARGE, - shouldTruncate: false, - }, - preferredSeriesType: 'bar_stacked', - valueLabels: 'hide', - fittingFunction: 'None', - minBarHeight: 2, - showCurrentTimeMarker: true, - axisTitlesVisibilitySettings: { - x: false, - yLeft: false, - yRight: false, - }, - gridlinesVisibilitySettings: { - x: true, - yLeft: true, - yRight: false, - }, - tickLabelsVisibilitySettings: { - x: true, - yLeft: true, - yRight: false, - }, - } as XYConfiguration); - const attributes = { - title: - title ?? - suggestion?.title ?? - i18n.translate('unifiedHistogram.lensTitle', { - defaultMessage: 'Edit visualization', - }), - references: [ - { - id: dataView.id ?? '', - name: 'indexpattern-datasource-current-indexpattern', - type: 'index-pattern', - }, - { - id: dataView.id ?? '', - name: 'indexpattern-datasource-layer-unifiedHistogram', - type: 'index-pattern', - }, - ], - state: { - datasourceStates, - filters, - query, - visualization, - ...(dataView && - dataView.id && - !dataView.isPersisted() && { - adHocDataViews: { - [dataView.id]: dataView.toSpec(false), - }, - }), - }, - visualizationType: suggestion ? suggestion.visualizationId : 'lnsXY', - } as TypedLensByValueInput['attributes']; - - return { - attributes, - requestData: { - dataViewId: dataView.id, - timeField: dataView.timeFieldName, - timeInterval, - breakdownField: breakdownField?.name, - }, - }; -}; diff --git a/src/plugins/unified_histogram/public/container/container.tsx b/src/plugins/unified_histogram/public/container/container.tsx index 3756b5da94e7b..ef18a2ba992e0 100644 --- a/src/plugins/unified_histogram/public/container/container.tsx +++ b/src/plugins/unified_histogram/public/container/container.tsx @@ -6,14 +6,18 @@ * Side Public License, v 1. */ -import React, { forwardRef, useEffect, useImperativeHandle, useState } from 'react'; +import React, { forwardRef, useEffect, useImperativeHandle, useMemo, useState } from 'react'; import { Subject } from 'rxjs'; import { pick } from 'lodash'; import useMount from 'react-use/lib/useMount'; import { LensSuggestionsApi } from '@kbn/lens-plugin/public'; -import type { Datatable } from '@kbn/expressions-plugin/common'; import { UnifiedHistogramLayout, UnifiedHistogramLayoutProps } from '../layout'; -import type { UnifiedHistogramInputMessage, UnifiedHistogramRequestContext } from '../types'; +import { + UnifiedHistogramExternalVisContextStatus, + UnifiedHistogramInputMessage, + UnifiedHistogramRequestContext, + UnifiedHistogramVisContext, +} from '../types'; import { createStateService, UnifiedHistogramStateOptions, @@ -21,7 +25,8 @@ import { } from './services/state_service'; import { useStateProps } from './hooks/use_state_props'; import { useStateSelector } from './utils/use_state_selector'; -import { topPanelHeightSelector, currentSuggestionSelector } from './utils/state_selectors'; +import { topPanelHeightSelector } from './utils/state_selectors'; +import { exportVisContext } from '../utils/external_vis_context'; type LayoutProps = Pick< UnifiedHistogramLayoutProps, @@ -44,7 +49,10 @@ export type UnifiedHistogramContainerProps = { searchSessionId?: UnifiedHistogramRequestContext['searchSessionId']; requestAdapter?: UnifiedHistogramRequestContext['adapter']; isChartLoading?: boolean; - table?: Datatable; + onVisContextChanged?: ( + nextVisContext: UnifiedHistogramVisContext | undefined, + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus + ) => void; } & Pick< UnifiedHistogramLayoutProps, | 'services' @@ -55,11 +63,13 @@ export type UnifiedHistogramContainerProps = { | 'timeRange' | 'relativeTimeRange' | 'columns' + | 'table' | 'container' | 'renderCustomChartToggleActions' | 'children' | 'onBrushEnd' | 'onFilter' + | 'externalVisContext' | 'withDefaultActions' | 'disabledActions' | 'abortController' @@ -86,7 +96,7 @@ export type UnifiedHistogramApi = { export const UnifiedHistogramContainer = forwardRef< UnifiedHistogramApi, UnifiedHistogramContainerProps ->((containerProps, ref) => { +>(({ onVisContextChanged, ...containerProps }, ref) => { const [layoutProps, setLayoutProps] = useState(); const [stateService, setStateService] = useState(); const [lensSuggestionsApi, setLensSuggestionsApi] = useState(); @@ -129,7 +139,6 @@ export const UnifiedHistogramContainer = forwardRef< }); }, [input$, stateService]); const { dataView, query, searchSessionId, requestAdapter, isChartLoading } = containerProps; - const currentSuggestion = useStateSelector(stateService?.state$, currentSuggestionSelector); const topPanelHeight = useStateSelector(stateService?.state$, topPanelHeightSelector); const stateProps = useStateProps({ stateService, @@ -139,6 +148,19 @@ export const UnifiedHistogramContainer = forwardRef< requestAdapter, }); + const handleVisContextChange: UnifiedHistogramLayoutProps['onVisContextChanged'] | undefined = + useMemo(() => { + if (!onVisContextChanged) { + return undefined; + } + + return (visContext, externalVisContextStatus) => { + const minifiedVisContext = exportVisContext(visContext); + + onVisContextChanged(minifiedVisContext, externalVisContextStatus); + }; + }, [onVisContextChanged]); + // Don't render anything until the container is initialized if (!layoutProps || !lensSuggestionsApi || !api) { return null; @@ -149,7 +171,7 @@ export const UnifiedHistogramContainer = forwardRef< {...containerProps} {...layoutProps} {...stateProps} - currentSuggestion={currentSuggestion} + onVisContextChanged={handleVisContextChange} isChartLoading={Boolean(isChartLoading)} topPanelHeight={topPanelHeight} input$={input$} diff --git a/src/plugins/unified_histogram/public/container/hooks/use_state_props.test.ts b/src/plugins/unified_histogram/public/container/hooks/use_state_props.test.ts index 15c1ef83a4b8c..44a216178f6d5 100644 --- a/src/plugins/unified_histogram/public/container/hooks/use_state_props.test.ts +++ b/src/plugins/unified_histogram/public/container/hooks/use_state_props.test.ts @@ -13,7 +13,6 @@ import { act } from 'react-test-renderer'; import { UnifiedHistogramFetchStatus } from '../../types'; import { dataViewMock } from '../../__mocks__/data_view'; import { dataViewWithTimefieldMock } from '../../__mocks__/data_view_with_timefield'; -import { currentSuggestionMock } from '../../__mocks__/suggestions'; import { lensAdaptersMock } from '../../__mocks__/lens_adapters'; import { unifiedHistogramServicesMock } from '../../__mocks__/services'; import { @@ -33,7 +32,7 @@ describe('useStateProps', () => { topPanelHeight: 100, totalHitsStatus: UnifiedHistogramFetchStatus.uninitialized, totalHitsResult: undefined, - currentSuggestion: undefined, + currentSuggestionContext: undefined, }; const getStateService = (options: Omit) => { @@ -47,7 +46,7 @@ describe('useStateProps', () => { jest.spyOn(stateService, 'setTimeInterval'); jest.spyOn(stateService, 'setLensRequestAdapter'); jest.spyOn(stateService, 'setTotalHits'); - jest.spyOn(stateService, 'setCurrentSuggestion'); + jest.spyOn(stateService, 'setCurrentSuggestionContext'); return stateService; }; @@ -122,7 +121,7 @@ describe('useStateProps', () => { "onBreakdownFieldChange": [Function], "onChartHiddenChange": [Function], "onChartLoad": [Function], - "onSuggestionChange": [Function], + "onSuggestionContextChange": [Function], "onTimeIntervalChange": [Function], "onTopPanelHeightChange": [Function], "onTotalHitsChange": [Function], @@ -203,7 +202,7 @@ describe('useStateProps', () => { "onBreakdownFieldChange": [Function], "onChartHiddenChange": [Function], "onChartLoad": [Function], - "onSuggestionChange": [Function], + "onSuggestionContextChange": [Function], "onTimeIntervalChange": [Function], "onTopPanelHeightChange": [Function], "onTotalHitsChange": [Function], @@ -226,7 +225,7 @@ describe('useStateProps', () => { const stateService = getStateService({ initialState: { ...initialState, - currentSuggestion: currentSuggestionMock, + currentSuggestionContext: undefined, }, }); const { result } = renderHook(() => @@ -305,7 +304,7 @@ describe('useStateProps', () => { "onBreakdownFieldChange": [Function], "onChartHiddenChange": [Function], "onChartLoad": [Function], - "onSuggestionChange": [Function], + "onSuggestionContextChange": [Function], "onTimeIntervalChange": [Function], "onTopPanelHeightChange": [Function], "onTotalHitsChange": [Function], @@ -383,7 +382,7 @@ describe('useStateProps', () => { "onBreakdownFieldChange": [Function], "onChartHiddenChange": [Function], "onChartLoad": [Function], - "onSuggestionChange": [Function], + "onSuggestionContextChange": [Function], "onTimeIntervalChange": [Function], "onTopPanelHeightChange": [Function], "onTotalHitsChange": [Function], @@ -420,7 +419,7 @@ describe('useStateProps', () => { onChartHiddenChange, onChartLoad, onBreakdownFieldChange, - onSuggestionChange, + onSuggestionContextChange, } = result.current; act(() => { onTopPanelHeightChange(200); @@ -452,9 +451,11 @@ describe('useStateProps', () => { expect(stateService.setBreakdownField).toHaveBeenLastCalledWith('field'); act(() => { - onSuggestionChange({ title: 'Stacked Bar' } as Suggestion); + onSuggestionContextChange({ title: 'Stacked Bar' } as Suggestion); + }); + expect(stateService.setCurrentSuggestionContext).toHaveBeenLastCalledWith({ + title: 'Stacked Bar', }); - expect(stateService.setCurrentSuggestion).toHaveBeenLastCalledWith({ title: 'Stacked Bar' }); }); it('should clear lensRequestAdapter when chart is hidden', () => { diff --git a/src/plugins/unified_histogram/public/container/hooks/use_state_props.ts b/src/plugins/unified_histogram/public/container/hooks/use_state_props.ts index d78afc50c15f5..7afdb029fd3cc 100644 --- a/src/plugins/unified_histogram/public/container/hooks/use_state_props.ts +++ b/src/plugins/unified_histogram/public/container/hooks/use_state_props.ts @@ -158,9 +158,9 @@ export const useStateProps = ({ [stateService] ); - const onSuggestionChange = useCallback( - (suggestion) => { - stateService?.setCurrentSuggestion(suggestion); + const onSuggestionContextChange = useCallback( + (suggestionContext) => { + stateService?.setCurrentSuggestionContext(suggestionContext); }, [stateService] ); @@ -190,6 +190,6 @@ export const useStateProps = ({ onChartHiddenChange, onChartLoad, onBreakdownFieldChange, - onSuggestionChange, + onSuggestionContextChange, }; }; diff --git a/src/plugins/unified_histogram/public/container/services/state_service.test.ts b/src/plugins/unified_histogram/public/container/services/state_service.test.ts index 40304a967243a..6249c3e423877 100644 --- a/src/plugins/unified_histogram/public/container/services/state_service.test.ts +++ b/src/plugins/unified_histogram/public/container/services/state_service.test.ts @@ -52,7 +52,7 @@ describe('UnifiedHistogramStateService', () => { topPanelHeight: 100, totalHitsStatus: UnifiedHistogramFetchStatus.uninitialized, totalHitsResult: undefined, - currentSuggestion: undefined, + currentSuggestionContext: undefined, }; it('should initialize state with default values', () => { @@ -67,8 +67,7 @@ describe('UnifiedHistogramStateService', () => { topPanelHeight: undefined, totalHitsResult: undefined, totalHitsStatus: UnifiedHistogramFetchStatus.uninitialized, - currentSuggestion: undefined, - allSuggestions: undefined, + currentSuggestionContext: undefined, }); }); diff --git a/src/plugins/unified_histogram/public/container/services/state_service.ts b/src/plugins/unified_histogram/public/container/services/state_service.ts index 1a79389e2bc6f..dd70dc646c9fb 100644 --- a/src/plugins/unified_histogram/public/container/services/state_service.ts +++ b/src/plugins/unified_histogram/public/container/services/state_service.ts @@ -7,7 +7,7 @@ */ import type { RequestAdapter } from '@kbn/inspector-plugin/common'; -import type { LensEmbeddableOutput, Suggestion } from '@kbn/lens-plugin/public'; +import type { LensEmbeddableOutput } from '@kbn/lens-plugin/public'; import { BehaviorSubject, Observable } from 'rxjs'; import { UnifiedHistogramFetchStatus } from '../..'; import type { UnifiedHistogramServices, UnifiedHistogramChartLoadEvent } from '../../types'; @@ -19,6 +19,7 @@ import { setChartHidden, setTopPanelHeight, } from '../utils/local_storage_utils'; +import type { UnifiedHistogramSuggestionContext } from '../../types'; /** * The current state of the container @@ -31,7 +32,7 @@ export interface UnifiedHistogramState { /** * The current Lens suggestion */ - currentSuggestion: Suggestion | undefined; + currentSuggestionContext: UnifiedHistogramSuggestionContext | undefined; /** * Whether or not the chart is hidden */ @@ -99,7 +100,9 @@ export interface UnifiedHistogramStateService { /** * Sets current Lens suggestion */ - setCurrentSuggestion: (suggestion: Suggestion | undefined) => void; + setCurrentSuggestionContext: ( + suggestionContext: UnifiedHistogramSuggestionContext | undefined + ) => void; /** * Sets the current top panel height */ @@ -150,7 +153,7 @@ export const createStateService = ( const state$ = new BehaviorSubject({ breakdownField: initialBreakdownField, chartHidden: initialChartHidden, - currentSuggestion: undefined, + currentSuggestionContext: undefined, lensRequestAdapter: undefined, timeInterval: 'auto', topPanelHeight: initialTopPanelHeight, @@ -193,9 +196,12 @@ export const createStateService = ( updateState({ breakdownField }); }, - setCurrentSuggestion: (suggestion: Suggestion | undefined) => { - updateState({ currentSuggestion: suggestion }); + setCurrentSuggestionContext: ( + suggestionContext: UnifiedHistogramSuggestionContext | undefined + ) => { + updateState({ currentSuggestionContext: suggestionContext }); }, + setTimeInterval: (timeInterval: string) => { updateState({ timeInterval }); }, diff --git a/src/plugins/unified_histogram/public/container/utils/state_selectors.ts b/src/plugins/unified_histogram/public/container/utils/state_selectors.ts index f0707cdbe747e..9c2c98b1aeae4 100644 --- a/src/plugins/unified_histogram/public/container/utils/state_selectors.ts +++ b/src/plugins/unified_histogram/public/container/utils/state_selectors.ts @@ -14,7 +14,6 @@ export const timeIntervalSelector = (state: UnifiedHistogramState) => state.time export const topPanelHeightSelector = (state: UnifiedHistogramState) => state.topPanelHeight; export const totalHitsResultSelector = (state: UnifiedHistogramState) => state.totalHitsResult; export const totalHitsStatusSelector = (state: UnifiedHistogramState) => state.totalHitsStatus; -export const currentSuggestionSelector = (state: UnifiedHistogramState) => state.currentSuggestion; export const lensAdaptersSelector = (state: UnifiedHistogramState) => state.lensAdapters; export const lensEmbeddableOutputSelector$ = (state: UnifiedHistogramState) => state.lensEmbeddableOutput$; diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_request_params.test.ts b/src/plugins/unified_histogram/public/hooks/use_request_params.test.ts similarity index 95% rename from src/plugins/unified_histogram/public/chart/hooks/use_request_params.test.ts rename to src/plugins/unified_histogram/public/hooks/use_request_params.test.ts index f3889d1de6a42..c49bcd4ce195b 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_request_params.test.ts +++ b/src/plugins/unified_histogram/public/hooks/use_request_params.test.ts @@ -7,7 +7,7 @@ */ import { renderHook } from '@testing-library/react-hooks'; -import { unifiedHistogramServicesMock } from '../../__mocks__/services'; +import { unifiedHistogramServicesMock } from '../__mocks__/services'; const getUseRequestParams = async () => { jest.doMock('@kbn/data-plugin/common', () => { diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_request_params.tsx b/src/plugins/unified_histogram/public/hooks/use_request_params.tsx similarity index 85% rename from src/plugins/unified_histogram/public/chart/hooks/use_request_params.tsx rename to src/plugins/unified_histogram/public/hooks/use_request_params.tsx index c5ea702f898f0..dfa58629903ef 100644 --- a/src/plugins/unified_histogram/public/chart/hooks/use_request_params.tsx +++ b/src/plugins/unified_histogram/public/hooks/use_request_params.tsx @@ -9,9 +9,17 @@ import { getAbsoluteTimeRange } from '@kbn/data-plugin/common'; import type { AggregateQuery, Filter, Query, TimeRange } from '@kbn/es-query'; import { useCallback, useMemo, useRef } from 'react'; -import type { UnifiedHistogramServices } from '../../types'; +import type { UnifiedHistogramServices } from '../types'; import { useStableCallback } from './use_stable_callback'; +export interface UseRequestParamsResult { + query: Query | AggregateQuery; + filters: Filter[]; + relativeTimeRange: TimeRange; + getTimeRange: () => TimeRange; + updateTimeRange: () => void; +} + export const useRequestParams = ({ services, query: originalQuery, @@ -22,7 +30,7 @@ export const useRequestParams = ({ query?: Query | AggregateQuery; filters?: Filter[]; timeRange?: TimeRange; -}) => { +}): UseRequestParamsResult => { const { data } = services; const filters = useMemo(() => originalFilters ?? [], [originalFilters]); diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_stable_callback.test.ts b/src/plugins/unified_histogram/public/hooks/use_stable_callback.test.ts similarity index 100% rename from src/plugins/unified_histogram/public/chart/hooks/use_stable_callback.test.ts rename to src/plugins/unified_histogram/public/hooks/use_stable_callback.test.ts diff --git a/src/plugins/unified_histogram/public/chart/hooks/use_stable_callback.ts b/src/plugins/unified_histogram/public/hooks/use_stable_callback.ts similarity index 100% rename from src/plugins/unified_histogram/public/chart/hooks/use_stable_callback.ts rename to src/plugins/unified_histogram/public/hooks/use_stable_callback.ts diff --git a/src/plugins/unified_histogram/public/index.ts b/src/plugins/unified_histogram/public/index.ts index 5b32836bfb258..08f79f7e2ee94 100644 --- a/src/plugins/unified_histogram/public/index.ts +++ b/src/plugins/unified_histogram/public/index.ts @@ -28,7 +28,9 @@ export type { UnifiedHistogramServices, UnifiedHistogramChartLoadEvent, UnifiedHistogramAdapters, + UnifiedHistogramVisContext, } from './types'; -export { UnifiedHistogramFetchStatus } from './types'; +export { UnifiedHistogramFetchStatus, UnifiedHistogramExternalVisContextStatus } from './types'; +export { canImportVisContext } from './utils/external_vis_context'; export const plugin = () => new UnifiedHistogramPublicPlugin(); diff --git a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts b/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts deleted file mode 100644 index f74cc8a3c5925..0000000000000 --- a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import { renderHook } from '@testing-library/react-hooks'; -import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; -import { calculateBounds } from '@kbn/data-plugin/public'; -import { deepMockedFields, buildDataViewMock } from '@kbn/discover-utils/src/__mocks__'; -import { allSuggestionsMock } from '../../__mocks__/suggestions'; -import { useLensSuggestions } from './use_lens_suggestions'; - -describe('useLensSuggestions', () => { - const dataMock = dataPluginMock.createStartContract(); - dataMock.query.timefilter.timefilter.calculateBounds = (timeRange) => { - return calculateBounds(timeRange); - }; - const dataViewMock = buildDataViewMock({ - name: 'the-data-view', - fields: deepMockedFields, - timeFieldName: '@timestamp', - }); - - test('should return empty suggestions for non aggregate query', async () => { - const { result } = renderHook(() => { - return useLensSuggestions({ - dataView: dataViewMock, - query: undefined, - isPlainRecord: false, - data: dataMock, - lensSuggestionsApi: jest.fn(), - }); - }); - const current = result.current; - expect(current).toStrictEqual({ - allSuggestions: [], - currentSuggestion: undefined, - isOnHistogramMode: false, - histogramQuery: undefined, - suggestionUnsupported: false, - }); - }); - - test('should return suggestions for aggregate query', async () => { - const { result } = renderHook(() => { - return useLensSuggestions({ - dataView: dataViewMock, - query: { esql: 'from the-data-view | stats maxB = max(bytes)' }, - isPlainRecord: true, - columns: [ - { - id: 'var0', - name: 'var0', - meta: { - type: 'number', - }, - }, - ], - data: dataMock, - lensSuggestionsApi: jest.fn(() => allSuggestionsMock), - }); - }); - const current = result.current; - expect(current).toStrictEqual({ - allSuggestions: allSuggestionsMock, - currentSuggestion: allSuggestionsMock[0], - isOnHistogramMode: false, - histogramQuery: undefined, - suggestionUnsupported: false, - }); - }); - - test('should return suggestionUnsupported if no timerange is provided and no suggestions returned by the api', async () => { - const { result } = renderHook(() => { - return useLensSuggestions({ - dataView: dataViewMock, - query: { esql: 'from the-data-view | stats maxB = max(bytes)' }, - isPlainRecord: true, - columns: [ - { - id: 'var0', - name: 'var0', - meta: { - type: 'number', - }, - }, - ], - data: dataMock, - lensSuggestionsApi: jest.fn(), - }); - }); - const current = result.current; - expect(current).toStrictEqual({ - allSuggestions: [], - currentSuggestion: undefined, - isOnHistogramMode: false, - histogramQuery: undefined, - suggestionUnsupported: true, - }); - }); - - test('should return histogramSuggestion if no suggestions returned by the api', async () => { - const firstMockReturn = undefined; - const secondMockReturn = allSuggestionsMock; - const lensSuggestionsApi = jest - .fn() - .mockReturnValueOnce(firstMockReturn) // will return to firstMockReturn object firstly - .mockReturnValueOnce(secondMockReturn); // will return to secondMockReturn object secondly - - const { result } = renderHook(() => { - return useLensSuggestions({ - dataView: dataViewMock, - query: { esql: 'from the-data-view | limit 100' }, - isPlainRecord: true, - columns: [ - { - id: 'var0', - name: 'var0', - meta: { - type: 'number', - }, - }, - ], - data: dataMock, - lensSuggestionsApi, - timeRange: { - from: '2023-09-03T08:00:00.000Z', - to: '2023-09-04T08:56:28.274Z', - }, - }); - }); - const current = result.current; - expect(current).toStrictEqual({ - allSuggestions: [], - currentSuggestion: allSuggestionsMock[0], - isOnHistogramMode: true, - histogramQuery: { - esql: 'from the-data-view | limit 100 | EVAL timestamp=DATE_TRUNC(30 minute, @timestamp) | stats results = count(*) by timestamp | rename timestamp as `@timestamp every 30 minute`', - }, - suggestionUnsupported: false, - }); - }); - - test('should return histogramSuggestion even if the ESQL query contains a DROP @timestamp statement', async () => { - const firstMockReturn = undefined; - const secondMockReturn = allSuggestionsMock; - const lensSuggestionsApi = jest - .fn() - .mockReturnValueOnce(firstMockReturn) // will return to firstMockReturn object firstly - .mockReturnValueOnce(secondMockReturn); // will return to secondMockReturn object secondly - - renderHook(() => { - return useLensSuggestions({ - dataView: dataViewMock, - query: { esql: 'from the-data-view | DROP @timestamp | limit 100' }, - isPlainRecord: true, - columns: [ - { - id: 'var0', - name: 'var0', - meta: { - type: 'number', - }, - }, - ], - data: dataMock, - lensSuggestionsApi, - timeRange: { - from: '2023-09-03T08:00:00.000Z', - to: '2023-09-04T08:56:28.274Z', - }, - }); - }); - expect(lensSuggestionsApi).toHaveBeenLastCalledWith( - expect.objectContaining({ - query: { esql: expect.stringMatching('from the-data-view | limit 100 ') }, - }), - expect.anything(), - ['lnsDatatable'] - ); - }); - - test('should not return histogramSuggestion if no suggestions returned by the api and transformational commands', async () => { - const firstMockReturn = undefined; - const secondMockReturn = allSuggestionsMock; - const lensSuggestionsApi = jest - .fn() - .mockReturnValueOnce(firstMockReturn) // will return to firstMockReturn object firstly - .mockReturnValueOnce(secondMockReturn); // will return to secondMockReturn object secondly - - const { result } = renderHook(() => { - return useLensSuggestions({ - dataView: dataViewMock, - query: { esql: 'from the-data-view | limit 100 | keep @timestamp' }, - isPlainRecord: true, - columns: [ - { - id: 'var0', - name: 'var0', - meta: { - type: 'number', - }, - }, - ], - data: dataMock, - lensSuggestionsApi, - timeRange: { - from: '2023-09-03T08:00:00.000Z', - to: '2023-09-04T08:56:28.274Z', - }, - }); - }); - const current = result.current; - expect(current).toStrictEqual({ - allSuggestions: [], - currentSuggestion: undefined, - isOnHistogramMode: false, - histogramQuery: undefined, - suggestionUnsupported: true, - }); - }); -}); diff --git a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts b/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts deleted file mode 100644 index c45a8c1d701a6..0000000000000 --- a/src/plugins/unified_histogram/public/layout/hooks/use_lens_suggestions.ts +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import { DataView } from '@kbn/data-views-plugin/common'; -import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import { removeDropCommandsFromESQLQuery } from '@kbn/esql-utils'; -import { - AggregateQuery, - isOfAggregateQueryType, - getAggregateQueryMode, - Query, - TimeRange, -} from '@kbn/es-query'; -import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/common'; -import { LensSuggestionsApi, Suggestion } from '@kbn/lens-plugin/public'; -import { isEqual } from 'lodash'; -import { useEffect, useMemo, useRef, useState } from 'react'; -import { computeInterval } from './compute_interval'; -import { shouldDisplayHistogram } from '../helpers'; - -export const useLensSuggestions = ({ - dataView, - query, - originalSuggestion, - isPlainRecord, - columns, - data, - timeRange, - lensSuggestionsApi, - onSuggestionChange, -}: { - dataView: DataView; - query?: Query | AggregateQuery; - originalSuggestion?: Suggestion; - isPlainRecord?: boolean; - columns?: DatatableColumn[]; - data: DataPublicPluginStart; - timeRange?: TimeRange; - lensSuggestionsApi: LensSuggestionsApi; - onSuggestionChange?: (suggestion: Suggestion | undefined) => void; - table?: Datatable; -}) => { - const suggestions = useMemo(() => { - const context = { - dataViewSpec: dataView?.toSpec(), - fieldName: '', - textBasedColumns: columns, - query: query && isOfAggregateQueryType(query) ? query : undefined, - }; - const allSuggestions = isPlainRecord - ? lensSuggestionsApi(context, dataView, ['lnsDatatable']) ?? [] - : []; - - const [firstSuggestion] = allSuggestions; - - return { firstSuggestion, allSuggestions }; - }, [dataView, columns, query, isPlainRecord, lensSuggestionsApi]); - - const [allSuggestions, setAllSuggestions] = useState(suggestions.allSuggestions); - const currentSuggestion = originalSuggestion || suggestions.firstSuggestion; - - const suggestionDeps = useRef(getSuggestionDeps({ dataView, query, columns })); - const histogramQuery = useRef(); - const histogramSuggestion = useMemo(() => { - if ( - !currentSuggestion && - dataView.isTimeBased() && - query && - isOfAggregateQueryType(query) && - getAggregateQueryMode(query) === 'esql' && - timeRange - ) { - const isOnHistogramMode = shouldDisplayHistogram(query); - if (!isOnHistogramMode) return undefined; - - const interval = computeInterval(timeRange, data); - const language = getAggregateQueryMode(query); - const safeQuery = removeDropCommandsFromESQLQuery(query[language]); - const esqlQuery = `${safeQuery} | EVAL timestamp=DATE_TRUNC(${interval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp | rename timestamp as \`${dataView.timeFieldName} every ${interval}\``; - const context = { - dataViewSpec: dataView?.toSpec(), - fieldName: '', - textBasedColumns: [ - { - id: `${dataView.timeFieldName} every ${interval}`, - name: `${dataView.timeFieldName} every ${interval}`, - meta: { - type: 'date', - }, - }, - { - id: 'results', - name: 'results', - meta: { - type: 'number', - }, - }, - ] as DatatableColumn[], - query: { - esql: esqlQuery, - }, - }; - const sug = lensSuggestionsApi(context, dataView, ['lnsDatatable']) ?? []; - if (sug.length) { - histogramQuery.current = { esql: esqlQuery }; - return sug[0]; - } - } - histogramQuery.current = undefined; - return undefined; - }, [currentSuggestion, dataView, query, timeRange, data, lensSuggestionsApi]); - - useEffect(() => { - const newSuggestionsDeps = getSuggestionDeps({ dataView, query, columns }); - - if (!isEqual(suggestionDeps.current, newSuggestionsDeps)) { - setAllSuggestions(suggestions.allSuggestions); - onSuggestionChange?.(suggestions.firstSuggestion); - - suggestionDeps.current = newSuggestionsDeps; - } - }, [ - columns, - dataView, - onSuggestionChange, - query, - suggestions.firstSuggestion, - suggestions.allSuggestions, - ]); - - return { - allSuggestions, - currentSuggestion: histogramSuggestion ?? currentSuggestion, - suggestionUnsupported: !currentSuggestion && !histogramSuggestion && isPlainRecord, - isOnHistogramMode: Boolean(histogramSuggestion), - histogramQuery: histogramQuery.current ? histogramQuery.current : undefined, - }; -}; - -const getSuggestionDeps = ({ - dataView, - query, - columns, -}: { - dataView: DataView; - query?: Query | AggregateQuery; - columns?: DatatableColumn[]; -}) => [dataView.id, columns, query]; diff --git a/src/plugins/unified_histogram/public/layout/layout.test.tsx b/src/plugins/unified_histogram/public/layout/layout.test.tsx index a10df63e7c328..dcb96b093cac7 100644 --- a/src/plugins/unified_histogram/public/layout/layout.test.tsx +++ b/src/plugins/unified_histogram/public/layout/layout.test.tsx @@ -76,6 +76,7 @@ describe('Layout', () => { to: '2020-05-14T11:20:13.590', }} lensSuggestionsApi={jest.fn()} + onSuggestionContextChange={jest.fn()} isChartLoading={false} {...rest} /> diff --git a/src/plugins/unified_histogram/public/layout/layout.tsx b/src/plugins/unified_histogram/public/layout/layout.tsx index aaeb67b15b101..5ceae61e13a9e 100644 --- a/src/plugins/unified_histogram/public/layout/layout.tsx +++ b/src/plugins/unified_histogram/public/layout/layout.tsx @@ -7,8 +7,9 @@ */ import { EuiSpacer, useEuiTheme, useIsWithinBreakpoints } from '@elastic/eui'; -import React, { PropsWithChildren, ReactElement, useMemo, useState } from 'react'; +import React, { PropsWithChildren, ReactElement, useEffect, useMemo, useState } from 'react'; import { Observable } from 'rxjs'; +import useObservable from 'react-use/lib/useObservable'; import { createHtmlPortalNode, InPortal, OutPortal } from 'react-reverse-portal'; import { css } from '@emotion/css'; import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/common'; @@ -18,28 +19,30 @@ import type { LensEmbeddableInput, LensEmbeddableOutput, LensSuggestionsApi, - Suggestion, } from '@kbn/lens-plugin/public'; -import { AggregateQuery, Filter, isOfAggregateQueryType, Query, TimeRange } from '@kbn/es-query'; +import { AggregateQuery, Filter, Query, TimeRange } from '@kbn/es-query'; import { ResizableLayout, - ResizableLayoutMode, ResizableLayoutDirection, + ResizableLayoutMode, } from '@kbn/resizable-layout'; -import { TextBasedPersistedState } from '@kbn/lens-plugin/public/datasources/text_based/types'; import { Chart, checkChartAvailability } from '../chart'; -import type { - UnifiedHistogramChartContext, - UnifiedHistogramServices, - UnifiedHistogramHitsContext, +import { + UnifiedHistogramVisContext, UnifiedHistogramBreakdownContext, - UnifiedHistogramFetchStatus, - UnifiedHistogramRequestContext, + UnifiedHistogramChartContext, UnifiedHistogramChartLoadEvent, + UnifiedHistogramFetchStatus, + UnifiedHistogramHitsContext, UnifiedHistogramInput$, + UnifiedHistogramRequestContext, + UnifiedHistogramServices, + UnifiedHistogramSuggestionContext, + UnifiedHistogramExternalVisContextStatus, } from '../types'; -import { useLensSuggestions } from './hooks/use_lens_suggestions'; -import { shouldDisplayHistogram } from './helpers'; +import { UnifiedHistogramSuggestionType } from '../types'; +import { LensVisService } from '../services/lens_vis_service'; +import { useRequestParams } from '../hooks/use_request_params'; const ChartMemoized = React.memo(Chart); @@ -67,9 +70,9 @@ export interface UnifiedHistogramLayoutProps extends PropsWithChildren */ filters?: Filter[]; /** - * The current Lens suggestion + * The external custom Lens vis */ - currentSuggestion?: Suggestion; + externalVisContext?: UnifiedHistogramVisContext; /** * Flag that indicates that a text based language is used */ @@ -159,7 +162,16 @@ export interface UnifiedHistogramLayoutProps extends PropsWithChildren /** * Callback to update the suggested chart */ - onSuggestionChange?: (suggestion: Suggestion | undefined) => void; + onSuggestionContextChange: ( + suggestionContext: UnifiedHistogramSuggestionContext | undefined + ) => void; + /** + * Callback to notify about the change in Lens attributes + */ + onVisContextChanged?: ( + visContext: UnifiedHistogramVisContext | undefined, + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus + ) => void; /** * Callback to update the total hits -- should set {@link UnifiedHistogramHitsContext.status} to status * and {@link UnifiedHistogramHitsContext.total} to result @@ -190,12 +202,12 @@ export const UnifiedHistogramLayout = ({ className, services, dataView, - query, - filters, - currentSuggestion: originalSuggestion, + query: originalQuery, + filters: originalFilters, + externalVisContext, isChartLoading, isPlainRecord, - timeRange, + timeRange: originalTimeRange, relativeTimeRange, columns, request, @@ -217,7 +229,8 @@ export const UnifiedHistogramLayout = ({ onChartHiddenChange, onTimeIntervalChange, onBreakdownFieldChange, - onSuggestionChange, + onSuggestionContextChange, + onVisContextChanged, onTotalHitsChange, onChartLoad, onFilter, @@ -226,55 +239,75 @@ export const UnifiedHistogramLayout = ({ withDefaultActions, abortController, }: UnifiedHistogramLayoutProps) => { - const { - allSuggestions, - currentSuggestion, - suggestionUnsupported, - isOnHistogramMode, - histogramQuery, - } = useLensSuggestions({ - dataView, - query, - originalSuggestion, - isPlainRecord, - columns, - timeRange, - data: services.data, - lensSuggestionsApi, - onSuggestionChange, - }); + const columnsMap = useMemo(() => { + if (!columns?.length) { + return undefined; + } - // apply table to current suggestion - const usedSuggestion = useMemo(() => { - if ( - currentSuggestion && - table && - query && - isOfAggregateQueryType(query) && - !shouldDisplayHistogram(query) - ) { - const { layers } = currentSuggestion.datasourceState as TextBasedPersistedState; + return columns.reduce((acc, column) => { + acc[column.id] = column; + return acc; + }, {} as Record); + }, [columns]); - const newState = { - ...currentSuggestion, - datasourceState: { - ...(currentSuggestion.datasourceState as TextBasedPersistedState), - layers: {} as Record, - }, - }; + const requestParams = useRequestParams({ + services, + query: originalQuery, + filters: originalFilters, + timeRange: originalTimeRange, + }); - for (const key of Object.keys(layers)) { - const newLayer = { ...layers[key], table }; - newState.datasourceState.layers[key] = newLayer; - } + const [lensVisService] = useState(() => new LensVisService({ services, lensSuggestionsApi })); + const lensVisServiceCurrentSuggestionContext = useObservable( + lensVisService.currentSuggestionContext$ + ); - return newState; - } else { - return currentSuggestion; + const originalChartTimeInterval = originalChart?.timeInterval; + useEffect(() => { + if (isChartLoading) { + return; } - }, [currentSuggestion, query, table]); - const chart = suggestionUnsupported ? undefined : originalChart; + lensVisService.update({ + externalVisContext, + queryParams: { + dataView, + query: requestParams.query, + filters: requestParams.filters, + timeRange: originalTimeRange, + isPlainRecord, + columns, + columnsMap, + }, + timeInterval: originalChartTimeInterval, + breakdownField: breakdown?.field, + table, + onSuggestionContextChange, + onVisContextChanged: isPlainRecord ? onVisContextChanged : undefined, + }); + }, [ + lensVisService, + dataView, + requestParams.query, + requestParams.filters, + originalTimeRange, + originalChartTimeInterval, + isPlainRecord, + columns, + columnsMap, + breakdown, + externalVisContext, + onSuggestionContextChange, + onVisContextChanged, + isChartLoading, + table, + ]); + + const chart = + !lensVisServiceCurrentSuggestionContext?.type || + lensVisServiceCurrentSuggestionContext.type === UnifiedHistogramSuggestionType.unsupported + ? undefined + : originalChart; const isChartAvailable = checkChartAvailability({ chart, dataView, isPlainRecord }); const [topPanelNode] = useState(() => @@ -315,15 +348,12 @@ export const UnifiedHistogramLayout = ({ className={chartClassName} services={services} dataView={dataView} - query={query} - filters={filters} - timeRange={timeRange} + requestParams={requestParams} relativeTimeRange={relativeTimeRange} request={request} hits={hits} - currentSuggestion={usedSuggestion} + lensVisService={lensVisService} isChartLoading={isChartLoading} - allSuggestions={allSuggestions} isPlainRecord={isPlainRecord} chart={chart} breakdown={breakdown} @@ -336,15 +366,12 @@ export const UnifiedHistogramLayout = ({ onChartHiddenChange={onChartHiddenChange} onTimeIntervalChange={onTimeIntervalChange} onBreakdownFieldChange={onBreakdownFieldChange} - onSuggestionChange={onSuggestionChange} onTotalHitsChange={onTotalHitsChange} onChartLoad={onChartLoad} onFilter={onFilter} onBrushEnd={onBrushEnd} lensAdapters={lensAdapters} lensEmbeddableOutput$={lensEmbeddableOutput$} - isOnHistogramMode={isOnHistogramMode} - histogramQuery={histogramQuery} withDefaultActions={withDefaultActions} /> diff --git a/src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.test.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.attributes.test.ts similarity index 87% rename from src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.test.ts rename to src/plugins/unified_histogram/public/services/lens_vis_service.attributes.test.ts index 3c049649d5c20..780069747a64a 100644 --- a/src/plugins/unified_histogram/public/chart/utils/get_lens_attributes.test.ts +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.attributes.test.ts @@ -6,13 +6,16 @@ * Side Public License, v 1. */ -import { getLensAttributes } from './get_lens_attributes'; import { AggregateQuery, Filter, FilterStateStore, Query } from '@kbn/es-query'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import { dataViewWithTimefieldMock } from '../../__mocks__/data_view_with_timefield'; -import { currentSuggestionMock } from '../../__mocks__/suggestions'; +import { + dataViewWithTimefieldMock, + dataViewWithAtTimefieldMock, +} from '../__mocks__/data_view_with_timefield'; +import { currentSuggestionMock } from '../__mocks__/suggestions'; +import { getLensVisMock } from '../__mocks__/lens_vis'; -describe('getLensAttributes', () => { +describe('LensVisService attributes', () => { const dataView: DataView = dataViewWithTimefieldMock; const filters: Filter[] = [ { @@ -41,29 +44,25 @@ describe('getLensAttributes', () => { }, ]; const query: Query | AggregateQuery = { language: 'kuery', query: 'extension : css' }; + const queryEsql: Query | AggregateQuery = { esql: 'from logstash-* | limit 10' }; const timeInterval = 'auto'; - it('should return correct attributes', () => { + it('should return correct attributes', async () => { const breakdownField: DataViewField | undefined = undefined; - expect( - getLensAttributes({ - title: 'test', - filters, - query, - dataView, - timeInterval, - breakdownField, - suggestion: undefined, - }) - ).toMatchInlineSnapshot(` + const lensVis = await getLensVisMock({ + filters, + query, + dataView, + timeInterval, + breakdownField, + columns: [], + isPlainRecord: false, + }); + + expect(lensVis.visContext).toMatchInlineSnapshot(` Object { "attributes": Object { "references": Array [ - Object { - "id": "index-pattern-with-timefield-id", - "name": "indexpattern-datasource-current-indexpattern", - "type": "index-pattern", - }, Object { "id": "index-pattern-with-timefield-id", "name": "indexpattern-datasource-layer-unifiedHistogram", @@ -187,7 +186,7 @@ describe('getLensAttributes', () => { "valueLabels": "hide", }, }, - "title": "test", + "title": "Edit visualization", "visualizationType": "lnsXY", }, "requestData": Object { @@ -196,33 +195,28 @@ describe('getLensAttributes', () => { "timeField": "timestamp", "timeInterval": "auto", }, + "suggestionType": "histogramForDataView", } `); }); - it('should return correct attributes with breakdown field', () => { + it('should return correct attributes with breakdown field', async () => { const breakdownField: DataViewField | undefined = dataView.fields.find( (f) => f.name === 'extension' ); - expect( - getLensAttributes({ - title: 'test', - filters, - query, - dataView, - timeInterval, - breakdownField, - suggestion: undefined, - }) - ).toMatchInlineSnapshot(` + const lensVis = await getLensVisMock({ + filters, + query, + dataView, + timeInterval, + breakdownField, + columns: [], + isPlainRecord: false, + }); + expect(lensVis.visContext).toMatchInlineSnapshot(` Object { "attributes": Object { "references": Array [ - Object { - "id": "index-pattern-with-timefield-id", - "name": "indexpattern-datasource-current-indexpattern", - "type": "index-pattern", - }, Object { "id": "index-pattern-with-timefield-id", "name": "indexpattern-datasource-layer-unifiedHistogram", @@ -364,7 +358,7 @@ describe('getLensAttributes', () => { "valueLabels": "hide", }, }, - "title": "test", + "title": "Edit visualization", "visualizationType": "lnsXY", }, "requestData": Object { @@ -373,33 +367,28 @@ describe('getLensAttributes', () => { "timeField": "timestamp", "timeInterval": "auto", }, + "suggestionType": "histogramForDataView", } `); }); - it('should return correct attributes with unsupported breakdown field', () => { + it('should return correct attributes with unsupported breakdown field', async () => { const breakdownField: DataViewField | undefined = dataView.fields.find( (f) => f.name === 'scripted' ); - expect( - getLensAttributes({ - title: 'test', - filters, - query, - dataView, - timeInterval, - breakdownField, - suggestion: undefined, - }) - ).toMatchInlineSnapshot(` + const lensVis = await getLensVisMock({ + filters, + query, + dataView, + timeInterval, + breakdownField, + columns: [], + isPlainRecord: false, + }); + expect(lensVis.visContext).toMatchInlineSnapshot(` Object { "attributes": Object { "references": Array [ - Object { - "id": "index-pattern-with-timefield-id", - "name": "indexpattern-datasource-current-indexpattern", - "type": "index-pattern", - }, Object { "id": "index-pattern-with-timefield-id", "name": "indexpattern-datasource-layer-unifiedHistogram", @@ -523,7 +512,7 @@ describe('getLensAttributes', () => { "valueLabels": "hide", }, }, - "title": "test", + "title": "Edit visualization", "visualizationType": "lnsXY", }, "requestData": Object { @@ -532,33 +521,28 @@ describe('getLensAttributes', () => { "timeField": "timestamp", "timeInterval": "auto", }, + "suggestionType": "histogramForDataView", } `); }); - it('should return correct attributes for text based languages', () => { - expect( - getLensAttributes({ - title: 'test', - filters, - query, - dataView, - timeInterval, - breakdownField: undefined, - suggestion: currentSuggestionMock, - }) - ).toMatchInlineSnapshot(` + it('should return correct attributes for text based languages', async () => { + const lensVis = await getLensVisMock({ + filters, + query: queryEsql, + dataView, + timeInterval, + breakdownField: undefined, + columns: [], + isPlainRecord: true, + }); + expect(lensVis.visContext).toMatchInlineSnapshot(` Object { "attributes": Object { "references": Array [ Object { "id": "index-pattern-with-timefield-id", - "name": "indexpattern-datasource-current-indexpattern", - "type": "index-pattern", - }, - Object { - "id": "index-pattern-with-timefield-id", - "name": "indexpattern-datasource-layer-unifiedHistogram", + "name": "textBasedLanguages-datasource-layer-suggestion", "type": "index-pattern", }, ], @@ -695,8 +679,7 @@ describe('getLensAttributes', () => { }, ], "query": Object { - "language": "kuery", - "query": "extension : css", + "esql": "from logstash-* | limit 10", }, "visualization": Object { "gridConfig": Object { @@ -719,34 +702,35 @@ describe('getLensAttributes', () => { "xAccessor": "81e332d6-ee37-42a8-a646-cea4fc75d2d3", }, }, - "title": "test", + "title": "Heat map", "visualizationType": "lnsHeatmap", }, "requestData": Object { "breakdownField": undefined, "dataViewId": "index-pattern-with-timefield-id", "timeField": "timestamp", - "timeInterval": "auto", + "timeInterval": undefined, }, + "suggestionType": "lensSuggestion", } `); }); - it('should return correct attributes for text based languages with adhoc dataview', () => { + it('should return correct attributes for text based languages with adhoc dataview', async () => { const adHocDataview = { ...dataView, isPersisted: () => false, } as DataView; - const lensAttrs = getLensAttributes({ - title: 'test', + const lensVis = await getLensVisMock({ filters, - query, + query: queryEsql, dataView: adHocDataview, timeInterval, breakdownField: undefined, - suggestion: currentSuggestionMock, + columns: [], + isPlainRecord: true, }); - expect(lensAttrs.attributes).toEqual({ + expect(lensVis.visContext?.attributes).toEqual({ state: expect.objectContaining({ adHocDataViews: { 'index-pattern-with-timefield-id': {}, @@ -755,31 +739,43 @@ describe('getLensAttributes', () => { references: [ { id: 'index-pattern-with-timefield-id', - name: 'indexpattern-datasource-current-indexpattern', - type: 'index-pattern', - }, - { - id: 'index-pattern-with-timefield-id', - name: 'indexpattern-datasource-layer-unifiedHistogram', + name: 'textBasedLanguages-datasource-layer-suggestion', type: 'index-pattern', }, ], - title: 'test', + title: 'Heat map', visualizationType: 'lnsHeatmap', }); }); - it('should return suggestion title if no title is given', () => { - expect( - getLensAttributes({ - title: undefined, - filters, - query, - dataView, - timeInterval, - breakdownField: undefined, - suggestion: currentSuggestionMock, - }).attributes.title - ).toBe(currentSuggestionMock.title); + it('should return suggestion title', async () => { + const lensVis = await getLensVisMock({ + filters, + query: queryEsql, + dataView, + timeInterval, + breakdownField: undefined, + columns: [], + isPlainRecord: true, + }); + expect(lensVis.visContext?.attributes.title).toBe(currentSuggestionMock.title); + }); + + it('should use the correct histogram query when no suggestion passed', async () => { + const histogramQuery = { + esql: 'from logstash-* | limit 10 | EVAL timestamp=DATE_TRUNC(10 minute, @timestamp) | stats results = count(*) by timestamp | rename timestamp as `@timestamp every 10 minute`', + }; + const lensVis = await getLensVisMock({ + filters, + query: queryEsql, + dataView: dataViewWithAtTimefieldMock, + timeInterval, + breakdownField: undefined, + columns: [], + isPlainRecord: true, + allSuggestions: [], // none available + hasHistogramSuggestionForESQL: true, + }); + expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); }); }); diff --git a/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts new file mode 100644 index 0000000000000..7993f933a8054 --- /dev/null +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.suggestions.test.ts @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import type { AggregateQuery, Query } from '@kbn/es-query'; +import { deepMockedFields, buildDataViewMock } from '@kbn/discover-utils/src/__mocks__'; +import { allSuggestionsMock } from '../__mocks__/suggestions'; +import { getLensVisMock } from '../__mocks__/lens_vis'; +import { UnifiedHistogramSuggestionType } from '../types'; + +describe('LensVisService suggestions', () => { + const dataViewMock = buildDataViewMock({ + name: 'the-data-view', + fields: deepMockedFields, + timeFieldName: '@timestamp', + }); + + test('should use a histogram fallback if suggestions are empty for non aggregate query', async () => { + const query: Query | AggregateQuery = { language: 'kuery', query: 'extension : css' }; + const lensVis = await getLensVisMock({ + filters: [], + query, + dataView: dataViewMock, + timeInterval: 'auto', + breakdownField: undefined, + columns: [], + isPlainRecord: false, + allSuggestions: [], + }); + + expect(lensVis.currentSuggestionContext?.type).toBe( + UnifiedHistogramSuggestionType.histogramForDataView + ); + expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined(); + }); + + test('should return suggestions for aggregate query', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | stats maxB = max(bytes)' }, + dataView: dataViewMock, + timeInterval: 'auto', + breakdownField: undefined, + columns: [ + { + id: 'var0', + name: 'var0', + meta: { + type: 'number', + }, + }, + ], + isPlainRecord: true, + allSuggestions: allSuggestionsMock, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe( + UnifiedHistogramSuggestionType.lensSuggestion + ); + expect(lensVis.currentSuggestionContext?.suggestion).toBe(allSuggestionsMock[0]); + }); + + test('should return suggestionUnsupported if no timerange is provided and no suggestions returned by the api', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | stats maxB = max(bytes)' }, + dataView: dataViewMock, + timeInterval: 'auto', + timeRange: null, + breakdownField: undefined, + columns: [ + { + id: 'var0', + name: 'var0', + meta: { + type: 'number', + }, + }, + ], + isPlainRecord: true, + allSuggestions: [], + hasHistogramSuggestionForESQL: false, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe(UnifiedHistogramSuggestionType.unsupported); + expect(lensVis.currentSuggestionContext?.suggestion).not.toBeDefined(); + }); + + test('should return histogramSuggestion if no suggestions returned by the api', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | limit 100' }, + dataView: dataViewMock, + timeInterval: 'auto', + timeRange: { + from: '2023-09-03T08:00:00.000Z', + to: '2023-09-04T08:56:28.274Z', + }, + breakdownField: undefined, + columns: [ + { + id: 'var0', + name: 'var0', + meta: { + type: 'number', + }, + }, + ], + isPlainRecord: true, + allSuggestions: [], + hasHistogramSuggestionForESQL: true, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe( + UnifiedHistogramSuggestionType.histogramForESQL + ); + expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined(); + + const histogramQuery = { + esql: 'from the-data-view | limit 100 | EVAL timestamp=DATE_TRUNC(30 minute, @timestamp) | stats results = count(*) by timestamp | rename timestamp as `@timestamp every 30 minute`', + }; + + expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); + }); + + test('should return histogramSuggestion even if the ESQL query contains a DROP @timestamp statement', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | DROP @timestamp | limit 100' }, + dataView: dataViewMock, + timeInterval: 'auto', + timeRange: { + from: '2023-09-03T08:00:00.000Z', + to: '2023-09-04T08:56:28.274Z', + }, + breakdownField: undefined, + columns: [ + { + id: 'var0', + name: 'var0', + meta: { + type: 'number', + }, + }, + ], + isPlainRecord: true, + allSuggestions: [], + hasHistogramSuggestionForESQL: true, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe( + UnifiedHistogramSuggestionType.histogramForESQL + ); + expect(lensVis.currentSuggestionContext?.suggestion).toBeDefined(); + + const histogramQuery = { + esql: 'from the-data-view | limit 100 | EVAL timestamp=DATE_TRUNC(30 minute, @timestamp) | stats results = count(*) by timestamp | rename timestamp as `@timestamp every 30 minute`', + }; + + expect(lensVis.visContext?.attributes.state.query).toStrictEqual(histogramQuery); + }); + + test('should not return histogramSuggestion if no suggestions returned by the api and transformational commands', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: { esql: 'from the-data-view | limit 100 | keep @timestamp' }, + dataView: dataViewMock, + timeInterval: 'auto', + timeRange: { + from: '2023-09-03T08:00:00.000Z', + to: '2023-09-04T08:56:28.274Z', + }, + breakdownField: undefined, + columns: [ + { + id: 'var0', + name: 'var0', + meta: { + type: 'number', + }, + }, + ], + isPlainRecord: true, + allSuggestions: [], + hasHistogramSuggestionForESQL: true, + }); + + expect(lensVis.currentSuggestionContext?.type).toBe(UnifiedHistogramSuggestionType.unsupported); + expect(lensVis.currentSuggestionContext?.suggestion).not.toBeDefined(); + }); +}); diff --git a/src/plugins/unified_histogram/public/services/lens_vis_service.ts b/src/plugins/unified_histogram/public/services/lens_vis_service.ts new file mode 100644 index 0000000000000..7b1cf7cdaf55e --- /dev/null +++ b/src/plugins/unified_histogram/public/services/lens_vis_service.ts @@ -0,0 +1,754 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { BehaviorSubject, distinctUntilChanged, map, Observable } from 'rxjs'; +import { isEqual } from 'lodash'; +import { removeDropCommandsFromESQLQuery } from '@kbn/esql-utils'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/common'; +import type { + CountIndexPatternColumn, + DateHistogramIndexPatternColumn, + GenericIndexPatternColumn, + LensSuggestionsApi, + Suggestion, + TermsIndexPatternColumn, + TypedLensByValueInput, +} from '@kbn/lens-plugin/public'; +import type { AggregateQuery, Query, TimeRange } from '@kbn/es-query'; +import { Filter, getAggregateQueryMode, isOfAggregateQueryType } from '@kbn/es-query'; +import { i18n } from '@kbn/i18n'; +import { getLensAttributesFromSuggestion } from '@kbn/visualization-utils'; +import { LegendSize } from '@kbn/visualizations-plugin/public'; +import { XYConfiguration } from '@kbn/visualizations-plugin/common'; +import type { Datatable, DatatableColumn } from '@kbn/expressions-plugin/common'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import { + UnifiedHistogramExternalVisContextStatus, + UnifiedHistogramSuggestionContext, + UnifiedHistogramSuggestionType, + UnifiedHistogramVisContext, +} from '../types'; +import { isSuggestionShapeAndVisContextCompatible } from '../utils/external_vis_context'; +import { computeInterval } from '../utils/compute_interval'; +import { fieldSupportsBreakdown } from '../utils/field_supports_breakdown'; +import { shouldDisplayHistogram } from '../layout/helpers'; +import { enrichLensAttributesWithTablesData } from '../utils/lens_vis_from_table'; + +const UNIFIED_HISTOGRAM_LAYER_ID = 'unifiedHistogram'; + +const stateSelectorFactory = + (state$: Observable) => + (selector: (state: S) => R, equalityFn?: (arg0: R, arg1: R) => boolean) => + state$.pipe(map(selector), distinctUntilChanged(equalityFn)); + +export enum LensVisServiceStatus { + 'initial' = 'initial', + 'completed' = 'completed', +} + +interface LensVisServiceState { + status: LensVisServiceStatus; + allSuggestions: Suggestion[] | undefined; + currentSuggestionContext: UnifiedHistogramSuggestionContext; + visContext: UnifiedHistogramVisContext | undefined; +} + +export interface QueryParams { + dataView: DataView; + query?: Query | AggregateQuery; + filters: Filter[] | undefined; + isPlainRecord?: boolean; + columns?: DatatableColumn[]; + columnsMap?: Record; + timeRange?: TimeRange; +} + +interface Services { + data: DataPublicPluginStart; +} + +interface LensVisServiceParams { + services: Services; + lensSuggestionsApi: LensSuggestionsApi; +} + +export class LensVisService { + private state$: BehaviorSubject; + private services: Services; + private lensSuggestionsApi: LensSuggestionsApi; + status$: Observable; + currentSuggestionContext$: Observable; + allSuggestions$: Observable; + visContext$: Observable; + prevUpdateContext: + | { + queryParams: QueryParams; + timeInterval: string | undefined; + breakdownField: DataViewField | undefined; + table: Datatable | undefined; + onSuggestionContextChange: ( + suggestionContext: UnifiedHistogramSuggestionContext | undefined + ) => void; + onVisContextChanged?: ( + visContext: UnifiedHistogramVisContext | undefined, + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus + ) => void; + } + | undefined; + + constructor({ services, lensSuggestionsApi }: LensVisServiceParams) { + this.services = services; + this.lensSuggestionsApi = lensSuggestionsApi; + + this.state$ = new BehaviorSubject({ + status: LensVisServiceStatus.initial, + allSuggestions: undefined, + currentSuggestionContext: { + suggestion: undefined, + type: UnifiedHistogramSuggestionType.unsupported, + }, + visContext: undefined, + }); + + const stateSelector = stateSelectorFactory(this.state$); + this.status$ = stateSelector((state) => state.status); + this.allSuggestions$ = stateSelector((state) => state.allSuggestions); + this.currentSuggestionContext$ = stateSelector( + (state) => state.currentSuggestionContext, + isEqual + ); + this.visContext$ = stateSelector((state) => state.visContext, isEqual); + this.prevUpdateContext = undefined; + } + + update = ({ + externalVisContext, + queryParams, + timeInterval, + breakdownField, + table, + onSuggestionContextChange, + onVisContextChanged, + }: { + externalVisContext: UnifiedHistogramVisContext | undefined; + queryParams: QueryParams; + timeInterval: string | undefined; + breakdownField: DataViewField | undefined; + table?: Datatable; + onSuggestionContextChange: ( + suggestionContext: UnifiedHistogramSuggestionContext | undefined + ) => void; + onVisContextChanged?: ( + visContext: UnifiedHistogramVisContext | undefined, + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus + ) => void; + }) => { + const allSuggestions = this.getAllSuggestions({ queryParams }); + + const suggestionState = this.getCurrentSuggestionState({ + externalVisContext, + queryParams, + allSuggestions, + timeInterval, + breakdownField, + }); + + const lensAttributesState = this.getLensAttributesState({ + currentSuggestionContext: suggestionState.currentSuggestionContext, + externalVisContext, + queryParams, + timeInterval, + breakdownField, + table, + }); + + onSuggestionContextChange(suggestionState.currentSuggestionContext); + onVisContextChanged?.( + lensAttributesState.visContext, + lensAttributesState.externalVisContextStatus + ); + + this.state$.next({ + status: LensVisServiceStatus.completed, + allSuggestions, + currentSuggestionContext: suggestionState.currentSuggestionContext, + visContext: lensAttributesState.visContext, + }); + + this.prevUpdateContext = { + queryParams, + timeInterval, + breakdownField, + table, + onSuggestionContextChange, + onVisContextChanged, + }; + }; + + onSuggestionEdited = ({ + editedSuggestionContext, + }: { + editedSuggestionContext: UnifiedHistogramSuggestionContext | undefined; + }): UnifiedHistogramVisContext | undefined => { + if (!editedSuggestionContext || !this.prevUpdateContext) { + return; + } + + const { queryParams, timeInterval, breakdownField, table, onVisContextChanged } = + this.prevUpdateContext; + + const lensAttributesState = this.getLensAttributesState({ + currentSuggestionContext: editedSuggestionContext, + externalVisContext: undefined, + queryParams, + timeInterval, + breakdownField, + table, + }); + + onVisContextChanged?.( + lensAttributesState.visContext, + UnifiedHistogramExternalVisContextStatus.manuallyCustomized + ); + }; + + private getCurrentSuggestionState = ({ + allSuggestions, + externalVisContext, + queryParams, + timeInterval, + breakdownField, + }: { + allSuggestions: Suggestion[]; + externalVisContext: UnifiedHistogramVisContext | undefined; + queryParams: QueryParams; + timeInterval: string | undefined; + breakdownField: DataViewField | undefined; + }): { + currentSuggestionContext: UnifiedHistogramSuggestionContext; + } => { + let type = UnifiedHistogramSuggestionType.unsupported; + let currentSuggestion: Suggestion | undefined; + + // takes lens suggestions if provided + const availableSuggestionsWithType = allSuggestions.map((lensSuggestion) => ({ + suggestion: lensSuggestion, + type: UnifiedHistogramSuggestionType.lensSuggestion, + })); + + if (queryParams.isPlainRecord) { + // appends an ES|QL histogram + const histogramSuggestionForESQL = this.getHistogramSuggestionForESQL({ queryParams }); + if (histogramSuggestionForESQL) { + availableSuggestionsWithType.push({ + suggestion: histogramSuggestionForESQL, + type: UnifiedHistogramSuggestionType.histogramForESQL, + }); + } + } else { + // appends histogram for the data view mode + const histogramSuggestionForDataView = this.getDefaultHistogramSuggestion({ + queryParams, + timeInterval, + breakdownField, + }); + if (histogramSuggestionForDataView) { + availableSuggestionsWithType.push({ + suggestion: histogramSuggestionForDataView, + type: UnifiedHistogramSuggestionType.histogramForDataView, + }); + } + } + + if (externalVisContext) { + // externalVisContext can be based on an unfamiliar suggestion, but it was saved somehow, so try to restore it too + const derivedSuggestion = deriveLensSuggestionFromLensAttributes({ + externalVisContext, + queryParams, + }); + + if ( + derivedSuggestion && + // it should be in a group of available lens suggestions + // for example, Pie is a subtype of Donut charts + allSuggestions.find((s) => s.visualizationId === derivedSuggestion.visualizationId) + ) { + availableSuggestionsWithType.push({ + suggestion: derivedSuggestion, + type: UnifiedHistogramSuggestionType.lensSuggestion, + }); + } + } + + if (externalVisContext) { + // try to find a suggestion that is compatible with the external vis context + const matchingItem = availableSuggestionsWithType.find((item) => + isSuggestionShapeAndVisContextCompatible(item.suggestion, externalVisContext) + ); + + if (matchingItem) { + currentSuggestion = matchingItem.suggestion; + type = matchingItem.type; + } + } + + if (!currentSuggestion && availableSuggestionsWithType.length) { + // otherwise pick any first available suggestion + currentSuggestion = availableSuggestionsWithType[0].suggestion; + type = availableSuggestionsWithType[0].type; + } + + return { + currentSuggestionContext: { + type: Boolean(currentSuggestion) ? type : UnifiedHistogramSuggestionType.unsupported, + suggestion: currentSuggestion, + }, + }; + }; + + private getDefaultHistogramSuggestion = ({ + queryParams, + timeInterval, + breakdownField, + }: { + queryParams: QueryParams; + timeInterval: string | undefined; + breakdownField: DataViewField | undefined; + }): Suggestion => { + const { dataView } = queryParams; + const showBreakdown = breakdownField && fieldSupportsBreakdown(breakdownField); + + let columnOrder = ['date_column', 'count_column']; + + if (showBreakdown) { + columnOrder = ['breakdown_column', ...columnOrder]; + } + + let columns: Record = { + date_column: { + dataType: 'date', + isBucketed: true, + label: dataView.timeFieldName ?? '', + operationType: 'date_histogram', + scale: 'interval', + sourceField: dataView.timeFieldName, + params: { + interval: timeInterval ?? 'auto', + }, + } as DateHistogramIndexPatternColumn, + count_column: { + dataType: 'number', + isBucketed: false, + label: i18n.translate('unifiedHistogram.countColumnLabel', { + defaultMessage: 'Count of records', + }), + operationType: 'count', + scale: 'ratio', + sourceField: '___records___', + params: { + format: { + id: 'number', + params: { + decimals: 0, + }, + }, + }, + } as CountIndexPatternColumn, + }; + + if (showBreakdown) { + columns = { + ...columns, + breakdown_column: { + dataType: 'string', + isBucketed: true, + label: i18n.translate('unifiedHistogram.breakdownColumnLabel', { + defaultMessage: 'Top 3 values of {fieldName}', + values: { fieldName: breakdownField?.displayName }, + }), + operationType: 'terms', + scale: 'ordinal', + sourceField: breakdownField.name, + params: { + size: 3, + orderBy: { + type: 'column', + columnId: 'count_column', + }, + orderDirection: 'desc', + otherBucket: true, + missingBucket: false, + parentFormat: { + id: 'terms', + }, + }, + } as TermsIndexPatternColumn, + }; + } + + const datasourceState = { + layers: { + [UNIFIED_HISTOGRAM_LAYER_ID]: { columnOrder, columns }, + }, + }; + + const visualizationState = { + layers: [ + { + accessors: ['count_column'], + layerId: UNIFIED_HISTOGRAM_LAYER_ID, + layerType: 'data', + seriesType: 'bar_stacked', + xAccessor: 'date_column', + ...(showBreakdown + ? { splitAccessor: 'breakdown_column' } + : { + yConfig: [ + { + forAccessor: 'count_column', + }, + ], + }), + }, + ], + legend: { + isVisible: true, + position: 'right', + legendSize: LegendSize.EXTRA_LARGE, + shouldTruncate: false, + }, + preferredSeriesType: 'bar_stacked', + valueLabels: 'hide', + fittingFunction: 'None', + minBarHeight: 2, + showCurrentTimeMarker: true, + axisTitlesVisibilitySettings: { + x: false, + yLeft: false, + yRight: false, + }, + gridlinesVisibilitySettings: { + x: true, + yLeft: true, + yRight: false, + }, + tickLabelsVisibilitySettings: { + x: true, + yLeft: true, + yRight: false, + }, + } as XYConfiguration; + + return { + visualizationId: 'lnsXY', + visualizationState, + datasourceState, + datasourceId: 'formBased', + columns: Object.keys(columns).length, + } as Suggestion; + }; + + private getHistogramSuggestionForESQL = ({ + queryParams, + }: { + queryParams: QueryParams; + }): Suggestion | undefined => { + const { dataView, query, timeRange } = queryParams; + if ( + dataView.isTimeBased() && + query && + isOfAggregateQueryType(query) && + getAggregateQueryMode(query) === 'esql' && + timeRange + ) { + const isOnHistogramMode = shouldDisplayHistogram(query); + if (!isOnHistogramMode) return undefined; + + const interval = computeInterval(timeRange, this.services.data); + const esqlQuery = this.getESQLHistogramQuery({ dataView, query, timeRange, interval }); + const context = { + dataViewSpec: dataView?.toSpec(), + fieldName: '', + textBasedColumns: [ + { + id: `${dataView.timeFieldName} every ${interval}`, + name: `${dataView.timeFieldName} every ${interval}`, + meta: { + type: 'date', + }, + }, + { + id: 'results', + name: 'results', + meta: { + type: 'number', + }, + }, + ] as DatatableColumn[], + query: { + esql: esqlQuery, + }, + }; + const suggestions = this.lensSuggestionsApi(context, dataView, ['lnsDatatable']) ?? []; + if (suggestions.length) { + return suggestions[0]; + } + } + + return undefined; + }; + + private getESQLHistogramQuery = ({ + dataView, + timeRange, + query, + interval, + }: { + dataView: DataView; + timeRange: TimeRange; + query: AggregateQuery; + interval?: string; + }): string => { + const queryInterval = interval ?? computeInterval(timeRange, this.services.data); + const language = getAggregateQueryMode(query); + const safeQuery = removeDropCommandsFromESQLQuery(query[language]); + return `${safeQuery} | EVAL timestamp=DATE_TRUNC(${queryInterval}, ${dataView.timeFieldName}) | stats results = count(*) by timestamp | rename timestamp as \`${dataView.timeFieldName} every ${queryInterval}\``; + }; + + private getAllSuggestions = ({ queryParams }: { queryParams: QueryParams }): Suggestion[] => { + const { dataView, columns, query, isPlainRecord } = queryParams; + + const context = { + dataViewSpec: dataView?.toSpec(), + fieldName: '', + textBasedColumns: columns, + query: query && isOfAggregateQueryType(query) ? query : undefined, + }; + const allSuggestions = isPlainRecord + ? this.lensSuggestionsApi(context, dataView, ['lnsDatatable']) ?? [] + : []; + + return allSuggestions; + }; + + private getLensAttributesState = ({ + currentSuggestionContext, + externalVisContext, + queryParams, + timeInterval, + breakdownField, + table, + }: { + currentSuggestionContext: UnifiedHistogramSuggestionContext; + externalVisContext: UnifiedHistogramVisContext | undefined; + queryParams: QueryParams; + timeInterval: string | undefined; + breakdownField: DataViewField | undefined; + table: Datatable | undefined; + }): { + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus; + visContext: UnifiedHistogramVisContext | undefined; + } => { + const { dataView, query, filters, timeRange } = queryParams; + const { type: suggestionType, suggestion } = currentSuggestionContext; + + if (!suggestion || !suggestion.datasourceId || !query || !filters) { + return { + externalVisContextStatus: UnifiedHistogramExternalVisContextStatus.unknown, + visContext: undefined, + }; + } + + const isTextBased = isOfAggregateQueryType(query); + const requestData = { + dataViewId: dataView.id, + timeField: dataView.timeFieldName, + timeInterval: isTextBased ? undefined : timeInterval, + breakdownField: isTextBased ? undefined : breakdownField?.name, + }; + + const currentQuery = + suggestionType === UnifiedHistogramSuggestionType.histogramForESQL && isTextBased && timeRange + ? { + esql: this.getESQLHistogramQuery({ dataView, query, timeRange }), + } + : query; + + let externalVisContextStatus: UnifiedHistogramExternalVisContextStatus; + let visContext: UnifiedHistogramVisContext | undefined; + + if (externalVisContext?.attributes) { + if ( + isEqual(currentQuery, externalVisContext.attributes?.state?.query) && + areSuggestionAndVisContextAndQueryParamsStillCompatible({ + suggestionType, + suggestion, + externalVisContext, + queryParams, + requestData, + }) + ) { + // using the external lens attributes + visContext = externalVisContext; + externalVisContextStatus = UnifiedHistogramExternalVisContextStatus.applied; + } else { + // external vis is not compatible with the current suggestion + externalVisContextStatus = UnifiedHistogramExternalVisContextStatus.automaticallyOverridden; + } + } else { + externalVisContextStatus = UnifiedHistogramExternalVisContextStatus.automaticallyCreated; + } + + if (!visContext) { + const attributes = getLensAttributesFromSuggestion({ + query: currentQuery, + filters, + suggestion, + dataView, + }) as TypedLensByValueInput['attributes']; + + if (suggestionType === UnifiedHistogramSuggestionType.histogramForDataView) { + attributes.title = i18n.translate('unifiedHistogram.lensTitle', { + defaultMessage: 'Edit visualization', + }); + attributes.references = [ + { + id: dataView.id ?? '', + name: `indexpattern-datasource-layer-${UNIFIED_HISTOGRAM_LAYER_ID}`, + type: 'index-pattern', + }, + ]; + } + + visContext = { + attributes, + requestData, + suggestionType, + }; + } + + if ( + table && // already fetched data + query && + isTextBased && + suggestionType === UnifiedHistogramSuggestionType.lensSuggestion && + visContext?.attributes + ) { + visContext = { + ...visContext, + attributes: enrichLensAttributesWithTablesData({ + attributes: visContext.attributes, + table, + }), + }; + } + + return { + externalVisContextStatus, + visContext, + }; + }; +} + +function deriveLensSuggestionFromLensAttributes({ + externalVisContext, + queryParams, +}: { + externalVisContext: UnifiedHistogramVisContext | undefined; + queryParams: QueryParams; +}): Suggestion | undefined { + if (!externalVisContext || !queryParams.isPlainRecord) { + return undefined; + } + + try { + if (externalVisContext.suggestionType === UnifiedHistogramSuggestionType.lensSuggestion) { + // should be based on same query + if (!isEqual(externalVisContext.attributes?.state?.query, queryParams.query)) { + return undefined; + } + + // it should be one of 'formBased'/'textBased' and have value + const datasourceId: 'formBased' | 'textBased' | undefined = [ + 'formBased' as const, + 'textBased' as const, + ].find((key) => Boolean(externalVisContext.attributes.state.datasourceStates[key])); + + if (!datasourceId) { + return undefined; + } + + const datasourceState = externalVisContext.attributes.state.datasourceStates[datasourceId]; + + // should be based on same columns + if ( + !datasourceState?.layers || + Object.values(datasourceState?.layers).some( + (layer) => + isEqual(layer.query, queryParams.query) && + layer.columns?.some( + // unknown column + (c: { fieldName: string }) => !queryParams.columnsMap?.[c.fieldName] + ) + ) + ) { + return undefined; + } + + return { + title: externalVisContext.attributes.title, + visualizationId: externalVisContext.attributes.visualizationType, + visualizationState: externalVisContext.attributes.state.visualization, + datasourceState, + datasourceId, + } as Suggestion; + } + } catch { + return undefined; + } + + return undefined; +} + +function areSuggestionAndVisContextAndQueryParamsStillCompatible({ + suggestionType, + suggestion, + externalVisContext, + queryParams, + requestData, +}: { + suggestionType: UnifiedHistogramSuggestionType; + suggestion: Suggestion; + externalVisContext: UnifiedHistogramVisContext; + queryParams: QueryParams; + requestData: UnifiedHistogramVisContext['requestData']; +}): boolean { + // requestData should match + if ( + (Object.keys(requestData) as Array).some( + (key) => !isEqual(requestData[key], externalVisContext.requestData[key]) + ) + ) { + return false; + } + + if ( + queryParams.isPlainRecord && + suggestionType === UnifiedHistogramSuggestionType.lensSuggestion && + !deriveLensSuggestionFromLensAttributes({ externalVisContext, queryParams }) + ) { + // can't retrieve back a suggestion with matching query and known columns + return false; + } + + return ( + suggestionType === externalVisContext.suggestionType && + // vis shape should match + isSuggestionShapeAndVisContextCompatible(suggestion, externalVisContext) + ); +} diff --git a/src/plugins/unified_histogram/public/types.ts b/src/plugins/unified_histogram/public/types.ts index 3ba27f7c5b26e..d19c4481f202f 100644 --- a/src/plugins/unified_histogram/public/types.ts +++ b/src/plugins/unified_histogram/public/types.ts @@ -9,7 +9,12 @@ import type { IUiSettingsClient, Capabilities } from '@kbn/core/public'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; -import type { LensEmbeddableOutput, LensPublicStart } from '@kbn/lens-plugin/public'; +import type { + LensEmbeddableOutput, + LensPublicStart, + TypedLensByValueInput, + Suggestion, +} from '@kbn/lens-plugin/public'; import type { DataViewField } from '@kbn/data-views-plugin/public'; import type { RequestAdapter } from '@kbn/inspector-plugin/public'; import type { DefaultInspectorAdapters } from '@kbn/expressions-plugin/common'; @@ -111,10 +116,6 @@ export interface UnifiedHistogramChartContext { * Controls the time interval of the chart */ timeInterval?: string; - /** - * The chart title -- sets the title property on the Lens chart input - */ - title?: string; } /** @@ -143,3 +144,39 @@ export type UnifiedHistogramInputMessage = UnifiedHistogramRefetchMessage; * Unified histogram input observable */ export type UnifiedHistogramInput$ = Subject; + +export enum UnifiedHistogramExternalVisContextStatus { + unknown = 'unknown', + applied = 'applied', + automaticallyCreated = 'automaticallyCreated', + automaticallyOverridden = 'automaticallyOverridden', + manuallyCustomized = 'manuallyCustomized', +} + +export enum UnifiedHistogramSuggestionType { + unsupported = 'unsupported', + lensSuggestion = 'lensSuggestion', + histogramForESQL = 'histogramForESQL', + histogramForDataView = 'histogramForDataView', +} + +export interface UnifiedHistogramSuggestionContext { + suggestion: Suggestion | undefined; + type: UnifiedHistogramSuggestionType; +} + +export interface LensRequestData { + dataViewId?: string; + timeField?: string; + timeInterval?: string; + breakdownField?: string; +} + +/** + * Unified Histogram type for recreating a stored Lens vis + */ +export interface UnifiedHistogramVisContext { + attributes: TypedLensByValueInput['attributes']; + requestData: LensRequestData; + suggestionType: UnifiedHistogramSuggestionType; +} diff --git a/src/plugins/unified_histogram/public/utils/__snapshots__/external_vis_context.test.ts.snap b/src/plugins/unified_histogram/public/utils/__snapshots__/external_vis_context.test.ts.snap new file mode 100644 index 0000000000000..fb4014f969700 --- /dev/null +++ b/src/plugins/unified_histogram/public/utils/__snapshots__/external_vis_context.test.ts.snap @@ -0,0 +1,342 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`external_vis_context exportVisContext should work correctly 1`] = ` +Object { + "attributes": Object { + "references": Array [ + Object { + "id": "index-pattern-with-timefield-id", + "name": "textBasedLanguages-datasource-layer-suggestion", + "type": "index-pattern", + }, + ], + "state": Object { + "datasourceStates": Object { + "textBased": Object { + "indexPatternRefs": Array [], + "initialContext": Object { + "contextualFields": Array [ + "Dest", + "AvgTicketPrice", + ], + "dataViewSpec": Object { + "allowNoIndex": false, + "fields": Object { + "AvgTicketPrice": Object { + "aggregatable": true, + "count": 0, + "esTypes": Array [ + "float", + ], + "format": Object { + "id": "number", + "params": Object { + "pattern": "$0,0.[00]", + }, + }, + "isMapped": true, + "name": "AvgTicketPrice", + "readFromDocValues": true, + "scripted": false, + "searchable": true, + "shortDotsEnable": false, + "type": "number", + }, + "Dest": Object { + "aggregatable": true, + "count": 0, + "esTypes": Array [ + "keyword", + ], + "format": Object { + "id": "string", + }, + "isMapped": true, + "name": "Dest", + "readFromDocValues": true, + "scripted": false, + "searchable": true, + "shortDotsEnable": false, + "type": "string", + }, + "timestamp": Object { + "aggregatable": true, + "count": 0, + "esTypes": Array [ + "date", + ], + "format": Object { + "id": "date", + }, + "isMapped": true, + "name": "timestamp", + "readFromDocValues": true, + "scripted": false, + "searchable": true, + "shortDotsEnable": false, + "type": "date", + }, + }, + "id": "d3d7af60-4c81-11e8-b3d7-01146121b73d", + "name": "Kibana Sample Data Flights", + "sourceFilters": Array [], + "timeFieldName": "timestamp", + "title": "kibana_sample_data_flights", + "version": "WzM1ODA3LDFd", + }, + "fieldName": "", + "query": Object { + "esql": "FROM \\"kibana_sample_data_flights\\"", + }, + }, + "layers": Object { + "46aa21fa-b747-4543-bf90-0b40007c546d": Object { + "columns": Array [ + Object { + "columnId": "81e332d6-ee37-42a8-a646-cea4fc75d2d3", + "fieldName": "Dest", + "meta": Object { + "type": "string", + }, + }, + Object { + "columnId": "5b9b8b76-0836-4a12-b9c0-980c9900502f", + "fieldName": "AvgTicketPrice", + "meta": Object { + "type": "number", + }, + }, + ], + "index": "d3d7af60-4c81-11e8-b3d7-01146121b73d", + "query": Object { + "esql": "FROM kibana_sample_data_flights | keep Dest, AvgTicketPrice", + }, + "table": Object { + "columns": Array [ + Object { + "id": "avg(bytes)", + "isNull": false, + "meta": Object { + "type": "number", + }, + "name": "avg(bytes)", + }, + Object { + "id": "extension.keyword", + "isNull": false, + "meta": Object { + "type": "string", + }, + "name": "extension.keyword", + }, + ], + "rows": Array [ + Object { + "avg(bytes)": 3850, + "extension.keyword": "", + }, + Object { + "avg(bytes)": 5393.5, + "extension.keyword": "css", + }, + Object { + "avg(bytes)": 3252, + "extension.keyword": "deb", + }, + ], + "type": "datatable", + }, + "timeField": "timestamp", + }, + }, + }, + }, + "filters": Array [], + "query": Object { + "esql": "from logstash | stats avg(bytes) by extension.keyword", + }, + "visualization": Object { + "gridConfig": Object { + "isCellLabelVisible": false, + "isXAxisLabelVisible": true, + "isXAxisTitleVisible": false, + "isYAxisLabelVisible": true, + "isYAxisTitleVisible": false, + "type": "heatmap_grid", + }, + "layerId": "46aa21fa-b747-4543-bf90-0b40007c546d", + "layerType": "data", + "legend": Object { + "isVisible": true, + "position": "right", + "type": "heatmap_legend", + }, + "shape": "heatmap", + "valueAccessor": "5b9b8b76-0836-4a12-b9c0-980c9900502f", + "xAccessor": "81e332d6-ee37-42a8-a646-cea4fc75d2d3", + }, + }, + "title": "Heat map", + "visualizationType": "lnsHeatmap", + }, + "requestData": Object { + "breakdownField": undefined, + "dataViewId": "index-pattern-with-timefield-id", + "timeField": "timestamp", + "timeInterval": undefined, + }, + "suggestionType": "lensSuggestion", +} +`; + +exports[`external_vis_context exportVisContext should work correctly 2`] = ` +Object { + "attributes": Object { + "references": Array [ + Object { + "id": "index-pattern-with-timefield-id", + "name": "textBasedLanguages-datasource-layer-suggestion", + "type": "index-pattern", + }, + ], + "state": Object { + "datasourceStates": Object { + "textBased": Object { + "indexPatternRefs": Array [], + "initialContext": Object { + "contextualFields": Array [ + "Dest", + "AvgTicketPrice", + ], + "dataViewSpec": Object { + "allowNoIndex": false, + "fields": Object { + "AvgTicketPrice": Object { + "aggregatable": true, + "count": 0, + "esTypes": Array [ + "float", + ], + "format": Object { + "id": "number", + "params": Object { + "pattern": "$0,0.[00]", + }, + }, + "isMapped": true, + "name": "AvgTicketPrice", + "readFromDocValues": true, + "scripted": false, + "searchable": true, + "shortDotsEnable": false, + "type": "number", + }, + "Dest": Object { + "aggregatable": true, + "count": 0, + "esTypes": Array [ + "keyword", + ], + "format": Object { + "id": "string", + }, + "isMapped": true, + "name": "Dest", + "readFromDocValues": true, + "scripted": false, + "searchable": true, + "shortDotsEnable": false, + "type": "string", + }, + "timestamp": Object { + "aggregatable": true, + "count": 0, + "esTypes": Array [ + "date", + ], + "format": Object { + "id": "date", + }, + "isMapped": true, + "name": "timestamp", + "readFromDocValues": true, + "scripted": false, + "searchable": true, + "shortDotsEnable": false, + "type": "date", + }, + }, + "id": "d3d7af60-4c81-11e8-b3d7-01146121b73d", + "name": "Kibana Sample Data Flights", + "sourceFilters": Array [], + "timeFieldName": "timestamp", + "title": "kibana_sample_data_flights", + "version": "WzM1ODA3LDFd", + }, + "fieldName": "", + "query": Object { + "esql": "FROM \\"kibana_sample_data_flights\\"", + }, + }, + "layers": Object { + "46aa21fa-b747-4543-bf90-0b40007c546d": Object { + "columns": Array [ + Object { + "columnId": "81e332d6-ee37-42a8-a646-cea4fc75d2d3", + "fieldName": "Dest", + "meta": Object { + "type": "string", + }, + }, + Object { + "columnId": "5b9b8b76-0836-4a12-b9c0-980c9900502f", + "fieldName": "AvgTicketPrice", + "meta": Object { + "type": "number", + }, + }, + ], + "index": "d3d7af60-4c81-11e8-b3d7-01146121b73d", + "query": Object { + "esql": "FROM kibana_sample_data_flights | keep Dest, AvgTicketPrice", + }, + "timeField": "timestamp", + }, + }, + }, + }, + "filters": Array [], + "query": Object { + "esql": "from logstash | stats avg(bytes) by extension.keyword", + }, + "visualization": Object { + "gridConfig": Object { + "isCellLabelVisible": false, + "isXAxisLabelVisible": true, + "isXAxisTitleVisible": false, + "isYAxisLabelVisible": true, + "isYAxisTitleVisible": false, + "type": "heatmap_grid", + }, + "layerId": "46aa21fa-b747-4543-bf90-0b40007c546d", + "layerType": "data", + "legend": Object { + "isVisible": true, + "position": "right", + "type": "heatmap_legend", + }, + "shape": "heatmap", + "valueAccessor": "5b9b8b76-0836-4a12-b9c0-980c9900502f", + "xAccessor": "81e332d6-ee37-42a8-a646-cea4fc75d2d3", + }, + }, + "title": "Heat map", + "visualizationType": "lnsHeatmap", + }, + "requestData": Object { + "dataViewId": "index-pattern-with-timefield-id", + "timeField": "timestamp", + }, + "suggestionType": "lensSuggestion", +} +`; diff --git a/src/plugins/unified_histogram/public/layout/hooks/compute_interval.test.ts b/src/plugins/unified_histogram/public/utils/compute_interval.test.ts similarity index 100% rename from src/plugins/unified_histogram/public/layout/hooks/compute_interval.test.ts rename to src/plugins/unified_histogram/public/utils/compute_interval.test.ts diff --git a/src/plugins/unified_histogram/public/layout/hooks/compute_interval.ts b/src/plugins/unified_histogram/public/utils/compute_interval.ts similarity index 100% rename from src/plugins/unified_histogram/public/layout/hooks/compute_interval.ts rename to src/plugins/unified_histogram/public/utils/compute_interval.ts diff --git a/src/plugins/unified_histogram/public/utils/external_vis_context.test.ts b/src/plugins/unified_histogram/public/utils/external_vis_context.test.ts new file mode 100644 index 0000000000000..a786bf102065a --- /dev/null +++ b/src/plugins/unified_histogram/public/utils/external_vis_context.test.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { DataView } from '@kbn/data-views-plugin/common'; +import type { Suggestion } from '@kbn/lens-plugin/public'; +import { + canImportVisContext, + exportVisContext, + isSuggestionShapeAndVisContextCompatible, +} from './external_vis_context'; +import { getLensVisMock } from '../__mocks__/lens_vis'; +import { dataViewWithTimefieldMock } from '../__mocks__/data_view_with_timefield'; +import { tableMock, tableQueryMock } from '../__mocks__/table'; +import { UnifiedHistogramSuggestionType, UnifiedHistogramVisContext } from '../types'; + +describe('external_vis_context', () => { + const dataView: DataView = dataViewWithTimefieldMock; + let exportedVisContext: UnifiedHistogramVisContext | undefined; + + describe('exportVisContext', () => { + it('should work correctly', async () => { + const lensVis = await getLensVisMock({ + filters: [], + query: tableQueryMock, + dataView, + timeInterval: 'auto', + breakdownField: undefined, + columns: [], + isPlainRecord: true, + table: tableMock, + }); + + const visContext = lensVis.visContext; + + expect(visContext).toMatchSnapshot(); + + exportedVisContext = exportVisContext(visContext); + expect(exportedVisContext).toMatchSnapshot(); + }); + }); + + describe('canImportVisContext', () => { + it('should work correctly for valid input', async () => { + expect(canImportVisContext(exportedVisContext)).toBe(true); + }); + + it('should work correctly for invalid input', async () => { + expect(canImportVisContext(undefined)).toBe(false); + expect(canImportVisContext({ attributes: {} })).toBe(false); + }); + }); + + describe('isSuggestionAndVisContextCompatible', () => { + it('should work correctly', async () => { + expect(isSuggestionShapeAndVisContextCompatible(undefined, undefined)).toBe(true); + + expect( + isSuggestionShapeAndVisContextCompatible( + { visualizationId: 'lnsPie', visualizationState: { shape: 'donut' } } as Suggestion, + { + suggestionType: UnifiedHistogramSuggestionType.lensSuggestion, + attributes: { + visualizationType: 'lnsPie', + state: { visualization: { shape: 'donut' } }, + }, + } as UnifiedHistogramVisContext + ) + ).toBe(true); + + expect( + isSuggestionShapeAndVisContextCompatible( + { visualizationId: 'lnsPie', visualizationState: { shape: 'donut' } } as Suggestion, + { + suggestionType: UnifiedHistogramSuggestionType.lensSuggestion, + attributes: { + visualizationType: 'lnsPie', + state: { visualization: { shape: 'waffle' } }, + }, + } as UnifiedHistogramVisContext + ) + ).toBe(false); + + expect( + isSuggestionShapeAndVisContextCompatible( + { visualizationId: 'lnsPie', visualizationState: { shape: 'donut' } } as Suggestion, + { + suggestionType: UnifiedHistogramSuggestionType.lensSuggestion, + attributes: { + visualizationType: 'lnsXY', + }, + } as UnifiedHistogramVisContext + ) + ).toBe(false); + + expect( + isSuggestionShapeAndVisContextCompatible( + { + visualizationId: 'lnsXY', + visualizationState: { preferredSeriesType: 'bar_stacked' }, + } as Suggestion, + { + attributes: { + visualizationType: 'lnsXY', + state: { visualization: { preferredSeriesType: 'bar_stacked' } }, + }, + } as UnifiedHistogramVisContext + ) + ).toBe(true); + + expect( + isSuggestionShapeAndVisContextCompatible( + { + visualizationId: 'lnsXY', + visualizationState: { preferredSeriesType: 'bar_stacked' }, + } as Suggestion, + { + suggestionType: UnifiedHistogramSuggestionType.histogramForESQL, + attributes: { + visualizationType: 'lnsXY', + state: { visualization: { preferredSeriesType: 'line' } }, + }, + } as UnifiedHistogramVisContext + ) + ).toBe(true); + + expect( + isSuggestionShapeAndVisContextCompatible( + { + visualizationId: 'lnsXY', + visualizationState: { preferredSeriesType: 'bar_stacked' }, + } as Suggestion, + { + suggestionType: UnifiedHistogramSuggestionType.lensSuggestion, + attributes: { + visualizationType: 'lnsXY', + state: { visualization: { preferredSeriesType: 'line' } }, + }, + } as UnifiedHistogramVisContext + ) + ).toBe(false); + + expect( + isSuggestionShapeAndVisContextCompatible( + { + visualizationId: 'lnsXY', + visualizationState: { preferredSeriesType: 'bar_stacked' }, + } as Suggestion, + { + suggestionType: UnifiedHistogramSuggestionType.histogramForDataView, + attributes: { + visualizationType: 'lnsXY', + state: { visualization: { preferredSeriesType: 'bar_stacked' } }, + }, + } as UnifiedHistogramVisContext + ) + ).toBe(true); + }); + }); +}); diff --git a/src/plugins/unified_histogram/public/utils/external_vis_context.ts b/src/plugins/unified_histogram/public/utils/external_vis_context.ts new file mode 100644 index 0000000000000..380b7dbc01094 --- /dev/null +++ b/src/plugins/unified_histogram/public/utils/external_vis_context.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { PieVisualizationState, Suggestion, XYState } from '@kbn/lens-plugin/public'; +import { UnifiedHistogramSuggestionType, UnifiedHistogramVisContext } from '../types'; +import { removeTablesFromLensAttributes } from './lens_vis_from_table'; + +export const exportVisContext = ( + visContext: UnifiedHistogramVisContext | undefined +): UnifiedHistogramVisContext | undefined => { + if ( + !visContext || + !visContext.requestData || + !visContext.attributes || + !visContext.suggestionType + ) { + return undefined; + } + + try { + const lightweightVisContext = visContext + ? { + suggestionType: visContext.suggestionType, + requestData: visContext.requestData, + attributes: removeTablesFromLensAttributes(visContext.attributes), + } + : undefined; + + const visContextWithoutUndefinedValues = lightweightVisContext + ? JSON.parse(JSON.stringify(lightweightVisContext)) + : undefined; + + return visContextWithoutUndefinedValues; + } catch { + return undefined; + } +}; + +export function canImportVisContext( + visContext: unknown | undefined +): visContext is UnifiedHistogramVisContext { + return ( + !!visContext && + typeof visContext === 'object' && + 'requestData' in visContext && + 'attributes' in visContext && + 'suggestionType' in visContext && + !!visContext.requestData && + !!visContext.attributes && + !!visContext.suggestionType && + typeof visContext.requestData === 'object' && + typeof visContext.attributes === 'object' && + typeof visContext.suggestionType === 'string' + ); +} + +export const isSuggestionShapeAndVisContextCompatible = ( + suggestion: Suggestion | undefined, + externalVisContext: UnifiedHistogramVisContext | undefined +): boolean => { + if (!suggestion && !externalVisContext) { + return true; + } + + if (suggestion?.visualizationId !== externalVisContext?.attributes?.visualizationType) { + return false; + } + + if (externalVisContext?.suggestionType !== UnifiedHistogramSuggestionType.lensSuggestion) { + return true; + } + + if (suggestion?.visualizationId === 'lnsXY') { + return ( + (suggestion?.visualizationState as XYState)?.preferredSeriesType === + (externalVisContext?.attributes?.state?.visualization as XYState)?.preferredSeriesType + ); + } + + return ( + (suggestion?.visualizationState as PieVisualizationState)?.shape === + (externalVisContext?.attributes?.state?.visualization as PieVisualizationState)?.shape + ); +}; diff --git a/src/plugins/unified_histogram/public/chart/utils/field_supports_breakdown.test.ts b/src/plugins/unified_histogram/public/utils/field_supports_breakdown.test.ts similarity index 100% rename from src/plugins/unified_histogram/public/chart/utils/field_supports_breakdown.test.ts rename to src/plugins/unified_histogram/public/utils/field_supports_breakdown.test.ts diff --git a/src/plugins/unified_histogram/public/chart/utils/field_supports_breakdown.ts b/src/plugins/unified_histogram/public/utils/field_supports_breakdown.ts similarity index 100% rename from src/plugins/unified_histogram/public/chart/utils/field_supports_breakdown.ts rename to src/plugins/unified_histogram/public/utils/field_supports_breakdown.ts diff --git a/src/plugins/unified_histogram/public/utils/lens_vis_from_table.ts b/src/plugins/unified_histogram/public/utils/lens_vis_from_table.ts new file mode 100644 index 0000000000000..565b52767022a --- /dev/null +++ b/src/plugins/unified_histogram/public/utils/lens_vis_from_table.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Datatable } from '@kbn/expressions-plugin/common'; +import type { LensAttributes } from '@kbn/lens-embeddable-utils'; +import type { TextBasedPersistedState } from '@kbn/lens-plugin/public/datasources/text_based/types'; + +export const enrichLensAttributesWithTablesData = ({ + attributes, + table, +}: { + attributes: LensAttributes; + table: Datatable | undefined; +}): LensAttributes => { + if (!attributes.state.datasourceStates.textBased) { + return attributes; + } + + const layers = attributes.state.datasourceStates.textBased?.layers; + + if (!layers) { + return attributes; + } + + const updatedAttributes = { + ...attributes, + state: { + ...attributes.state, + datasourceStates: { + ...attributes.state.datasourceStates, + textBased: { + ...attributes.state.datasourceStates.textBased, + layers: {} as TextBasedPersistedState['layers'], + }, + }, + }, + }; + + for (const key of Object.keys(layers)) { + const newLayer = { ...layers[key], table }; + if (!table) { + delete newLayer.table; + } + updatedAttributes.state.datasourceStates.textBased.layers[key] = newLayer; + } + + return updatedAttributes; +}; + +export const removeTablesFromLensAttributes = (attributes: LensAttributes): LensAttributes => { + return enrichLensAttributesWithTablesData({ attributes, table: undefined }); +}; diff --git a/src/plugins/unified_histogram/tsconfig.json b/src/plugins/unified_histogram/tsconfig.json index fa266de08ecbf..a1c15026479cc 100644 --- a/src/plugins/unified_histogram/tsconfig.json +++ b/src/plugins/unified_histogram/tsconfig.json @@ -23,7 +23,6 @@ "@kbn/ui-actions-plugin", "@kbn/kibana-utils-plugin", "@kbn/visualizations-plugin", - "@kbn/discover-utils", "@kbn/resizable-layout", "@kbn/shared-ux-button-toolbar", "@kbn/calculate-width-from-char-count", @@ -31,6 +30,8 @@ "@kbn/i18n-react", "@kbn/field-utils", "@kbn/esql-utils", + "@kbn/discover-utils", + "@kbn/visualization-utils", ], "exclude": [ "target/**/*", diff --git a/test/functional/apps/discover/group3/_lens_vis.ts b/test/functional/apps/discover/group3/_lens_vis.ts new file mode 100644 index 0000000000000..5747dbd85de64 --- /dev/null +++ b/test/functional/apps/discover/group3/_lens_vis.ts @@ -0,0 +1,675 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const testSubjects = getService('testSubjects'); + const monacoEditor = getService('monacoEditor'); + const retry = getService('retry'); + const find = getService('find'); + const browser = getService('browser'); + const toasts = getService('toasts'); + const PageObjects = getPageObjects([ + 'settings', + 'common', + 'discover', + 'header', + 'timePicker', + 'dashboard', + 'unifiedFieldList', + 'unifiedSearch', + ]); + const security = getService('security'); + const defaultSettings = { + defaultIndex: 'logstash-*', + hideAnnouncements: true, + }; + + const defaultTimespan = + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000 (interval: Auto - 3 hours)'; + const defaultTimespanESQL = 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000'; + const defaultTotalCount = '14,004'; + + async function checkNoVis(totalCount: string) { + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.isChartVisible()).to.be(false); + expect(await PageObjects.discover.getHitCount()).to.be(totalCount); + } + + async function checkHistogramVis(timespan: string, totalCount: string) { + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.existOrFail('xyVisChart'); + await testSubjects.existOrFail('unifiedHistogramEditVisualization'); + await testSubjects.existOrFail('unifiedHistogramBreakdownSelectorButton'); + await testSubjects.existOrFail('unifiedHistogramTimeIntervalSelectorButton'); + expect(await PageObjects.discover.getChartTimespan()).to.be(timespan); + expect(await PageObjects.discover.getHitCount()).to.be(totalCount); + } + + async function checkESQLHistogramVis(timespan: string, totalCount: string) { + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.existOrFail('xyVisChart'); + await testSubjects.existOrFail('unifiedHistogramSaveVisualization'); + await testSubjects.existOrFail('unifiedHistogramEditFlyoutVisualization'); + await testSubjects.missingOrFail('unifiedHistogramEditVisualization'); + await testSubjects.missingOrFail('unifiedHistogramBreakdownSelectorButton'); + await testSubjects.missingOrFail('unifiedHistogramTimeIntervalSelectorButton'); + expect(await PageObjects.discover.getChartTimespan()).to.be(timespan); + expect(await PageObjects.discover.getHitCount()).to.be(totalCount); + } + + async function changeVisSeriesType(seriesType: string) { + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + await retry.waitFor('flyout', async () => { + return await testSubjects.exists('lnsChartSwitchPopover'); + }); + await testSubjects.click('lnsChartSwitchPopover'); + await testSubjects.setValue('lnsChartSwitchSearch', seriesType, { + clearWithKeyboard: true, + }); + await testSubjects.click(`lnsChartSwitchPopover_${seriesType.toLowerCase()}`); + await retry.try(async () => { + expect(await testSubjects.getVisibleText('lnsChartSwitchPopover')).to.be(seriesType); + }); + + await toasts.dismissAll(); + await testSubjects.scrollIntoView('applyFlyoutButton'); + await testSubjects.click('applyFlyoutButton'); + } + + async function getCurrentVisSeriesTypeLabel() { + await toasts.dismissAll(); + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + const seriesType = await testSubjects.getVisibleText('lnsChartSwitchPopover'); + await testSubjects.click('cancelFlyoutButton'); + return seriesType; + } + + async function getCurrentVisChartTitle() { + const chartElement = await find.byCssSelector( + '[data-test-subj="unifiedHistogramChart"] [data-render-complete="true"]' + ); + return await chartElement.getAttribute('data-title'); + } + + describe('discover lens vis', function describeIndexTests() { + before(async () => { + await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']); + await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); + await browser.setWindowSize(1300, 1000); + }); + + after(async () => { + await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover'); + await kibanaServer.uiSettings.replace({}); + await kibanaServer.savedObjects.cleanStandardList(); + }); + + beforeEach(async function () { + await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings(); + await kibanaServer.uiSettings.update(defaultSettings); + await PageObjects.common.navigateToApp('discover'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + }); + + it('should show histogram by default', async () => { + await checkHistogramVis(defaultTimespan, defaultTotalCount); + + await PageObjects.timePicker.setAbsoluteRange( + 'Sep 20, 2015 @ 00:00:00.000', + 'Sep 20, 2015 @ 23:50:13.253' + ); + + const savedSearchTimeSpan = + 'Sep 20, 2015 @ 00:00:00.000 - Sep 20, 2015 @ 23:50:13.253 (interval: Auto - 30 minutes)'; + const savedSearchTotalCount = '4,756'; + await checkHistogramVis(savedSearchTimeSpan, savedSearchTotalCount); + + await PageObjects.discover.saveSearch('testDefault'); + + await checkHistogramVis(savedSearchTimeSpan, savedSearchTotalCount); + + await browser.refresh(); + + await checkHistogramVis(savedSearchTimeSpan, savedSearchTotalCount); + }); + + it('should show no histogram for no results view and recover when time range expanded', async () => { + await PageObjects.timePicker.setAbsoluteRange( + 'Sep 19, 2015 @ 00:00:00.000', + 'Sep 19, 2015 @ 00:00:00.000' + ); + + expect(await PageObjects.discover.hasNoResults()).to.be(true); + + await PageObjects.timePicker.setAbsoluteRange( + 'Sep 20, 2015 @ 00:00:00.000', + 'Sep 20, 2015 @ 00:00:00.000' + ); + + await checkHistogramVis( + 'Sep 20, 2015 @ 00:00:00.000 - Sep 20, 2015 @ 00:00:00.000 (interval: Auto - millisecond)', + '1' + ); + }); + + it('should show no histogram for non-time-based data views and recover for time-based data views', async () => { + await PageObjects.discover.createAdHocDataView('logs*', false); + + await checkNoVis(defaultTotalCount); + + await PageObjects.discover.clickIndexPatternActions(); + await PageObjects.unifiedSearch.editDataView('logs*', '@timestamp'); + + await checkHistogramVis(defaultTimespan, defaultTotalCount); + }); + + it('should show ESQL histogram for text-based query', async () => { + await PageObjects.discover.selectTextBaseLang(); + + await monacoEditor.setCodeEditorValue('from logstash-* | limit 10'); + await testSubjects.click('querySubmitButton'); + + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + + await PageObjects.timePicker.setAbsoluteRange( + 'Sep 20, 2015 @ 00:00:00.000', + 'Sep 20, 2015 @ 00:00:00.000' + ); + + await checkESQLHistogramVis('Sep 20, 2015 @ 00:00:00.000 - Sep 20, 2015 @ 00:00:00.000', '1'); + }); + + it('should be able to customize ESQL histogram and save it', async () => { + await PageObjects.discover.selectTextBaseLang(); + + await monacoEditor.setCodeEditorValue('from logstash-* | limit 10'); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await changeVisSeriesType('Line'); + + await PageObjects.discover.saveSearch('testCustomESQLHistogram'); + + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + }); + + it('should be able to load a saved search with custom histogram vis, edit vis and revert changes', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLHistogram'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + + await changeVisSeriesType('Area'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Area'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + + await PageObjects.discover.revertUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + }); + + it('should be able to load a saved search with custom histogram vis, edit query and revert changes', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLHistogram'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 10'); + + // by changing the query we reset the histogram customization + await monacoEditor.setCodeEditorValue('from logstash-* | limit 100'); + await testSubjects.click('querySubmitButton'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Bar vertical stacked'); + + await checkESQLHistogramVis(defaultTimespanESQL, '100'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 100'); + + await PageObjects.discover.revertUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + + await checkESQLHistogramVis(defaultTimespanESQL, '10'); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 10'); + + // now we are changing to a different query to check lens suggestion logic too + await monacoEditor.setCodeEditorValue( + 'from logstash-* | stats averageA = avg(bytes) by extension' + ); + await testSubjects.click('querySubmitButton'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await checkESQLHistogramVis(defaultTimespanESQL, '5'); + await PageObjects.discover.chooseLensChart('Donut'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + expect(await monacoEditor.getCodeEditorValue()).to.contain('averageA'); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await PageObjects.discover.revertUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + expect(await getCurrentVisChartTitle()).to.be('Bar vertical stacked'); + + await checkESQLHistogramVis(defaultTimespanESQL, '10'); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 10'); + }); + + it('should be able to load a saved search with custom histogram vis and handle invalidations', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLHistogram'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 10'); + + // now we are changing to a different query to check invalidation logic + await monacoEditor.setCodeEditorValue( + 'from logstash-* | stats averageA = avg(bytes) by extension' + ); + await testSubjects.click('querySubmitButton'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await checkESQLHistogramVis(defaultTimespanESQL, '5'); + await PageObjects.discover.chooseLensChart('Donut'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + expect(await monacoEditor.getCodeEditorValue()).to.contain('averageA'); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await PageObjects.discover.saveSearch('testCustomESQLHistogramInvalidation', true); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await browser.refresh(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + }); + + it('should be able to load a saved search with custom histogram vis and save new customization', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLHistogram'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + await checkESQLHistogramVis( + 'Sep 19, 2015 @ 06:31:44.000 - Sep 23, 2015 @ 18:31:44.000', + '10' + ); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 10'); + + // now we are changing to a different query to check invalidation logic + await monacoEditor.setCodeEditorValue( + 'from logstash-* | stats averageA = avg(bytes) by extension' + ); + await testSubjects.click('querySubmitButton'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await checkESQLHistogramVis(defaultTimespanESQL, '5'); + await PageObjects.discover.chooseLensChart('Donut'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + expect(await monacoEditor.getCodeEditorValue()).to.contain('averageA'); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + // now we customize the vis again + await PageObjects.discover.chooseLensChart('Waffle'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Waffle'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Waffle'); + expect(await getCurrentVisChartTitle()).to.be('Waffle'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + + await PageObjects.discover.saveSearch( + 'testCustomESQLHistogramInvalidationPlusCustomization', + true + ); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Waffle'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Waffle'); + expect(await getCurrentVisChartTitle()).to.be('Waffle'); + + await browser.refresh(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Waffle'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Waffle'); + expect(await getCurrentVisChartTitle()).to.be('Waffle'); + }); + + it('should be able to customize ESQL vis and save it', async () => { + await PageObjects.discover.selectTextBaseLang(); + + await monacoEditor.setCodeEditorValue( + 'from logstash-* | stats averageB = avg(bytes) by extension' + ); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await checkESQLHistogramVis(defaultTimespanESQL, '5'); + await PageObjects.discover.chooseLensChart('Donut'); + + await PageObjects.discover.saveSearch('testCustomESQLVis'); + await PageObjects.discover.saveSearch('testCustomESQLVisDonut', true); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await browser.refresh(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + }); + + it('should be able to load a saved search with custom vis, edit query and revert changes', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLVisDonut'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + + // by changing the query we reset the vis customization to histogram + await monacoEditor.setCodeEditorValue('from logstash-* | limit 100'); + await testSubjects.click('querySubmitButton'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await getCurrentVisSeriesTypeLabel()).to.be('Bar vertical stacked'); + + await checkESQLHistogramVis(defaultTimespanESQL, '100'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + expect(await monacoEditor.getCodeEditorValue()).to.be('from logstash-* | limit 100'); + + await PageObjects.discover.revertUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + expect(await monacoEditor.getCodeEditorValue()).to.contain('averageB'); + + // should be still Donut after reverting and saving again + await PageObjects.discover.saveSearch('testCustomESQLVisDonut'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + }); + + it('should be able to change to an unfamiliar vis type via lens flyout', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLVisDonut'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + + await changeVisSeriesType('Pie'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Customized'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Pie'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await PageObjects.discover.saveSearch('testCustomESQLVisPie', true); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Customized'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Pie'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await browser.refresh(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Customized'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Pie'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await monacoEditor.setCodeEditorValue( + 'from logstash-* | stats averageB = avg(bytes) by extension.raw' + ); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Bar vertical stacked'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + + await PageObjects.discover.revertUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Customized'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Pie'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + }); + + it('should be able to load a saved search with custom vis, edit vis and revert changes', async () => { + await PageObjects.discover.loadSavedSearch('testCustomESQLVis'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + + await PageObjects.discover.chooseLensChart('Waffle'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Waffle'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Waffle'); + expect(await getCurrentVisChartTitle()).to.be('Waffle'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + + await PageObjects.discover.revertUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await PageObjects.discover.chooseLensChart('Bar vertical stacked'); + await changeVisSeriesType('Line'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + await PageObjects.discover.saveUnsavedChanges(); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Customized'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Line'); + expect(await getCurrentVisChartTitle()).to.be('Bar vertical stacked'); + }); + + it('should close lens flyout on revert changes', async () => { + await PageObjects.discover.selectTextBaseLang(); + + await monacoEditor.setCodeEditorValue( + 'from logstash-* | stats averageB = avg(bytes) by extension' + ); + await testSubjects.click('querySubmitButton'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Bar vertical stacked'); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + + await PageObjects.discover.chooseLensChart('Treemap'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Treemap'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Treemap'); + expect(await getCurrentVisChartTitle()).to.be('Treemap'); + + await PageObjects.discover.saveSearch('testCustomESQLVisRevert'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + await testSubjects.missingOrFail('unsavedChangesBadge'); + + await PageObjects.discover.chooseLensChart('Donut'); + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Donut'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Donut'); + expect(await getCurrentVisChartTitle()).to.be('Donut'); + + await testSubjects.click('unifiedHistogramEditFlyoutVisualization'); // open the flyout + await testSubjects.existOrFail('lnsEditOnFlyFlyout'); + + await testSubjects.existOrFail('unsavedChangesBadge'); + await PageObjects.discover.revertUnsavedChanges(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.discover.waitUntilSearchingHasFinished(); + + await testSubjects.missingOrFail('unsavedChangesBadge'); + await testSubjects.missingOrFail('lnsEditOnFlyFlyout'); // it should close the flyout + expect(await PageObjects.discover.getCurrentLensChart()).to.be('Treemap'); + expect(await getCurrentVisSeriesTypeLabel()).to.be('Treemap'); + expect(await getCurrentVisChartTitle()).to.be('Treemap'); + }); + }); +} diff --git a/test/functional/apps/discover/group3/index.ts b/test/functional/apps/discover/group3/index.ts index 2fe5a4ebb1db1..a80ae44e49801 100644 --- a/test/functional/apps/discover/group3/index.ts +++ b/test/functional/apps/discover/group3/index.ts @@ -30,5 +30,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_view_mode_toggle')); loadTestFile(require.resolve('./_unsaved_changes_badge')); loadTestFile(require.resolve('./_panels_toggle')); + loadTestFile(require.resolve('./_lens_vis')); }); } diff --git a/test/functional/page_objects/discover_page.ts b/test/functional/page_objects/discover_page.ts index 07f82309c321b..47165f90952ee 100644 --- a/test/functional/page_objects/discover_page.ts +++ b/test/functional/page_objects/discover_page.ts @@ -243,6 +243,12 @@ export class DiscoverPageObject extends FtrService { await this.comboBox.set('unifiedHistogramSuggestionSelector', chart); } + public async getCurrentLensChart() { + return ( + await this.comboBox.getComboBoxSelectedOptions('unifiedHistogramSuggestionSelector') + )?.[0]; + } + public async getHistogramLegendList() { const unifiedHistogram = await this.testSubjects.find('unifiedHistogramChart'); const list = await unifiedHistogram.findAllByClassName('echLegendItem__label'); diff --git a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/flyout_wrapper.tsx b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/flyout_wrapper.tsx index 009d21d65eb57..f8ee1c5779693 100644 --- a/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/flyout_wrapper.tsx +++ b/x-pack/plugins/lens/public/app_plugin/shared/edit_on_the_fly/flyout_wrapper.tsx @@ -133,6 +133,7 @@ export const FlyoutWrapper = ({ { title: 'foo', id: 'foo', toSpec: jest.fn(), + toMinimalSpec: jest.fn(), isPersisted: jest.fn().mockReturnValue(false), }) ), From 069d814fe40d3c74c1ec1696835b8ccf757b29c7 Mon Sep 17 00:00:00 2001 From: Sandra G Date: Mon, 8 Apr 2024 10:07:16 -0400 Subject: [PATCH 09/11] [Infra] add apm synthtrace kibana service and cleanup package install (#179764) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes https://github.com/elastic/kibana/issues/175064 - Creates a service for ApmSynthtraceKibanaClient to easily access in tests and other plugins for managing the installation of the APM package needed for indexing apm documents with synthtrace's elasticsearch client - Updates the Infra api integration and functional tests to use the service - Updates Infra tests to cleanup and uninstall the apm package - Updates ApmSynthtraceKibanaClient.installApmPackage to install the latest version if no version was passed in - Updates ApmSynthtraceKibanaClient.installApmPackage to return the version that was installed - Updates ApmSynthtraceKibanaClient to have an uninstallApmPackage method https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5599 --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Felix Stürmer --- .../client/apm_synthtrace_kibana_client.ts | 78 +++++++++++++++++-- .../api_integration/apis/metrics_ui/config.ts | 53 +------------ .../apis/metrics_ui/services.ts | 36 ++++++--- .../api_integration/apis/metrics_ui/types.ts | 19 ----- .../services/apm_synthtrace_kibana_client.ts | 30 +++++++ x-pack/test/common/services/index.ts | 2 + .../common/utils/synthtrace/apm_es_client.ts | 26 +++++++ .../test/functional/apps/infra/hosts_view.ts | 37 ++------- 8 files changed, 163 insertions(+), 118 deletions(-) delete mode 100644 x-pack/test/api_integration/apis/metrics_ui/types.ts create mode 100644 x-pack/test/common/services/apm_synthtrace_kibana_client.ts create mode 100644 x-pack/test/common/utils/synthtrace/apm_es_client.ts diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts index 13b4ee07aad24..caf6f47be45ce 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts @@ -20,10 +20,18 @@ export class ApmSynthtraceKibanaClient { this.target = options.target; } + getFleetApmPackagePath(packageVersion?: string): string { + let path = `${this.target}/api/fleet/epm/packages/apm`; + if (packageVersion) { + path = `${path}/${packageVersion}`; + } + return path; + } + async fetchLatestApmPackageVersion() { this.logger.debug(`Fetching latest APM package version`); - const fleetPackageApiUrl = `${this.target}/api/fleet/epm/packages/apm?prerelease=true`; - const response = await fetch(fleetPackageApiUrl, { + const url = `${this.getFleetApmPackagePath()}?prerelease=true`; + const response = await fetch(url, { method: 'GET', headers: kibanaHeaders(), }); @@ -41,10 +49,13 @@ export class ApmSynthtraceKibanaClient { return latestVersion as string; } - async installApmPackage(packageVersion: string) { + async installApmPackage(packageVersion?: string) { this.logger.debug(`Installing APM package ${packageVersion}`); + if (!packageVersion) { + packageVersion = await this.fetchLatestApmPackageVersion(); + } - const url = `${this.target}/api/fleet/epm/packages/apm/${packageVersion}`; + const url = this.getFleetApmPackagePath(packageVersion); const response = await pRetry( async () => { const res = await fetch(url, { @@ -53,10 +64,14 @@ export class ApmSynthtraceKibanaClient { body: '{"force":true}', }); - if (res.status >= 400) { + if (!res.ok) { const errorJson = await res.json(); + const errorMessage = + typeof errorJson.message === 'string' + ? errorJson.message + : 'An error occurred during APM package installation.'; throw new Error( - `APM package installation returned ${res.status} status code\nError: ${errorJson}` + `APM package installation returned ${res.status} status code\nError: ${errorMessage}` ); } return res; @@ -75,10 +90,59 @@ export class ApmSynthtraceKibanaClient { if (!responseJson.items) { throw new Error( - `Failed to install APM package version ${packageVersion}, received HTTP ${response.status} and message: ${responseJson.message} for url ${url}` + `No installed assets received for APM package version ${packageVersion}, received HTTP ${response.status} for url ${url}` ); } this.logger.info(`Installed APM package ${packageVersion}`); + return { version: packageVersion }; + } + + async uninstallApmPackage() { + this.logger.debug('Uninstalling APM package'); + const latestApmPackageVersion = await this.fetchLatestApmPackageVersion(); + + const url = this.getFleetApmPackagePath(latestApmPackageVersion); + const response = await pRetry( + async () => { + const res = await fetch(url, { + method: 'DELETE', + headers: kibanaHeaders(), + body: '{"force":true}', + }); + + if (!res.ok) { + const errorJson = await res.json(); + const errorMessage = + typeof errorJson.message === 'string' + ? errorJson.message + : 'An error occurred during APM package uninstallation.'; + throw new Error( + `APM package uninstallation returned ${res.status} status code\nError: ${errorMessage}` + ); + } + return res; + }, + { + retries: 5, + onFailedAttempt: (error) => { + this.logger.debug( + `APM package version ${latestApmPackageVersion} uninstallation failure. ${ + error.retriesLeft >= 1 ? 'Retrying' : 'Aborting' + }` + ); + }, + } + ); + + const responseJson = await response.json(); + + if (!responseJson.items) { + throw new Error( + `No uninstalled assets received for APM package version ${latestApmPackageVersion}, received HTTP ${response.status} for url ${url}` + ); + } + + this.logger.info(`Uninstalled APM package ${latestApmPackageVersion}`); } } diff --git a/x-pack/test/api_integration/apis/metrics_ui/config.ts b/x-pack/test/api_integration/apis/metrics_ui/config.ts index ffca87e276ef1..c737db9499836 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/config.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/config.ts @@ -5,61 +5,12 @@ * 2.0. */ -import { - ApmSynthtraceEsClient, - ApmSynthtraceKibanaClient, - createLogger, - LogLevel, -} from '@kbn/apm-synthtrace'; -import url from 'url'; -import { FtrConfigProviderContext, kbnTestConfig } from '@kbn/test'; -import { FtrProviderContext as InheritedFtrProviderContext } from '../../ftr_provider_context'; -import { InheritedServices } from './types'; +import { FtrConfigProviderContext } from '@kbn/test'; -interface MetricsUIConfig { - services: InheritedServices & { - apmSynthtraceEsClient: (context: InheritedFtrProviderContext) => Promise; - }; -} -export default async function createTestConfig({ - readConfigFile, -}: FtrConfigProviderContext): Promise { +export default async function ({ readConfigFile }: FtrConfigProviderContext) { const baseIntegrationTestsConfig = await readConfigFile(require.resolve('../../config.ts')); - const services = baseIntegrationTestsConfig.get('services'); return { ...baseIntegrationTestsConfig.getAll(), testFiles: [require.resolve('.')], - services: { - ...services, - apmSynthtraceEsClient: async (context: InheritedFtrProviderContext) => { - const servers = baseIntegrationTestsConfig.get('servers'); - - const kibanaServer = servers.kibana as url.UrlObject; - const kibanaServerUrl = url.format(kibanaServer); - const kibanaServerUrlWithAuth = url - .format({ - ...url.parse(kibanaServerUrl), - auth: `elastic:${kbnTestConfig.getUrlParts().password}`, - }) - .slice(0, -1); - - const kibanaClient = new ApmSynthtraceKibanaClient({ - target: kibanaServerUrlWithAuth, - logger: createLogger(LogLevel.debug), - }); - const kibanaVersion = await kibanaClient.fetchLatestApmPackageVersion(); - await kibanaClient.installApmPackage(kibanaVersion); - - return new ApmSynthtraceEsClient({ - client: context.getService('es'), - logger: createLogger(LogLevel.info), - version: kibanaVersion, - refreshAfterIndex: true, - }); - }, - }, }; } -export type CreateTestConfig = Awaited>; - -export type MetricsUIServices = CreateTestConfig['services']; diff --git a/x-pack/test/api_integration/apis/metrics_ui/services.ts b/x-pack/test/api_integration/apis/metrics_ui/services.ts index 92ea26934603e..959e6f1109e74 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/services.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/services.ts @@ -7,24 +7,40 @@ import expect from '@kbn/expect'; import { ServicesAPIResponseRT } from '@kbn/infra-plugin/common/http_api/host_details'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { decodeOrThrow } from '@kbn/infra-plugin/common/runtime_types'; -import { FtrProviderContext } from './types'; +import { FtrProviderContext } from '../../ftr_provider_context'; import { generateServicesData, generateServicesLogsOnlyData } from './helpers'; +import { getApmSynthtraceEsClient } from '../../../common/utils/synthtrace/apm_es_client'; const SERVICES_ENDPOINT = '/api/infra/services'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const synthtrace = getService('apmSynthtraceEsClient'); + const apmSynthtraceKibanaClient = getService('apmSynthtraceKibanaClient'); + const esClient = getService('es'); + describe('GET /infra/services', () => { + let synthtraceApmClient: ApmSynthtraceEsClient; const from = new Date(Date.now() - 1000 * 60 * 2).toISOString(); const to = new Date().toISOString(); + before(async () => { + const version = (await apmSynthtraceKibanaClient.installApmPackage()).version; + synthtraceApmClient = await getApmSynthtraceEsClient({ + client: esClient, + packageVersion: version, + }); + }); + after(async () => apmSynthtraceKibanaClient.uninstallApmPackage()); describe('with transactions', () => { before(async () => - synthtrace.index(generateServicesData({ from, to, instanceCount: 3, servicesPerHost: 3 })) + synthtraceApmClient.index( + generateServicesData({ from, to, instanceCount: 3, servicesPerHost: 3 }) + ) ); - after(async () => synthtrace.clean()); + after(async () => synthtraceApmClient.clean()); + it('returns no services with no data', async () => { const filters = JSON.stringify({ 'host.name': 'some-host', @@ -81,14 +97,12 @@ export default function ({ getService }: FtrProviderContext) { }); }); describe('with logs only', () => { - before(async () => { - await synthtrace.index( + before(async () => + synthtraceApmClient.index( generateServicesLogsOnlyData({ from, to, instanceCount: 1, servicesPerHost: 2 }) - ); - }); - after(async () => { - await synthtrace.clean(); - }); + ) + ); + after(async () => synthtraceApmClient.clean()); it('should return services with logs only data', async () => { const filters = JSON.stringify({ 'host.name': 'host-0', diff --git a/x-pack/test/api_integration/apis/metrics_ui/types.ts b/x-pack/test/api_integration/apis/metrics_ui/types.ts deleted file mode 100644 index c0e4a84741bd2..0000000000000 --- a/x-pack/test/api_integration/apis/metrics_ui/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { GenericFtrProviderContext } from '@kbn/test'; -import { FtrProviderContext as InheritedFtrProviderContext } from '../../ftr_provider_context'; -import { MetricsUIServices } from './config'; - -export type InheritedServices = InheritedFtrProviderContext extends GenericFtrProviderContext< - infer TServices, - {} -> - ? TServices - : {}; - -export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/common/services/apm_synthtrace_kibana_client.ts b/x-pack/test/common/services/apm_synthtrace_kibana_client.ts new file mode 100644 index 0000000000000..63bbd917f93ea --- /dev/null +++ b/x-pack/test/common/services/apm_synthtrace_kibana_client.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import url from 'url'; +import { kbnTestConfig } from '@kbn/test'; +import { ApmSynthtraceKibanaClient, createLogger, LogLevel } from '@kbn/apm-synthtrace'; + +const getKibanaServerUrlWithAuth = () => { + const kibanaServerUrl = url.format(kbnTestConfig.getUrlParts() as url.UrlObject); + const kibanaServerUrlWithAuth = url + .format({ + ...url.parse(kibanaServerUrl), + auth: `elastic:${kbnTestConfig.getUrlParts().password}`, + }) + .slice(0, -1); + return kibanaServerUrlWithAuth; +}; +export function ApmSynthtraceKibanaClientProvider() { + const kibanaServerUrlWithAuth = getKibanaServerUrlWithAuth(); + const kibanaClient = new ApmSynthtraceKibanaClient({ + target: kibanaServerUrlWithAuth, + logger: createLogger(LogLevel.debug), + }); + + return kibanaClient; +} diff --git a/x-pack/test/common/services/index.ts b/x-pack/test/common/services/index.ts index 0f247ad743edf..a613fb8899c6c 100644 --- a/x-pack/test/common/services/index.ts +++ b/x-pack/test/common/services/index.ts @@ -10,6 +10,7 @@ import { services as kibanaCommonServices } from '../../../../test/common/servic import { InfraLogViewsServiceProvider } from './infra_log_views'; import { SpacesServiceProvider } from './spaces'; import { BsearchSecureService } from './bsearch_secure'; +import { ApmSynthtraceKibanaClientProvider } from './apm_synthtrace_kibana_client'; export const services = { ...kibanaCommonServices, @@ -17,4 +18,5 @@ export const services = { supertest: kibanaApiIntegrationServices.supertest, spaces: SpacesServiceProvider, secureBsearch: BsearchSecureService, + apmSynthtraceKibanaClient: ApmSynthtraceKibanaClientProvider, }; diff --git a/x-pack/test/common/utils/synthtrace/apm_es_client.ts b/x-pack/test/common/utils/synthtrace/apm_es_client.ts new file mode 100644 index 0000000000000..9bdc258c1e1be --- /dev/null +++ b/x-pack/test/common/utils/synthtrace/apm_es_client.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; +import { ApmSynthtraceEsClient, createLogger, LogLevel } from '@kbn/apm-synthtrace'; + +interface GetApmSynthtraceEsClientParams { + client: Client; + packageVersion: string; +} + +export async function getApmSynthtraceEsClient({ + client, + packageVersion, +}: GetApmSynthtraceEsClientParams) { + return new ApmSynthtraceEsClient({ + client, + logger: createLogger(LogLevel.info), + version: packageVersion, + refreshAfterIndex: true, + }); +} diff --git a/x-pack/test/functional/apps/infra/hosts_view.ts b/x-pack/test/functional/apps/infra/hosts_view.ts index 1f8f86017af9c..b452241739163 100644 --- a/x-pack/test/functional/apps/infra/hosts_view.ts +++ b/x-pack/test/functional/apps/infra/hosts_view.ts @@ -7,17 +7,10 @@ import moment from 'moment'; import expect from '@kbn/expect'; -import { - ApmSynthtraceEsClient, - ApmSynthtraceKibanaClient, - createLogger, - LogLevel, -} from '@kbn/apm-synthtrace'; -import url from 'url'; -import { kbnTestConfig } from '@kbn/test'; import { enableInfrastructureHostsView } from '@kbn/observability-plugin/common'; import { ALERT_STATUS_ACTIVE, ALERT_STATUS_RECOVERED } from '@kbn/rule-data-utils'; import { WebElementWrapper } from '@kbn/ftr-common-functional-ui-services'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { FtrProviderContext } from '../../ftr_provider_context'; import { DATES, @@ -26,6 +19,7 @@ import { DATE_PICKER_FORMAT, } from './constants'; import { generateAddServicesToExistingHost } from './helpers'; +import { getApmSynthtraceEsClient } from '../../../common/utils/synthtrace/apm_es_client'; const START_DATE = moment.utc(DATES.metricsAndLogs.hosts.min); const END_DATE = moment.utc(DATES.metricsAndLogs.hosts.max); @@ -110,6 +104,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const observability = getService('observability'); const retry = getService('retry'); const testSubjects = getService('testSubjects'); + const apmSynthtraceKibanaClient = getService('apmSynthtraceKibanaClient'); const pageObjects = getPageObjects([ 'assetDetails', 'common', @@ -123,17 +118,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { // Helpers - const getKibanaServerUrl = () => { - const kibanaServerUrl = url.format(kbnTestConfig.getUrlParts() as url.UrlObject); - const kibanaServerUrlWithAuth = url - .format({ - ...url.parse(kibanaServerUrl), - auth: `elastic:${kbnTestConfig.getUrlParts().password}`, - }) - .slice(0, -1); - return kibanaServerUrlWithAuth; - }; - const setHostViewEnabled = (value: boolean = true) => kibanaServer.uiSettings.update({ [enableInfrastructureHostsView]: value }); @@ -156,17 +140,10 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { describe('Hosts View', function () { let synthtraceApmClient: ApmSynthtraceEsClient; before(async () => { - const kibanaClient = new ApmSynthtraceKibanaClient({ - target: getKibanaServerUrl(), - logger: createLogger(LogLevel.debug), - }); - const kibanaVersion = await kibanaClient.fetchLatestApmPackageVersion(); - await kibanaClient.installApmPackage(kibanaVersion); - synthtraceApmClient = new ApmSynthtraceEsClient({ + const version = (await apmSynthtraceKibanaClient.installApmPackage()).version; + synthtraceApmClient = await getApmSynthtraceEsClient({ client: esClient, - logger: createLogger(LogLevel.info), - version: kibanaVersion, - refreshAfterIndex: true, + packageVersion: version, }); const services = generateAddServicesToExistingHost({ @@ -183,12 +160,12 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { esArchiver.load('x-pack/test/functional/es_archives/infra/alerts'), esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs'), esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_hosts_processes'), - kibanaServer.savedObjects.cleanStandardList(), ]); }); after(async () => { return Promise.all([ + apmSynthtraceKibanaClient.uninstallApmPackage(), synthtraceApmClient.clean(), esArchiver.unload('x-pack/test/functional/es_archives/infra/alerts'), esArchiver.unload('x-pack/test/functional/es_archives/infra/metrics_and_logs'), From c2adb13ee97399ee8d8070bfb6dc39de94e975f7 Mon Sep 17 00:00:00 2001 From: Marco Liberati Date: Mon, 8 Apr 2024 16:07:54 +0200 Subject: [PATCH 10/11] [ES|QL] Make `getActions` more resilient to lack of callbacks (#180260) ## Summary Now `getActions` provides some more fixes in case of lack of callabacks, like with unquoted fields. The feature is still experimental and applies only to unquoted fields (disabling the existence check on quoted fields). ### Checklist - [x] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../kbn-esql-ast/src/antlr_error_listener.ts | 4 +- .../README.md | 28 ++ .../src/code_actions/actions.test.ts | 457 ++++++++++++------ .../src/code_actions/actions.ts | 144 +++++- .../src/code_actions/types.ts | 4 + .../src/esql/lib/esql_ast_provider.ts | 1 + 6 files changed, 475 insertions(+), 163 deletions(-) diff --git a/packages/kbn-esql-ast/src/antlr_error_listener.ts b/packages/kbn-esql-ast/src/antlr_error_listener.ts index 0bb2912e2d235..add9a7bcace0f 100644 --- a/packages/kbn-esql-ast/src/antlr_error_listener.ts +++ b/packages/kbn-esql-ast/src/antlr_error_listener.ts @@ -25,8 +25,8 @@ export class ESQLErrorListener extends ErrorListener { const textMessage = `SyntaxError: ${message}`; const tokenPosition = getPosition(offendingSymbol); - const startColumn = tokenPosition?.min + 1 || column; - const endColumn = tokenPosition?.max + 1 || column + 1; + const startColumn = offendingSymbol && tokenPosition ? tokenPosition.min + 1 : column + 1; + const endColumn = offendingSymbol && tokenPosition ? tokenPosition.max + 1 : column + 2; this.errors.push({ startLineNumber: line, diff --git a/packages/kbn-esql-validation-autocomplete/README.md b/packages/kbn-esql-validation-autocomplete/README.md index e521bbe7839d7..f146ff876df40 100644 --- a/packages/kbn-esql-validation-autocomplete/README.md +++ b/packages/kbn-esql-validation-autocomplete/README.md @@ -116,6 +116,7 @@ const {title, edits} = await getActions( queryString, errors, getAstAndSyntaxErrors, + undefined, myCallbacks ); @@ -124,6 +125,33 @@ const {title, edits} = await getActions( console.log({ title, edits }); ``` +Like with validation also `getActions` can 'relax' its internal checks when no callbacks, either all or specific ones, are passed. + +```js +import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import { validateQuery, getActions } from '@kbn/esql-validation-autocomplete'; + +const queryString = "from index2 | keep unquoted-field" + +const myCallbacks = { + getSources: async () => [{name: 'index', hidden: false}], + ... +}; +const { errors, warnings } = await validateQuery(queryString, getAstAndSyntaxErrors, undefined, myCallbacks); + +const {title, edits} = await getActions( + queryString, + errors, + getAstAndSyntaxErrors, + { relaxOnMissingCallbacks: true }, + myCallbacks +); + +console.log(edits[0].text); // => `unquoted-field` +``` + +**Note**: this behaviour is still experimental, and applied for few error types, like the unquoted fields case. + ### getAstContext This is an important function in order to build more features on top of the existing one. diff --git a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts index 1f5af9239e686..ac38e8bcb79b9 100644 --- a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts +++ b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.test.ts @@ -10,6 +10,7 @@ import { getActions } from './actions'; import { validateQuery } from '../validation/validation'; import { getAllFunctions } from '../shared/helpers'; import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import { CodeActionOptions } from './types'; function getCallbackMocks() { return { @@ -60,6 +61,8 @@ function getCallbackMocks() { }; } +export type Simplify = { [KeyType in keyof T]: T[KeyType] } & {}; + /** * There are different wats to test the code here: one is a direct unit test of the feature, another is * an integration test passing from the query statement validation. The latter is more realistic, but @@ -68,27 +71,41 @@ function getCallbackMocks() { function testQuickFixesFn( statement: string, expectedFixes: string[] = [], - options: { equalityCheck?: 'include' | 'equal' } = {}, + options: Simplify<{ equalityCheck?: 'include' | 'equal' } & CodeActionOptions> = {}, { only, skip }: { only?: boolean; skip?: boolean } = {} ) { const testFn = only ? it.only : skip ? it.skip : it; - testFn(`${statement} => ["${expectedFixes.join('","')}"]`, async () => { - const callbackMocks = getCallbackMocks(); - const { errors } = await validateQuery( - statement, - getAstAndSyntaxErrors, - undefined, - callbackMocks - ); + testFn( + `${statement} => ["${expectedFixes.join('","')}"]${ + options.relaxOnMissingCallbacks != null + ? ` (Relaxed = ${options.relaxOnMissingCallbacks})` + : '' + } `, + async () => { + const callbackMocks = getCallbackMocks(); + const { errors } = await validateQuery( + statement, + getAstAndSyntaxErrors, + undefined, + callbackMocks + ); + const { equalityCheck, ...fnOptions } = options || {}; - const actions = await getActions(statement, errors, getAstAndSyntaxErrors, callbackMocks); - const edits = actions.map(({ edits: actionEdits }) => actionEdits[0].text); - expect(edits).toEqual( - !options || !options.equalityCheck || options.equalityCheck === 'equal' - ? expectedFixes - : expect.arrayContaining(expectedFixes) - ); - }); + const actions = await getActions( + statement, + errors, + getAstAndSyntaxErrors, + fnOptions, + callbackMocks + ); + const edits = actions.map(({ edits: actionEdits }) => actionEdits[0].text); + expect(edits).toEqual( + !equalityCheck || equalityCheck === 'equal' + ? expectedFixes + : expect.arrayContaining(expectedFixes) + ); + } + ); } type TestArgs = [string, string[], { equalityCheck?: 'include' | 'equal' }?]; @@ -111,84 +128,128 @@ const testQuickFixes = Object.assign(testQuickFixesFn, { describe('quick fixes logic', () => { describe('fixing index spellchecks', () => { - // No error, no quick action - testQuickFixes('FROM index', []); - testQuickFixes('FROM index2', ['index']); - testQuickFixes('FROM myindex', ['index', 'my-index']); - // wildcards - testQuickFixes('FROM index*', []); - testQuickFixes('FROM ind*', []); - testQuickFixes('FROM end*', ['ind*']); - testQuickFixes('FROM endex*', ['index']); - // Too far for the levenstein distance and should not fix with a hidden index - testQuickFixes('FROM secretIndex', []); - testQuickFixes('FROM secretIndex2', []); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + // No error, no quick action + testQuickFixes('FROM index', [], options); + testQuickFixes('FROM index2', ['index'], options); + testQuickFixes('FROM myindex', ['index', 'my-index'], options); + // wildcards + testQuickFixes('FROM index*', [], options); + testQuickFixes('FROM ind*', [], options); + testQuickFixes('FROM end*', ['ind*']); + testQuickFixes('FROM endex*', ['index'], options); + // Too far for the levenstein distance and should not fix with a hidden index + testQuickFixes('FROM secretIndex', [], options); + testQuickFixes('FROM secretIndex2', [], options); + } }); describe('fixing fields spellchecks', () => { - for (const command of ['KEEP', 'DROP', 'EVAL']) { - testQuickFixes(`FROM index | ${command} stringField`, []); - // strongField => stringField - testQuickFixes(`FROM index | ${command} strongField`, ['stringField']); - testQuickFixes(`FROM index | ${command} numberField, strongField`, ['stringField']); - } - testQuickFixes(`FROM index | EVAL round(strongField)`, ['stringField']); - testQuickFixes(`FROM index | EVAL var0 = round(strongField)`, ['stringField']); - testQuickFixes(`FROM index | WHERE round(strongField) > 0`, ['stringField']); - testQuickFixes(`FROM index | WHERE 0 < round(strongField)`, ['stringField']); - testQuickFixes(`FROM index | RENAME strongField as newField`, ['stringField']); - // This levarage the knowledge of the enrich policy fields to suggest the right field - testQuickFixes(`FROM index | ENRICH policy | KEEP yetAnotherField2`, ['yetAnotherField']); - testQuickFixes(`FROM index | ENRICH policy ON strongField`, ['stringField']); - testQuickFixes(`FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, [ - 'yetAnotherField', - ]); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + for (const command of ['KEEP', 'DROP', 'EVAL']) { + testQuickFixes(`FROM index | ${command} stringField`, [], options); + // strongField => stringField + testQuickFixes(`FROM index | ${command} strongField`, ['stringField'], options); + testQuickFixes( + `FROM index | ${command} numberField, strongField`, + ['stringField'], + options + ); + } + testQuickFixes(`FROM index | EVAL round(strongField)`, ['stringField'], options); + testQuickFixes(`FROM index | EVAL var0 = round(strongField)`, ['stringField'], options); + testQuickFixes(`FROM index | WHERE round(strongField) > 0`, ['stringField'], options); + testQuickFixes(`FROM index | WHERE 0 < round(strongField)`, ['stringField'], options); + testQuickFixes(`FROM index | RENAME strongField as newField`, ['stringField'], options); + // This levarage the knowledge of the enrich policy fields to suggest the right field + testQuickFixes( + `FROM index | ENRICH policy | KEEP yetAnotherField2`, + ['yetAnotherField'], + options + ); + testQuickFixes(`FROM index | ENRICH policy ON strongField`, ['stringField'], options); + testQuickFixes( + `FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, + ['yetAnotherField'], + options + ); - describe('metafields spellchecks', () => { - for (const isWrapped of [true, false]) { - function setWrapping(text: string) { - return isWrapped ? `[${text}]` : text; + describe('metafields spellchecks', () => { + for (const isWrapped of [true, false]) { + function setWrapping(text: string) { + return isWrapped ? `[${text}]` : text; + } + testQuickFixes(`FROM index ${setWrapping('metadata _i_ndex')}`, ['_index'], options); + testQuickFixes(`FROM index ${setWrapping('metadata _id, _i_ndex')}`, ['_index'], options); + testQuickFixes(`FROM index ${setWrapping('METADATA _id, _i_ndex')}`, ['_index'], options); } - testQuickFixes(`FROM index ${setWrapping('metadata _i_ndex')}`, ['_index']); - testQuickFixes(`FROM index ${setWrapping('metadata _id, _i_ndex')}`, ['_index']); - testQuickFixes(`FROM index ${setWrapping('METADATA _id, _i_ndex')}`, ['_index']); - } - }); + }); + } }); describe('fixing meta fields spellchecks', () => { - for (const command of ['KEEP', 'DROP', 'EVAL']) { - testQuickFixes(`FROM index | ${command} stringField`, []); - // strongField => stringField - testQuickFixes(`FROM index | ${command} strongField`, ['stringField']); - testQuickFixes(`FROM index | ${command} numberField, strongField`, ['stringField']); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + for (const command of ['KEEP', 'DROP', 'EVAL']) { + testQuickFixes(`FROM index | ${command} stringField`, [], options); + // strongField => stringField + testQuickFixes(`FROM index | ${command} strongField`, ['stringField'], options); + testQuickFixes( + `FROM index | ${command} numberField, strongField`, + ['stringField'], + options + ); + } + testQuickFixes(`FROM index | EVAL round(strongField)`, ['stringField'], options); + testQuickFixes(`FROM index | EVAL var0 = round(strongField)`, ['stringField'], options); + testQuickFixes(`FROM index | WHERE round(strongField) > 0`, ['stringField'], options); + testQuickFixes(`FROM index | WHERE 0 < round(strongField)`, ['stringField'], options); + testQuickFixes(`FROM index | RENAME strongField as newField`, ['stringField'], options); + // This levarage the knowledge of the enrich policy fields to suggest the right field + testQuickFixes( + `FROM index | ENRICH policy | KEEP yetAnotherField2`, + ['yetAnotherField'], + options + ); + testQuickFixes(`FROM index | ENRICH policy ON strongField`, ['stringField'], options); + testQuickFixes( + `FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, + ['yetAnotherField'], + options + ); } - testQuickFixes(`FROM index | EVAL round(strongField)`, ['stringField']); - testQuickFixes(`FROM index | EVAL var0 = round(strongField)`, ['stringField']); - testQuickFixes(`FROM index | WHERE round(strongField) > 0`, ['stringField']); - testQuickFixes(`FROM index | WHERE 0 < round(strongField)`, ['stringField']); - testQuickFixes(`FROM index | RENAME strongField as newField`, ['stringField']); - // This levarage the knowledge of the enrich policy fields to suggest the right field - testQuickFixes(`FROM index | ENRICH policy | KEEP yetAnotherField2`, ['yetAnotherField']); - testQuickFixes(`FROM index | ENRICH policy ON strongField`, ['stringField']); - testQuickFixes(`FROM index | ENRICH policy ON stringField WITH yetAnotherField2`, [ - 'yetAnotherField', - ]); }); describe('fixing policies spellchecks', () => { - testQuickFixes(`FROM index | ENRICH poli`, ['policy']); - testQuickFixes(`FROM index | ENRICH mypolicy`, ['policy']); - testQuickFixes(`FROM index | ENRICH policy[`, ['policy', 'policy[]']); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + testQuickFixes(`FROM index | ENRICH poli`, ['policy'], options); + testQuickFixes(`FROM index | ENRICH mypolicy`, ['policy'], options); + testQuickFixes(`FROM index | ENRICH policy[`, ['policy', 'policy[]'], options); - describe('modes', () => { - testQuickFixes(`FROM index | ENRICH _ann:policy`, ['_any']); - const modes = ['_any', '_coordinator', '_remote']; - for (const mode of modes) { - testQuickFixes(`FROM index | ENRICH ${mode.replace('_', '@')}:policy`, [mode]); - } - testQuickFixes(`FROM index | ENRICH unknown:policy`, modes); - }); + describe('modes', () => { + testQuickFixes(`FROM index | ENRICH _ann:policy`, ['_any'], options); + const modes = ['_any', '_coordinator', '_remote']; + for (const mode of modes) { + testQuickFixes(`FROM index | ENRICH ${mode.replace('_', '@')}:policy`, [mode], options); + } + testQuickFixes(`FROM index | ENRICH unknown:policy`, modes, options); + }); + } }); describe('fixing function spellchecks', () => { @@ -197,68 +258,147 @@ describe('quick fixes logic', () => { } // it should be strange enough to make the function invalid const BROKEN_PREFIX = 'Q'; - for (const fn of getAllFunctions({ type: 'eval' })) { - // add an A to the function name to make it invalid - testQuickFixes( - `FROM index | EVAL ${BROKEN_PREFIX}${fn.name}()`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); - testQuickFixes( - `FROM index | EVAL var0 = ${BROKEN_PREFIX}${fn.name}()`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); - testQuickFixes( - `FROM index | STATS avg(${BROKEN_PREFIX}${fn.name}())`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); - testQuickFixes( - `FROM index | STATS avg(numberField) BY ${BROKEN_PREFIX}${fn.name}()`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); - testQuickFixes( - `FROM index | STATS avg(numberField) BY var0 = ${BROKEN_PREFIX}${fn.name}()`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); - } - for (const fn of getAllFunctions({ type: 'agg' })) { - // add an A to the function name to make it invalid - testQuickFixes( - `FROM index | STATS ${BROKEN_PREFIX}${fn.name}()`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); - testQuickFixes( - `FROM index | STATS var0 = ${BROKEN_PREFIX}${fn.name}()`, - [fn.name].map(toFunctionSignature), - { equalityCheck: 'include' } - ); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + for (const fn of getAllFunctions({ type: 'eval' })) { + // add an A to the function name to make it invalid + testQuickFixes( + `FROM index | EVAL ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + testQuickFixes( + `FROM index | EVAL var0 = ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + testQuickFixes( + `FROM index | STATS avg(${BROKEN_PREFIX}${fn.name}())`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + testQuickFixes( + `FROM index | STATS avg(numberField) BY ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + testQuickFixes( + `FROM index | STATS avg(numberField) BY var0 = ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + } + for (const fn of getAllFunctions({ type: 'agg' })) { + // add an A to the function name to make it invalid + testQuickFixes( + `FROM index | STATS ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + testQuickFixes( + `FROM index | STATS var0 = ${BROKEN_PREFIX}${fn.name}()`, + [fn.name].map(toFunctionSignature), + { equalityCheck: 'include', ...options } + ); + } + // it should preserve the arguments + testQuickFixes(`FROM index | EVAL rAund(numberField)`, ['round(numberField)'], { + equalityCheck: 'include', + ...options, + }); + testQuickFixes(`FROM index | STATS AVVG(numberField)`, ['avg(numberField)'], { + equalityCheck: 'include', + ...options, + }); } - // it should preserve the arguments - testQuickFixes(`FROM index | EVAL rAund(numberField)`, ['round(numberField)'], { - equalityCheck: 'include', - }); - testQuickFixes(`FROM index | STATS AVVG(numberField)`, ['avg(numberField)'], { - equalityCheck: 'include', - }); }); describe('fixing wrong quotes', () => { - testQuickFixes(`FROM index | WHERE stringField like 'asda'`, ['"asda"']); - testQuickFixes(`FROM index | WHERE stringField not like 'asda'`, ['"asda"']); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + testQuickFixes(`FROM index | WHERE stringField like 'asda'`, ['"asda"'], options); + testQuickFixes(`FROM index | WHERE stringField not like 'asda'`, ['"asda"'], options); + } }); describe('fixing unquoted field names', () => { - testQuickFixes('FROM index | DROP any#Char$Field', ['`any#Char$Field`']); - testQuickFixes('FROM index | DROP numberField, any#Char$Field', ['`any#Char$Field`']); + for (const options of [ + undefined, + { relaxOnMissingCallbacks: false }, + { relaxOnMissingCallbacks: false }, + ]) { + testQuickFixes('FROM index | DROP any#Char$Field', ['`any#Char$Field`'], options); + testQuickFixes( + 'FROM index | DROP numberField, any#Char$Field', + ['`any#Char$Field`'], + options + ); + } + describe('with no callbacks', () => { + describe('with no relaxed option', () => { + it('return no result without callbacks and relaxed option', async () => { + const statement = `FROM index | DROP any#Char$Field`; + const { errors } = await validateQuery(statement, getAstAndSyntaxErrors); + const edits = await getActions(statement, errors, getAstAndSyntaxErrors); + expect(edits.length).toBe(0); + }); + + it('return no result without specific callback and relaxed option', async () => { + const callbackMocks = getCallbackMocks(); + const statement = `FROM index | DROP any#Char$Field`; + const { errors } = await validateQuery(statement, getAstAndSyntaxErrors, undefined, { + ...callbackMocks, + getFieldsFor: undefined, + }); + const edits = await getActions(statement, errors, getAstAndSyntaxErrors, undefined, { + ...callbackMocks, + getFieldsFor: undefined, + }); + expect(edits.length).toBe(0); + }); + }); + describe('with relaxed option', () => { + it('return a result without callbacks and relaxed option', async () => { + const statement = `FROM index | DROP any#Char$Field`; + const { errors } = await validateQuery(statement, getAstAndSyntaxErrors); + const actions = await getActions(statement, errors, getAstAndSyntaxErrors, { + relaxOnMissingCallbacks: true, + }); + const edits = actions.map(({ edits: actionEdits }) => actionEdits[0].text); + expect(edits).toEqual(['`any#Char$Field`']); + }); + + it('return a result without specific callback and relaxed option', async () => { + const callbackMocks = getCallbackMocks(); + const statement = `FROM index | DROP any#Char$Field`; + const { errors } = await validateQuery(statement, getAstAndSyntaxErrors, undefined, { + ...callbackMocks, + getFieldsFor: undefined, + }); + const actions = await getActions( + statement, + errors, + getAstAndSyntaxErrors, + { + relaxOnMissingCallbacks: true, + }, + { ...callbackMocks, getFieldsFor: undefined } + ); + const edits = actions.map(({ edits: actionEdits }) => actionEdits[0].text); + expect(edits).toEqual(['`any#Char$Field`']); + }); + }); + }); }); describe('callbacks', () => { - it('should not crash if callback functions are not passed', async () => { + it('should not crash if specific callback functions are not passed', async () => { const callbackMocks = getCallbackMocks(); const statement = `from a | eval b = a | enrich policy | dissect stringField "%{firstWord}"`; const { errors } = await validateQuery( @@ -268,7 +408,7 @@ describe('quick fixes logic', () => { callbackMocks ); try { - await getActions(statement, errors, getAstAndSyntaxErrors, { + await getActions(statement, errors, getAstAndSyntaxErrors, undefined, { getFieldsFor: undefined, getSources: undefined, getPolicies: undefined, @@ -279,6 +419,33 @@ describe('quick fixes logic', () => { } }); + it('should not crash if specific callback functions are not passed with relaxed option', async () => { + const callbackMocks = getCallbackMocks(); + const statement = `from a | eval b = a | enrich policy | dissect stringField "%{firstWord}"`; + const { errors } = await validateQuery( + statement, + getAstAndSyntaxErrors, + undefined, + callbackMocks + ); + try { + await getActions( + statement, + errors, + getAstAndSyntaxErrors, + { relaxOnMissingCallbacks: true }, + { + getFieldsFor: undefined, + getSources: undefined, + getPolicies: undefined, + getMetaFields: undefined, + } + ); + } catch { + fail('Should not throw'); + } + }); + it('should not crash no callbacks are passed', async () => { const callbackMocks = getCallbackMocks(); const statement = `from a | eval b = a | enrich policy | dissect stringField "%{firstWord}"`; @@ -289,7 +456,25 @@ describe('quick fixes logic', () => { callbackMocks ); try { - await getActions(statement, errors, getAstAndSyntaxErrors, undefined); + await getActions(statement, errors, getAstAndSyntaxErrors); + } catch { + fail('Should not throw'); + } + }); + + it('should not crash no callbacks are passed with relaxed option', async () => { + const callbackMocks = getCallbackMocks(); + const statement = `from a | eval b = a | enrich policy | dissect stringField "%{firstWord}"`; + const { errors } = await validateQuery( + statement, + getAstAndSyntaxErrors, + undefined, + callbackMocks + ); + try { + await getActions(statement, errors, getAstAndSyntaxErrors, { + relaxOnMissingCallbacks: true, + }); } catch { fail('Should not throw'); } diff --git a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.ts b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.ts index 47694d1721fab..0fe7c1d0e544a 100644 --- a/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.ts +++ b/packages/kbn-esql-validation-autocomplete/src/code_actions/actions.ts @@ -8,6 +8,7 @@ import { i18n } from '@kbn/i18n'; import levenshtein from 'js-levenshtein'; import type { AstProviderFn, ESQLAst, ESQLCommand, EditorError, ESQLMessage } from '@kbn/esql-ast'; +import { uniqBy } from 'lodash'; import { getFieldsByTypeHelper, getPolicyHelper, @@ -16,13 +17,15 @@ import { import { getAllFunctions, getCommandDefinition, + isColumnItem, isSourceItem, shouldBeQuotedText, } from '../shared/helpers'; import { ESQLCallbacks } from '../shared/types'; import { buildQueryForFieldsFromSource } from '../validation/helpers'; import { DOUBLE_BACKTICK, SINGLE_TICK_REGEX } from '../shared/constants'; -import type { CodeAction, Callbacks } from './types'; +import type { CodeAction, Callbacks, CodeActionOptions } from './types'; +import { getAstContext } from '../shared/context'; import { wrapAsEditorMessage } from './utils'; function getFieldsByTypeRetriever(queryString: string, resourceRetriever?: ESQLCallbacks) { @@ -114,9 +117,13 @@ async function getSpellingActionForColumns( error: EditorError, queryString: string, ast: ESQLAst, - { getFieldsByType, getPolicies, getPolicyFields }: Callbacks + options: CodeActionOptions, + { getFieldsByType, getPolicies, getPolicyFields }: Partial ) { const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); + if (!getFieldsByType || !getPolicyFields) { + return []; + } // @TODO add variables support const possibleFields = await getSpellingPossibilities(async () => { const availableFields = await getFieldsByType('any'); @@ -133,11 +140,32 @@ async function getSpellingActionForColumns( return wrapIntoSpellingChangeAction(error, possibleFields); } +function extractUnquotedFieldText( + query: string, + errorType: string, + ast: ESQLAst, + possibleStart: number, + end: number +) { + if (errorType === 'syntaxError') { + // scope it down to column items for now + const { node } = getAstContext(query, ast, possibleStart - 1); + if (node && isColumnItem(node)) { + return { + start: node.location.min + 1, + name: query.substring(node.location.min, end).trimEnd(), + }; + } + } + return { start: possibleStart + 1, name: query.substring(possibleStart, end).trimEnd() }; +} + async function getQuotableActionForColumns( error: EditorError, queryString: string, ast: ESQLAst, - { getFieldsByType }: Callbacks + options: CodeActionOptions, + { getFieldsByType }: Partial ): Promise { const commandEndIndex = ast.find((command) => command.location.max > error.endColumn)?.location .max; @@ -159,14 +187,20 @@ async function getQuotableActionForColumns( error.endColumn - 1, error.endColumn + stopIndex ); - const errorText = queryString - .substring(error.startColumn - 1, error.endColumn + possibleUnquotedText.length) - .trimEnd(); + const { start, name: errorText } = extractUnquotedFieldText( + queryString, + error.code || 'syntaxError', + ast, + error.startColumn - 1, + error.endColumn + possibleUnquotedText.length + ); const actions: CodeAction[] = []; if (shouldBeQuotedText(errorText)) { - const availableFields = new Set(await getFieldsByType('any')); const solution = `\`${errorText.replace(SINGLE_TICK_REGEX, DOUBLE_BACKTICK)}\``; - if (availableFields.has(errorText) || availableFields.has(solution)) { + if (!getFieldsByType) { + if (!options.relaxOnMissingCallbacks) { + return []; + } actions.push( createAction( i18n.translate('kbn-esql-validation-autocomplete.esql.quickfix.replaceWithSolution', { @@ -176,9 +210,25 @@ async function getQuotableActionForColumns( }, }), solution, - { ...error, endColumn: error.startColumn + errorText.length } // override the location + { ...error, startColumn: start, endColumn: start + errorText.length } // override the location ) ); + } else { + const availableFields = new Set(await getFieldsByType('any')); + if (availableFields.has(errorText) || availableFields.has(solution)) { + actions.push( + createAction( + i18n.translate('kbn-esql-validation-autocomplete.esql.quickfix.replaceWithSolution', { + defaultMessage: 'Did you mean {solution} ?', + values: { + solution, + }, + }), + solution, + { ...error, startColumn: start, endColumn: start + errorText.length } // override the location + ) + ); + } } } return actions; @@ -188,8 +238,12 @@ async function getSpellingActionForIndex( error: EditorError, queryString: string, ast: ESQLAst, - { getSources }: Callbacks + options: CodeActionOptions, + { getSources }: Partial ) { + if (!getSources) { + return []; + } const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); const possibleSources = await getSpellingPossibilities(async () => { // Handle fuzzy names via truncation to test levenstein distance @@ -208,8 +262,12 @@ async function getSpellingActionForPolicies( error: EditorError, queryString: string, ast: ESQLAst, - { getPolicies }: Callbacks + options: CodeActionOptions, + { getPolicies }: Partial ) { + if (!getPolicies) { + return []; + } const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); const possiblePolicies = await getSpellingPossibilities(getPolicies, errorText); return wrapIntoSpellingChangeAction(error, possiblePolicies); @@ -245,8 +303,12 @@ async function getSpellingActionForMetadata( error: EditorError, queryString: string, ast: ESQLAst, - { getMetaFields }: Callbacks + options: CodeActionOptions, + { getMetaFields }: Partial ) { + if (!getMetaFields) { + return []; + } const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); const possibleMetafields = await getSpellingPossibilities(getMetaFields, errorText); return wrapIntoSpellingChangeAction(error, possibleMetafields); @@ -256,7 +318,8 @@ async function getSpellingActionForEnrichMode( error: EditorError, queryString: string, ast: ESQLAst, - _callbacks: Callbacks + options: CodeActionOptions, + _callbacks: Partial ) { const errorText = queryString.substring(error.startColumn - 1, error.endColumn - 1); const commandContext = @@ -300,17 +363,27 @@ function extractQuotedText(rawText: string, error: EditorError) { return rawText.substring(error.startColumn - 2, error.endColumn); } -function inferCodeFromError(error: EditorError & { owner?: string }, rawText: string) { +function inferCodeFromError( + error: EditorError & { owner?: string }, + ast: ESQLAst, + rawText: string +) { if (error.message.endsWith('expecting QUOTED_STRING')) { const value = extractQuotedText(rawText, error); return /^'(.)*'$/.test(value) ? 'wrongQuotes' : undefined; } + if (error.message.startsWith('SyntaxError: token recognition error at:')) { + // scope it down to column items for now + const { node } = getAstContext(rawText, ast, error.startColumn - 2); + return node && isColumnItem(node) ? 'quotableFields' : undefined; + } } export async function getActions( innerText: string, markers: Array, astProvider: AstProviderFn, + options: CodeActionOptions = {}, resourceRetriever?: ESQLCallbacks ): Promise { const actions: CodeAction[] = []; @@ -327,28 +400,46 @@ export async function getActions( const getMetaFields = getMetaFieldsRetriever(innerText, ast, resourceRetriever); const callbacks = { - getFieldsByType, - getSources, - getPolicies, - getPolicyFields, - getMetaFields, + getFieldsByType: resourceRetriever?.getFieldsFor ? getFieldsByType : undefined, + getSources: resourceRetriever?.getSources ? getSources : undefined, + getPolicies: resourceRetriever?.getPolicies ? getPolicies : undefined, + getPolicyFields: resourceRetriever?.getPolicies ? getPolicyFields : undefined, + getMetaFields: resourceRetriever?.getMetaFields ? getMetaFields : undefined, }; // Markers are sent only on hover and are limited to the hovered area // so unless there are multiple error/markers for the same area, there's just one // in some cases, like syntax + semantic errors (i.e. unquoted fields eval field-1 ), there might be more than one for (const error of editorMarkers) { - const code = error.code ?? inferCodeFromError(error, innerText); + const code = error.code ?? inferCodeFromError(error, ast, innerText); switch (code) { - case 'unknownColumn': + case 'unknownColumn': { const [columnsSpellChanges, columnsQuotedChanges] = await Promise.all([ - getSpellingActionForColumns(error, innerText, ast, callbacks), - getQuotableActionForColumns(error, innerText, ast, callbacks), + getSpellingActionForColumns(error, innerText, ast, options, callbacks), + getQuotableActionForColumns(error, innerText, ast, options, callbacks), ]); actions.push(...(columnsQuotedChanges.length ? columnsQuotedChanges : columnsSpellChanges)); break; + } + case 'quotableFields': { + const columnsQuotedChanges = await getQuotableActionForColumns( + error, + innerText, + ast, + options, + callbacks + ); + actions.push(...columnsQuotedChanges); + break; + } case 'unknownIndex': - const indexSpellChanges = await getSpellingActionForIndex(error, innerText, ast, callbacks); + const indexSpellChanges = await getSpellingActionForIndex( + error, + innerText, + ast, + options, + callbacks + ); actions.push(...indexSpellChanges); break; case 'unknownPolicy': @@ -356,6 +447,7 @@ export async function getActions( error, innerText, ast, + options, callbacks ); actions.push(...policySpellChanges); @@ -369,6 +461,7 @@ export async function getActions( error, innerText, ast, + options, callbacks ); actions.push(...metadataSpellChanges); @@ -396,6 +489,7 @@ export async function getActions( error, innerText, ast, + options, callbacks ); actions.push(...enrichModeSpellChanges); @@ -404,5 +498,5 @@ export async function getActions( break; } } - return actions; + return uniqBy(actions, ({ edits }) => edits[0].text); } diff --git a/packages/kbn-esql-validation-autocomplete/src/code_actions/types.ts b/packages/kbn-esql-validation-autocomplete/src/code_actions/types.ts index 854f19ff279c1..949e734eb1794 100644 --- a/packages/kbn-esql-validation-autocomplete/src/code_actions/types.ts +++ b/packages/kbn-esql-validation-autocomplete/src/code_actions/types.ts @@ -31,3 +31,7 @@ export interface CodeAction { text: string; }>; } + +export interface CodeActionOptions { + relaxOnMissingCallbacks?: boolean; +} diff --git a/packages/kbn-monaco/src/esql/lib/esql_ast_provider.ts b/packages/kbn-monaco/src/esql/lib/esql_ast_provider.ts index 4b4d54de39e8b..8aa2e16979c5b 100644 --- a/packages/kbn-monaco/src/esql/lib/esql_ast_provider.ts +++ b/packages/kbn-monaco/src/esql/lib/esql_ast_provider.ts @@ -89,6 +89,7 @@ export class ESQLAstAdapter { model.getValue(), context.markers as EditorError[], getAstFn, + undefined, this.callbacks ); return codeActions; From 87491d63a39ab62e76d4273602061f089356b8a0 Mon Sep 17 00:00:00 2001 From: Matthew Kime Date: Mon, 8 Apr 2024 09:12:21 -0500 Subject: [PATCH 11/11] [data views] Reenable runtime field creation test on serverless (#180241) ## Summary Manually ran against a qa instance I set up myself - and it passed! Closes https://github.com/elastic/kibana/issues/178939 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Dzmitry Lemechko --- .../common/management/data_views/_runtime_fields_composite.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_runtime_fields_composite.ts b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_runtime_fields_composite.ts index 79d190295eec8..74bd6312807a0 100644 --- a/x-pack/test_serverless/functional/test_suites/common/management/data_views/_runtime_fields_composite.ts +++ b/x-pack/test_serverless/functional/test_suites/common/management/data_views/_runtime_fields_composite.ts @@ -17,8 +17,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const testSubjects = getService('testSubjects'); describe('runtime fields', function () { - // Bug: https://github.com/elastic/kibana/issues/178939 - this.tags('failsOnMKI'); before(async function () { await browser.setWindowSize(1200, 800); await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');