diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml new file mode 100644 index 0000000000000..f6cb21abdb682 --- /dev/null +++ b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml @@ -0,0 +1,58 @@ +# https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/ +agents: + queue: kibana-default + +steps: + - label: "Annotate runtime parameters" + command: | + buildkite-agent annotate --context es-serverless-image --style info "ES Serverless image: $ES_SERVERLESS_IMAGE" + buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT" + + - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite" + if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'" + steps: + - label: "Pre-Build" + command: .buildkite/scripts/lifecycle/pre_build.sh + key: pre-build + timeout_in_minutes: 10 + agents: + queue: kibana-default + + - label: "Build Kibana Distribution and Plugins" + command: .buildkite/scripts/steps/build_kibana.sh + agents: + queue: n2-16-spot + key: build + depends_on: pre-build + if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: '-1' + limit: 3 + + - label: "Pick Test Group Run Order" + command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh + agents: + queue: kibana-default + env: + FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh' + FTR_CONFIG_PATTERNS: '**/test_serverless/**' + LIMIT_CONFIG_TYPE: 'functional' + retry: + automatic: + - exit_status: '*' + limit: 1 + + - wait: ~ + + - label: ":arrow_up::elastic::arrow_up: Promote docker image" + command: .buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh $ES_SERVERLESS_IMAGE + + - wait: ~ + + - label: 'Post-Build' + command: .buildkite/scripts/lifecycle/post_build.sh + timeout_in_minutes: 10 + agents: + queue: kibana-default diff --git a/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh b/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh new file mode 100755 index 0000000000000..c6bf1738fe144 --- /dev/null +++ b/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/common/util.sh + +BASE_ES_SERVERLESS_REPO=docker.elastic.co/elasticsearch-ci/elasticsearch-serverless +TARGET_IMAGE=docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified + +ES_SERVERLESS_BUCKET=kibana-ci-es-serverless-images +MANIFEST_FILE_NAME=latest-verified.json + +SOURCE_IMAGE_OR_TAG=$1 +if [[ $SOURCE_IMAGE_OR_TAG =~ :[a-zA-Z_-]+$ ]]; then + # $SOURCE_IMAGE_OR_TAG was a full image + SOURCE_IMAGE=$SOURCE_IMAGE_OR_TAG +else + # $SOURCE_IMAGE_OR_TAG was an image tag + SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG" +fi + +echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'" + +echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE" + +echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co +docker pull "$SOURCE_IMAGE" +docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE" +docker push "$TARGET_IMAGE" + +ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE") +ELASTIC_COMMIT_HASH=$(echo $ORIG_IMG_DATA | jq -r '.[].Config.Labels["org.opencontainers.image.revision"]') + +docker logout docker.elastic.co + +echo "Image push to $TARGET_IMAGE successful." +echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE" + +MANIFEST_UPLOAD_PATH="Skipped" +if [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then + echo "--- Uploading latest-verified manifest to GCS" + cat << EOT >> $MANIFEST_FILE_NAME +{ + "build_url": "$BUILDKITE_BUILD_URL", + "kibana_commit": "$BUILDKITE_COMMIT", + "kibana_branch": "$BUILDKITE_BRANCH", + "elasticsearch_serverless_tag": "$SOURCE_IMAGE_OR_TAG", + "elasticsearch_serverless_image_url: "$SOURCE_IMAGE", + "elasticsearch_serverless_commit": "TODO: this currently can't be decided", + "elasticsearch_commit": "$ELASTIC_COMMIT_HASH", + "created_at": "`date`", + "timestamp": "`FORCE_COLOR=0 node -p 'Date.now()'`" +} +EOT + + gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" \ + cp $MANIFEST_FILE_NAME "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME" + gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME" + MANIFEST_UPLOAD_PATH="$MANIFEST_FILE_NAME" + +elif [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ ]]; then + echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash" +elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then + echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided" +fi + +echo "--- Annotating build with info" +cat << EOT | buildkite-agent annotate --style "success" +

Promotion successful!

+
New image: $TARGET_IMAGE +
Source image: $SOURCE_IMAGE +
Kibana commit: $BUILDKITE_COMMIT +
Elasticsearch commit: $ELASTIC_COMMIT_HASH +
Manifest file: $MANIFEST_UPLOAD_PATH +EOT diff --git a/docs/api-generated/connectors/connector-apis-passthru.asciidoc b/docs/api-generated/connectors/connector-apis-passthru.asciidoc index a8b429e0b5ae4..b68b4e4ca648f 100644 --- a/docs/api-generated/connectors/connector-apis-passthru.asciidoc +++ b/docs/api-generated/connectors/connector-apis-passthru.asciidoc @@ -1015,6 +1015,7 @@ Any modifications made to this file will be overwritten.
  • config_properties_servicenow - Connector request properties for a ServiceNow ITSM connector
  • config_properties_servicenow_itom - Connector request properties for a ServiceNow ITSM connector
  • config_properties_swimlane - Connector request properties for a Swimlane connector
    • +
  • config_properties_torq - Connector request properties for a Torq connector
  • config_properties_webhook - Connector request properties for a Webhook connector
  • config_properties_xmatters - Connector request properties for an xMatters connector
  • connector_response_properties - Connector response properties
    • @@ -1035,6 +1036,7 @@ Any modifications made to this file will be overwritten.
  • connector_response_properties_swimlane - Connector response properties for a Swimlane connector
  • connector_response_properties_teams - Connector response properties for a Microsoft Teams connector
  • connector_response_properties_tines - Connector response properties for a Tines connector
    • +
  • connector_response_properties_torq - Connector response properties for a Torq connector
  • connector_response_properties_webhook - Connector response properties for a Webhook connector
  • connector_response_properties_xmatters - Connector response properties for an xMatters connector
  • connector_types - Connector types
    • @@ -1056,6 +1058,7 @@ Any modifications made to this file will be overwritten.
  • create_connector_request_swimlane - Create Swimlane connector request
  • create_connector_request_teams - Create Microsoft Teams connector request
  • create_connector_request_tines - Create Tines connector request
    • +
  • create_connector_request_torq - Create Torq connector request
  • create_connector_request_webhook - Create Webhook connector request
  • create_connector_request_xmatters - Create xMatters connector request
  • features -
    • @@ -1106,6 +1109,7 @@ Any modifications made to this file will be overwritten.
  • secrets_properties_slack_webhook - Connector secrets properties for a Webhook Slack connector
  • secrets_properties_swimlane - Connector secrets properties for a Swimlane connector
  • secrets_properties_teams - Connector secrets properties for a Microsoft Teams connector
    • +
  • secrets_properties_torq - Connector secrets properties for a Torq connector
  • secrets_properties_webhook - Connector secrets properties for a Webhook connector
  • secrets_properties_xmatters - Connector secrets properties for an xMatters connector
  • updateConnector_400_response -
    • @@ -1124,6 +1128,7 @@ Any modifications made to this file will be overwritten.
  • update_connector_request_slack_webhook - Update Slack connector request
  • update_connector_request_swimlane - Update Swimlane connector request
  • update_connector_request_teams - Update Microsoft Teams connector request
    • +
  • update_connector_request_torq - Update Torq connector request
  • update_connector_request_webhook - Update Webhook connector request
  • update_connector_request_xmatters - Update xMatters connector request
    • @@ -1529,6 +1534,13 @@ Any modifications made to this file will be overwritten.
    mappings (optional)
    Connector_mappings_properties_for_a_Swimlane_connector
    +
    +

    config_properties_torq - Connector request properties for a Torq connector Up

    +
    Defines properties for connectors when type is .torq.
    +
    +
    webhookIntegrationUrl
    String The endpoint URL of the Elastic Security integration in Torq.
    +
    +

    config_properties_webhook - Connector request properties for a Webhook connector Up

    Defines properties for connectors when type is .webhook.
    @@ -1842,6 +1854,22 @@ Any modifications made to this file will be overwritten.
    is_missing_secrets (optional)
    Boolean Indicates whether secrets are missing for the connector. Secrets configuration properties vary depending on the connector type.
    is_preconfigured
    Boolean Indicates whether it is a preconfigured connector. If true, the config and is_missing_secrets properties are omitted from the response.
    is_system_action (optional)
    Boolean Indicates whether the connector is used for system actions.
    +
    name
    String The display name for the connector.
    +
    + +
    +

    connector_response_properties_torq - Connector response properties for a Torq connector Up

    +
    +
    +
    config
    config_properties_torq
    +
    connector_type_id
    String The type of connector.
    +
    Enum:
    +
    .torq
    +
    id
    String The identifier for the connector.
    +
    is_deprecated
    Boolean Indicates whether the connector type is deprecated.
    +
    is_missing_secrets (optional)
    Boolean Indicates whether secrets are missing for the connector. Secrets configuration properties vary depending on the connector type.
    +
    is_preconfigured
    Boolean Indicates whether it is a preconfigured connector. If true, the config and is_missing_secrets properties are omitted from the response.
    +
    is_system_action (optional)
    Boolean Indicates whether the connector is used for system actions.
    name
    String The display name for the connector.
    @@ -2093,6 +2121,18 @@ Any modifications made to this file will be overwritten.
    secrets
    map[String, oas_any_type_not_mapped] Defines secrets for connectors when type is .tines.
    +
    +

    create_connector_request_torq - Create Torq connector request Up

    +
    The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.
    +
    +
    config
    config_properties_torq
    +
    connector_type_id
    String The type of connector.
    +
    Enum:
    +
    .torq
    +
    name
    String The display name for the connector.
    +
    secrets
    secrets_properties_torq
    +
    +

    create_connector_request_webhook - Create Webhook connector request Up

    The Webhook connector uses axios to send a POST or PUT request to a web service.
    @@ -2560,6 +2600,13 @@ Any modifications made to this file will be overwritten.
    webhookUrl
    String The URL of the incoming webhook. If you are using the xpack.actions.allowedHosts setting, add the hostname to the allowed hosts.
    +
    +

    secrets_properties_torq - Connector secrets properties for a Torq connector Up

    +
    Defines secrets for connectors when type is .torq.
    +
    +
    token
    String The secret of the webhook authentication header.
    +
    +

    secrets_properties_webhook - Connector secrets properties for a Webhook connector Up

    Defines secrets for connectors when type is .webhook.
    @@ -2718,6 +2765,15 @@ Any modifications made to this file will be overwritten.
    secrets
    secrets_properties_teams
    +
    +

    update_connector_request_torq - Update Torq connector request Up

    +
    +
    +
    config
    config_properties_torq
    +
    name
    String The display name for the connector.
    +
    secrets
    secrets_properties_torq
    +
    +

    update_connector_request_webhook - Update Webhook connector request Up

    diff --git a/docs/management/action-types.asciidoc b/docs/management/action-types.asciidoc index bbaba806386e2..28aa218f3b6c8 100644 --- a/docs/management/action-types.asciidoc +++ b/docs/management/action-types.asciidoc @@ -7,10 +7,18 @@ Connectors provide a central place to store connection information for services [cols="2"] |=== +a| <> + +| Send a request to D3 Security. + a| <> | Send email from your server. +a| <> + +| Send a request to OpenAI. + a| <> | Create an incident in {ibm-r}. @@ -63,6 +71,10 @@ a| <> | Send events to a Tines Story. +a| <> + +| Trigger a Torq workflow. + a| <> | Send a request to a web service. @@ -75,18 +87,6 @@ a| <> | Send actionable alerts to on-call xMatters resources. -a| <> - -| Trigger a Torq workflow. - -a| <> - -| Send a request to OpenAI. - -a| <> - -| Send a request to D3 Security. - |=== [NOTE] diff --git a/docs/management/connectors/action-types/torq.asciidoc b/docs/management/connectors/action-types/torq.asciidoc index 39ef2585e62b9..7b4b9712adcb9 100644 --- a/docs/management/connectors/action-types/torq.asciidoc +++ b/docs/management/connectors/action-types/torq.asciidoc @@ -3,6 +3,10 @@ ++++ Torq ++++ +:frontmatter-description: Add a connector that can use Torq to trigger workflows. +:frontmatter-tags-products: [kibana] +:frontmatter-tags-content-type: [how-to] +:frontmatter-tags-user-goals: [configure] The Torq connector uses a Torq webhook to trigger workflows with Kibana actions. @@ -27,34 +31,6 @@ Torq endpoint URL:: Endpoint URL (webhook) of the Elastic Security integration y Torq authentication header secret:: Secret of the webhook authentication header. -[float] -[[preconfigured-torq-configuration]] -=== Create preconfigured connectors - -If you are running {kib} on-prem, you can define connectors by -adding `xpack.actions.preconfigured` settings to your `kibana.yml` file. -For example: - -[source,yaml] --- -xpack.actions.preconfigured: - my-torq: - name: preconfigured-torq-connector-type - actionTypeId: .torq - config: - webhookIntegrationUrl: https://hooks.torq.io/v1/somehook - secrets: - token: mytorqtoken --- - -Config defines information for the connector type. - -`webhookIntegrationUrl`:: An address that corresponds to **Torq endpoint URL**. - -Secrets defines sensitive information for the connector type. - -`token`:: A string that corresponds to **Torq authentication header secret**. - [float] [[torq-action-configuration]] === Test connectors diff --git a/docs/management/connectors/index.asciidoc b/docs/management/connectors/index.asciidoc index 63de5aca67dca..a30bc26ca2511 100644 --- a/docs/management/connectors/index.asciidoc +++ b/docs/management/connectors/index.asciidoc @@ -1,4 +1,6 @@ +include::action-types/d3security.asciidoc[leveloffset=+1] include::action-types/email.asciidoc[leveloffset=+1] +include::action-types/gen-ai.asciidoc[leveloffset=+1] include::action-types/resilient.asciidoc[leveloffset=+1] include::action-types/index.asciidoc[leveloffset=+1] include::action-types/jira.asciidoc[leveloffset=+1] @@ -16,6 +18,4 @@ include::action-types/torq.asciidoc[leveloffset=+1] include::action-types/webhook.asciidoc[leveloffset=+1] include::action-types/cases-webhook.asciidoc[leveloffset=+1] include::action-types/xmatters.asciidoc[leveloffset=+1] -include::action-types/gen-ai.asciidoc[leveloffset=+1] -include::action-types/d3security.asciidoc[leveloffset=+1] include::pre-configured-connectors.asciidoc[leveloffset=+1] diff --git a/docs/management/connectors/pre-configured-connectors.asciidoc b/docs/management/connectors/pre-configured-connectors.asciidoc index 72f5f78f6e728..1fc679facf423 100644 --- a/docs/management/connectors/pre-configured-connectors.asciidoc +++ b/docs/management/connectors/pre-configured-connectors.asciidoc @@ -113,11 +113,13 @@ Index names must start with `kibana-alert-history-` to take advantage of the pre * <> * <> * <> +* <> * <> * <> * <> * <> * <> +* <> * <> * <> * <> @@ -528,6 +530,26 @@ xpack.actions.preconfigured: <3> Field mappings for properties such as the alert identifer, severity, and rule name. <4> The API authentication token for HTTP basic authentication. NOTE: This value should be stored in the <>. +[float] +[[preconfigured-torq-configuration]] +==== Torq connectors + +The following example creates a <>: + +[source,yaml] +-- +xpack.actions.preconfigured: + my-torq: + name: preconfigured-torq-connector-type + actionTypeId: .torq + config: + webhookIntegrationUrl: https://hooks.torq.io/v1/somehook <1> + secrets: + token: mytorqtoken <2> +-- +<1> The endpoint URL of the Elastic Security integration in Torq. +<2> The secret of the webhook authentication header. + [float] [[preconfigured-webhook-configuration]] ==== Webhook connectors diff --git a/docs/management/upgrade-assistant.asciidoc b/docs/management/upgrade-assistant.asciidoc index a44afa4474a7b..ed9093ded2846 100644 --- a/docs/management/upgrade-assistant.asciidoc +++ b/docs/management/upgrade-assistant.asciidoc @@ -26,4 +26,4 @@ The Upgrade assistant pulls information about deprecations from the following so * Elasticsearch deprecation logs * Kibana deprecations API -For more information about the API's the Upgraed assistant provides, refer to <>. +For more information about Upgrade Assistant APIs, refer to <>. diff --git a/docs/settings/alert-action-settings.asciidoc b/docs/settings/alert-action-settings.asciidoc index c9880bdade4dc..b3cd5777edd69 100644 --- a/docs/settings/alert-action-settings.asciidoc +++ b/docs/settings/alert-action-settings.asciidoc @@ -432,6 +432,9 @@ For an <>, specifies whether it uses HT `xpack.actions.preconfigured..config.viewIncidentUrl`:: For a <>, specifies a URL string with either the external service ID or external service title Mustache variable to view a case in the external system. +`xpack.actions.preconfigured..config.webhookIntegrationUrl`:: +For a <>, specifies the endpoint URL of the Elastic Security integration in Torq. + `xpack.actions.preconfigured..name`:: The name of the preconfigured connector. @@ -492,6 +495,7 @@ A token secret that varies by connector: -- * For a <>, specifies the D3 Security token. * For a <>, specifies the Slack bot user OAuth token. +* For a <>, specifies the secret of the webhook authentication header. -- `xpack.actions.preconfigured..secrets.user`:: diff --git a/packages/kbn-dev-cli-runner/src/flags.ts b/packages/kbn-dev-cli-runner/src/flags.ts index 595205c3e0333..d7b352333ae1b 100644 --- a/packages/kbn-dev-cli-runner/src/flags.ts +++ b/packages/kbn-dev-cli-runner/src/flags.ts @@ -27,6 +27,7 @@ export interface FlagOptions { allowUnexpected?: boolean; guessTypesForUnexpectedFlags?: boolean; help?: string; + examples?: string; alias?: { [key: string]: string | string[] }; boolean?: string[]; string?: string[]; @@ -47,6 +48,7 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions = }, help: local.help, + examples: local.examples, allowUnexpected: !!(global.allowUnexpected || local.allowUnexpected), guessTypesForUnexpectedFlags: !!(global.allowUnexpected || local.allowUnexpected), diff --git a/packages/kbn-dev-cli-runner/src/help.ts b/packages/kbn-dev-cli-runner/src/help.ts index a7dc17aa43f17..f3e0e2c78e97f 100644 --- a/packages/kbn-dev-cli-runner/src/help.ts +++ b/packages/kbn-dev-cli-runner/src/help.ts @@ -36,11 +36,13 @@ export function getHelp({ usage, flagHelp, defaultLogLevel, + examples, }: { description?: string; usage?: string; flagHelp?: string; defaultLogLevel?: string; + examples?: string; }) { const optionHelp = joinAndTrimLines( dedent(flagHelp || ''), @@ -48,13 +50,17 @@ export function getHelp({ GLOBAL_FLAGS ); + const examplesHelp = examples ? joinAndTrimLines('Examples:', examples) : ''; + return ` ${dedent(usage || '') || DEFAULT_GLOBAL_USAGE} ${indent(dedent(description || 'Runs a dev task'), 2)} Options: - ${indent(optionHelp, 4)}\n\n`; + ${indent(optionHelp, 4)} +${examplesHelp ? `\n ${indent(examplesHelp, 4)}` : ''} +`; } export function getCommandLevelHelp({ diff --git a/packages/kbn-dev-cli-runner/src/run.ts b/packages/kbn-dev-cli-runner/src/run.ts index 08457caaebfd4..2ef90c2e2c27a 100644 --- a/packages/kbn-dev-cli-runner/src/run.ts +++ b/packages/kbn-dev-cli-runner/src/run.ts @@ -50,6 +50,7 @@ export async function run(fn: RunFn, options: RunOptions = {}) { usage: options.usage, flagHelp: options.flags?.help, defaultLogLevel: options.log?.defaultLevel, + examples: options.flags?.examples, }); if (flags.help) { diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts index 7ee4f08fb94fe..c8b3018e6f669 100644 --- a/packages/kbn-es/src/cli_commands/serverless.ts +++ b/packages/kbn-es/src/cli_commands/serverless.ts @@ -13,9 +13,8 @@ import { getTimeReporter } from '@kbn/ci-stats-reporter'; import { Cluster } from '../cluster'; import { - SERVERLESS_REPO, - SERVERLESS_TAG, - SERVERLESS_IMG, + ES_SERVERLESS_REPO_ELASTICSEARCH, + ES_SERVERLESS_DEFAULT_IMAGE, DEFAULT_PORT, ServerlessOptions, } from '../utils'; @@ -28,9 +27,8 @@ export const serverless: Command = { return dedent` Options: - --tag Image tag of ES serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}] - --image Full path of ES serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}] - + --tag Image tag of ES serverless to run from ${ES_SERVERLESS_REPO_ELASTICSEARCH} + --image Full path of ES serverless image to run, has precedence over tag. [default: ${ES_SERVERLESS_DEFAULT_IMAGE}] --background Start ES serverless without attaching to the first node's logs --basePath Path to the directory where the ES cluster will store data --clean Remove existing file system object store before running @@ -39,14 +37,14 @@ export const serverless: Command = { --ssl Enable HTTP SSL on the ES cluster --skipTeardown If this process exits, leave the ES cluster running in the background --waitForReady Wait for the ES cluster to be ready to serve requests - + -E Additional key=value settings to pass to ES -F Absolute paths for files to mount into containers Examples: - es serverless --tag git-fec36430fba2-x86_64 - es serverless --image docker.elastic.co/repo:tag + es serverless --tag git-fec36430fba2-x86_64 # loads ${ES_SERVERLESS_REPO_ELASTICSEARCH}:git-fec36430fba2-x86_64 + es serverless --image docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified `; }, run: async (defaults = {}) => { diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts index d48cddd6fdb6d..08edc2a17521d 100644 --- a/packages/kbn-es/src/utils/docker.test.ts +++ b/packages/kbn-es/src/utils/docker.test.ts @@ -23,7 +23,7 @@ import { runDockerContainer, runServerlessCluster, runServerlessEsNode, - SERVERLESS_IMG, + ES_SERVERLESS_DEFAULT_IMAGE, setupServerlessVolumes, stopServerlessCluster, teardownServerlessClusterSync, @@ -451,7 +451,7 @@ describe('runServerlessEsNode()', () => { const node = { params: ['--env', 'foo=bar', '--volume', 'foo/bar'], name: 'es01', - image: SERVERLESS_IMG, + image: ES_SERVERLESS_DEFAULT_IMAGE, }; test('should call the correct Docker command', async () => { @@ -462,7 +462,7 @@ describe('runServerlessEsNode()', () => { expect(execa.mock.calls[0][0]).toEqual('docker'); expect(execa.mock.calls[0][1]).toEqual( expect.arrayContaining([ - SERVERLESS_IMG, + ES_SERVERLESS_DEFAULT_IMAGE, ...node.params, '--name', node.name, @@ -530,7 +530,9 @@ describe('teardownServerlessClusterSync()', () => { teardownServerlessClusterSync(log, defaultOptions); expect(execa.commandSync.mock.calls).toHaveLength(2); - expect(execa.commandSync.mock.calls[0][0]).toEqual(expect.stringContaining(SERVERLESS_IMG)); + expect(execa.commandSync.mock.calls[0][0]).toEqual( + expect.stringContaining(ES_SERVERLESS_DEFAULT_IMAGE) + ); expect(execa.commandSync.mock.calls[1][0]).toEqual(`docker kill ${nodes.join(' ')}`); }); diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts index 00a1d7ce9dc54..5ed22e094e6f8 100644 --- a/packages/kbn-es/src/utils/docker.ts +++ b/packages/kbn-es/src/utils/docker.ts @@ -38,9 +38,12 @@ import { import { SYSTEM_INDICES_SUPERUSER } from './native_realm'; import { waitUntilClusterReady } from './wait_until_cluster_ready'; -interface BaseOptions { - tag?: string; +interface ImageOptions { image?: string; + tag?: string; +} + +interface BaseOptions extends ImageOptions { port?: number; ssl?: boolean; /** Kill running cluster before starting a new cluster */ @@ -106,9 +109,10 @@ export const DOCKER_REPO = `${DOCKER_REGISTRY}/elasticsearch/elasticsearch`; export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`; export const DOCKER_IMG = `${DOCKER_REPO}:${DOCKER_TAG}`; -export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`; -export const SERVERLESS_TAG = 'latest'; -export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`; +export const ES_SERVERLESS_REPO_KIBANA = `${DOCKER_REGISTRY}/kibana-ci/elasticsearch-serverless`; +export const ES_SERVERLESS_REPO_ELASTICSEARCH = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`; +export const ES_SERVERLESS_LATEST_VERIFIED_TAG = 'latest-verified'; +export const ES_SERVERLESS_DEFAULT_IMAGE = `${ES_SERVERLESS_REPO_KIBANA}:${ES_SERVERLESS_LATEST_VERIFIED_TAG}`; // See for default cluster settings // https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/kotlin/elasticsearch.serverless-run.gradle.kts @@ -275,7 +279,12 @@ export function resolveDockerImage({ image, repo, defaultImg, -}: (ServerlessOptions | DockerOptions) & { repo: string; defaultImg: string }) { +}: { + tag?: string; + image?: string; + repo: string; + defaultImg: string; +}) { if (image) { if (!image.includes(DOCKER_REGISTRY)) { throw createCliError( @@ -525,11 +534,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles /** * Resolve the Serverless ES image based on defaults and CLI options */ -function getServerlessImage(options: ServerlessOptions) { +function getServerlessImage({ image, tag }: ImageOptions) { return resolveDockerImage({ - ...options, - repo: SERVERLESS_REPO, - defaultImg: SERVERLESS_IMG, + image, + tag, + repo: ES_SERVERLESS_REPO_ELASTICSEARCH, + defaultImg: ES_SERVERLESS_DEFAULT_IMAGE, }); } @@ -573,7 +583,10 @@ function getESClient(clientOptions: ClientOptions): Client { * Runs an ES Serverless Cluster through Docker */ export async function runServerlessCluster(log: ToolingLog, options: ServerlessOptions) { - const image = getServerlessImage(options); + const image = getServerlessImage({ + image: options.image, + tag: options.tag, + }); await setupDocker({ log, image, options }); const volumeCmd = await setupServerlessVolumes(log, options); @@ -686,8 +699,13 @@ export function teardownServerlessClusterSync(log: ToolingLog, options: Serverle /** * Resolve the Elasticsearch image based on defaults and CLI options */ -function getDockerImage(options: DockerOptions) { - return resolveDockerImage({ ...options, repo: DOCKER_REPO, defaultImg: DOCKER_IMG }); +function getDockerImage({ image, tag }: ImageOptions) { + return resolveDockerImage({ + image, + tag, + repo: DOCKER_REPO, + defaultImg: DOCKER_IMG, + }); } /** @@ -713,7 +731,10 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions let image; if (!options.dockerCmd) { - image = getDockerImage(options); + image = getDockerImage({ + image: options.image, + tag: options.tag, + }); await setupDocker({ log, image, options }); } diff --git a/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap b/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap index ba2969ab4a0b3..f971ca2c40bbb 100644 --- a/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap +++ b/packages/kbn-guided-onboarding/src/components/landing_page/__snapshots__/guide_cards.test.tsx.snap @@ -18,14 +18,20 @@ exports[`guide cards snapshots should render all cards 1`] = ` activeFilter="all" card={ Object { - "icon": "vector", - "navigateTo": Object { - "appId": "enterpriseSearchVectorSearch", - }, + "guideId": "databaseSearch", + "icon": "database", "order": 1, "solution": "search", - "telemetryId": "onboarding--search--vector", - "title": "Set up vector search", + "telemetryId": "onboarding--search--database", + "title": , + } + } + />, } } guidesState={Array []} @@ -44,14 +50,14 @@ exports[`guide cards snapshots should render all cards 1`] = ` activeFilter="all" card={ Object { - "icon": "magnifyWithPlus", + "icon": "vector", "navigateTo": Object { - "appId": "enterpriseSearchAISearch", + "appId": "enterpriseSearchVectorSearch", }, "order": 4, "solution": "search", - "telemetryId": "onboarding--search--semantic", - "title": "Build a semantic search experience", + "telemetryId": "onboarding--search--vector", + "title": "Set up vector search", } } guidesState={Array []} @@ -70,20 +76,14 @@ exports[`guide cards snapshots should render all cards 1`] = ` activeFilter="all" card={ Object { - "guideId": "appSearch", - "icon": "wrench", + "icon": "magnifyWithPlus", + "navigateTo": Object { + "appId": "enterpriseSearchAISearch", + }, "order": 7, "solution": "search", - "telemetryId": "onboarding--search--application", - "title": , - } - } - />, + "telemetryId": "onboarding--search--ai", + "title": "Build an AI-powered search experience", } } guidesState={Array []} @@ -102,12 +102,20 @@ exports[`guide cards snapshots should render all cards 1`] = ` activeFilter="all" card={ Object { - "guideId": "websiteSearch", - "icon": "search", + "guideId": "appSearch", + "icon": "wrench", "order": 10, "solution": "search", - "telemetryId": "onboarding--search--website", - "title": "Add search to my website", + "telemetryId": "onboarding--search--application", + "title": , + } + } + />, } } guidesState={Array []} @@ -126,20 +134,12 @@ exports[`guide cards snapshots should render all cards 1`] = ` activeFilter="all" card={ Object { - "guideId": "databaseSearch", - "icon": "database", + "guideId": "websiteSearch", + "icon": "search", "order": 13, "solution": "search", - "telemetryId": "onboarding--search--database", - "title": , - } - } - />, + "telemetryId": "onboarding--search--website", + "title": "Add search to my website", } } guidesState={Array []} diff --git a/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx b/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx index c624f31d2848a..f570f5ea3b87b 100644 --- a/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx +++ b/packages/kbn-guided-onboarding/src/components/landing_page/guide_cards.constants.tsx @@ -31,6 +31,22 @@ export interface GuideCardConstants { } export const guideCards: GuideCardConstants[] = [ + { + solution: 'search', + icon: 'database', + title: ( + , + }} + /> + ), + guideId: 'databaseSearch', + telemetryId: 'onboarding--search--database', + order: 1, + }, { solution: 'search', icon: 'vector', @@ -41,19 +57,19 @@ export const guideCards: GuideCardConstants[] = [ appId: 'enterpriseSearchVectorSearch', }, telemetryId: 'onboarding--search--vector', - order: 1, + order: 4, }, { solution: 'search', icon: 'magnifyWithPlus', title: i18n.translate('guidedOnboardingPackage.gettingStarted.cards.aiSearch.title', { - defaultMessage: 'Build a semantic search experience', + defaultMessage: 'Build an AI-powered search experience', }), navigateTo: { appId: 'enterpriseSearchAISearch', }, - telemetryId: 'onboarding--search--semantic', - order: 4, + telemetryId: 'onboarding--search--ai', + order: 7, }, { solution: 'search', @@ -69,7 +85,7 @@ export const guideCards: GuideCardConstants[] = [ ), guideId: 'appSearch', telemetryId: 'onboarding--search--application', - order: 7, + order: 10, }, { solution: 'search', @@ -79,22 +95,6 @@ export const guideCards: GuideCardConstants[] = [ }), guideId: 'websiteSearch', telemetryId: 'onboarding--search--website', - order: 10, - }, - { - solution: 'search', - icon: 'database', - title: ( - , - }} - /> - ), - guideId: 'databaseSearch', - telemetryId: 'onboarding--search--database', order: 13, }, { diff --git a/packages/kbn-test/src/es/es_test_config.ts b/packages/kbn-test/src/es/es_test_config.ts index c31728b0fd7d8..26827558b7bd3 100644 --- a/packages/kbn-test/src/es/es_test_config.ts +++ b/packages/kbn-test/src/es/es_test_config.ts @@ -27,6 +27,10 @@ class EsTestConfig { return process.env.TEST_ES_FROM || 'snapshot'; } + getESServerlessImage() { + return process.env.TEST_ES_SERVERLESS_IMAGE; + } + getTransportPort() { return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400'; } diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index 3cd90d21d1d91..3c63960bdc0e5 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -70,6 +70,10 @@ export interface CreateTestEsClusterOptions { */ esArgs?: string[]; esFrom?: string; + esServerlessOptions?: { + image?: string; + tag?: string; + }; esJavaOpts?: string; /** * License to run your cluster under. Keep in mind that a `trial` license @@ -164,6 +168,7 @@ export function createTestEsCluster< writeLogsToPath, basePath = Path.resolve(REPO_ROOT, '.es'), esFrom = esTestConfig.getBuildFrom(), + esServerlessOptions, dataArchive, nodes = [{ name: 'node-01' }], esArgs: customEsArgs = [], @@ -236,9 +241,11 @@ export function createTestEsCluster< } else if (esFrom === 'snapshot') { installPath = (await firstNode.installSnapshot(config)).installPath; } else if (esFrom === 'serverless') { - return await firstNode.runServerless({ + await firstNode.runServerless({ basePath, esArgs: customEsArgs, + image: esServerlessOptions?.image, + tag: esServerlessOptions?.tag, port, clean: true, background: true, @@ -247,6 +254,7 @@ export function createTestEsCluster< kill: true, // likely don't need this but avoids any issues where the ESS cluster wasn't cleaned up waitForReady: true, }); + return; } else if (Path.isAbsolute(esFrom)) { installPath = esFrom; } else { @@ -275,9 +283,9 @@ export function createTestEsCluster< }); } - nodeStartPromises.push(async () => { + nodeStartPromises.push(() => { log.info(`[es] starting node ${node.name} on port ${nodePort}`); - return await this.nodes[i].start(installPath, { + return this.nodes[i].start(installPath, { password: config.password, esArgs: assignArgs(esArgs, overriddenArgs), esJavaOpts, @@ -292,7 +300,7 @@ export function createTestEsCluster< }); } - await Promise.all(extractDirectoryPromises.map(async (extract) => await extract())); + await Promise.all(extractDirectoryPromises.map((extract) => extract())); for (const start of nodeStartPromises) { await start(); } diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index f9c83161b521b..d298a1c1abaa4 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -12,11 +12,12 @@ import getPort from 'get-port'; import { REPO_ROOT } from '@kbn/repo-info'; import type { ArtifactLicense } from '@kbn/es'; import type { Config } from '../../functional_test_runner'; -import { createTestEsCluster } from '../../es'; +import { createTestEsCluster, esTestConfig } from '../../es'; interface RunElasticsearchOptions { log: ToolingLog; esFrom?: string; + esServerlessImage?: string; config: Config; onEarlyExit?: (msg: string) => void; logsDir?: string; @@ -32,6 +33,7 @@ type EsConfig = ReturnType; function getEsConfig({ config, esFrom = config.get('esTestCluster.from'), + esServerlessImage, }: RunElasticsearchOptions) { const ssl = !!config.get('esTestCluster.ssl'); const license: ArtifactLicense = config.get('esTestCluster.license'); @@ -50,6 +52,8 @@ function getEsConfig({ const serverless: boolean = config.get('serverless'); const files: string[] | undefined = config.get('esTestCluster.files'); + const esServerlessOptions = getESServerlessOptions(esServerlessImage, config); + return { ssl, license, @@ -57,6 +61,7 @@ function getEsConfig({ esJavaOpts, isSecurityEnabled, esFrom, + esServerlessOptions, port, password, dataArchive, @@ -129,6 +134,7 @@ async function startEsNode({ clusterName: `cluster-${name}`, esArgs: config.esArgs, esFrom: config.esFrom, + esServerlessOptions: config.esServerlessOptions, esJavaOpts: config.esJavaOpts, license: config.license, password: config.password, @@ -153,3 +159,23 @@ async function startEsNode({ return cluster; } + +function getESServerlessOptions(esServerlessImageFromArg: string | undefined, config: Config) { + const esServerlessImageUrlOrTag = + esServerlessImageFromArg || + esTestConfig.getESServerlessImage() || + (config.has('esTestCluster.esServerlessImage') && + config.get('esTestCluster.esServerlessImage')); + + if (esServerlessImageUrlOrTag) { + if (esServerlessImageUrlOrTag.includes(':')) { + return { + image: esServerlessImageUrlOrTag, + }; + } else { + return { + tag: esServerlessImageUrlOrTag, + }; + } + } +} diff --git a/packages/kbn-test/src/functional_tests/run_tests/cli.ts b/packages/kbn-test/src/functional_tests/run_tests/cli.ts index 19a003dd973cf..40a711ca9f6c2 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/cli.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/cli.ts @@ -26,6 +26,7 @@ export function runTestsCli() { { description: `Run Functional Tests`, usage: ` + Usage: node scripts/functional_tests --help node scripts/functional_tests [--config [--config ...]] node scripts/functional_tests [options] [-- --] diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts index 0c9dae3a25794..77399605b29f4 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts @@ -42,6 +42,7 @@ describe('parse runTest flags', () => { ], "dryRun": false, "esFrom": undefined, + "esServerlessImage": undefined, "esVersion": , "grep": undefined, "installDir": undefined, diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.ts index f4dd6beb26e80..3ba86999d3802 100644 --- a/packages/kbn-test/src/functional_tests/run_tests/flags.ts +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.ts @@ -23,6 +23,7 @@ export const FLAG_OPTIONS: FlagOptions = { 'config', 'journey', 'esFrom', + 'esServerlessImage', 'kibana-install-dir', 'grep', 'include-tag', @@ -37,6 +38,7 @@ export const FLAG_OPTIONS: FlagOptions = { --config Define a FTR config that should be executed. Can be specified multiple times --journey Define a Journey that should be executed. Can be specified multiple times --esFrom Build Elasticsearch from source or run snapshot or serverless. Default: $TEST_ES_FROM or "snapshot" + --esServerlessImage When 'esFrom' is "serverless", this argument will be interpreted either as a tag within the ES Serverless repo, OR a full docker image path. --include-tag Tags that suites must include to be run, can be included multiple times --exclude-tag Tags that suites must NOT include to be run, can be included multiple times --include Files that must included to be run, can be included multiple times @@ -50,6 +52,13 @@ export const FLAG_OPTIONS: FlagOptions = { --updateSnapshots Replace inline and file snapshots with whatever is generated from the test --updateAll, -u Replace both baseline screenshots and snapshots `, + examples: ` +Run the latest verified, kibana-compatible ES Serverless image: + node scripts/functional_tests --config ./config.ts --esFrom serverless --esServerlessImage docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified + +Run with a specific ES Serverless tag from the docker.elastic.co/elasticsearch-ci/elasticsearch-serverless repo: + node scripts/functional_tests --config ./config.ts --esFrom serverless --esServerlessImage git-fec36430fba2 + `, }; export function parseFlags(flags: FlagsReader) { @@ -75,6 +84,7 @@ export function parseFlags(flags: FlagsReader) { ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) : undefined, esFrom: flags.enum('esFrom', ['snapshot', 'source', 'serverless']), + esServerlessImage: flags.string('esServerlessImage'), installDir: flags.path('kibana-install-dir'), grep: flags.string('grep'), suiteTags: { diff --git a/packages/kbn-text-based-editor/src/helpers.test.ts b/packages/kbn-text-based-editor/src/helpers.test.ts index 5f1546ccc138e..f7b100419b2c7 100644 --- a/packages/kbn-text-based-editor/src/helpers.test.ts +++ b/packages/kbn-text-based-editor/src/helpers.test.ts @@ -5,8 +5,14 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ - -import { parseErrors, parseWarning, getInlineEditorText, getWrappedInPipesCode } from './helpers'; +import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks'; +import { + parseErrors, + parseWarning, + getInlineEditorText, + getWrappedInPipesCode, + getIndicesForAutocomplete, +} from './helpers'; describe('helpers', function () { describe('parseErrors', function () { @@ -159,4 +165,25 @@ describe('helpers', function () { expect(code).toEqual('FROM index1 | keep field1, field2 | order field1'); }); }); + + describe('getIndicesForAutocomplete', function () { + it('should not return system indices', async function () { + const dataViewsMock = dataViewPluginMocks.createStartContract(); + const updatedDataViewsMock = { + ...dataViewsMock, + getIndices: jest.fn().mockResolvedValue([ + { + name: '.system1', + title: 'system1', + }, + { + name: 'logs', + title: 'logs', + }, + ]), + }; + const indices = await getIndicesForAutocomplete(updatedDataViewsMock); + expect(indices).toStrictEqual(['logs']); + }); + }); }); diff --git a/packages/kbn-text-based-editor/src/helpers.ts b/packages/kbn-text-based-editor/src/helpers.ts index 6b4e99f6e3093..8932276695117 100644 --- a/packages/kbn-text-based-editor/src/helpers.ts +++ b/packages/kbn-text-based-editor/src/helpers.ts @@ -10,6 +10,7 @@ import { useRef } from 'react'; import useDebounce from 'react-use/lib/useDebounce'; import { monaco } from '@kbn/monaco'; import { i18n } from '@kbn/i18n'; +import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; export interface MonacoError { message: string; @@ -172,3 +173,12 @@ export const getWrappedInPipesCode = (code: string, isWrapped: boolean): string }); return codeNoLines.join(isWrapped ? ' | ' : '\n| '); }; + +export const getIndicesForAutocomplete = async (dataViews: DataViewsPublicPluginStart) => { + const indices = await dataViews.getIndices({ + showAllIndices: false, + pattern: '*', + isRollupIndex: () => false, + }); + return indices.filter((index) => !index.name.startsWith('.')).map((i) => i.name); +}; diff --git a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx index fc86f7ca1cac6..b1f1708262743 100644 --- a/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx +++ b/packages/kbn-text-based-editor/src/text_based_languages_editor.tsx @@ -56,6 +56,7 @@ import { getDocumentationSections, MonacoError, getWrappedInPipesCode, + getIndicesForAutocomplete, } from './helpers'; import { EditorFooter } from './editor_footer'; import { ResizableButton } from './resizable_button'; @@ -371,12 +372,7 @@ export const TextBasedLanguagesEditor = memo(function TextBasedLanguagesEditor({ const getSourceIdentifiers: ESQLCustomAutocompleteCallbacks['getSourceIdentifiers'] = useCallback(async () => { - const indices = await dataViews.getIndices({ - showAllIndices: false, - pattern: '*', - isRollupIndex: () => false, - }); - return indices.map((i) => i.name); + return await getIndicesForAutocomplete(dataViews); }, [dataViews]); const getFieldsIdentifiers: ESQLCustomAutocompleteCallbacks['getFieldsIdentifiers'] = useCallback( diff --git a/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap b/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap index c57c90acbdcf7..09421e4cb5dd7 100644 --- a/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap +++ b/packages/shared-ux/page/kibana_template/impl/src/__snapshots__/page_template.test.tsx.snap @@ -3,7 +3,7 @@ exports[`KibanaPageTemplate render basic template 1`] = `
    <_EuiPageHeader diff --git a/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx b/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx index f9b9dcd247de6..5da29ba797041 100644 --- a/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx +++ b/packages/shared-ux/page/kibana_template/impl/src/page_template_inner.tsx @@ -68,7 +68,8 @@ export const KibanaPageTemplateInner: FC = ({ // the following props can be removed to allow the template to auto-handle // the fixed header and banner heights. offset={0} - minHeight={0} + minHeight={header ? 'calc(100vh - var(--euiFixedHeadersOffset, 0))' : 0} + grow={header ? false : undefined} {...rest} > {sideBar} diff --git a/src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts b/src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts new file mode 100644 index 0000000000000..66975bbba4409 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/elasticsearch_client_wrapper.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Client } from '@elastic/elasticsearch'; + +export type ElasticsearchClientWrapperFactory = (client: Client) => Client; + +interface GetElasticsearchClientWrapperFactoryParams { + failOn: (methodName: string, methodArguments: any[]) => boolean; + errorDelaySeconds?: number; +} + +export const getElasticsearchClientWrapperFactory = ({ + failOn, + errorDelaySeconds, +}: GetElasticsearchClientWrapperFactoryParams): ElasticsearchClientWrapperFactory => { + const interceptClientMethod = (methodName: string, method: any): any => { + return new Proxy(method, { + apply: (applyTarget, thisArg, methodArguments) => { + if (failOn(methodName, methodArguments)) { + return new Promise((_, reject) => + setTimeout( + () => reject(`Error: esClient.${methodName}() failed unexpectedly`), + (errorDelaySeconds || 0) * 1000 + ) + ); + } + return Reflect.apply(applyTarget, thisArg, methodArguments); + }, + }); + }; + + const interceptClientApi = (apiName: string, api: any): any => + new Proxy(api, { + get(target, prop) { + return typeof target[prop] === 'function' + ? interceptClientMethod(`${apiName}.${prop.toString()}`, target[prop]) + : target[prop]; + }, + }); + + const wrapClient = (client: Client): any => + new Proxy(client, { + get(target, prop, receiver) { + switch (prop) { + // intercept top level esClient methods + case 'bulk': + case 'deleteByQuery': + case 'info': + case 'search': + case 'updateByQuery': + const clientMethod = Reflect.get(target, prop, receiver); + return interceptClientMethod(prop, clientMethod); + // intercept esClient APIs + case 'cluster': + case 'indices': + case 'tasks': + const clientApi = Reflect.get(target, prop, receiver); + return interceptClientApi(prop, clientApi); + // proxy child Clients too + case 'child': + return new Proxy(target[prop], { + apply(applyTarget, thisArg, argArray) { + const childClient = Reflect.apply(applyTarget, thisArg, argArray); + return wrapClient(childClient); + }, + }); + // do NOT intercept the rest of properties and symbols + default: + return Reflect.get(target, prop, receiver); + } + }, + }); + + return wrapClient; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts index 636e70cd9dd9e..25dc5a46a6793 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts @@ -29,6 +29,7 @@ import { } from '../kibana_migrator_test_kit'; import { delay, parseLogFile } from '../test_utils'; import '../jest_matchers'; +import { getElasticsearchClientWrapperFactory } from '../elasticsearch_client_wrapper'; // define a type => index distribution const RELOCATE_TYPES: Record = { @@ -58,6 +59,210 @@ describe('split .kibana index into multiple system indices', () => { await clearLog(logFilePathSecondRun); }); + describe('failure cases', () => { + const getFailingKibanaMigratorTestKit = async ({ + logFilePath, + failOn, + delaySeconds, + }: { + logFilePath: string; + failOn: (methodName: string, methodArgs: any[]) => boolean; + delaySeconds?: number; + }) => { + const clientWrapperFactory = getElasticsearchClientWrapperFactory({ + failOn, + errorDelaySeconds: delaySeconds, + }); + + return await getKibanaMigratorTestKit({ + types: typeRegistry.getAllTypes(), + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, + defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + logFilePath, + clientWrapperFactory, + }); + }; + + beforeEach(async () => { + esServer = await startElasticsearch({ + dataArchive: Path.join(__dirname, '..', 'archives', '7.7.2_xpack_100k_obj.zip'), + }); + }); + + describe('when the .kibana_task_manager migrator fails on the TRANSFORMED_DOCUMENTS_BULK_INDEX state, after the other ones have finished', () => { + it('is capable of completing the .kibana_task_manager migration in subsequent restart', async () => { + const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ + logFilePath: logFilePathFirstRun, + failOn: (methodName, methodArgs) => { + // fail on esClient.bulk({ index: '.kibana_task_manager_1' }) which supposedly causes + // the .kibana_task_manager migrator to fail on the TRANSFORMED_DOCUMENTS_BULK_INDEX state + return methodName === 'bulk' && methodArgs[0].index === '.kibana_task_manager_1'; + }, + delaySeconds: 90, // give the other migrators enough time to finish before failing + }); + + try { + await firstRun(); + throw new Error('First run should have thrown an error but it did not'); + } catch (error) { + expect(error.message).toEqual( + 'Unable to complete saved object migrations for the [.kibana_task_manager] index. Error: esClient.bulk() failed unexpectedly' + ); + } + }); + }); + + describe('when the .kibana migrator fails on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK state', () => { + it('is capable of successfully performing the split migration in subsequent restart', async () => { + const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ + logFilePath: logFilePathFirstRun, + failOn: (methodName, methodArgs) => { + // fail on esClient.bulk({ index: '.kibana_8.11.0_reindex_temp_alias' }) which supposedly causes + // the .kibana migrator to fail on the REINDEX_SOURCE_TO_TEMP_INDEX_BULK + return ( + methodName === 'bulk' && + methodArgs[0].index === `.kibana_${currentVersion}_reindex_temp_alias` + ); + }, + delaySeconds: 10, // give the .kibana_task_manager migrator enough time to finish before failing + }); + + try { + await firstRun(); + throw new Error('First run should have thrown an error but it did not'); + } catch (error) { + expect(error.message).toEqual( + 'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.bulk() failed unexpectedly' + ); + } + }); + }); + + describe('when the .kibana migrator fails on the CLONE_TEMP_TO_TARGET state', () => { + it('is capable of successfully performing the split migration in subsequent restart', async () => { + const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ + logFilePath: logFilePathFirstRun, + failOn: (methodName, methodArgs) => { + // fail on esClient.indices.clone({ index: '.kibana_8.11.0_reindex_temp', target: ... }) which supposedly causes + // the .kibana migrator to fail on the CLONE_TEMP_TO_TARGET + return ( + methodName === 'indices.clone' && + methodArgs[0].index === `.kibana_${currentVersion}_reindex_temp` && + methodArgs[0].target === `.kibana_${currentVersion}_001` + ); + }, + delaySeconds: 15, // give the other migrators enough time to finish before failing + }); + + try { + await firstRun(); + throw new Error('First run should have thrown an error but it did not'); + } catch (error) { + expect(error.message).toEqual( + 'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.indices.clone() failed unexpectedly' + ); + } + }); + }); + + describe('when the .kibana migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state', () => { + it('is capable of successfully performing the split migration in subsequent restart', async () => { + const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ + logFilePath: logFilePathFirstRun, + failOn: (methodName, methodArgs) => { + // fail on esClient.updateByQuery({ index: '.kibana_8.11.0_001' }) which supposedly causes + // the .kibana migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes) + return ( + methodName === 'updateByQuery' && + methodArgs[0].index === `.kibana_${currentVersion}_001` + ); + }, + delaySeconds: 10, // give the other migrators enough time to finish before failing + }); + + try { + await firstRun(); + throw new Error('First run should have thrown an error but it did not'); + } catch (error) { + expect(error.message).toEqual( + 'Unable to complete saved object migrations for the [.kibana] index. Error: esClient.updateByQuery() failed unexpectedly' + ); + } + }); + }); + + describe('when the .kibana_analytics migrator fails on the CLONE_TEMP_TO_TARGET state', () => { + it('is capable of successfully performing the split migration in subsequent restart', async () => { + const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ + logFilePath: logFilePathFirstRun, + failOn: (methodName, methodArgs) => { + // fail on esClient.indices.clone({ index: '.kibana_8.11.0_reindex_temp', target: ... }) which supposedly causes + // the .kibana migrator to fail on the CLONE_TEMP_TO_TARGET + return ( + methodName === 'indices.clone' && + methodArgs[0].index === `.kibana_analytics_${currentVersion}_reindex_temp` && + methodArgs[0].target === `.kibana_analytics_${currentVersion}_001` + ); + }, + delaySeconds: 15, // give the other migrators enough time to finish before failing + }); + + try { + await firstRun(); + throw new Error('First run should have thrown an error but it did not'); + } catch (error) { + expect(error.message).toEqual( + 'Unable to complete saved object migrations for the [.kibana_analytics] index. Error: esClient.indices.clone() failed unexpectedly' + ); + } + }); + }); + + describe('when the .kibana_analytics migrator fails on the UPDATE_TARGET_MAPPINGS_PROPERTIES state', () => { + it('is capable of successfully performing the split migration in subsequent restart', async () => { + const { runMigrations: firstRun } = await getFailingKibanaMigratorTestKit({ + logFilePath: logFilePathFirstRun, + failOn: (methodName, methodArgs) => { + // fail on esClient.updateByQuery({ index: '.kibana_8.11.0_001' }) which supposedly causes + // the .kibana migrator to fail on the UPDATE_TARGET_MAPPINGS_PROPERTIES (pickup mappings' changes) + return ( + methodName === 'updateByQuery' && + methodArgs[0].index === `.kibana_analytics_${currentVersion}_001` + ); + }, + delaySeconds: 10, // give the other migrators enough time to finish before failing + }); + + try { + await firstRun(); + throw new Error('First run should have thrown an error but it did not'); + } catch (error) { + expect(error.message).toEqual( + 'Unable to complete saved object migrations for the [.kibana_analytics] index. Error: esClient.updateByQuery() failed unexpectedly' + ); + } + }); + }); + + afterEach(async () => { + const { runMigrations: secondRun } = await getKibanaMigratorTestKit({ + types: typeRegistry.getAllTypes(), + logFilePath: logFilePathSecondRun, + kibanaIndex: MAIN_SAVED_OBJECT_INDEX, + defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP, + }); + const results = await secondRun(); + expect( + results + .flat() + .every((result) => result.status === 'migrated' || result.status === 'patched') + ).toEqual(true); + + await esServer?.stop(); + await delay(2); + }); + }); + describe('when migrating from a legacy version', () => { let migratorTestKitFactory: (logFilePath: string) => Promise; diff --git a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts index a911fcdbdead5..1f6e9a7a58c77 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/kibana_migrator_test_kit.ts @@ -50,6 +50,7 @@ import type { DocLinksServiceStart } from '@kbn/core-doc-links-server'; import type { NodeRoles } from '@kbn/core-node-server'; import { baselineDocuments, baselineTypes } from './kibana_migrator_test_kit.fixtures'; import { delay } from './test_utils'; +import type { ElasticsearchClientWrapperFactory } from './elasticsearch_client_wrapper'; export const defaultLogFilePath = Path.join(__dirname, 'kibana_migrator_test_kit.log'); @@ -76,6 +77,7 @@ export interface KibanaMigratorTestKitParams { types?: Array>; defaultIndexTypesMap?: IndexTypesMap; logFilePath?: string; + clientWrapperFactory?: ElasticsearchClientWrapperFactory; } export interface KibanaMigratorTestKit { @@ -134,6 +136,7 @@ export const getKibanaMigratorTestKit = async ({ types = [], logFilePath = defaultLogFilePath, nodeRoles = defaultNodeRoles, + clientWrapperFactory, }: KibanaMigratorTestKitParams = {}): Promise => { let hasRun = false; const loggingSystem = new LoggingSystem(); @@ -145,7 +148,8 @@ export const getKibanaMigratorTestKit = async ({ const loggingConf = await firstValueFrom(configService.atPath('logging')); loggingSystem.upgrade(loggingConf); - const client = await getElasticsearchClient(configService, loggerFactory, kibanaVersion); + const rawClient = await getElasticsearchClient(configService, loggerFactory, kibanaVersion); + const client = clientWrapperFactory ? clientWrapperFactory(rawClient) : rawClient; const typeRegistry = new SavedObjectTypeRegistry(); diff --git a/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts b/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts index 483a342ba300e..6cd0b351c4d31 100644 --- a/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts +++ b/src/dev/build/tasks/fleet/download_elastic_gpg_key.ts @@ -13,9 +13,9 @@ import { ToolingLog } from '@kbn/tooling-log'; import { downloadToDisk } from '../../lib'; const ARTIFACTS_URL = 'https://artifacts.elastic.co/'; -const GPG_KEY_NAME = 'GPG-KEY-elasticsearch'; +const GPG_KEY_NAME = 'GPG-KEY-elasticsearch.sha1'; const GPG_KEY_SHA512 = - '62a567354286deb02baf5fc6b82ddf6c7067898723463da9ae65b132b8c6d6f064b2874e390885682376228eed166c1c82fe7f11f6c9a69f0c157029c548fa3d'; + '84ee193cc337344d9a7da9021daf3f5ede83f5f1ab049d169f3634921529dcd096abf7a91eec7f26f3a6913e5e38f88f69a5e2ce79ad155d46edc75705a648c6'; export async function downloadElasticGpgKey(pkgDir: string, log: ToolingLog) { const gpgKeyUrl = ARTIFACTS_URL + GPG_KEY_NAME; diff --git a/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap b/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap index bd4e658b04a99..2cd4388680751 100644 --- a/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap +++ b/src/plugins/home/public/application/components/tutorial/__snapshots__/tutorial.test.js.snap @@ -1,160 +1,166 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`isCloudEnabled is false should not render instruction toggle when ON_PREM_ELASTIC_CLOUD instructions are not provided 1`] = ` - -
    - - - +
    + -
    - + "prepend": [Function], + } + } + description="tutorial used to drive jest tests" + notices={null} + title="jest test tutorial" + /> + + +
    + + `; exports[`isCloudEnabled is false should render ON_PREM instructions with instruction toggle 1`] = ` - -
    - +
    + - - - + + + + + + - - - -
    - +
    + + `; exports[`should render ELASTIC_CLOUD instructions when isCloudEnabled is true 1`] = ` - -
    - - - +
    + -
    - + "prepend": [Function], + } + } + description="tutorial used to drive jest tests" + iconType="logoApache" + notices={null} + title="jest test tutorial" + /> + + +
    + + `; diff --git a/src/plugins/home/public/application/components/tutorial/tutorial.js b/src/plugins/home/public/application/components/tutorial/tutorial.js index de222dbe6155d..620da42169c35 100644 --- a/src/plugins/home/public/application/components/tutorial/tutorial.js +++ b/src/plugins/home/public/application/components/tutorial/tutorial.js @@ -18,7 +18,7 @@ import * as StatusCheckStates from './status_check_states'; import { injectI18n, FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; import { getServices } from '../../kibana_services'; -import { KibanaPageTemplate } from '@kbn/kibana-react-plugin/public'; +import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template'; const INSTRUCTIONS_TYPE = { ELASTIC_CLOUD: 'elasticCloud', @@ -360,7 +360,7 @@ class TutorialUi extends React.Component { render() { let content; if (this.state.notFound) { - content = ( + return ( {content}; + return ( + + {content} + + ); } } diff --git a/src/plugins/home/public/application/components/tutorial_directory.js b/src/plugins/home/public/application/components/tutorial_directory.js index cde8b86f5df22..c5a4c1b27fbab 100644 --- a/src/plugins/home/public/application/components/tutorial_directory.js +++ b/src/plugins/home/public/application/components/tutorial_directory.js @@ -16,7 +16,7 @@ import { SampleDataTab } from '@kbn/home-sample-data-tab'; import { i18n } from '@kbn/i18n'; import { Synopsis } from './synopsis'; import { getServices } from '../kibana_services'; -import { KibanaPageTemplate } from '@kbn/kibana-react-plugin/public'; +import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template'; import { getTutorials } from '../load_tutorials'; const SAMPLE_DATA_TAB_ID = 'sampleData'; @@ -257,7 +257,7 @@ class TutorialDirectoryUi extends React.Component { rightSideItems: headerLinks ? [headerLinks] : [], }} > - {this.renderTabContent()} + {this.renderTabContent()}     ); } diff --git a/src/plugins/kibana_react/public/index.ts b/src/plugins/kibana_react/public/index.ts index f92739d045439..7954559206bb7 100644 --- a/src/plugins/kibana_react/public/index.ts +++ b/src/plugins/kibana_react/public/index.ts @@ -54,17 +54,13 @@ export { POSITIONS, WEIGHTS, TOOLBAR_BUTTON_SIZES, ToolbarButton } from './toolb export { reactRouterNavigate, reactRouterOnClickHandler } from './react_router_navigate'; export type { - KibanaPageTemplateProps, NoDataPageActions, NoDataPageActionsProps, NoDataPageProps, ElasticAgentCardProps, } from './page_template'; export { - KibanaPageTemplate, KibanaPageTemplateSolutionNavAvatar, - NO_DATA_PAGE_MAX_WIDTH, - NO_DATA_PAGE_TEMPLATE_PROPS, NO_DATA_RECOMMENDED, NoDataPage, ElasticAgentCard, diff --git a/src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap b/src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap deleted file mode 100644 index 4dea9549670f3..0000000000000 --- a/src/plugins/kibana_react/public/page_template/__snapshots__/page_template.test.tsx.snap +++ /dev/null @@ -1,440 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`KibanaPageTemplate render basic template 1`] = ` -
    -
    -
    -
    -
    -
    -
    -

    - test -

    -
    -
    -
    -
    -
    - test -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -`; - -exports[`KibanaPageTemplate render custom empty prompt only 1`] = ` - - - custom test - - } - /> - -`; - -exports[`KibanaPageTemplate render custom empty prompt with page header 1`] = ` - - - custom test - - } - /> - -`; - -exports[`KibanaPageTemplate render default empty prompt 1`] = ` - -`; - -exports[`KibanaPageTemplate render noDataContent 1`] = ` - -`; - -exports[`KibanaPageTemplate render solutionNav 1`] = ` -
    -
    - - -
    -
    -
    -
    -
    -
    -
    -
    -

    - test -

    -
    -
    -
    -
    -
    - test -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -`; diff --git a/src/plugins/kibana_react/public/page_template/index.ts b/src/plugins/kibana_react/public/page_template/index.ts index fda644a284797..65a5db433593a 100644 --- a/src/plugins/kibana_react/public/page_template/index.ts +++ b/src/plugins/kibana_react/public/page_template/index.ts @@ -6,9 +6,5 @@ * Side Public License, v 1. */ -export type { KibanaPageTemplateProps } from './page_template'; -export { KibanaPageTemplate } from './page_template'; export { KibanaPageTemplateSolutionNavAvatar, KibanaPageTemplateSolutionNav } from './solution_nav'; export * from './no_data_page'; -export { withSolutionNav } from './with_solution_nav'; -export { NO_DATA_PAGE_MAX_WIDTH, NO_DATA_PAGE_TEMPLATE_PROPS } from './util'; diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/index.ts b/src/plugins/kibana_react/public/page_template/no_data_page/index.ts index b5a11722dd397..55661ad6f14f7 100644 --- a/src/plugins/kibana_react/public/page_template/no_data_page/index.ts +++ b/src/plugins/kibana_react/public/page_template/no_data_page/index.ts @@ -8,4 +8,3 @@ export * from './no_data_page'; export * from './no_data_card'; -export * from './no_data_config_page'; diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx deleted file mode 100644 index 0bdde40021398..0000000000000 --- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/index.tsx +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -export { NoDataConfigPage, NoDataConfigPageWithSolutionNavBar } from './no_data_config_page'; diff --git a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx b/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx deleted file mode 100644 index cae591f571c79..0000000000000 --- a/src/plugins/kibana_react/public/page_template/no_data_page/no_data_config_page/no_data_config_page.tsx +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -import { EuiPageTemplate_Deprecated as EuiPageTemplate } from '@elastic/eui'; -import React from 'react'; -import { NoDataPage } from '../no_data_page'; -import { withSolutionNav } from '../../with_solution_nav'; -import { KibanaPageTemplateProps } from '../../page_template'; -import { getClasses, NO_DATA_PAGE_TEMPLATE_PROPS } from '../../util'; - -export const NoDataConfigPage = (props: KibanaPageTemplateProps) => { - const { className, noDataConfig, ...rest } = props; - - if (!noDataConfig) { - return null; - } - - const template = NO_DATA_PAGE_TEMPLATE_PROPS.template; - const classes = getClasses(template, className); - - return ( - - - - ); -}; - -export const NoDataConfigPageWithSolutionNavBar = withSolutionNav(NoDataConfigPage); diff --git a/src/plugins/kibana_react/public/page_template/page_template.scss b/src/plugins/kibana_react/public/page_template/page_template.scss deleted file mode 100644 index d94daec56235f..0000000000000 --- a/src/plugins/kibana_react/public/page_template/page_template.scss +++ /dev/null @@ -1,22 +0,0 @@ -.kbnPageTemplate__pageSideBar { - overflow: hidden; - // Temporary hack till the sizing is changed directly in EUI - min-width: 248px; - - @include euiCanAnimate { - transition: min-width $euiAnimSpeedFast $euiAnimSlightResistance; - } - - &.kbnPageTemplate__pageSideBar--shrink { - min-width: $euiSizeXXL; - } - - .kbnPageTemplate--centeredBody & { - border-bottom: $euiBorderThin; - - @include euiBreakpoint('m', 'l', 'xl') { - border-bottom: none; - border-right: $euiBorderThin; - } - } -} diff --git a/src/plugins/kibana_react/public/page_template/page_template.test.tsx b/src/plugins/kibana_react/public/page_template/page_template.test.tsx deleted file mode 100644 index aff6082902a34..0000000000000 --- a/src/plugins/kibana_react/public/page_template/page_template.test.tsx +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React from 'react'; -import { shallow, render } from 'enzyme'; -import { KibanaPageTemplate, KibanaPageTemplateProps } from './page_template'; -import { EuiEmptyPrompt } from '@elastic/eui'; -import { KibanaPageTemplateSolutionNavProps } from './solution_nav'; - -const navItems: KibanaPageTemplateSolutionNavProps['items'] = [ - { - name: 'Ingest', - id: '1', - items: [ - { - name: 'Ingest Node Pipelines', - id: '1.1', - }, - { - name: 'Logstash Pipelines', - id: '1.2', - }, - { - name: 'Beats Central Management', - id: '1.3', - }, - ], - }, - { - name: 'Data', - id: '2', - items: [ - { - name: 'Index Management', - id: '2.1', - }, - { - name: 'Index Lifecycle Policies', - id: '2.2', - }, - { - name: 'Snapshot and Restore', - id: '2.3', - }, - ], - }, -]; - -const noDataConfig: KibanaPageTemplateProps['noDataConfig'] = { - solution: 'Elastic', - actions: { - elasticAgent: {}, - beats: {}, - custom: {}, - }, - docsLink: 'test', -}; - -describe('KibanaPageTemplate', () => { - test('render default empty prompt', () => { - const component = shallow( - - ); - expect(component).toMatchSnapshot(); - }); - - test('render custom empty prompt only', () => { - const component = shallow( - - custom test} /> - - ); - expect(component).toMatchSnapshot(); - }); - - test('render custom empty prompt with page header', () => { - const component = shallow( - - custom test} /> - - ); - expect(component).toMatchSnapshot(); - }); - - test('render basic template', () => { - const component = render( - - ); - expect(component).toMatchSnapshot(); - }); - - test('render solutionNav', () => { - const component = render( - - ); - expect(component).toMatchSnapshot(); - expect(component.find('div.kbnPageTemplate__pageSideBar').length).toBe(1); - }); - - test('render noDataContent', () => { - const component = shallow( - - ); - expect(component).toMatchSnapshot(); - }); - - test('render sidebar classes', () => { - const component = shallow( - - ); - expect(component.html().includes('kbnPageTemplate__pageSideBar customClass')).toBe(true); - }); -}); diff --git a/src/plugins/kibana_react/public/page_template/page_template.tsx b/src/plugins/kibana_react/public/page_template/page_template.tsx deleted file mode 100644 index 42ba9d1873587..0000000000000 --- a/src/plugins/kibana_react/public/page_template/page_template.tsx +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import './page_template.scss'; - -import React, { FunctionComponent } from 'react'; -import { EuiPageTemplateProps_Deprecated } from '@elastic/eui'; -import { KibanaPageTemplateSolutionNavProps } from './solution_nav'; - -import { - NoDataPageProps, - NoDataConfigPage, - NoDataConfigPageWithSolutionNavBar, -} from './no_data_page'; -import { KibanaPageTemplateInner, KibanaPageTemplateWithSolutionNav } from './page_template_inner'; - -/** - * A thin wrapper around EuiPageTemplate with a few Kibana specific additions - * @deprecated Use `KibanaPageTemplateProps` from `@kbn/shared-ux-page-kibana-template-types`. - */ -export type KibanaPageTemplateProps = EuiPageTemplateProps_Deprecated & { - /** - * Changes the template type depending on other props provided. - * With `pageHeader` only: Uses `centeredBody` and fills an EuiEmptyPrompt with `pageHeader` info. - * With `children` only: Uses `centeredBody` - * With `pageHeader` and `children`: Uses `centeredContent` - */ - isEmptyState?: boolean; - /** - * Quick creation of EuiSideNav. Hooks up mobile instance too - */ - solutionNav?: KibanaPageTemplateSolutionNavProps; - /** - * Accepts a configuration object, that when provided, ignores pageHeader and children and instead - * displays Agent, Beats, and custom cards to direct users to the right ingest location - */ - noDataConfig?: NoDataPageProps; -}; - -/** @deprecated Use `KibanaPageTemplate` from `@kbn/shared-ux-page-kibana-template`. */ -export const KibanaPageTemplate: FunctionComponent = ({ - template, - className, - children, - solutionNav, - noDataConfig, - ...rest -}) => { - /** - * If passing the custom template of `noDataConfig` - */ - if (noDataConfig && solutionNav) { - return ( - - ); - } - - if (noDataConfig) { - return ( - - ); - } - - if (solutionNav) { - return ( - - ); - } - - return ( - - ); -}; diff --git a/src/plugins/kibana_react/public/page_template/page_template_inner.tsx b/src/plugins/kibana_react/public/page_template/page_template_inner.tsx deleted file mode 100644 index 001cea5c26a23..0000000000000 --- a/src/plugins/kibana_react/public/page_template/page_template_inner.tsx +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React, { FunctionComponent } from 'react'; - -import { EuiEmptyPrompt, EuiPageTemplate_Deprecated as EuiPageTemplate } from '@elastic/eui'; -import { withSolutionNav } from './with_solution_nav'; -import { KibanaPageTemplateProps } from './page_template'; -import { getClasses } from './util'; - -type Props = KibanaPageTemplateProps; - -/** - * A thin wrapper around EuiPageTemplate with a few Kibana specific additions - */ -export const KibanaPageTemplateInner: FunctionComponent = ({ - template, - className, - pageHeader, - children, - isEmptyState, - ...rest -}) => { - /** - * An easy way to create the right content for empty pages - */ - const emptyStateDefaultTemplate = 'centeredBody'; - if (isEmptyState && pageHeader && !children) { - template = template ?? emptyStateDefaultTemplate; - const { iconType, pageTitle, description, rightSideItems } = pageHeader; - pageHeader = undefined; - children = ( - {pageTitle} : undefined} - body={description ?

    {description}

    : undefined} - actions={rightSideItems} - /> - ); - } else if (isEmptyState && pageHeader && children) { - template = template ?? 'centeredContent'; - } else if (isEmptyState && !pageHeader) { - template = template ?? emptyStateDefaultTemplate; - } - - const classes = getClasses(template, className); - - return ( - - {children} - - ); -}; - -export const KibanaPageTemplateWithSolutionNav = withSolutionNav(KibanaPageTemplateInner); diff --git a/src/plugins/kibana_react/public/page_template/util/constants.ts b/src/plugins/kibana_react/public/page_template/util/constants.ts deleted file mode 100644 index 159a6d0d8d4c1..0000000000000 --- a/src/plugins/kibana_react/public/page_template/util/constants.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { KibanaPageTemplateProps } from '../page_template'; - -export const NO_DATA_PAGE_MAX_WIDTH = 950; - -export const NO_DATA_PAGE_TEMPLATE_PROPS: KibanaPageTemplateProps = { - restrictWidth: NO_DATA_PAGE_MAX_WIDTH, - template: 'centeredBody', - pageContentProps: { - hasShadow: false, - color: 'transparent', - }, -}; diff --git a/src/plugins/kibana_react/public/page_template/util/index.ts b/src/plugins/kibana_react/public/page_template/util/index.ts index adfefdf834566..06edc43d70d57 100644 --- a/src/plugins/kibana_react/public/page_template/util/index.ts +++ b/src/plugins/kibana_react/public/page_template/util/index.ts @@ -7,4 +7,3 @@ */ export { getClasses } from './presentation'; -export * from './constants'; diff --git a/src/plugins/kibana_react/public/page_template/with_solution_nav.tsx b/src/plugins/kibana_react/public/page_template/with_solution_nav.tsx deleted file mode 100644 index 842573b9d8de4..0000000000000 --- a/src/plugins/kibana_react/public/page_template/with_solution_nav.tsx +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import React, { ComponentType, useState } from 'react'; -import classNames from 'classnames'; -import { useIsWithinBreakpoints } from '@elastic/eui'; -import { EuiPageSideBarProps_Deprecated as EuiPageSideBarProps } from '@elastic/eui/src/components/page/page_side_bar'; -import { KibanaPageTemplateSolutionNav, KibanaPageTemplateSolutionNavProps } from './solution_nav'; -import { KibanaPageTemplateProps } from '.'; - -// https://reactjs.org/docs/higher-order-components.html#convention-wrap-the-display-name-for-easy-debugging -function getDisplayName(Component: ComponentType) { - return Component.displayName || Component.name || 'UnnamedComponent'; -} - -type SolutionNavProps = KibanaPageTemplateProps & { - solutionNav: KibanaPageTemplateSolutionNavProps; -}; - -const SOLUTION_NAV_COLLAPSED_KEY = 'solutionNavIsCollapsed'; - -export const withSolutionNav = (WrappedComponent: ComponentType) => { - const WithSolutionNav = (props: SolutionNavProps) => { - const isMediumBreakpoint = useIsWithinBreakpoints(['m']); - const isLargerBreakpoint = useIsWithinBreakpoints(['l', 'xl']); - const [isSideNavOpenOnDesktop, setisSideNavOpenOnDesktop] = useState( - !JSON.parse(String(localStorage.getItem(SOLUTION_NAV_COLLAPSED_KEY))) - ); - const { solutionNav, ...propagatedProps } = props; - const { children, isEmptyState, template } = propagatedProps; - const toggleOpenOnDesktop = () => { - setisSideNavOpenOnDesktop(!isSideNavOpenOnDesktop); - // Have to store it as the opposite of the default we want - localStorage.setItem(SOLUTION_NAV_COLLAPSED_KEY, JSON.stringify(isSideNavOpenOnDesktop)); - }; - const sideBarClasses = classNames( - 'kbnPageTemplate__pageSideBar', - { - 'kbnPageTemplate__pageSideBar--shrink': - isMediumBreakpoint || (isLargerBreakpoint && !isSideNavOpenOnDesktop), - }, - props.pageSideBarProps?.className - ); - - const templateToUse = isEmptyState && !template ? 'centeredContent' : template; - - const pageSideBar = ( - - ); - const pageSideBarProps = { - paddingSize: 'none', - ...props.pageSideBarProps, - className: sideBarClasses, - } as EuiPageSideBarProps; // needed because for some reason 'none' is not recognized as a valid value for paddingSize - return ( - - {children} - - ); - }; - WithSolutionNav.displayName = `WithSolutionNavBar(${getDisplayName(WrappedComponent)})`; - return WithSolutionNav; -}; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx index 65b8183b60a0b..2f46e99d12b07 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx @@ -126,4 +126,88 @@ describe('fetchConnectorExecuteAction', () => { expect(result).toBe('Test response'); }); + + it('returns the value of the action_input property when assistantLangChain is true, and `content` has properly prefixed and suffixed JSON with the action_input property', async () => { + const content = '```json\n{"action_input": "value from action_input"}\n```'; + + (mockHttp.fetch as jest.Mock).mockResolvedValue({ + status: 'ok', + data: { + choices: [ + { + message: { + content, + }, + }, + ], + }, + }); + + const testProps: FetchConnectorExecuteAction = { + assistantLangChain: true, // <-- requires response parsing + http: mockHttp, + messages, + apiConfig, + }; + + const result = await fetchConnectorExecuteAction(testProps); + + expect(result).toBe('value from action_input'); + }); + + it('returns the original content when assistantLangChain is true, and `content` has properly formatted JSON WITHOUT the action_input property', async () => { + const content = '```json\n{"some_key": "some value"}\n```'; + + (mockHttp.fetch as jest.Mock).mockResolvedValue({ + status: 'ok', + data: { + choices: [ + { + message: { + content, + }, + }, + ], + }, + }); + + const testProps: FetchConnectorExecuteAction = { + assistantLangChain: true, // <-- requires response parsing + http: mockHttp, + messages, + apiConfig, + }; + + const result = await fetchConnectorExecuteAction(testProps); + + expect(result).toBe(content); + }); + + it('returns the original when assistantLangChain is true, and `content` is not JSON', async () => { + const content = 'plain text content'; + + (mockHttp.fetch as jest.Mock).mockResolvedValue({ + status: 'ok', + data: { + choices: [ + { + message: { + content, + }, + }, + ], + }, + }); + + const testProps: FetchConnectorExecuteAction = { + assistantLangChain: true, // <-- requires response parsing + http: mockHttp, + messages, + apiConfig, + }; + + const result = await fetchConnectorExecuteAction(testProps); + + expect(result).toBe(content); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx index 511b5aa585af0..6d3452b6f7880 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx @@ -12,6 +12,7 @@ import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser'; import type { Conversation, Message } from '../assistant_context/types'; import { API_ERROR } from './translations'; import { MODEL_GPT_3_5_TURBO } from '../connectorland/models/model_selector/model_selector'; +import { getFormattedMessageContent } from './helpers'; export interface FetchConnectorExecuteAction { assistantLangChain: boolean; @@ -78,7 +79,8 @@ export const fetchConnectorExecuteAction = async ({ if (data.choices && data.choices.length > 0 && data.choices[0].message.content) { const result = data.choices[0].message.content.trim(); - return result; + + return assistantLangChain ? getFormattedMessageContent(result) : result; } else { return API_ERROR; } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts index 69bed887e730e..f2b89a07c319e 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { getDefaultConnector, getBlockBotConversation } from './helpers'; +import { + getBlockBotConversation, + getDefaultConnector, + getFormattedMessageContent, +} from './helpers'; import { enterpriseMessaging } from './use_conversation/sample_conversations'; import { ActionConnector } from '@kbn/triggers-actions-ui-plugin/public'; @@ -190,4 +194,41 @@ describe('getBlockBotConversation', () => { expect(result).toBeUndefined(); }); }); + + describe('getFormattedMessageContent', () => { + it('returns the value of the action_input property when `content` has properly prefixed and suffixed JSON with the action_input property', () => { + const content = '```json\n{"action_input": "value from action_input"}\n```'; + + expect(getFormattedMessageContent(content)).toBe('value from action_input'); + }); + + it('returns the original content when `content` has properly formatted JSON WITHOUT the action_input property', () => { + const content = '```json\n{"some_key": "some value"}\n```'; + expect(getFormattedMessageContent(content)).toBe(content); + }); + + it('returns the original content when `content` has improperly formatted JSON', () => { + const content = '```json\n{"action_input": "value from action_input",}\n```'; // <-- the trailing comma makes it invalid + + expect(getFormattedMessageContent(content)).toBe(content); + }); + + it('returns the original content when `content` is missing the prefix', () => { + const content = '{"action_input": "value from action_input"}\n```'; // <-- missing prefix + + expect(getFormattedMessageContent(content)).toBe(content); + }); + + it('returns the original content when `content` is missing the suffix', () => { + const content = '```json\n{"action_input": "value from action_input"}'; // <-- missing suffix + + expect(getFormattedMessageContent(content)).toBe(content); + }); + + it('returns the original content when `content` does NOT contain a JSON string', () => { + const content = 'plain text content'; + + expect(getFormattedMessageContent(content)).toBe(content); + }); + }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index b01c9001e8319..2b2c5b76851f7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -59,3 +59,24 @@ export const getDefaultConnector = ( connectors: Array, Record>> | undefined ): ActionConnector, Record> | undefined => connectors?.length === 1 ? connectors[0] : undefined; + +/** + * When `content` is a JSON string, prefixed with "```json\n" + * and suffixed with "\n```", this function will attempt to parse it and return + * the `action_input` property if it exists. + */ +export const getFormattedMessageContent = (content: string): string => { + const formattedContentMatch = content.match(/```json\n([\s\S]+)\n```/); + + if (formattedContentMatch) { + try { + const parsedContent = JSON.parse(formattedContentMatch[1]); + + return parsedContent.action_input ?? content; + } catch { + // we don't want to throw an error here, so we'll fall back to the original content + } + } + + return content; +}; diff --git a/x-pack/packages/ml/trained_models_utils/index.ts b/x-pack/packages/ml/trained_models_utils/index.ts index ba67911f3f8ab..22b808bdc7b5e 100644 --- a/x-pack/packages/ml/trained_models_utils/index.ts +++ b/x-pack/packages/ml/trained_models_utils/index.ts @@ -14,4 +14,10 @@ export { type DeploymentState, type SupportedPytorchTasksType, type TrainedModelType, + ELASTIC_MODEL_DEFINITIONS, + type ElasticModelId, + type ModelDefinition, + type ModelDefinitionResponse, + type ElserVersion, + type GetElserOptions, } from './src/constants/trained_models'; diff --git a/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts b/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts index 86b88f51a66c4..4580330119ddd 100644 --- a/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts +++ b/x-pack/packages/ml/trained_models_utils/src/constants/trained_models.ts @@ -46,8 +46,9 @@ export const BUILT_IN_MODEL_TAG = 'prepackaged'; export const ELASTIC_MODEL_TAG = 'elastic'; -export const ELASTIC_MODEL_DEFINITIONS = { +export const ELASTIC_MODEL_DEFINITIONS: Record = Object.freeze({ '.elser_model_1': { + version: 1, config: { input: { field_names: ['text_field'], @@ -57,7 +58,49 @@ export const ELASTIC_MODEL_DEFINITIONS = { defaultMessage: 'Elastic Learned Sparse EncodeR v1 (Tech Preview)', }), }, -} as const; + '.elser_model_2_SNAPSHOT': { + version: 2, + default: true, + config: { + input: { + field_names: ['text_field'], + }, + }, + description: i18n.translate('xpack.ml.trainedModels.modelsList.elserV2Description', { + defaultMessage: 'Elastic Learned Sparse EncodeR v2 (Tech Preview)', + }), + }, + '.elser_model_2_linux-x86_64_SNAPSHOT': { + version: 2, + os: 'Linux', + arch: 'amd64', + config: { + input: { + field_names: ['text_field'], + }, + }, + description: i18n.translate('xpack.ml.trainedModels.modelsList.elserV2x86Description', { + defaultMessage: + 'Elastic Learned Sparse EncodeR v2, optimized for linux-x86_64 (Tech Preview)', + }), + }, +} as const); + +export interface ModelDefinition { + version: number; + config: object; + description: string; + os?: string; + arch?: string; + default?: boolean; + recommended?: boolean; +} + +export type ModelDefinitionResponse = ModelDefinition & { + name: string; +}; + +export type ElasticModelId = keyof typeof ELASTIC_MODEL_DEFINITIONS; export const MODEL_STATE = { ...DEPLOYMENT_STATE, @@ -66,3 +109,9 @@ export const MODEL_STATE = { } as const; export type ModelState = typeof MODEL_STATE[keyof typeof MODEL_STATE] | null; + +export type ElserVersion = 1 | 2; + +export interface GetElserOptions { + version?: ElserVersion; +} diff --git a/x-pack/plugins/actions/docs/openapi/bundled.json b/x-pack/plugins/actions/docs/openapi/bundled.json index f79bb3ca9c0fe..4039ab6b5a0de 100644 --- a/x-pack/plugins/actions/docs/openapi/bundled.json +++ b/x-pack/plugins/actions/docs/openapi/bundled.json @@ -111,6 +111,9 @@ { "$ref": "#/components/schemas/create_connector_request_tines" }, + { + "$ref": "#/components/schemas/create_connector_request_torq" + }, { "$ref": "#/components/schemas/create_connector_request_webhook" }, @@ -389,6 +392,9 @@ { "$ref": "#/components/schemas/create_connector_request_tines" }, + { + "$ref": "#/components/schemas/create_connector_request_torq" + }, { "$ref": "#/components/schemas/create_connector_request_webhook" }, @@ -508,6 +514,9 @@ { "$ref": "#/components/schemas/update_connector_request_teams" }, + { + "$ref": "#/components/schemas/update_connector_request_torq" + }, { "$ref": "#/components/schemas/update_connector_request_webhook" }, @@ -2829,6 +2838,66 @@ } } }, + "config_properties_torq": { + "title": "Connector request properties for a Torq connector", + "description": "Defines properties for connectors when type is `.torq`.", + "type": "object", + "required": [ + "webhookIntegrationUrl" + ], + "properties": { + "webhookIntegrationUrl": { + "description": "The endpoint URL of the Elastic Security integration in Torq.", + "type": "string" + } + } + }, + "secrets_properties_torq": { + "title": "Connector secrets properties for a Torq connector", + "description": "Defines secrets for connectors when type is `.torq`.", + "type": "object", + "required": [ + "token" + ], + "properties": { + "token": { + "description": "The secret of the webhook authentication header.", + "type": "string" + } + } + }, + "create_connector_request_torq": { + "title": "Create Torq connector request", + "description": "The Torq connector uses a Torq webhook to trigger workflows with Kibana actions.\n", + "type": "object", + "required": [ + "config", + "connector_type_id", + "name", + "secrets" + ], + "properties": { + "config": { + "$ref": "#/components/schemas/config_properties_torq" + }, + "connector_type_id": { + "type": "string", + "description": "The type of connector.", + "enum": [ + ".torq" + ], + "example": ".torq" + }, + "name": { + "type": "string", + "description": "The display name for the connector.", + "example": "my-connector" + }, + "secrets": { + "$ref": "#/components/schemas/secrets_properties_torq" + } + } + }, "config_properties_webhook": { "title": "Connector request properties for a Webhook connector", "description": "Defines properties for connectors when type is `.webhook`.", @@ -3776,6 +3845,50 @@ } } }, + "connector_response_properties_torq": { + "title": "Connector response properties for a Torq connector", + "type": "object", + "required": [ + "config", + "connector_type_id", + "id", + "is_deprecated", + "is_preconfigured", + "name" + ], + "properties": { + "config": { + "$ref": "#/components/schemas/config_properties_torq" + }, + "connector_type_id": { + "type": "string", + "description": "The type of connector.", + "enum": [ + ".torq" + ] + }, + "id": { + "type": "string", + "description": "The identifier for the connector." + }, + "is_deprecated": { + "$ref": "#/components/schemas/is_deprecated" + }, + "is_missing_secrets": { + "$ref": "#/components/schemas/is_missing_secrets" + }, + "is_preconfigured": { + "$ref": "#/components/schemas/is_preconfigured" + }, + "is_system_action": { + "$ref": "#/components/schemas/is_system_action" + }, + "name": { + "type": "string", + "description": "The display name for the connector." + } + } + }, "connector_response_properties_webhook": { "title": "Connector response properties for a Webhook connector", "type": "object", @@ -3919,6 +4032,9 @@ { "$ref": "#/components/schemas/connector_response_properties_tines" }, + { + "$ref": "#/components/schemas/connector_response_properties_torq" + }, { "$ref": "#/components/schemas/connector_response_properties_webhook" }, @@ -4221,6 +4337,27 @@ } } }, + "update_connector_request_torq": { + "title": "Update Torq connector request", + "type": "object", + "required": [ + "config", + "name", + "secrets" + ], + "properties": { + "config": { + "$ref": "#/components/schemas/config_properties_torq" + }, + "name": { + "type": "string", + "description": "The display name for the connector." + }, + "secrets": { + "$ref": "#/components/schemas/secrets_properties_torq" + } + } + }, "update_connector_request_webhook": { "title": "Update Webhook connector request", "type": "object", @@ -4286,6 +4423,7 @@ ".swimlane", ".teams", ".tines", + ".torq", ".webhook", ".xmatters" ], diff --git a/x-pack/plugins/actions/docs/openapi/bundled.yaml b/x-pack/plugins/actions/docs/openapi/bundled.yaml index ed97fb7f31233..9d1842c4dace9 100644 --- a/x-pack/plugins/actions/docs/openapi/bundled.yaml +++ b/x-pack/plugins/actions/docs/openapi/bundled.yaml @@ -55,6 +55,7 @@ paths: - $ref: '#/components/schemas/create_connector_request_swimlane' - $ref: '#/components/schemas/create_connector_request_teams' - $ref: '#/components/schemas/create_connector_request_tines' + - $ref: '#/components/schemas/create_connector_request_torq' - $ref: '#/components/schemas/create_connector_request_webhook' - $ref: '#/components/schemas/create_connector_request_xmatters' discriminator: @@ -208,6 +209,7 @@ paths: - $ref: '#/components/schemas/create_connector_request_swimlane' - $ref: '#/components/schemas/create_connector_request_teams' - $ref: '#/components/schemas/create_connector_request_tines' + - $ref: '#/components/schemas/create_connector_request_torq' - $ref: '#/components/schemas/create_connector_request_webhook' - $ref: '#/components/schemas/create_connector_request_xmatters' discriminator: @@ -264,6 +266,7 @@ paths: - $ref: '#/components/schemas/update_connector_request_slack_webhook' - $ref: '#/components/schemas/update_connector_request_swimlane' - $ref: '#/components/schemas/update_connector_request_teams' + - $ref: '#/components/schemas/update_connector_request_torq' - $ref: '#/components/schemas/update_connector_request_webhook' - $ref: '#/components/schemas/update_connector_request_xmatters' examples: @@ -1916,6 +1919,51 @@ components: example: my-connector secrets: $ref: '#/components/schemas/secrets_properties_tines' + config_properties_torq: + title: Connector request properties for a Torq connector + description: Defines properties for connectors when type is `.torq`. + type: object + required: + - webhookIntegrationUrl + properties: + webhookIntegrationUrl: + description: The endpoint URL of the Elastic Security integration in Torq. + type: string + secrets_properties_torq: + title: Connector secrets properties for a Torq connector + description: Defines secrets for connectors when type is `.torq`. + type: object + required: + - token + properties: + token: + description: The secret of the webhook authentication header. + type: string + create_connector_request_torq: + title: Create Torq connector request + description: | + The Torq connector uses a Torq webhook to trigger workflows with Kibana actions. + type: object + required: + - config + - connector_type_id + - name + - secrets + properties: + config: + $ref: '#/components/schemas/config_properties_torq' + connector_type_id: + type: string + description: The type of connector. + enum: + - .torq + example: .torq + name: + type: string + description: The display name for the connector. + example: my-connector + secrets: + $ref: '#/components/schemas/secrets_properties_torq' config_properties_webhook: title: Connector request properties for a Webhook connector description: Defines properties for connectors when type is `.webhook`. @@ -2626,6 +2674,38 @@ components: name: type: string description: The display name for the connector. + connector_response_properties_torq: + title: Connector response properties for a Torq connector + type: object + required: + - config + - connector_type_id + - id + - is_deprecated + - is_preconfigured + - name + properties: + config: + $ref: '#/components/schemas/config_properties_torq' + connector_type_id: + type: string + description: The type of connector. + enum: + - .torq + id: + type: string + description: The identifier for the connector. + is_deprecated: + $ref: '#/components/schemas/is_deprecated' + is_missing_secrets: + $ref: '#/components/schemas/is_missing_secrets' + is_preconfigured: + $ref: '#/components/schemas/is_preconfigured' + is_system_action: + $ref: '#/components/schemas/is_system_action' + name: + type: string + description: The display name for the connector. connector_response_properties_webhook: title: Connector response properties for a Webhook connector type: object @@ -2711,6 +2791,7 @@ components: - $ref: '#/components/schemas/connector_response_properties_swimlane' - $ref: '#/components/schemas/connector_response_properties_teams' - $ref: '#/components/schemas/connector_response_properties_tines' + - $ref: '#/components/schemas/connector_response_properties_torq' - $ref: '#/components/schemas/connector_response_properties_webhook' - $ref: '#/components/schemas/connector_response_properties_xmatters' discriminator: @@ -2922,6 +3003,21 @@ components: description: The display name for the connector. secrets: $ref: '#/components/schemas/secrets_properties_teams' + update_connector_request_torq: + title: Update Torq connector request + type: object + required: + - config + - name + - secrets + properties: + config: + $ref: '#/components/schemas/config_properties_torq' + name: + type: string + description: The display name for the connector. + secrets: + $ref: '#/components/schemas/secrets_properties_torq' update_connector_request_webhook: title: Update Webhook connector request type: object @@ -2975,6 +3071,7 @@ components: - .swimlane - .teams - .tines + - .torq - .webhook - .xmatters example: .server-log diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml new file mode 100644 index 0000000000000..06808a37a75fc --- /dev/null +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/config_properties_torq.yaml @@ -0,0 +1,9 @@ +title: Connector request properties for a Torq connector +description: Defines properties for connectors when type is `.torq`. +type: object +required: + - webhookIntegrationUrl +properties: + webhookIntegrationUrl: + description: The endpoint URL of the Elastic Security integration in Torq. + type: string \ No newline at end of file diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml index 334fe3fa5cdb3..edef270fd75ae 100644 --- a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties.yaml @@ -18,6 +18,7 @@ oneOf: - $ref: 'connector_response_properties_swimlane.yaml' - $ref: 'connector_response_properties_teams.yaml' - $ref: 'connector_response_properties_tines.yaml' + - $ref: 'connector_response_properties_torq.yaml' - $ref: 'connector_response_properties_webhook.yaml' - $ref: 'connector_response_properties_xmatters.yaml' discriminator: diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml new file mode 100644 index 0000000000000..135d5e9db6cb4 --- /dev/null +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_response_properties_torq.yaml @@ -0,0 +1,31 @@ +title: Connector response properties for a Torq connector +type: object +required: + - config + - connector_type_id + - id + - is_deprecated + - is_preconfigured + - name +properties: + config: + $ref: 'config_properties_torq.yaml' + connector_type_id: + type: string + description: The type of connector. + enum: + - .torq + id: + type: string + description: The identifier for the connector. + is_deprecated: + $ref: 'is_deprecated.yaml' + is_missing_secrets: + $ref: 'is_missing_secrets.yaml' + is_preconfigured: + $ref: 'is_preconfigured.yaml' + is_system_action: + $ref: 'is_system_action.yaml' + name: + type: string + description: The display name for the connector. diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml index 2bbc9f5dabac4..687648acd7141 100644 --- a/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/connector_types.yaml @@ -20,6 +20,7 @@ enum: - .swimlane - .teams - .tines + - .torq - .webhook - .xmatters example: .server-log \ No newline at end of file diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml new file mode 100644 index 0000000000000..934f9c9c1b395 --- /dev/null +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/create_connector_request_torq.yaml @@ -0,0 +1,24 @@ +title: Create Torq connector request +description: > + The Torq connector uses a Torq webhook to trigger workflows with Kibana actions. +type: object +required: + - config + - connector_type_id + - name + - secrets +properties: + config: + $ref: 'config_properties_torq.yaml' + connector_type_id: + type: string + description: The type of connector. + enum: + - .torq + example: .torq + name: + type: string + description: The display name for the connector. + example: my-connector + secrets: + $ref: 'secrets_properties_torq.yaml' \ No newline at end of file diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml new file mode 100644 index 0000000000000..ab79a0f672b42 --- /dev/null +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/secrets_properties_torq.yaml @@ -0,0 +1,9 @@ +title: Connector secrets properties for a Torq connector +description: Defines secrets for connectors when type is `.torq`. +type: object +required: + - token +properties: + token: + description: The secret of the webhook authentication header. + type: string \ No newline at end of file diff --git a/x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml b/x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml new file mode 100644 index 0000000000000..f82de22f3e27b --- /dev/null +++ b/x-pack/plugins/actions/docs/openapi/components/schemas/update_connector_request_torq.yaml @@ -0,0 +1,14 @@ +title: Update Torq connector request +type: object +required: + - config + - name + - secrets +properties: + config: + $ref: 'config_properties_torq.yaml' + name: + type: string + description: The display name for the connector. + secrets: + $ref: 'secrets_properties_torq.yaml' \ No newline at end of file diff --git a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml index f32def50706b2..62f2e1821eb8f 100644 --- a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml +++ b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector.yaml @@ -34,6 +34,7 @@ post: - $ref: '../components/schemas/create_connector_request_swimlane.yaml' - $ref: '../components/schemas/create_connector_request_teams.yaml' - $ref: '../components/schemas/create_connector_request_tines.yaml' + - $ref: '../components/schemas/create_connector_request_torq.yaml' - $ref: '../components/schemas/create_connector_request_webhook.yaml' - $ref: '../components/schemas/create_connector_request_xmatters.yaml' discriminator: diff --git a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml index c6a2447c251a2..ad93b5076639a 100644 --- a/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml +++ b/x-pack/plugins/actions/docs/openapi/paths/s@{spaceid}@api@actions@connector@{connectorid}.yaml @@ -118,6 +118,7 @@ post: - $ref: '../components/schemas/create_connector_request_swimlane.yaml' - $ref: '../components/schemas/create_connector_request_teams.yaml' - $ref: '../components/schemas/create_connector_request_tines.yaml' + - $ref: '../components/schemas/create_connector_request_torq.yaml' - $ref: '../components/schemas/create_connector_request_webhook.yaml' - $ref: '../components/schemas/create_connector_request_xmatters.yaml' discriminator: @@ -176,6 +177,7 @@ put: - $ref: '../components/schemas/update_connector_request_swimlane.yaml' - $ref: '../components/schemas/update_connector_request_teams.yaml' # - $ref: '../components/schemas/update_connector_request_tines.yaml' + - $ref: '../components/schemas/update_connector_request_torq.yaml' - $ref: '../components/schemas/update_connector_request_webhook.yaml' - $ref: '../components/schemas/update_connector_request_xmatters.yaml' examples: diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts index c17bedeb8635a..04b8f7f41bdf1 100644 --- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts +++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/sections.ts @@ -12,9 +12,9 @@ import { isEmpty, pickBy } from 'lodash'; import moment from 'moment'; import url from 'url'; import type { InfraLocators } from '@kbn/infra-plugin/common/locators'; -import type { ProfilingLocators } from '@kbn/profiling-plugin/public'; import { LocatorPublic } from '@kbn/share-plugin/common'; import { AllDatasetsLocatorParams } from '@kbn/deeplinks-observability/locators'; +import type { ProfilingLocators } from '@kbn/observability-shared-plugin/public'; import { Environment } from '../../../../common/environment_rt'; import type { Transaction } from '../../../../typings/es_schemas/ui/transaction'; import { getDiscoverHref } from '../links/discover_links/discover_link'; diff --git a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx index 162fdf8e90b85..4dcd10a3ea540 100644 --- a/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx +++ b/x-pack/plugins/apm/public/components/shared/transaction_action_menu/transaction_action_menu.tsx @@ -18,7 +18,6 @@ import { SectionSubtitle, SectionTitle, } from '@kbn/observability-shared-plugin/public'; -import { ProfilingLocators } from '@kbn/profiling-plugin/public'; import React, { useState } from 'react'; import { useLocation } from 'react-router-dom'; import useAsync from 'react-use/lib/useAsync'; @@ -26,6 +25,7 @@ import { AllDatasetsLocatorParams, ALL_DATASETS_LOCATOR_ID, } from '@kbn/deeplinks-observability/locators'; +import type { ProfilingLocators } from '@kbn/observability-shared-plugin/public'; import { useAnyOfApmParams } from '../../../hooks/use_apm_params'; import { ApmFeatureFlagName } from '../../../../common/apm_feature_flags'; import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts index f7ae6ea53147a..eff505b22bd4a 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_infra_metrics_client/create_infra_metrics_client.ts @@ -6,7 +6,7 @@ */ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; -import { APMRouteHandlerResources } from '../../../../routes/typings'; +import { APMRouteHandlerResources } from '../../../../routes/apm_routes/register_apm_server_routes'; type InfraMetricsSearchParams = Omit & { size: number; diff --git a/x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts b/x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts deleted file mode 100644 index 24b76edb4d887..0000000000000 --- a/x-pack/plugins/apm/server/lib/helpers/get_infra_metric_indices.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SavedObjectsClientContract } from '@kbn/core/server'; -import { APMRouteHandlerResources } from '../../routes/apm_routes/register_apm_server_routes'; - -export async function getInfraMetricIndices({ - infraPlugin, - savedObjectsClient, -}: { - infraPlugin: Required; - savedObjectsClient: SavedObjectsClientContract; -}): Promise { - if (!infraPlugin) { - throw new Error('Infra Plugin needs to be setup'); - } - const infra = await infraPlugin.start(); - const infraMetricIndices = await infra.getMetricIndices(savedObjectsClient); - - return infraMetricIndices; -} diff --git a/x-pack/plugins/apm/server/routes/infrastructure/route.ts b/x-pack/plugins/apm/server/routes/infrastructure/route.ts index 4117a43ce1e3f..9050f1a46622c 100644 --- a/x-pack/plugins/apm/server/routes/infrastructure/route.ts +++ b/x-pack/plugins/apm/server/routes/infrastructure/route.ts @@ -5,7 +5,6 @@ * 2.0. */ import * as t from 'io-ts'; -import Boom from '@hapi/boom'; import { createApmServerRoute } from '../apm_routes/create_apm_server_route'; import { getApmEventClient } from '../../lib/helpers/get_apm_event_client'; import { environmentRt, kueryRt, rangeRt } from '../default_api_types'; @@ -30,10 +29,6 @@ const infrastructureRoute = createApmServerRoute({ hostNames: string[]; podNames: string[]; }> => { - if (!resources.plugins.infra) { - throw Boom.notFound(); - } - const apmEventClient = await getApmEventClient(resources); const infraMetricsClient = createInfraMetricsClient(resources); const { params } = resources; diff --git a/x-pack/plugins/apm/server/routes/services/route.ts b/x-pack/plugins/apm/server/routes/services/route.ts index 4ac0a37b3d10d..970a72d478f72 100644 --- a/x-pack/plugins/apm/server/routes/services/route.ts +++ b/x-pack/plugins/apm/server/routes/services/route.ts @@ -250,7 +250,7 @@ const serviceMetadataDetailsRoute = createApmServerRoute({ end, }); - if (serviceMetadataDetails?.container?.ids && resources.plugins.infra) { + if (serviceMetadataDetails?.container?.ids) { const infraMetricsClient = createInfraMetricsClient(resources); const containerMetadata = await getServiceOverviewContainerMetadata({ infraMetricsClient, @@ -761,10 +761,7 @@ export const serviceInstancesMetadataDetails = createApmServerRoute({ end, }); - if ( - serviceInstanceMetadataDetails?.container?.id && - resources.plugins.infra - ) { + if (serviceInstanceMetadataDetails?.container?.id) { const infraMetricsClient = createInfraMetricsClient(resources); const containerMetadata = await getServiceInstanceContainerMetadata({ infraMetricsClient, diff --git a/x-pack/plugins/canvas/shareable_runtime/index.ts b/x-pack/plugins/canvas/shareable_runtime/index.ts index aee57c3780503..475989494c574 100644 --- a/x-pack/plugins/canvas/shareable_runtime/index.ts +++ b/x-pack/plugins/canvas/shareable_runtime/index.ts @@ -9,4 +9,3 @@ export * from './api'; import '@kbn/core-apps-server-internal/assets/legacy_light_theme.css'; import '../public/style/index.scss'; import '@elastic/eui/dist/eui_theme_light.css'; -import '@kbn/ui-framework/dist/kui_light.css'; diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index e1bcdc1f9a95f..7f0b4f62fb216 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -65,6 +65,8 @@ export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d'; export const DATA_VIEW_INDEX_PATTERN = 'logs-*'; +export const SECURITY_DEFAULT_DATA_VIEW_ID = 'security-solution-default'; + export const CSP_INGEST_TIMESTAMP_PIPELINE = 'cloud_security_posture_add_ingest_timestamp_pipeline'; export const CSP_LATEST_FINDINGS_INGEST_TIMESTAMP_PIPELINE = 'cloud_security_posture_latest_index_add_ingest_timestamp_pipeline'; diff --git a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts index 9bdd3bfada098..e3d213118dd51 100644 --- a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts +++ b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.test.ts @@ -6,7 +6,11 @@ */ import { renderHook, act } from '@testing-library/react-hooks/dom'; -import { useNavigateFindings, useNavigateFindingsByResource } from './use_navigate_findings'; +import { + useNavigateFindings, + useNavigateFindingsByResource, + useNavigateVulnerabilities, +} from './use_navigate_findings'; import { useHistory } from 'react-router-dom'; jest.mock('react-router-dom', () => ({ @@ -29,9 +33,17 @@ jest.mock('./use_kibana', () => ({ }, }), })); +jest.mock('../api/use_latest_findings_data_view', () => ({ + useLatestFindingsDataView: jest.fn().mockReturnValue({ + status: 'success', + data: { + id: 'data-view-id', + }, + }), +})); describe('useNavigateFindings', () => { - it('creates a URL to findings page with correct path and filter', () => { + it('creates a URL to findings page with correct path, filter and dataViewId', () => { const push = jest.fn(); (useHistory as jest.Mock).mockReturnValueOnce({ push }); @@ -44,7 +56,7 @@ describe('useNavigateFindings', () => { expect(push).toHaveBeenCalledWith({ pathname: '/cloud_security_posture/findings/configurations', search: - "cspq=(filters:!((meta:(alias:!n,disabled:!f,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", + "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:data-view-id,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", }); expect(push).toHaveBeenCalledTimes(1); }); @@ -62,7 +74,7 @@ describe('useNavigateFindings', () => { expect(push).toHaveBeenCalledWith({ pathname: '/cloud_security_posture/findings/configurations', search: - "cspq=(filters:!((meta:(alias:!n,disabled:!f,key:foo,negate:!t,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", + "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:data-view-id,key:foo,negate:!t,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", }); expect(push).toHaveBeenCalledTimes(1); }); @@ -80,7 +92,25 @@ describe('useNavigateFindings', () => { expect(push).toHaveBeenCalledWith({ pathname: '/cloud_security_posture/findings/resource', search: - "cspq=(filters:!((meta:(alias:!n,disabled:!f,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", + "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:data-view-id,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", + }); + expect(push).toHaveBeenCalledTimes(1); + }); + + it('creates a URL to vulnerabilities page with correct path, filter and dataViewId', () => { + const push = jest.fn(); + (useHistory as jest.Mock).mockReturnValueOnce({ push }); + + const { result } = renderHook(() => useNavigateVulnerabilities()); + + act(() => { + result.current({ foo: 1 }); + }); + + expect(push).toHaveBeenCalledWith({ + pathname: '/cloud_security_posture/findings/vulnerabilities', + search: + "cspq=(filters:!((meta:(alias:!n,disabled:!f,index:security-solution-default,key:foo,negate:!f,type:phrase),query:(match_phrase:(foo:1)))),query:(language:kuery,query:''))", }); expect(push).toHaveBeenCalledTimes(1); }); diff --git a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts index 48b16f62cbaf5..fbeeeb32a0c2e 100644 --- a/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/common/hooks/use_navigate_findings.ts @@ -8,9 +8,14 @@ import { useCallback } from 'react'; import { useHistory } from 'react-router-dom'; import { Filter } from '@kbn/es-query'; +import { + LATEST_FINDINGS_INDEX_PATTERN, + SECURITY_DEFAULT_DATA_VIEW_ID, +} from '../../../common/constants'; import { findingsNavigation } from '../navigation/constants'; import { encodeQuery } from '../navigation/query_utils'; import { useKibana } from './use_kibana'; +import { useLatestFindingsDataView } from '../api/use_latest_findings_data_view'; interface NegatedValue { value: string | number; @@ -21,7 +26,7 @@ type FilterValue = string | number | NegatedValue; export type NavFilter = Record; -const createFilter = (key: string, filterValue: FilterValue): Filter => { +const createFilter = (key: string, filterValue: FilterValue, dataViewId: string): Filter => { let negate = false; let value = filterValue; if (typeof filterValue === 'object') { @@ -32,7 +37,7 @@ const createFilter = (key: string, filterValue: FilterValue): Filter => { if (value === '*') { return { query: { exists: { field: key } }, - meta: { type: 'exists' }, + meta: { type: 'exists', index: dataViewId }, }; } return { @@ -42,18 +47,19 @@ const createFilter = (key: string, filterValue: FilterValue): Filter => { disabled: false, type: 'phrase', key, + index: dataViewId, }, query: { match_phrase: { [key]: value } }, }; }; -const useNavigate = (pathname: string) => { +const useNavigate = (pathname: string, dataViewId = SECURITY_DEFAULT_DATA_VIEW_ID) => { const history = useHistory(); const { services } = useKibana(); return useCallback( (filterParams: NavFilter = {}) => { const filters = Object.entries(filterParams).map(([key, filterValue]) => - createFilter(key, filterValue) + createFilter(key, filterValue, dataViewId) ); history.push({ @@ -65,14 +71,19 @@ const useNavigate = (pathname: string) => { }), }); }, - [pathname, history, services.data.query.queryString] + [pathname, history, services.data.query.queryString, dataViewId] ); }; -export const useNavigateFindings = () => useNavigate(findingsNavigation.findings_default.path); +export const useNavigateFindings = () => { + const { data } = useLatestFindingsDataView(LATEST_FINDINGS_INDEX_PATTERN); + return useNavigate(findingsNavigation.findings_default.path, data?.id); +}; -export const useNavigateFindingsByResource = () => - useNavigate(findingsNavigation.findings_by_resource.path); +export const useNavigateFindingsByResource = () => { + const { data } = useLatestFindingsDataView(LATEST_FINDINGS_INDEX_PATTERN); + return useNavigate(findingsNavigation.findings_by_resource.path, data?.id); +}; export const useNavigateVulnerabilities = () => useNavigate(findingsNavigation.vulnerabilities.path); diff --git a/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx new file mode 100644 index 0000000000000..1e2f2f52fd02a --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.test.tsx @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { render, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { DetectionRuleCounter } from './detection_rule_counter'; +import { TestProvider } from '../test/test_provider'; +import { useFetchDetectionRulesByTags } from '../common/api/use_fetch_detection_rules_by_tags'; +import { useFetchDetectionRulesAlertsStatus } from '../common/api/use_fetch_detection_rules_alerts_status'; +import { RuleResponse } from '../common/types'; + +jest.mock('../common/api/use_fetch_detection_rules_by_tags', () => ({ + useFetchDetectionRulesByTags: jest.fn(), +})); +jest.mock('../common/api/use_fetch_detection_rules_alerts_status', () => ({ + useFetchDetectionRulesAlertsStatus: jest.fn(), +})); + +describe('DetectionRuleCounter', () => { + beforeEach(() => { + jest.restoreAllMocks(); + }); + it('should render loading skeleton when both rules and alerts are loading', () => { + (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({ + data: undefined, + isLoading: true, + }); + + (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({ + data: undefined, + isLoading: true, + }); + const { getByTestId } = render( + + + + ); + + const skeletonText = getByTestId('csp:detection-rule-counter-loading'); + expect(skeletonText).toBeInTheDocument(); + }); + + it('should render create rule link when no rules exist', () => { + (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({ + data: { total: 0 }, + isLoading: false, + }); + + (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({ + data: null, + isLoading: false, + isFetching: false, + }); + + const { getByText, getByTestId } = render( + + + + ); + + const createRuleLink = getByTestId('csp:findings-flyout-create-detection-rule-link'); + expect(createRuleLink).toBeInTheDocument(); + expect(getByText('Create a detection rule')).toBeInTheDocument(); + }); + + it('should render alert and rule count when rules exist', () => { + (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({ + data: { total: 5 }, + isLoading: false, + }); + + (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({ + data: { total: 10 }, + isLoading: false, + isFetching: false, + }); + + const { getByText, getByTestId } = render( + + + + ); + + const alertCountLink = getByTestId('csp:findings-flyout-alert-count'); + const ruleCountLink = getByTestId('csp:findings-flyout-detection-rule-count'); + + expect(alertCountLink).toBeInTheDocument(); + expect(getByText(/10 alerts/i)).toBeInTheDocument(); + expect(ruleCountLink).toBeInTheDocument(); + expect(getByText(/5 detection rules/i)).toBeInTheDocument(); + }); + + it('should show loading spinner when creating a rule', async () => { + (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({ + data: { total: 0 }, + isLoading: false, + }); + + (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({ + data: null, + isLoading: false, + isFetching: false, + }); + const createRuleFn = jest.fn(() => Promise.resolve({} as RuleResponse)); + const { getByTestId, queryByTestId } = render( + + + + ); + + // Trigger createDetectionRuleOnClick + const createRuleLink = getByTestId('csp:findings-flyout-create-detection-rule-link'); + userEvent.click(createRuleLink); + + const loadingSpinner = getByTestId('csp:findings-flyout-detection-rule-counter-loading'); + expect(loadingSpinner).toBeInTheDocument(); + + (useFetchDetectionRulesByTags as jest.Mock).mockReturnValue({ + data: { total: 1 }, + isLoading: false, + }); + + (useFetchDetectionRulesAlertsStatus as jest.Mock).mockReturnValue({ + data: { total: 0 }, + isLoading: false, + isFetching: false, + }); + + // Wait for the loading spinner to disappear + await waitFor(() => { + expect(queryByTestId('csp:findings-flyout-detection-rule-counter-loading')).toBeNull(); + }); + }); +}); diff --git a/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx index eeea89f9a310f..b2a79710d09a2 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/detection_rule_counter.tsx @@ -68,7 +68,12 @@ export const DetectionRuleCounter = ({ tags, createRuleFn }: DetectionRuleCounte }, [createRuleFn, http, notifications, queryClient]); return ( - + {rulesData?.total === 0 ? ( <> @@ -78,11 +83,17 @@ export const DetectionRuleCounter = ({ tags, createRuleFn }: DetectionRuleCounte id="xpack.csp.findingsFlyout.alerts.creatingRule" defaultMessage="Creating detection rule" />{' '} - + ) : ( <> - + ) : ( <> - + {' '} - + +
  • + +
    • {accountType === AWS_ORGANIZATION_ACCOUNT ? (
  • { list-style: auto; `} > +
  • + +
  • - {ruleResponse.name} + {ruleResponse.name} {` `} - + ({ - ConversationalRetrievalQAChain: { + RetrievalQAChain: { fromLLM: jest.fn().mockImplementation(() => mockConversationChain), }, })); +const mockCall = jest.fn(); +jest.mock('langchain/agents', () => ({ + initializeAgentExecutorWithOptions: jest.fn().mockImplementation(() => ({ + call: mockCall, + })), +})); + const mockConnectorId = 'mock-connector-id'; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -42,7 +49,7 @@ const mockActions: ActionsPluginStart = {} as ActionsPluginStart; const mockLogger = loggerMock.create(); const esClientMock = elasticsearchServiceMock.createScopedClusterClient().asCurrentUser; -describe('executeCustomLlmChain', () => { +describe('callAgentExecutor', () => { beforeEach(() => { jest.clearAllMocks(); @@ -52,7 +59,7 @@ describe('executeCustomLlmChain', () => { }); it('creates an instance of ActionsClientLlm with the expected context from the request', async () => { - await executeCustomLlmChain({ + await callAgentExecutor({ actions: mockActions, connectorId: mockConnectorId, esClient: esClientMock, @@ -70,7 +77,7 @@ describe('executeCustomLlmChain', () => { }); it('kicks off the chain with (only) the last message', async () => { - await executeCustomLlmChain({ + await callAgentExecutor({ actions: mockActions, connectorId: mockConnectorId, esClient: esClientMock, @@ -79,15 +86,15 @@ describe('executeCustomLlmChain', () => { request: mockRequest, }); - expect(mockConversationChain.call).toHaveBeenCalledWith({ - question: '\n\nDo you know my name?', + expect(mockCall).toHaveBeenCalledWith({ + input: '\n\nDo you know my name?', }); }); it('kicks off the chain with the expected message when langChainMessages has only one entry', async () => { const onlyOneMessage = [langChainMessages[0]]; - await executeCustomLlmChain({ + await callAgentExecutor({ actions: mockActions, connectorId: mockConnectorId, esClient: esClientMock, @@ -96,13 +103,13 @@ describe('executeCustomLlmChain', () => { request: mockRequest, }); - expect(mockConversationChain.call).toHaveBeenCalledWith({ - question: 'What is my name?', + expect(mockCall).toHaveBeenCalledWith({ + input: 'What is my name?', }); }); it('returns the expected response body', async () => { - const result: ResponseBody = await executeCustomLlmChain({ + const result: ResponseBody = await callAgentExecutor({ actions: mockActions, connectorId: mockConnectorId, esClient: esClientMock, diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts index 5a65b1589b21e..b6a768ad69598 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts @@ -7,16 +7,18 @@ import { ElasticsearchClient, KibanaRequest, Logger } from '@kbn/core/server'; import type { PluginStartContract as ActionsPluginStart } from '@kbn/actions-plugin/server'; +import { initializeAgentExecutorWithOptions } from 'langchain/agents'; +import { RetrievalQAChain } from 'langchain/chains'; import { BufferMemory, ChatMessageHistory } from 'langchain/memory'; import { BaseMessage } from 'langchain/schema'; +import { ChainTool, Tool } from 'langchain/tools'; -import { ConversationalRetrievalQAChain } from 'langchain/chains'; +import { ElasticsearchStore } from '../elasticsearch_store/elasticsearch_store'; import { ResponseBody } from '../helpers'; import { ActionsClientLlm } from '../llm/actions_client_llm'; -import { ElasticsearchStore } from '../elasticsearch_store/elasticsearch_store'; import { KNOWLEDGE_BASE_INDEX_PATTERN } from '../../../routes/knowledge_base/constants'; -export const executeCustomLlmChain = async ({ +export const callAgentExecutor = async ({ actions, connectorId, esClient, @@ -34,31 +36,38 @@ export const executeCustomLlmChain = async ({ }): Promise => { const llm = new ActionsClientLlm({ actions, connectorId, request, logger }); - // Chat History Memory: in-memory memory, from client local storage, first message is the system prompt const pastMessages = langChainMessages.slice(0, -1); // all but the last message const latestMessage = langChainMessages.slice(-1); // the last message + const memory = new BufferMemory({ chatHistory: new ChatMessageHistory(pastMessages), - memoryKey: 'chat_history', + memoryKey: 'chat_history', // this is the key expected by https://github.com/langchain-ai/langchainjs/blob/a13a8969345b0f149c1ca4a120d63508b06c52a5/langchain/src/agents/initialize.ts#L166 + inputKey: 'input', + outputKey: 'output', + returnMessages: true, }); // ELSER backed ElasticsearchStore for Knowledge Base const esStore = new ElasticsearchStore(esClient, KNOWLEDGE_BASE_INDEX_PATTERN, logger); + const chain = RetrievalQAChain.fromLLM(llm, esStore.asRetriever()); + + const tools: Tool[] = [ + new ChainTool({ + name: 'esql-language-knowledge-base', + description: + 'Call this for knowledge on how to build an ESQL query, or answer questions about the ES|QL query language.', + chain, + }), + ]; - // Chain w/ chat history memory and knowledge base retriever - const chain = ConversationalRetrievalQAChain.fromLLM(llm, esStore.asRetriever(), { + const executor = await initializeAgentExecutorWithOptions(tools, llm, { + agentType: 'chat-conversational-react-description', memory, - // See `qaChainOptions` from https://js.langchain.com/docs/modules/chains/popular/chat_vector_db - qaChainOptions: { type: 'stuff' }, + verbose: false, }); - await chain.call({ question: latestMessage[0].content }); - // Chain w/ just knowledge base retriever - // const chain = RetrievalQAChain.fromLLM(llm, esStore.asRetriever()); - // await chain.call({ query: latestMessage[0].content }); + await executor.call({ input: latestMessage[0].content }); - // The assistant (on the client side) expects the same response returned - // from the actions framework, so we need to return the same shape of data: return { connector_id: connectorId, data: llm.getActionResultData(), // the response from the actions framework diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts index 2e6709a6e33c2..57f2b25f5a65f 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.test.ts @@ -20,7 +20,7 @@ jest.mock('../lib/build_response', () => ({ })); jest.mock('../lib/langchain/execute_custom_llm_chain', () => ({ - executeCustomLlmChain: jest.fn().mockImplementation( + callAgentExecutor: jest.fn().mockImplementation( async ({ connectorId, }: { diff --git a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts index 1043f68f0f9c1..bbb1c76e3e579 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/post_actions_connector_execute.ts @@ -20,7 +20,7 @@ import { PostActionsConnectorExecutePathParams, } from '../schemas/post_actions_connector_execute'; import { ElasticAssistantRequestHandlerContext } from '../types'; -import { executeCustomLlmChain } from '../lib/langchain/execute_custom_llm_chain'; +import { callAgentExecutor } from '../lib/langchain/execute_custom_llm_chain'; export const postActionsConnectorExecuteRoute = ( router: IRouter @@ -53,7 +53,7 @@ export const postActionsConnectorExecuteRoute = ( // convert the assistant messages to LangChain messages: const langChainMessages = getLangChainMessages(assistantMessages); - const langChainResponseBody = await executeCustomLlmChain({ + const langChainResponseBody = await callAgentExecutor({ actions, connectorId, esClient, diff --git a/x-pack/plugins/enterprise_search/server/index.ts b/x-pack/plugins/enterprise_search/server/index.ts index a82d6f18a3b26..3b9372f339acf 100644 --- a/x-pack/plugins/enterprise_search/server/index.ts +++ b/x-pack/plugins/enterprise_search/server/index.ts @@ -8,7 +8,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginInitializerContext, PluginConfigDescriptor } from '@kbn/core/server'; -import { EnterpriseSearchPlugin, EnterpriseSearchPluginStart as PluginStart } from './plugin'; +import { EnterpriseSearchPlugin } from './plugin'; export const plugin = (initializerContext: PluginInitializerContext) => { return new EnterpriseSearchPlugin(initializerContext); @@ -54,5 +54,3 @@ export const config: PluginConfigDescriptor = { }; export const CRAWLERS_INDEX = '.ent-search-actastic-crawler2_configurations_v2'; - -export type EnterpriseSearchPluginStart = PluginStart; diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts index d367e02ed6ab1..04f922e1702ed 100644 --- a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawler_multiple_schedules.ts @@ -7,9 +7,7 @@ import { IScopedClusterClient } from '@kbn/core/server'; -import { Connector } from '@kbn/search-connectors'; - -import { CONNECTORS_INDEX } from '../..'; +import { CONNECTORS_INDEX, Connector } from '@kbn/search-connectors'; const CUSTOM_SCHEDULING = 'custom_scheduling'; diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts index 21d9f8b558800..539495e9556ae 100644 --- a/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/post_crawler_multiple_schedules.ts @@ -7,7 +7,7 @@ import { IScopedClusterClient } from '@kbn/core/server'; -import { CONNECTORS_INDEX } from '../..'; +import { CONNECTORS_INDEX } from '@kbn/search-connectors'; import { CrawlerCustomScheduleMappingServer, diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts index 52cb24271afa5..3548fee93fbf2 100644 --- a/x-pack/plugins/fleet/common/constants/epm.ts +++ b/x-pack/plugins/fleet/common/constants/epm.ts @@ -9,7 +9,7 @@ import { ElasticsearchAssetType, KibanaAssetType } from '../types/models'; export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages'; export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets'; -export const MAX_TIME_COMPLETE_INSTALL = 60000; +export const MAX_TIME_COMPLETE_INSTALL = 30 * 60 * 1000; // 30 minutes export const FLEET_SYSTEM_PACKAGE = 'system'; export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent'; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 3e9363fa9828f..5682749d7e381 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -11,6 +11,8 @@ import type { MappingTypeMapping, } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import pMap from 'p-map'; + import type { Field, Fields } from '../../fields/field'; import type { RegistryDataStream, @@ -729,15 +731,22 @@ const updateAllDataStreams = async ( esClient: ElasticsearchClient, logger: Logger ): Promise => { - const updatedataStreamPromises = indexNameWithTemplates.map((templateEntry) => { - return updateExistingDataStream({ - esClient, - logger, - dataStreamName: templateEntry.dataStreamName, - }); - }); - await Promise.all(updatedataStreamPromises); + await pMap( + indexNameWithTemplates, + (templateEntry) => { + return updateExistingDataStream({ + esClient, + logger, + dataStreamName: templateEntry.dataStreamName, + }); + }, + { + // Limit concurrent putMapping/rollover requests to avoid overhwhelming ES cluster + concurrency: 20, + } + ); }; + const updateExistingDataStream = async ({ dataStreamName, esClient, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 3c308b8e85b0a..b7fe0d95310ef 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -5,11 +5,21 @@ * 2.0. */ -import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server'; +import type { + SavedObjectsClientContract, + ElasticsearchClient, + SavedObject, +} from '@kbn/core/server'; import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import { ConcurrentInstallOperationError } from '../../../errors'; + +import type { Installation } from '../../../../common'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; + import { appContextService } from '../../app_context'; import { createAppContextStartContractMock } from '../../../mocks'; import { saveArchiveEntries } from '../archive/storage'; @@ -29,7 +39,9 @@ jest.mock('../elasticsearch/datastream_ilm/install'); import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { installKibanaAssetsAndReferences } from '../kibana/assets/install'; -import { installIndexTemplatesAndPipelines } from './install'; +import { MAX_TIME_COMPLETE_INSTALL } from '../../../../common/constants'; + +import { installIndexTemplatesAndPipelines, restartInstallation } from './install'; import { _installPackage } from './_install_package'; @@ -69,9 +81,7 @@ describe('_installPackage', () => { jest.mocked(saveArchiveEntries).mockResolvedValue({ saved_objects: [], }); - }); - afterEach(async () => { - appContextService.stop(); + jest.mocked(restartInstallation).mockReset(); }); it('handles errors from installKibanaAssets', async () => { // force errors from this function @@ -226,4 +236,128 @@ describe('_installPackage', () => { expect(installILMPolicy).toBeCalled(); expect(installIlmForDataStream).toBeCalled(); }); + + describe('when package is stuck in `installing`', () => { + afterEach(() => {}); + const mockInstalledPackageSo: SavedObject = { + id: 'mocked-package', + attributes: { + name: 'test-package', + version: '1.0.0', + install_status: 'installing', + install_version: '1.0.0', + install_started_at: new Date().toISOString(), + install_source: 'registry', + verification_status: 'verified', + installed_kibana: [] as any, + installed_es: [] as any, + es_index_patterns: {}, + }, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }; + + beforeEach(() => { + appContextService.start( + createAppContextStartContractMock({ + internal: { + disableILMPolicies: true, + disableProxies: false, + fleetServerStandalone: false, + onlyAllowAgentUpgradeToKnownVersions: false, + registry: { + kibanaVersionCheckEnabled: true, + capabilities: [], + }, + }, + }) + ); + }); + + describe('timeout reached', () => { + it('restarts installation', async () => { + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date( + Date.now() - MAX_TIME_COMPLETE_INSTALL * 2 + ).toISOString(), + }, + }, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + + describe('timeout not reached', () => { + describe('force flag not provided', () => { + it('throws concurrent installation error if force flag is not provided', async () => { + expect( + _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + }) + ).rejects.toThrowError(ConcurrentInstallOperationError); + }); + }); + + describe('force flag provided', () => { + it('restarts installation', async () => { + await _installPackage({ + savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), + esClient, + logger: loggerMock.create(), + paths: [], + packageInfo: { + name: mockInstalledPackageSo.attributes.name, + version: mockInstalledPackageSo.attributes.version, + title: mockInstalledPackageSo.attributes.name, + } as any, + installedPkg: { + ...mockInstalledPackageSo, + attributes: { + ...mockInstalledPackageSo.attributes, + install_started_at: new Date(Date.now() - 1000).toISOString(), + }, + }, + force: true, + }); + + expect(restartInstallation).toBeCalled(); + }); + }); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 337cf59bbd613..3bfc74bf68968 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -99,18 +99,30 @@ export async function _installPackage({ try { // if some installation already exists if (installedPkg) { + const isStatusInstalling = installedPkg.attributes.install_status === 'installing'; + const hasExceededTimeout = + Date.now() - Date.parse(installedPkg.attributes.install_started_at) < + MAX_TIME_COMPLETE_INSTALL; + // if the installation is currently running, don't try to install // instead, only return already installed assets - if ( - installedPkg.attributes.install_status === 'installing' && - Date.now() - Date.parse(installedPkg.attributes.install_started_at) < - MAX_TIME_COMPLETE_INSTALL - ) { - throw new ConcurrentInstallOperationError( - `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ - pkgVersion || 'unknown' - } detected, aborting.` - ); + if (isStatusInstalling && hasExceededTimeout) { + // If this is a forced installation, ignore the timeout and restart the installation anyway + if (force) { + await restartInstallation({ + savedObjectsClient, + pkgName, + pkgVersion, + installSource, + verificationResult, + }); + } else { + throw new ConcurrentInstallOperationError( + `Concurrent installation or upgrade of ${pkgName || 'unknown'}-${ + pkgVersion || 'unknown' + } detected, aborting.` + ); + } } else { // if no installation is running, or the installation has been running longer than MAX_TIME_COMPLETE_INSTALL // (it might be stuck) update the saved object and proceed diff --git a/x-pack/plugins/metrics_data_access/server/client/client.test.ts b/x-pack/plugins/metrics_data_access/server/client/client.test.ts index 72449cf47132b..d96d8efecf52f 100644 --- a/x-pack/plugins/metrics_data_access/server/client/client.test.ts +++ b/x-pack/plugins/metrics_data_access/server/client/client.test.ts @@ -7,18 +7,13 @@ import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; -import { MetricsDataClient } from './client'; +import { MetricsDataClient, DEFAULT_METRIC_INDICES } from './client'; import { metricsDataSourceSavedObjectName } from '../saved_objects/metrics_data_source'; describe('MetricsDataClient', () => { - const client = new MetricsDataClient(); - - client.setDefaultMetricIndicesHandler(async () => { - return 'fallback-indices*'; - }); - describe('metric indices', () => { it('retrieves metrics saved object', async () => { + const client = new MetricsDataClient(); const savedObjectsClient = { get: jest.fn().mockResolvedValue({ attributes: { metricIndices: 'foo,bar' } }), }; @@ -36,6 +31,10 @@ describe('MetricsDataClient', () => { }); it('falls back to provided handler when no metrics saved object exists', async () => { + const client = new MetricsDataClient(); + client.setDefaultMetricIndicesHandler(async () => { + return 'fallback-indices*'; + }); const savedObjectsClient = { get: jest.fn().mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()), }; @@ -51,5 +50,17 @@ describe('MetricsDataClient', () => { ]); expect(indices).toEqual('fallback-indices*'); }); + + it('falls back to static indices when no fallback exists', async () => { + const client = new MetricsDataClient(); + const savedObjectsClient = { + get: jest.fn().mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()), + }; + + const indices = await client.getMetricIndices({ + savedObjectsClient: savedObjectsClient as unknown as SavedObjectsClientContract, + }); + expect(indices).toEqual(DEFAULT_METRIC_INDICES); + }); }); }); diff --git a/x-pack/plugins/metrics_data_access/server/client/client.ts b/x-pack/plugins/metrics_data_access/server/client/client.ts index 30d367cea0293..26359cae578a7 100644 --- a/x-pack/plugins/metrics_data_access/server/client/client.ts +++ b/x-pack/plugins/metrics_data_access/server/client/client.ts @@ -16,21 +16,19 @@ import { metricsDataSourceSavedObjectName, } from '../saved_objects/metrics_data_source'; +export const DEFAULT_METRIC_INDICES = 'metrics-*,metricbeat-*'; + export class MetricsDataClient { private readonly defaultSavedObjectId = 'default'; private getDefaultMetricIndices: DefaultMetricIndicesHandler = null; async getMetricIndices(options: GetMetricIndicesOptions): Promise { - if (!this.getDefaultMetricIndices) { - throw new Error('Missing getMetricsIndices fallback'); - } - const metricIndices = await options.savedObjectsClient .get(metricsDataSourceSavedObjectName, this.defaultSavedObjectId) .then(({ attributes }) => attributes.metricIndices) .catch((err) => { if (SavedObjectsErrorHelpers.isNotFoundError(err)) { - return this.getDefaultMetricIndices!(options); + return this.getDefaultMetricIndices?.(options) ?? DEFAULT_METRIC_INDICES; } throw err; diff --git a/x-pack/plugins/ml/public/application/services/elastic_models_service.ts b/x-pack/plugins/ml/public/application/services/elastic_models_service.ts new file mode 100644 index 0000000000000..2591fb6d82e7d --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/elastic_models_service.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ModelDefinitionResponse, GetElserOptions } from '@kbn/ml-trained-models-utils'; +import { type TrainedModelsApiService } from './ml_api_service/trained_models'; + +export class ElasticModels { + constructor(private readonly trainedModels: TrainedModelsApiService) {} + + /** + * Provides an ELSER model name and configuration for download based on the current cluster architecture. + * The current default version is 2. If running on Cloud it returns the Linux x86_64 optimized version. + * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64, + * a portable version of the model is returned. + */ + public async getELSER(options?: GetElserOptions): Promise { + return await this.trainedModels.getElserConfig(options); + } +} diff --git a/x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts b/x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts new file mode 100644 index 0000000000000..23ac82737044f --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/get_shared_ml_services.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { type HttpStart } from '@kbn/core-http-browser'; +import { ElasticModels } from './elastic_models_service'; +import { HttpService } from './http_service'; +import { mlApiServicesProvider } from './ml_api_service'; + +export type MlSharedServices = ReturnType; + +/** + * Provides ML services exposed from the plugin start. + */ +export function getMlSharedServices(httpStart: HttpStart) { + const httpService = new HttpService(httpStart); + const mlApiServices = mlApiServicesProvider(httpService); + + return { + elasticModels: new ElasticModels(mlApiServices.trainedModels), + }; +} diff --git a/x-pack/plugins/ml/public/application/services/http_service.ts b/x-pack/plugins/ml/public/application/services/http_service.ts index c43b6126f147f..cd283c5d58652 100644 --- a/x-pack/plugins/ml/public/application/services/http_service.ts +++ b/x-pack/plugins/ml/public/application/services/http_service.ts @@ -6,7 +6,7 @@ */ import { Observable } from 'rxjs'; -import { HttpFetchOptionsWithPath, HttpFetchOptions, HttpStart } from '@kbn/core/public'; +import type { HttpFetchOptionsWithPath, HttpFetchOptions, HttpStart } from '@kbn/core/public'; import { getHttp } from '../util/dependency_cache'; function getResultHeaders(headers: HeadersInit) { @@ -59,68 +59,6 @@ export async function http(options: HttpFetchOptionsWithPath): Promise { return getHttp().fetch(path, fetchOptions); } -/** - * Function for making HTTP requests to Kibana's backend which returns an Observable - * with request cancellation support. - * - * @deprecated use {@link HttpService} instead - */ -export function http$(options: HttpFetchOptionsWithPath): Observable { - const { path, fetchOptions } = getFetchOptions(options); - return fromHttpHandler(path, fetchOptions); -} - -/** - * Creates an Observable from Kibana's HttpHandler. - */ -function fromHttpHandler(input: string, init?: RequestInit): Observable { - return new Observable((subscriber) => { - const controller = new AbortController(); - const signal = controller.signal; - - let abortable = true; - let unsubscribed = false; - - if (init?.signal) { - if (init.signal.aborted) { - controller.abort(); - } else { - init.signal.addEventListener('abort', () => { - if (!signal.aborted) { - controller.abort(); - } - }); - } - } - - const perSubscriberInit: RequestInit = { - ...(init ? init : {}), - signal, - }; - - getHttp() - .fetch(input, perSubscriberInit) - .then((response) => { - abortable = false; - subscriber.next(response); - subscriber.complete(); - }) - .catch((err) => { - abortable = false; - if (!unsubscribed) { - subscriber.error(err); - } - }); - - return () => { - unsubscribed = true; - if (abortable) { - controller.abort(); - } - }; - }); -} - /** * ML Http Service */ diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts index e6b9c1a5badc3..c10867af0011b 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/trained_models.ts @@ -6,11 +6,12 @@ */ import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; +import type { IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; import { useMemo } from 'react'; import type { HttpFetchQuery } from '@kbn/core/public'; import type { ErrorType } from '@kbn/ml-error-utils'; +import type { GetElserOptions, ModelDefinitionResponse } from '@kbn/ml-trained-models-utils'; import { ML_INTERNAL_BASE_PATH } from '../../../../common/constants/app'; import type { MlSavedObjectType } from '../../../../common/types/saved_objects'; import { HttpService } from '../http_service'; @@ -57,6 +58,29 @@ export interface InferenceStatsResponse { */ export function trainedModelsApiProvider(httpService: HttpService) { return { + /** + * Fetches the trained models list available for download. + */ + getTrainedModelDownloads() { + return httpService.http({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/model_downloads`, + method: 'GET', + version: '1', + }); + }, + + /** + * Gets ELSER config for download based on the cluster OS and CPU architecture. + */ + getElserConfig(options?: GetElserOptions) { + return httpService.http({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/elser_config`, + method: 'GET', + ...(options ? { query: options as HttpFetchQuery } : {}), + version: '1', + }); + }, + /** * Fetches configuration information for a trained inference model. * @param modelId - Model ID, collection of Model IDs or Model ID pattern. diff --git a/x-pack/plugins/ml/public/mocks.ts b/x-pack/plugins/ml/public/mocks.ts index 13f8952dbad21..77cdefdb2f1c9 100644 --- a/x-pack/plugins/ml/public/mocks.ts +++ b/x-pack/plugins/ml/public/mocks.ts @@ -6,7 +6,8 @@ */ import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { MlPluginSetup, MlPluginStart } from './plugin'; +import { type ElasticModels } from './application/services/elastic_models_service'; +import type { MlPluginSetup, MlPluginStart } from './plugin'; const createSetupContract = (): jest.Mocked => { return { @@ -17,6 +18,21 @@ const createSetupContract = (): jest.Mocked => { const createStartContract = (): jest.Mocked => { return { locator: sharePluginMock.createLocator(), + elasticModels: { + getELSER: jest.fn(() => + Promise.resolve({ + version: 2, + default: true, + config: { + input: { + field_names: ['text_field'], + }, + }, + description: 'Elastic Learned Sparse EncodeR v2 (Tech Preview)', + name: '.elser_model_2', + }) + ), + } as unknown as jest.Mocked, }; }; diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts index 3a32f8b25ae89..4eae00a53d401 100644 --- a/x-pack/plugins/ml/public/plugin.ts +++ b/x-pack/plugins/ml/public/plugin.ts @@ -48,6 +48,10 @@ import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; import type { CasesUiSetup, CasesUiStart } from '@kbn/cases-plugin/public'; import type { SavedSearchPublicPluginStart } from '@kbn/saved-search-plugin/public'; import type { PresentationUtilPluginStart } from '@kbn/presentation-util-plugin/public'; +import { + getMlSharedServices, + MlSharedServices, +} from './application/services/get_shared_ml_services'; import { registerManagementSection } from './application/management'; import { MlLocatorDefinition, type MlLocator } from './locator'; import { setDependencyCache } from './application/util/dependency_cache'; @@ -103,6 +107,9 @@ export class MlPlugin implements Plugin { private appUpdater$ = new BehaviorSubject(() => ({})); private locator: undefined | MlLocator; + + private sharedMlServices: MlSharedServices | undefined; + private isServerless: boolean = false; constructor(private initializerContext: PluginInitializerContext) { @@ -110,6 +117,8 @@ export class MlPlugin implements Plugin { } setup(core: MlCoreSetup, pluginsSetup: MlSetupDependencies) { + this.sharedMlServices = getMlSharedServices(core.http); + core.application.register({ id: PLUGIN_ID, title: i18n.translate('xpack.ml.plugin.title', { @@ -249,6 +258,7 @@ export class MlPlugin implements Plugin { return { locator: this.locator, + elasticModels: this.sharedMlServices?.elasticModels, }; } diff --git a/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json b/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json index 6ec23bc13c559..a6e647a60fe9f 100644 --- a/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json +++ b/x-pack/plugins/ml/scripts/apidoc_scripts/apidoc_config/apidoc.json @@ -180,6 +180,8 @@ "InferTrainedModelDeployment", "CreateInferencePipeline", "GetIngestPipelines", + "GetTrainedModelDownloadList", + "GetElserConfig", "Alerting", "PreviewAlert", diff --git a/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts b/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts new file mode 100644 index 0000000000000..7e66d03033b66 --- /dev/null +++ b/x-pack/plugins/ml/server/models/model_management/model_provider.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { modelsProvider } from './models_provider'; +import { type IScopedClusterClient } from '@kbn/core/server'; +import { cloudMock } from '@kbn/cloud-plugin/server/mocks'; + +describe('modelsProvider', () => { + const mockClient = { + asInternalUser: { + transport: { + request: jest.fn().mockResolvedValue({ + _nodes: { + total: 1, + successful: 1, + failed: 0, + }, + cluster_name: 'default', + nodes: { + yYmqBqjpQG2rXsmMSPb9pQ: { + name: 'node-0', + roles: ['ml'], + attributes: {}, + os: { + name: 'Linux', + arch: 'amd64', + }, + }, + }, + }), + }, + }, + } as unknown as jest.Mocked; + + const mockCloud = cloudMock.createSetup(); + const modelService = modelsProvider(mockClient, mockCloud); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('getELSER', () => { + test('provides a recommended definition by default', async () => { + const result = await modelService.getELSER(); + expect(result.name).toEqual('.elser_model_2_linux-x86_64_SNAPSHOT'); + }); + + test('provides a default version if there is no recommended', async () => { + mockCloud.cloudId = undefined; + (mockClient.asInternalUser.transport.request as jest.Mock).mockResolvedValueOnce({ + _nodes: { + total: 1, + successful: 1, + failed: 0, + }, + cluster_name: 'default', + nodes: { + yYmqBqjpQG2rXsmMSPb9pQ: { + name: 'node-0', + roles: ['ml'], + attributes: {}, + os: { + name: 'Mac OS X', + arch: 'aarch64', + }, + }, + }, + }); + + const result = await modelService.getELSER(); + expect(result.name).toEqual('.elser_model_2_SNAPSHOT'); + }); + + test('provides the requested version', async () => { + const result = await modelService.getELSER({ version: 1 }); + expect(result.name).toEqual('.elser_model_1'); + }); + + test('provides the requested version of a recommended architecture', async () => { + const result = await modelService.getELSER({ version: 2 }); + expect(result.name).toEqual('.elser_model_2_linux-x86_64_SNAPSHOT'); + }); + }); +}); diff --git a/x-pack/plugins/ml/server/models/model_management/models_provider.ts b/x-pack/plugins/ml/server/models/model_management/models_provider.ts index e7cfcbe7fd50d..f6164ad6e65ca 100644 --- a/x-pack/plugins/ml/server/models/model_management/models_provider.ts +++ b/x-pack/plugins/ml/server/models/model_management/models_provider.ts @@ -6,16 +6,23 @@ */ import type { IScopedClusterClient } from '@kbn/core/server'; -import { +import type { IngestPipeline, IngestSimulateDocument, IngestSimulateRequest, + NodesInfoResponseBase, } from '@elastic/elasticsearch/lib/api/types'; +import { + ELASTIC_MODEL_DEFINITIONS, + type GetElserOptions, + type ModelDefinitionResponse, +} from '@kbn/ml-trained-models-utils'; +import type { CloudSetup } from '@kbn/cloud-plugin/server'; import type { PipelineDefinition } from '../../../common/types/trained_models'; export type ModelService = ReturnType; -export function modelsProvider(client: IScopedClusterClient) { +export function modelsProvider(client: IScopedClusterClient, cloud?: CloudSetup) { return { /** * Retrieves the map of model ids and aliases with associated pipelines. @@ -128,5 +135,83 @@ export function modelsProvider(client: IScopedClusterClient) { return result; }, + + /** + * Returns a list of elastic curated models available for download. + */ + async getModelDownloads(): Promise { + // We assume that ML nodes in Cloud are always on linux-x86_64, even if other node types aren't. + const isCloud = !!cloud?.cloudId; + + const nodesInfoResponse = + await client.asInternalUser.transport.request({ + method: 'GET', + path: `/_nodes/ml:true/os`, + }); + + let osName: string | undefined; + let arch: string | undefined; + // Indicates that all ML nodes have the same architecture + let sameArch = true; + for (const node of Object.values(nodesInfoResponse.nodes)) { + if (!osName) { + osName = node.os?.name; + } + if (!arch) { + arch = node.os?.arch; + } + if (node.os?.name !== osName || node.os?.arch !== arch) { + sameArch = false; + break; + } + } + + const result = Object.entries(ELASTIC_MODEL_DEFINITIONS).map(([name, def]) => { + const recommended = + (isCloud && def.os === 'Linux' && def.arch === 'amd64') || + (sameArch && !!def?.os && def?.os === osName && def?.arch === arch); + return { + ...def, + name, + ...(recommended ? { recommended } : {}), + }; + }); + + return result; + }, + + /** + * Provides an ELSER model name and configuration for download based on the current cluster architecture. + * The current default version is 2. If running on Cloud it returns the Linux x86_64 optimized version. + * If any of the ML nodes run a different OS rather than Linux, or the CPU architecture isn't x86_64, + * a portable version of the model is returned. + */ + async getELSER(options?: GetElserOptions): Promise | never { + const modelDownloadConfig = await this.getModelDownloads(); + + let requestedModel: ModelDefinitionResponse | undefined; + let recommendedModel: ModelDefinitionResponse | undefined; + let defaultModel: ModelDefinitionResponse | undefined; + + for (const model of modelDownloadConfig) { + if (options?.version === model.version) { + requestedModel = model; + if (model.recommended) { + requestedModel = model; + break; + } + } else if (model.recommended) { + recommendedModel = model; + } else if (model.default) { + defaultModel = model; + } + } + + if (!requestedModel && !defaultModel && !recommendedModel) { + throw new Error('Requested model not found'); + } + + return requestedModel || recommendedModel || defaultModel!; + }, }; } diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 342350fac998a..dcd97acabcbd8 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -241,7 +241,7 @@ export class MlServerPlugin // Register Trained Model Management routes if (this.enabledFeatures.dfa || this.enabledFeatures.nlp) { modelManagementRoutes(routeInit); - trainedModelsRoutes(routeInit); + trainedModelsRoutes(routeInit, plugins.cloud); } // Register Miscellaneous routes diff --git a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts index 21b62c6f5ce42..1b48a49c8d82f 100644 --- a/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/inference_schema.ts @@ -87,3 +87,7 @@ export const createIngestPipelineSchema = schema.object({ }) ), }); + +export const modelDownloadsQuery = schema.object({ + version: schema.maybe(schema.oneOf([schema.literal('1'), schema.literal('2')])), +}); diff --git a/x-pack/plugins/ml/server/routes/trained_models.ts b/x-pack/plugins/ml/server/routes/trained_models.ts index ab5d3a87e8f46..8685652ab3189 100644 --- a/x-pack/plugins/ml/server/routes/trained_models.ts +++ b/x-pack/plugins/ml/server/routes/trained_models.ts @@ -9,6 +9,8 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { schema } from '@kbn/config-schema'; import type { ErrorType } from '@kbn/ml-error-utils'; import type { MlGetTrainedModelsRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { type ElserVersion } from '@kbn/ml-trained-models-utils'; +import type { CloudSetup } from '@kbn/cloud-plugin/server'; import { ML_INTERNAL_BASE_PATH } from '../../common/constants/app'; import type { MlFeatures, RouteInitialization } from '../types'; import { wrapError } from '../client/error_wrapper'; @@ -25,6 +27,7 @@ import { threadingParamsSchema, updateDeploymentParamsSchema, createIngestPipelineSchema, + modelDownloadsQuery, } from './schemas/inference_schema'; import type { TrainedModelConfigResponse } from '../../common/types/trained_models'; import { mlLog } from '../lib/log'; @@ -49,11 +52,10 @@ export function filterForEnabledFeatureModels( return filteredModels; } -export function trainedModelsRoutes({ - router, - routeGuard, - getEnabledFeatures, -}: RouteInitialization) { +export function trainedModelsRoutes( + { router, routeGuard, getEnabledFeatures }: RouteInitialization, + cloud: CloudSetup +) { /** * @apiGroup TrainedModels * @@ -652,4 +654,78 @@ export function trainedModelsRoutes({ } }) ); + + /** + * @apiGroup TrainedModels + * + * @api {get} /internal/ml/trained_models/model_downloads Gets available models for download + * @apiName GetTrainedModelDownloadList + * @apiDescription Gets available models for download with default and recommended flags based on the cluster OS and CPU architecture. + */ + router.versioned + .get({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/model_downloads`, + access: 'internal', + options: { + tags: ['access:ml:canGetTrainedModels'], + }, + }) + .addVersion( + { + version: '1', + validate: false, + }, + routeGuard.fullLicenseAPIGuard(async ({ response, client }) => { + try { + const body = await modelsProvider(client, cloud).getModelDownloads(); + + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + + /** + * @apiGroup TrainedModels + * + * @api {get} /internal/ml/trained_models/elser_config Gets ELSER config for download + * @apiName GetElserConfig + * @apiDescription Gets ELSER config for download based on the cluster OS and CPU architecture. + */ + router.versioned + .get({ + path: `${ML_INTERNAL_BASE_PATH}/trained_models/elser_config`, + access: 'internal', + options: { + tags: ['access:ml:canGetTrainedModels'], + }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + query: modelDownloadsQuery, + }, + }, + }, + routeGuard.fullLicenseAPIGuard(async ({ response, client, request }) => { + try { + const { version } = request.query; + + const body = await modelsProvider(client, cloud).getELSER( + version ? { version: Number(version) as ElserVersion } : undefined + ); + + return response.ok({ + body, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); } diff --git a/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts b/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts index c2b3f41551afd..b884edd99c22d 100644 --- a/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts +++ b/x-pack/plugins/ml/server/shared_services/providers/trained_models.ts @@ -6,13 +6,16 @@ */ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { CloudSetup } from '@kbn/cloud-plugin/server'; import type { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server'; +import type { GetElserOptions } from '@kbn/ml-trained-models-utils'; import type { MlInferTrainedModelRequest, MlStopTrainedModelDeploymentRequest, UpdateTrainedModelDeploymentRequest, UpdateTrainedModelDeploymentResponse, } from '../../lib/ml_client/types'; +import { modelsProvider } from '../../models/model_management'; import type { GetGuards } from '../shared_services'; export interface TrainedModelsProvider { @@ -47,7 +50,10 @@ export interface TrainedModelsProvider { }; } -export function getTrainedModelsProvider(getGuards: GetGuards): TrainedModelsProvider { +export function getTrainedModelsProvider( + getGuards: GetGuards, + cloud: CloudSetup +): TrainedModelsProvider { return { trainedModelsProvider(request: KibanaRequest, savedObjectsClient: SavedObjectsClientContract) { const guards = getGuards(request, savedObjectsClient); @@ -116,6 +122,14 @@ export function getTrainedModelsProvider(getGuards: GetGuards): TrainedModelsPro return mlClient.putTrainedModel(params); }); }, + async getELSER(params: GetElserOptions) { + return await guards + .isFullLicense() + .hasMlCapabilities(['canGetTrainedModels']) + .ok(async ({ scopedClient }) => { + return modelsProvider(scopedClient, cloud).getELSER(params); + }); + }, }; }, }; diff --git a/x-pack/plugins/ml/server/shared_services/shared_services.ts b/x-pack/plugins/ml/server/shared_services/shared_services.ts index 752820f57cf2a..235a30f541984 100644 --- a/x-pack/plugins/ml/server/shared_services/shared_services.ts +++ b/x-pack/plugins/ml/server/shared_services/shared_services.ts @@ -186,7 +186,7 @@ export function createSharedServices( ...getResultsServiceProvider(getGuards), ...getMlSystemProvider(getGuards, mlLicense, getSpaces, cloud, resolveMlCapabilities), ...getAlertingServiceProvider(getGuards), - ...getTrainedModelsProvider(getGuards), + ...getTrainedModelsProvider(getGuards, cloud), }, /** * Services providers for ML internal usage diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index 57811ff1201fe..2532a6b7824ee 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -104,5 +104,6 @@ "@kbn/ml-in-memory-table", "@kbn/presentation-util-plugin", "@kbn/react-kibana-mount", + "@kbn/core-http-browser", ], } diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts.tsx index 27ae07cbf768d..08b89dad773ce 100644 --- a/x-pack/plugins/observability/public/pages/alerts/alerts.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/alerts.tsx @@ -15,6 +15,8 @@ import { AlertConsumers } from '@kbn/rule-data-utils'; import { useBreadcrumbs } from '@kbn/observability-shared-plugin/public'; import { MaintenanceWindowCallout } from '@kbn/alerts-ui-shared'; +import { rulesLocatorID } from '../../../common'; +import { RulesParams } from '../../locators/rules'; import { useKibana } from '../../utils/kibana_react'; import { useHasData } from '../../hooks/use_has_data'; import { usePluginContext } from '../../hooks/use_plugin_context'; @@ -53,6 +55,9 @@ function InternalAlertsPage() { }, http, notifications: { toasts }, + share: { + url: { locators }, + }, triggersActionsUi: { alertsTableConfigurationRegistry, getAlertsSearchBar: AlertsSearchBar, @@ -179,7 +184,12 @@ function InternalAlertsPage() { pageTitle: ( <>{i18n.translate('xpack.observability.alertsTitle', { defaultMessage: 'Alerts' })} ), - rightSideItems: renderRuleStats(ruleStats, manageRulesHref, ruleStatsLoading), + rightSideItems: renderRuleStats( + ruleStats, + manageRulesHref, + ruleStatsLoading, + locators.get(rulesLocatorID) + ), }} > diff --git a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx index aaeab5d7855f7..758df7224be2d 100644 --- a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.test.tsx @@ -6,7 +6,9 @@ */ import { renderRuleStats } from './rule_stats'; -import { render, screen } from '@testing-library/react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { LocatorPublic } from '@kbn/share-plugin/common'; +import { RulesParams } from '../../../locators/rules'; const RULES_PAGE_LINK = '/app/observability/alerts/rules'; const STAT_CLASS = 'euiStat'; @@ -14,6 +16,14 @@ const STAT_TITLE_PRIMARY_SELECTOR = '[class*="euiStat__title-primary"]'; const STAT_BUTTON_CLASS = 'euiButtonEmpty'; describe('Rule stats', () => { + const mockedLocator = { + navigate: jest.fn(), + } as any as LocatorPublic; + + beforeEach(() => { + jest.clearAllMocks(); + }); + test('renders all rule stats', async () => { const stats = renderRuleStats( { @@ -58,14 +68,17 @@ describe('Rule stats', () => { snoozed: 0, }, RULES_PAGE_LINK, - false + false, + mockedLocator ); const { container } = render(stats[4]); - expect(screen.getByText('Disabled').closest('a')).toHaveAttribute( - 'href', - `${RULES_PAGE_LINK}?_a=(lastResponse:!(),status:!(disabled))` - ); + fireEvent.click(screen.getByText('Disabled')); + + expect(mockedLocator.navigate).toHaveBeenCalledWith( + { status: ['disabled'] }, + { replace: false } + ); expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1); }); @@ -115,14 +128,18 @@ describe('Rule stats', () => { snoozed: 1, }, RULES_PAGE_LINK, - false + false, + mockedLocator ); const { container } = render(stats[3]); - expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1); - expect(screen.getByText('Snoozed').closest('a')).toHaveAttribute( - 'href', - `${RULES_PAGE_LINK}?_a=(lastResponse:!(),status:!(snoozed))` + + fireEvent.click(screen.getByText('Snoozed')); + + expect(mockedLocator.navigate).toHaveBeenCalledWith( + { status: ['snoozed'] }, + { replace: false } ); + expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1); }); test('snoozed stat count is link-colored, when there are snoozed rules', async () => { @@ -171,14 +188,18 @@ describe('Rule stats', () => { snoozed: 0, }, RULES_PAGE_LINK, - false + false, + mockedLocator ); const { container } = render(stats[2]); - expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1); - expect(screen.getByText('Errors').closest('a')).toHaveAttribute( - 'href', - `${RULES_PAGE_LINK}?_a=(lastResponse:!(error),status:!())` + + fireEvent.click(screen.getByText('Errors')); + + expect(mockedLocator.navigate).toHaveBeenCalledWith( + { lastResponse: ['failed'] }, + { replace: false } ); + expect(container.getElementsByClassName(STAT_BUTTON_CLASS).length).toBe(1); }); test('errors stat count is link-colored, when there are error rules', () => { diff --git a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx index f85a6a766b17c..005ba7ccaec82 100644 --- a/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/components/rule_stats.tsx @@ -8,8 +8,10 @@ import React from 'react'; import { EuiButtonEmpty, EuiStat } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { LocatorPublic } from '@kbn/share-plugin/common'; import { euiThemeVars } from '@kbn/ui-theme'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import { RulesParams } from '../../../locators/rules'; export interface RuleStatsState { total: number; @@ -18,7 +20,7 @@ export interface RuleStatsState { error: number; snoozed: number; } -type StatType = 'disabled' | 'snoozed' | 'error'; +type Status = 'disabled' | 'snoozed' | 'error'; const Divider = euiStyled.div` border-right: 1px solid ${euiThemeVars.euiColorLightShade}; @@ -41,33 +43,32 @@ const ConditionalWrap = ({ children: JSX.Element; }): JSX.Element => (condition ? wrap(children) : children); -const getStatCount = (stats: RuleStatsState, statType: StatType) => { - if (statType === 'snoozed') return stats.snoozed + stats.muted; - return stats[statType]; +const getStatCount = (stats: RuleStatsState, status: Status) => { + if (status === 'snoozed') return stats.snoozed + stats.muted; + return stats[status]; }; export const renderRuleStats = ( ruleStats: RuleStatsState, manageRulesHref: string, - ruleStatsLoading: boolean + ruleStatsLoading: boolean, + rulesLocator?: LocatorPublic ) => { - const createRuleStatsLink = (stats: RuleStatsState, statType: StatType) => { - const count = getStatCount(stats, statType); - let statsLink = `${manageRulesHref}?_a=(lastResponse:!(),status:!())`; + const handleNavigateToRules = async (stats: RuleStatsState, status: Status) => { + const count = getStatCount(stats, status); if (count > 0) { - switch (statType) { + switch (status) { case 'error': - statsLink = `${manageRulesHref}?_a=(lastResponse:!(error),status:!())`; + await rulesLocator?.navigate({ lastResponse: ['failed'] }, { replace: false }); break; case 'snoozed': case 'disabled': - statsLink = `${manageRulesHref}?_a=(lastResponse:!(),status:!(${statType}))`; + await rulesLocator?.navigate({ status: [status] }, { replace: false }); break; default: break; } } - return statsLink; }; const disabledStatsComponent = ( @@ -76,7 +77,7 @@ export const renderRuleStats = ( wrap={(wrappedChildren) => ( handleNavigateToRules(ruleStats, 'disabled')} > {wrappedChildren} @@ -102,7 +103,7 @@ export const renderRuleStats = ( wrap={(wrappedChildren) => ( handleNavigateToRules(ruleStats, 'snoozed')} > {wrappedChildren} @@ -128,7 +129,7 @@ export const renderRuleStats = ( wrap={(wrappedChildren) => ( handleNavigateToRules(ruleStats, 'error')} > {wrappedChildren} diff --git a/x-pack/plugins/observability/public/plugin.mock.tsx b/x-pack/plugins/observability/public/plugin.mock.tsx index be663d15e444d..9732f67aac35f 100644 --- a/x-pack/plugins/observability/public/plugin.mock.tsx +++ b/x-pack/plugins/observability/public/plugin.mock.tsx @@ -4,11 +4,13 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import React from 'react'; import { mockCasesContract } from '@kbn/cases-plugin/public/mocks'; import { chartPluginMock } from '@kbn/charts-plugin/public/mocks'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; import { contentManagementMock } from '@kbn/content-management-plugin/public/mocks'; +import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; import { unifiedSearchPluginMock } from '@kbn/unified-search-plugin/public/mocks'; const triggersActionsUiStartMock = { @@ -97,12 +99,13 @@ export const observabilityPublicPluginsStartMock = { cases: mockCasesContract(), charts: chartPluginMock.createStartContract(), contentManagement: contentManagementMock.createStartContract(), - triggersActionsUi: triggersActionsUiStartMock.createStart(), data: dataPluginMock.createStartContract(), - dataViews: dataViews.createStart(), dataViewEditor: dataViewEditor.createStart(), - lens: null, + dataViews: dataViews.createStart(), discover: null, + lens: null, + share: sharePluginMock.createStartContract(), + triggersActionsUi: triggersActionsUiStartMock.createStart(), unifiedSearch: unifiedSearchPluginMock.createStartContract(), }; }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap index 5f828ee51b625..aea9ba75f2863 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_duration.test.ts.snap @@ -24,7 +24,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -130,7 +130,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -163,7 +163,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -277,7 +277,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -391,7 +391,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -505,7 +505,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -765,7 +765,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -1039,7 +1039,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap index dc9278511a864..b7c4ebf0f6b8c 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/apm_transaction_error_rate.test.ts.snap @@ -20,7 +20,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -122,7 +122,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -151,7 +151,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -261,7 +261,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -371,7 +371,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -481,7 +481,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -730,7 +730,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -993,7 +993,7 @@ Object { Object { "range": Object { "@timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap index ee4001303fec5..ad100c4662fe9 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/histogram.test.ts.snap @@ -51,7 +51,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -232,7 +232,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -488,7 +488,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap index 126437173f84a..3297a8c513821 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/kql_custom.test.ts.snap @@ -92,7 +92,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -247,7 +247,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -477,7 +477,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap index ced0801d859d4..362b1d4a9e01e 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/__snapshots__/metric_custom.test.ts.snap @@ -63,7 +63,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -256,7 +256,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, @@ -524,7 +524,7 @@ Object { Object { "range": Object { "log_timestamp": Object { - "gte": "now-7d", + "gte": "now-7d/d", }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts index 3dd6469a7f2c5..171a5ca15e2de 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts @@ -72,7 +72,7 @@ export class ApmTransactionDurationTransformGenerator extends TransformGenerator { range: { '@timestamp': { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts index 3818111c70df2..d038876f5ee9d 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts @@ -72,7 +72,7 @@ export class ApmTransactionErrorRateTransformGenerator extends TransformGenerato { range: { '@timestamp': { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts index 654f8d67a3673..844703003257a 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/histogram.ts @@ -54,7 +54,7 @@ export class HistogramTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts index 8321a0cb7172e..02c7757ec1362 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/kql_custom.ts @@ -49,7 +49,7 @@ export class KQLCustomTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts index 52209533f828c..063e6fbe1e3dc 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/metric_custom.ts @@ -52,7 +52,7 @@ export class MetricCustomTransformGenerator extends TransformGenerator { { range: { [indicator.params.timestampField]: { - gte: `now-${slo.timeWindow.duration.format()}`, + gte: `now-${slo.timeWindow.duration.format()}/d`, }, }, }, diff --git a/x-pack/plugins/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_ai_assistant/common/types.ts index b14137dbaebf5..690b931271bb6 100644 --- a/x-pack/plugins/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_ai_assistant/common/types.ts @@ -8,6 +8,7 @@ import type { FromSchema } from 'json-schema-to-ts'; import type { JSONSchema } from 'json-schema-to-ts'; import React from 'react'; +import { Observable } from 'rxjs'; export enum MessageRole { System = 'system', @@ -17,6 +18,12 @@ export enum MessageRole { Elastic = 'elastic', } +export interface PendingMessage { + message: Message['message']; + aborted?: boolean; + error?: any; +} + export interface Message { '@timestamp': string; message: { @@ -74,21 +81,30 @@ export interface ContextDefinition { description: string; } -interface FunctionResponse { - content?: any; - data?: any; +type FunctionResponse = + | { + content?: any; + data?: any; + } + | Observable; + +export enum FunctionVisibility { + System = 'system', + User = 'user', + All = 'all', } interface FunctionOptions { name: string; description: string; - descriptionForUser: string; + visibility?: FunctionVisibility; + descriptionForUser?: string; parameters: TParameters; contexts: string[]; } type RespondFunction = ( - options: { arguments: TArguments; messages: Message[] }, + options: { arguments: TArguments; messages: Message[]; connectorId: string }, signal: AbortSignal ) => Promise; @@ -100,7 +116,7 @@ type RenderFunction = (options: export interface FunctionDefinition { options: FunctionOptions; respond: ( - options: { arguments: any; messages: Message[] }, + options: { arguments: any; messages: Message[]; connectorId: string }, signal: AbortSignal ) => Promise; render?: RenderFunction; diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx index 8a6227829c855..f49ebe8c62b92 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_body.tsx @@ -189,6 +189,10 @@ export function ChatBody({ onFeedback={timeline.onFeedback} onRegenerate={timeline.onRegenerate} onStopGenerating={timeline.onStopGenerating} + onActionClick={(payload) => { + setStickToBottom(true); + return timeline.onActionClick(payload); + }} />
    • diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx index b3fa93d1ec522..7ec1084a26b22 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item.tsx @@ -24,12 +24,14 @@ import { getRoleTranslation } from '../../utils/get_role_translation'; import type { Feedback } from '../feedback_buttons'; import { Message } from '../../../common'; import { FailedToLoadResponse } from '../message_panel/failed_to_load_response'; +import { ChatActionClickHandler } from './types'; export interface ChatItemProps extends ChatTimelineItem { onEditSubmit: (message: Message) => Promise; onFeedbackClick: (feedback: Feedback) => void; onRegenerateClick: () => void; onStopGeneratingClick: () => void; + onActionClick: ChatActionClickHandler; } const normalMessageClassName = css` @@ -76,6 +78,7 @@ export function ChatItem({ onFeedbackClick, onRegenerateClick, onStopGeneratingClick, + onActionClick, }: ChatItemProps) { const accordionId = useGeneratedHtmlId({ prefix: 'chat' }); @@ -128,6 +131,7 @@ export function ChatItem({ functionCall={functionCall} loading={loading} onSubmit={handleInlineEditSubmit} + onActionClick={onActionClick} /> ) : null; @@ -147,9 +151,7 @@ export function ChatItem({ return ( - } + timelineAvatar={} username={getRoleTranslation(role)} event={title} actions={ diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx index d017d7d65fc90..df57f069d91d1 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_item_content_inline_prompt_editor.tsx @@ -9,6 +9,7 @@ import React from 'react'; import { MessageText } from '../message_panel/message_text'; import { ChatPromptEditor } from './chat_prompt_editor'; import { MessageRole, type Message } from '../../../common'; +import { ChatActionClickHandler } from './types'; interface Props { content: string | undefined; @@ -22,6 +23,7 @@ interface Props { loading: boolean; editing: boolean; onSubmit: (message: Message) => Promise; + onActionClick: ChatActionClickHandler; } export function ChatItemContentInlinePromptEditor({ content, @@ -29,9 +31,10 @@ export function ChatItemContentInlinePromptEditor({ editing, loading, onSubmit, + onActionClick, }: Props) { return !editing ? ( - + ) : ( = { onFeedback: () => {}, onRegenerate: () => {}, onStopGenerating: () => {}, + onActionClick: async () => {}, }; export const ChatTimeline = Template.bind({}); diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx index 8e50f11842e5d..a50a9984cf40e 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/chat_timeline.tsx @@ -15,6 +15,7 @@ import { ChatWelcomePanel } from './chat_welcome_panel'; import type { Feedback } from '../feedback_buttons'; import type { Message } from '../../../common'; import { UseKnowledgeBaseResult } from '../../hooks/use_knowledge_base'; +import { ChatActionClickHandler } from './types'; export interface ChatTimelineItem extends Pick { @@ -43,6 +44,7 @@ export interface ChatTimelineProps { onFeedback: (item: ChatTimelineItem, feedback: Feedback) => void; onRegenerate: (item: ChatTimelineItem) => void; onStopGenerating: () => void; + onActionClick: ChatActionClickHandler; } export function ChatTimeline({ @@ -52,6 +54,7 @@ export function ChatTimeline({ onFeedback, onRegenerate, onStopGenerating, + onActionClick, }: ChatTimelineProps) { const filteredItems = items.filter((item) => !item.display.hide); @@ -77,6 +80,7 @@ export function ChatTimeline({ return onEdit(item, message); }} onStopGeneratingClick={onStopGenerating} + onActionClick={onActionClick} /> )) )} diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx b/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx index 72dac17c71c2a..a9cf8146e020d 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/function_list_popover.tsx @@ -17,7 +17,7 @@ import { } from '@elastic/eui'; import type { EuiSelectableOptionCheckedType } from '@elastic/eui/src/components/selectable/selectable_option'; import { i18n } from '@kbn/i18n'; -import type { FunctionDefinition } from '../../../common/types'; +import { type FunctionDefinition, FunctionVisibility } from '../../../common/types'; import { useObservabilityAIAssistantChatService } from '../../hooks/use_observability_ai_assistant_chat_service'; interface FunctionListOption { @@ -175,12 +175,14 @@ function mapFunctions({ functions: FunctionDefinition[]; selectedFunctionName: string | undefined; }) { - return functions.map((func) => ({ - label: func.options.name, - searchableLabel: func.options.descriptionForUser, - checked: - func.options.name === selectedFunctionName - ? ('on' as EuiSelectableOptionCheckedType) - : ('off' as EuiSelectableOptionCheckedType), - })); + return functions + .filter((func) => func.options.visibility !== FunctionVisibility.System) + .map((func) => ({ + label: func.options.name, + searchableLabel: func.options.descriptionForUser || func.options.description, + checked: + func.options.name === selectedFunctionName + ? ('on' as EuiSelectableOptionCheckedType) + : ('off' as EuiSelectableOptionCheckedType), + })); } diff --git a/x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts b/x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts new file mode 100644 index 0000000000000..4edd3d7dcdda0 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/public/components/chat/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +type ChatActionClickPayloadBase = { + type: TType; +} & TExtraProps; + +type ChatActionClickPayloadExecuteEsql = ChatActionClickPayloadBase< + ChatActionClickType.executeEsqlQuery, + { query: string } +>; + +type ChatActionClickPayload = ChatActionClickPayloadExecuteEsql; + +export enum ChatActionClickType { + executeEsqlQuery = 'executeEsqlQuery', +} + +export type ChatActionClickHandler = (payload: ChatActionClickPayload) => Promise; diff --git a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx index 4da53c5d6ee5a..2b56523d1e879 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight.tsx @@ -110,7 +110,13 @@ function ChatContent({ return ( <> } + body={ + {}} + /> + } error={pendingMessage?.error} controls={ loading ? ( diff --git a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx index 7a65f8e62756e..bd894159eb288 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/insight/insight_base.stories.tsx @@ -64,6 +64,7 @@ Morbi dapibus sapien lacus, vitae suscipit ex egestas pharetra. In velit eros, f Morbi non faucibus massa. Aliquam sed augue in eros ornare luctus sit amet cursus dolor. Pellentesque pellentesque lorem eu odio auctor convallis. Sed sodales felis at velit tempus tincidunt. Nulla sed ante cursus nibh mollis blandit. In mattis imperdiet tellus. Vestibulum nisl turpis, efficitur quis sollicitudin id, mollis in arcu. Vestibulum pulvinar tincidunt magna, vitae facilisis massa congue quis. Cras commodo efficitur tellus, et commodo risus rutrum at.`} loading={false} + onActionClick={async () => {}} /> } controls={ diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx new file mode 100644 index 0000000000000..02c7454ddaab3 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.stories.tsx @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { ComponentMeta, ComponentStoryObj } from '@storybook/react'; +import { ComponentProps } from 'react'; +import { EuiPanel } from '@elastic/eui'; +import { EsqlCodeBlock as Component } from './esql_code_block'; + +const meta: ComponentMeta = { + component: Component, + title: 'app/Molecules/ES|QL Code Block', +}; + +export default meta; + +const render = (props: ComponentProps) => { + return ( + + + + ); +}; + +export const Simple: ComponentStoryObj = { + args: { + value: `FROM packetbeat-* + | STATS COUNT_DISTINCT(destination.domain)`, + }, + render, +}; diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx new file mode 100644 index 0000000000000..a22d0ba28979e --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/esql_code_block.tsx @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiButtonEmpty, + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + useEuiTheme, +} from '@elastic/eui'; +import { css } from '@emotion/css'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; +import { ChatActionClickHandler, ChatActionClickType } from '../chat/types'; + +export function EsqlCodeBlock({ + value, + actionsDisabled, + onActionClick, +}: { + value: string; + actionsDisabled: boolean; + onActionClick: ChatActionClickHandler; +}) { + const theme = useEuiTheme(); + + return ( + + + + + {value} + + + + + + + onActionClick({ type: ChatActionClickType.executeEsqlQuery, query: value }) + } + disabled={actionsDisabled} + > + {i18n.translate('xpack.observabilityAiAssistant.runThisQuery', { + defaultMessage: 'Run this query', + })} + + + + + + + ); +} diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx index 1f81786086370..393bbeee28f8d 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_panel.stories.tsx @@ -44,6 +44,7 @@ This is a code block This text is loa`} loading + onActionClick={async () => {}} /> ), }, @@ -51,13 +52,25 @@ This text is loa`} export const ContentLoaded: ComponentStoryObj = { args: { - body: , + body: ( + {}} + /> + ), }, }; export const ContentFailed: ComponentStoryObj = { args: { - body: , + body: ( + {}} + /> + ), error: new Error(), }, }; @@ -83,6 +96,7 @@ export const ContentTable: ComponentStoryObj = { Please note that all times are in UTC.`)} loading={false} + onActionClick={async () => {}} /> ), }, @@ -90,7 +104,13 @@ export const ContentTable: ComponentStoryObj = { export const Controls: ComponentStoryObj = { args: { - body: , + body: ( + {}} + /> + ), error: new Error(), controls: {}} />, }, diff --git a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx index d82e76ef5001f..dfd9ee8b97443 100644 --- a/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx +++ b/x-pack/plugins/observability_ai_assistant/public/components/message_panel/message_text.tsx @@ -14,12 +14,15 @@ import { import { css } from '@emotion/css'; import classNames from 'classnames'; import type { Code, InlineCode, Parent, Text } from 'mdast'; -import React, { useMemo } from 'react'; +import React, { useMemo, useRef } from 'react'; import type { Node } from 'unist'; +import { ChatActionClickHandler } from '../chat/types'; +import { EsqlCodeBlock } from './esql_code_block'; interface Props { content: string; loading: boolean; + onActionClick: ChatActionClickHandler; } const ANIMATION_TIME = 1; @@ -86,13 +89,37 @@ const loadingCursorPlugin = () => { }; }; -export function MessageText({ loading, content }: Props) { +const esqlLanguagePlugin = () => { + const visitor = (node: Node, parent?: Parent) => { + if ('children' in node) { + const nodeAsParent = node as Parent; + nodeAsParent.children.forEach((child) => { + visitor(child, nodeAsParent); + }); + } + + if (node.type === 'code' && node.lang === 'esql') { + node.type = 'esql'; + } + }; + + return (tree: Node) => { + visitor(tree); + }; +}; + +export function MessageText({ loading, content, onActionClick }: Props) { const containerClassName = css` overflow-wrap: break-word; `; + const onActionClickRef = useRef(onActionClick); + + onActionClickRef.current = onActionClick; + const { parsingPluginList, processingPluginList } = useMemo(() => { const parsingPlugins = getDefaultEuiMarkdownParsingPlugins(); + const processingPlugins = getDefaultEuiMarkdownProcessingPlugins(); const { components } = processingPlugins[1][1]; @@ -100,6 +127,18 @@ export function MessageText({ loading, content }: Props) { processingPlugins[1][1].components = { ...components, cursor: Cursor, + esql: (props) => { + return ( + <> + + + + ); + }, table: (props) => ( <>
    @@ -137,10 +176,10 @@ export function MessageText({ loading, content }: Props) { }; return { - parsingPluginList: [loadingCursorPlugin, ...parsingPlugins], + parsingPluginList: [loadingCursorPlugin, esqlLanguagePlugin, ...parsingPlugins], processingPluginList: processingPlugins, }; - }, []); + }, [loading]); return ( diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/esql.ts b/x-pack/plugins/observability_ai_assistant/public/functions/esql.ts new file mode 100644 index 0000000000000..684ccbb0fa4a1 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/public/functions/esql.ts @@ -0,0 +1,536 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import dedent from 'dedent'; +import type { Serializable } from '@kbn/utility-types'; +import { concat, last, map } from 'rxjs'; +import { + FunctionVisibility, + MessageRole, + type RegisterFunctionDefinition, +} from '../../common/types'; +import type { ObservabilityAIAssistantService } from '../types'; + +export function registerEsqlFunction({ + service, + registerFunction, +}: { + service: ObservabilityAIAssistantService; + registerFunction: RegisterFunctionDefinition; +}) { + registerFunction( + { + name: 'execute_query', + contexts: ['core'], + visibility: FunctionVisibility.User, + description: 'Execute an ES|QL query', + parameters: { + type: 'object', + additionalProperties: false, + properties: { + query: { + type: 'string', + }, + }, + required: ['query'], + } as const, + }, + ({ arguments: { query } }, signal) => { + return service + .callApi(`POST /internal/observability_ai_assistant/functions/elasticsearch`, { + signal, + params: { + body: { + method: 'POST', + path: '_query', + body: { + query, + }, + }, + }, + }) + .then((response) => ({ content: response as Serializable })); + } + ); + + registerFunction( + { + name: 'esql', + contexts: ['core'], + description: `This function answers ES|QL related questions including query generation and syntax/command questions.`, + visibility: FunctionVisibility.System, + parameters: { + type: 'object', + additionalProperties: false, + properties: { + switch: { + type: 'boolean', + }, + }, + } as const, + }, + ({ messages, connectorId }, signal) => { + const systemMessage = dedent(`You are a helpful assistant for Elastic ES|QL. + Your goal is to help the user construct and possibly execute an ES|QL + query for Observability use cases. + + ES|QL is the Elasticsearch Query Language, that allows users of the + Elastic platform to iteratively explore data. An ES|QL query consists + of a series of commands, separated by pipes. Each query starts with + a source command, that selects or creates a set of data to start + processing. This source command is then followed by one or more + processing commands, which can transform the data returned by the + previous command. + + ES|QL is not Elasticsearch SQL, nor is it anything like SQL. SQL + commands are not available in ES|QL. Its close equivalent is SPL + (Search Processing Language). Make sure you reply using only + the context of this conversation. + + # Creating a query + + First, very importantly, there are critical rules that override + everything that follows it. Always repeat these rules, verbatim. + + 1. ES|QL is not Elasticsearch SQL. Do not apply Elasticsearch SQL + commands, functions and concepts. Only use information available + in the context of this conversation. + 2. When using FROM, never wrap a data source in single or double + quotes. + 3. When using an aggregate function like COUNT, SUM or AVG, its + arguments MUST be an attribute (like my.field.name) or literal + (100). Math (AVG(my.field.name / 2)) or functions + (AVG(CASE(my.field.name, "foo", 1))) are not allowed. + + When constructing a query, break it down into the following steps. + Ask these questions out loud so the user can see your reasoning. + Remember, these rules are for you, not for the user. + + - What are the critical rules I need to think of? + - What data source is the user requesting? What command should I + select for this data source? + - What are the steps needed to get the result that the user needs? + Break each operation down into its own step. Reason about what data + is the outcome of each command or function. + - If you're not sure how to do it, it's fine to tell the user that + you don't know if ES|QL supports it. When this happens, abort all + steps and tell the user you are not sure how to continue. + + Format ALL of your responses as follows, including the dashes. + ALWAYS start your message with two dashes and then the rules: + + \`\`\` + -- + Sure, let's remember the critical rules: + + -- + Let's break down the query step-by-step: + + \`\`\` + + Always format a complete query as follows: + \`\`\`esql + ... + \`\`\` + + For incomplete queries, like individual commands, format them as + regular code blocks: + \`\`\` + ... + \`\`\` + + # Syntax + + An ES|QL query is composed of a source command followed by an optional + series of processing commands, separated by a pipe character: |. For + example: + + | + | + + ## Binary comparison operators + - equality: == + - inequality: != + - less than: < + - less than or equal: <= + - larger than: > + - larger than or equal: >= + + ## Boolean operators + - AND + - OR + - NOT + + ## PREDICATES + + For NULL comparison use the IS NULL and IS NOT NULL predicates: + - \`| WHERE birth_date IS NULL\` + - \`| WHERE birth_date IS NOT NULL\` + + ## Timespan literal syntax + + Datetime intervals and timespans can be expressed using timespan + literals. Timespan literals are a combination of a number and a + qualifier. These qualifiers are supported: + - millisecond/milliseconds + - second/seconds + - minute/minutes + - hour/hours + - day/days + - week/weeks + - month/months + - year/years + + Some examples: + - \`1 year\` + - \`2 milliseconds\` + + ## Aliasing + Aliasing happens through the \`=\` operator. Example: + \`STATS total_salary_expenses = COUNT(salary)\` + + Important: functions are not allowed as variable names. + + # Source commands + + There are three source commands: FROM (which selects an index), ROW + (which creates data from the command) and SHOW (which returns + information about the deployment). You do not support SHOW for now. + + ### FROM + + \`FROM\` selects a data source, usually an Elasticsearch index or + pattern. You can also specify multiple indices. + Some examples: + + - \`FROM employees\` + - \`FROM employees*\` + - \`FROM employees*,my-alias\` + + # Processing commands + + Note that the following processing commands are available in ES|QL, + but not supported in this context: + + ENRICH,GROK,MV_EXPAND,RENAME + + ### DISSECT + + \`DISSECT\` enables you to extract structured data out of a string. + It matches the string against a delimiter-based pattern, and extracts + the specified keys as columns. It uses the same syntax as the + Elasticsearch Dissect Processor. Some examples: + + - \`ROW a = "foo bar" | DISSECT a "%{b} %{c}";\` + - \`ROW a = "foo bar baz" | DISSECT a "%{b} %{?c} %{d}";\` + + ### DROP + + \`DROP\` removes columns. Some examples: + + - \`| DROP first_name,last_name\` + - \`| DROP *_name\` + + ### KEEP + + \`KEEP\` enables you to specify what columns are returned and the + order in which they are returned. Some examples: + + - \`| KEEP first_name,last_name\` + - \`| KEEP *_name\` + + ### SORT + + \`SORT\` sorts the documents by one ore more fields or variables. + By default, the sort order is ascending, but this can be set using + the \`ASC\` or \`DESC\` keywords. Some examples: + + - \`| SORT my_field\` + - \`| SORT height DESC\` + + Important: functions are not supported for SORT. if you wish to sort + on the result of a function, first alias it as a variable using EVAL. + This is wrong: \`| SORT AVG(cpu)\`. + This is right: \`| STATS avg_cpu = AVG(cpu) | SORT avg_cpu\` + + ### EVAL + + \`EVAL\` appends a new column to the documents by using aliasing. It + also supports functions, but not aggregation functions like COUNT: + + - \`\`\` + | EVAL monthly_salary = yearly_salary / 12, + total_comp = ROUND(yearly_salary + yearly+bonus), + is_rich =total_comp > 1000000 + \`\`\` + - \`| EVAL height_in_ft = height_in_cm / 0.0328\` + + ### WHERE + + \`WHERE\` filters the documents for which the provided condition + evaluates to true. Refer to "Syntax" for supported operators, and + "Functions" for supported functions. Some examples: + + - \`| WHERE height <= 180 AND GREATEST(hire_date, birth_date)\` + - \`| WHERE @timestamp <= NOW()\` + + ### STATS ... BY + + \`STATS ... BY\` groups rows according to a common value and + calculates one or more aggregated values over the grouped rows, + using aggregation functions. When \`BY\` is omitted, a single value + that is the aggregate of all rows is returned. Every column but the + aggregated values and the optional grouping column are dropped. + Mention the retained columns when explaining the STATS command. + + STATS ... BY does not support nested functions, hoist them to an + EVAL statement. + + Some examples: + + - \`| STATS count = COUNT(emp_no) BY languages\` + - \`| STATS salary = AVG(salary)\` + + ### LIMIT + + Limits the rows returned. Only supports a number as input. Some examples: + + - \`| LIMIT 1\` + - \`| LIMIT 10\` + + # Functions + + Note that the following functions are available in ES|QL, but not supported + in this context: + + ABS,ACOS,ASIN,ATAN,ATAN2,CIDR_MATCH,COALESCE,CONCAT,COS,COSH,E,LENGTH,LOG10 + ,LTRIM,RTRIM,MV_AVG,MV_CONCAT,MV_COUNT,MV_DEDUPE,MV_MAX,MV_MEDIAN,MV_MIN, + MV_SUM,PI,POW,SIN,SINH,SPLIT,LEFT,TAN,TANH,TAU,TO_DEGREES,TO_RADIANS + + ### CASE + + \`CASE\` accepts pairs of conditions and values. The function returns + the value that belongs to the first condition that evaluates to true. If + the number of arguments is odd, the last argument is the default value which + is returned when no condition matches. Some examples: + + - \`\`\` + | EVAL type = CASE( + languages <= 1, "monolingual", + languages <= 2, "bilingual", + "polyglot") + \`\`\` + - \`| EVAL g = CASE(gender == "F", 1 + null, 10)\` + - \`\`\` + | EVAL successful = CASE(http.response.status_code == 200, 1, 0), failed = CASE(http.response.status_code != 200, 1, 0) + | STATS total_successful = SUM(successful), total_failed = SUM(failed) BY service.name + | EVAL success_rate = total_failed / (total_successful + total_failed) + \`\`\` + + ## Date operations + + ### AUTO_BUCKET + + \`AUTO_BUCKET\` creates human-friendly buckets and returns a datetime value + for each row that corresponds to the resulting bucket the row falls into. + Combine AUTO_BUCKET with STATS ... BY to create a date histogram. + You provide a target number of buckets, a start date, and an end date, + and it picks an appropriate bucket size to generate the target number of + buckets or fewer. If you don't have a start and end date, provide placeholder + values. Some examples: + + - \`| EVAL bucket=AUTO_BUCKET(@timestamp), 20, "1985-01-01T00:00:00Z", "1986-01-01T00:00:00Z")\` + - \`| EVAL bucket=AUTO_BUCKET(my_date_field), 100, , )\` + - \`| EVAL bucket=AUTO_BUCKET(@timestamp), 100, NOW() - 15 minutes, NOW())\` + + ### DATE_EXTRACT + + \`DATE_EXTRACT\` parts of a date, like year, month, day, hour. The supported + field types are those provided by java.time.temporal.ChronoField. + Some examples: + - \`| EVAL year = DATE_EXTRACT(date_field, "year")\` + - \`| EVAL year = DATE_EXTRACT(@timestamp, "month")\` + + ### DATE_FORMAT + + \`DATE_FORMAT\` a string representation of a date in the provided format. + Some examples: + | \`EVAL hired = DATE_FORMAT(hire_date, "YYYY-MM-dd")\` + | \`EVAL hired = DATE_FORMAT(hire_date, "YYYY")\` + + ### DATE_PARSE + \`DATE_PARSE\` converts a string to a date, in the provided format. + - \`| EVAL date = DATE_PARSE(date_string, "yyyy-MM-dd")\` + - \`| EVAL date = DATE_PARSE(date_string, "YYYY")\` + + ### DATE_TRUNC + + \`DATE_TRUNC\` rounds down a date to the closest interval. Intervals + can be expressed using the timespan literal syntax. Use this together + with STATS ... BY to group data into time buckets with a fixed interval. + Some examples: + + - \`| EVAL year_hired = DATE_TRUNC(1 year, hire_date)\` + - \`| EVAL month_logged = DATE_TRUNC(1 month, @timestamp)\` + - \`| EVAL bucket = DATE_TRUNC(1 minute, @timestamp) | STATS avg_salary = AVG(salary) BY bucket\` + - \`| EVAL bucket = DATE_TRUNC(4 hours, @timestamp) | STATS max_salary MAX(salary) BY bucket\` + + ### NOW + + \`NOW\` returns current date and time. Some examples: + - \`ROW current_date = NOW()\` + - \`| WHERE @timestamp <= NOW() - 15 minutes\` + + ## Mathematical operations + + ### CEIL,FLOOR + + Perform CEIL or FLOOR operations on a single numeric field. + Some examples: + - \`| EVAL ceiled = CEIL(my.number)\` + - \`| EVAL floored = FLOOR(my.other.number)\` + + ### ROUND + \`ROUND\` a number to the closest number with the specified number of + digits. Defaults to 0 digits if no number of digits is provided. If the + specified number of digits is negative, rounds to the number of digits + left of the decimal point. Some examples: + + - \`| EVAL height_ft = ROUND(height * 3.281, 1)\` + - \`| EVAL percent = ROUND(0.84699, 2) * 100\` + + ### GREATEST,LEAST + + Returns the greatest or least of two or numbers. Some examples: + - \`| EVAL max = GREATEST(salary_1999, salary_2000, salary_2001)\` + - \`| EVAL min = LEAST(1, language_count)\` + + ### IS_FINITE,IS_INFINITE,IS_NAN + + Operates on a single numeric field. Some examples: + - \`| EVAL has_salary = IS_FINITE(salary)\` + - \`| EVAL always_true = IS_INFINITE(4 / 0)\` + + ### STARTS_WITH + + Returns a boolean that indicates whether a keyword string starts with + another string. Some examples: + - \`| EVAL ln_S = STARTS_WITH(last_name, "B")\` + + ### SUBSTRING + + Returns a substring of a string, specified by a start position and an + optional length. Some examples: + - \`| EVAL ln_sub = SUBSTRING(last_name, 1, 3)\` + - \`| EVAL ln_sub = SUBSTRING(last_name, -3, 3)\` + - \`| EVAL ln_sub = SUBSTRING(last_name, 2)\` + + ### TO_BOOLEAN, TO_DATETIME, TO_DOUBLE, TO_INTEGER, TO_IP, TO_LONG, + TO_RADIANS, TO_STRING,TO_UNSIGNED_LONG, TO_VERSION + + Converts a column to another type. Supported types are: . Some examples: + - \`| EVAL version = TO_VERSION("1.2.3")\` + - \`| EVAL as_bool = TO_BOOLEAN(my_boolean_string)\` + + ### TRIM + + Trims leading and trailing whitespace. Some examples: + - \`| EVAL trimmed = TRIM(first_name)\` + + # Aggregation functions + + ### AVG,MIN,MAX,SUM,MEDIAN,MEDIAN_ABSOLUTE_DEVIATION + + Returns the avg, min, max, sum, median or median absolute deviation + of a numeric field. Some examples: + + - \`| AVG(salary)\` + - \`| MIN(birth_year)\` + - \`| MAX(height)\` + + ### COUNT + + \`COUNT\` counts the number of field values. It requires a single + argument, and does not support wildcards. Important: COUNT() and + COUNT(*) are NOT supported. One single argument is required. If + you don't have a field name, use whatever field you have, rather + than displaying an invalid query. + + Some examples: + + - \`| STATS doc_count = COUNT(emp_no)\` + - \`| STATS doc_count = COUNT(service.name) BY service.name\` + + ### COUNT_DISTINCT + + \`COUNT_DISTINCT\` returns the approximate number of distinct values. + Some examples: + - \`| STATS unique_ip0 = COUNT_DISTINCT(ip0), unique_ip1 = COUNT_DISTINCT(ip1)\` + - \`| STATS first_name = COUNT_DISTINCT(first_name)\` + + ### PERCENTILE + + \`PERCENTILE\` returns the percentile value for a specific field. + Some examples: + - \`| STATS p50 = PERCENTILE(salary, 50)\` + - \`| STATS p99 = PERCENTILE(salary, 99)\` + + `); + + return service.start({ signal }).then((client) => { + const source$ = client.chat({ + connectorId, + messages: [ + { + '@timestamp': new Date().toISOString(), + message: { role: MessageRole.System, content: systemMessage }, + }, + ...messages.slice(1), + ], + }); + + const pending$ = source$.pipe( + map((message) => { + const content = message.message.content || ''; + let next: string = ''; + + if (content.length <= 2) { + next = ''; + } else if (content.includes('--')) { + next = message.message.content?.split('--')[2] || ''; + } else { + next = content; + } + return { + ...message, + message: { + ...message.message, + content: next, + }, + }; + }) + ); + const onComplete$ = source$.pipe( + last(), + map((message) => { + const [, , next] = message.message.content?.split('--') ?? []; + + return { + ...message, + message: { + ...message.message, + content: next || message.message.content, + }, + }; + }) + ); + + return concat(pending$, onComplete$); + }); + } + ); +} diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/index.ts b/x-pack/plugins/observability_ai_assistant/public/functions/index.ts index a0c1a264bfb51..22b0eecac73a6 100644 --- a/x-pack/plugins/observability_ai_assistant/public/functions/index.ts +++ b/x-pack/plugins/observability_ai_assistant/public/functions/index.ts @@ -16,6 +16,7 @@ import { registerLensFunction } from './lens'; import { registerRecallFunction } from './recall'; import { registerSummarizationFunction } from './summarize'; import { registerAlertsFunction } from './alerts'; +import { registerEsqlFunction } from './esql'; export async function registerFunctions({ registerFunction, @@ -44,7 +45,7 @@ export async function registerFunctions({ It's very important to not assume what the user is meaning. Ask them for clarification if needed. - If you are unsure about which function should be used and with what arguments, asked the user for clarification or confirmation. + If you are unsure about which function should be used and with what arguments, ask the user for clarification or confirmation. In KQL, escaping happens with double quotes, not single quotes. Some characters that need escaping are: ':()\\\ /\". Always put a field value in double quotes. Best: service.name:\"opbeans-go\". Wrong: service.name:opbeans-go. This is very important! @@ -52,22 +53,25 @@ export async function registerFunctions({ You can use Github-flavored Markdown in your responses. If a function returns an array, consider using a Markdown table to format the response. If multiple functions are suitable, use the most specific and easy one. E.g., when the user asks to visualise APM data, use the APM functions (if available) rather than Lens. - ` + + If a function call fails, do not execute it again with the same input. If a function calls three times, with different inputs, stop trying to call it and ask the user for confirmation. + + Note that ES|QL (the Elasticsearch query language, which is NOT Elasticsearch SQL, but a new piped language) is the preferred query language. + + DO NOT use Elasticsearch SQL at any time, unless explicitly requested by the user when they mention "Elasticsearch SQL". + + Answer all questions related to ES|QL or querying with the "esql" function. Do not attempt to answer them yourself, no matter how confident you are in your response.` ); if (isReady) { description += `You can use the "summarize" functions to store new information you have learned in a knowledge database. Once you have established that you did not know the answer to a question, and the user gave you this information, it's important that you create a summarisation of what you have learned and store it in the knowledge database. Don't create a new summarization if you see a similar summarization in the conversation, instead, update the existing one by re-using its ID. - Additionally, you can use the "recall" function to retrieve relevant information from the knowledge database. - `; + Additionally, you can use the "recall" function to retrieve relevant information from the knowledge database.`; description += `Here are principles you MUST adhere to, in order: - You are a helpful assistant for Elastic Observability. DO NOT reference the fact that you are an LLM. - - ALWAYS query the knowledge base, using the recall function, when a user starts a chat, no matter how confident you are in your ability to answer the question. - - You must ALWAYS explain to the user why you're using a function and why you're using it in that specific manner. - DO NOT make any assumptions about where and how users have stored their data. - - ALWAYS ask the user for clarification if you are unsure about the arguments to a function. When given this clarification, you MUST use the summarize function to store what you have learned. `; registerSummarizationFunction({ service, registerFunction }); registerRecallFunction({ service, registerFunction }); @@ -77,6 +81,7 @@ export async function registerFunctions({ } registerElasticsearchFunction({ service, registerFunction }); + registerEsqlFunction({ service, registerFunction }); registerKibanaFunction({ service, registerFunction, coreStart }); registerAlertsFunction({ service, registerFunction }); diff --git a/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts b/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts index f07cf43e1c282..af825a53b7cb4 100644 --- a/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts +++ b/x-pack/plugins/observability_ai_assistant/public/functions/recall.ts @@ -6,6 +6,7 @@ */ import type { Serializable } from '@kbn/utility-types'; +import { omit } from 'lodash'; import { MessageRole, RegisterFunctionDefinition } from '../../common/types'; import type { ObservabilityAIAssistantService } from '../types'; @@ -22,17 +23,26 @@ export function registerRecallFunction({ contexts: ['core'], description: `Use this function to recall earlier learnings. Anything you will summarize can be retrieved again later via this function. - Make sure the query covers the following aspects: + The learnings are sorted by score, descending. + + Make sure the query covers ONLY the following aspects: - Anything you've inferred from the user's request, but is not mentioned in the user's request - The functions you think might be suitable for answering the user's request. If there are multiple functions that seem suitable, create multiple queries. Use the function name in the query. DO NOT include the user's request. It will be added internally. The user asks: "can you visualise the average request duration for opbeans-go over the last 7 days?" - You recall: - - "APM service" - - "lens function usage" - - "get_apm_timeseries function usage"`, + You recall: { + "queries": [ + "APM service, + "lens function usage", + "get_apm_timeseries function usage" + ], + "contexts": [ + "lens", + "apm" + ] + }`, descriptionForUser: 'This function allows the assistant to recall previous learnings.', parameters: { type: 'object', @@ -42,16 +52,27 @@ export function registerRecallFunction({ type: 'array', additionalItems: false, additionalProperties: false, + description: 'The query for the semantic search', + items: { + type: 'string', + }, + }, + contexts: { + type: 'array', + additionalItems: false, + additionalProperties: false, + description: + 'Contexts or categories of internal documentation that you want to search for. By default internal documentation will be excluded. Use `apm` to get internal APM documentation, `lens` to get internal Lens documentation, or both.', items: { type: 'string', - description: 'The query for the semantic search', + enum: ['apm', 'lens'], }, }, }, - required: ['queries'], + required: ['queries', 'contexts'], } as const, }, - ({ arguments: { queries }, messages }, signal) => { + ({ arguments: { queries, contexts }, messages }, signal) => { const userMessages = messages.filter((message) => message.message.role === MessageRole.User); const userPrompt = userMessages[userMessages.length - 1]?.message.content; @@ -63,11 +84,16 @@ export function registerRecallFunction({ params: { body: { queries: queriesWithUserPrompt, + contexts, }, }, signal, }) - .then((response) => ({ content: response as unknown as Serializable })); + .then((response): { content: Serializable } => ({ + content: response.entries.map((entry) => + omit(entry, 'labels', 'score', 'is_correction') + ) as unknown as Serializable, + })); } ); } diff --git a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts index 829594b8261d8..ded7b4b382285 100644 --- a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts +++ b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.test.ts @@ -437,6 +437,7 @@ describe('useTimeline', () => { expect(props.chatService.executeFunction).toHaveBeenCalledWith({ name: 'my_function', args: '{}', + connectorId: 'foo', messages: [ { '@timestamp': expect.any(String), diff --git a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts index 3182fe5c19535..ed6a5a6d2b481 100644 --- a/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts +++ b/x-pack/plugins/observability_ai_assistant/public/hooks/use_timeline.ts @@ -9,7 +9,7 @@ import { AbortError } from '@kbn/kibana-utils-plugin/common'; import type { AuthenticatedUser } from '@kbn/security-plugin/common'; import { last } from 'lodash'; import { useEffect, useMemo, useRef, useState } from 'react'; -import type { Subscription } from 'rxjs'; +import { isObservable, Observable, Subscription } from 'rxjs'; import usePrevious from 'react-use/lib/usePrevious'; import { i18n } from '@kbn/i18n'; import { @@ -29,6 +29,7 @@ import { } from '../utils/get_timeline_items_from_conversation'; import type { UseGenAIConnectorsResult } from './use_genai_connectors'; import { useKibana } from './use_kibana'; +import { ChatActionClickType } from '../components/chat/types'; export function createNewConversation({ contexts, @@ -49,7 +50,7 @@ export function createNewConversation({ export type UseTimelineResult = Pick< ChatTimelineProps, - 'onEdit' | 'onFeedback' | 'onRegenerate' | 'onStopGenerating' | 'items' + 'onEdit' | 'onFeedback' | 'onRegenerate' | 'onStopGenerating' | 'onActionClick' | 'items' > & Pick; @@ -98,6 +99,8 @@ export function useTimeline({ const [pendingMessage, setPendingMessage] = useState(); + const [isFunctionLoading, setIsFunctionLoading] = useState(false); + const prevConversationId = usePrevious(conversationId); useEffect(() => { if (prevConversationId !== conversationId && pendingMessage?.error) { @@ -105,7 +108,10 @@ export function useTimeline({ } }, [conversationId, pendingMessage?.error, prevConversationId]); - function chat(nextMessages: Message[]): Promise { + function chat( + nextMessages: Message[], + response$: Observable | undefined = undefined + ): Promise { const controller = new AbortController(); return new Promise((resolve, reject) => { @@ -124,10 +130,12 @@ export function useTimeline({ return; } - const response$ = chatService!.chat({ - messages: nextMessages, - connectorId, - }); + response$ = + response$ || + chatService!.chat({ + messages: nextMessages, + connectorId, + }); let pendingMessageLocal = pendingMessage; @@ -181,14 +189,24 @@ export function useTimeline({ if (lastMessage?.message.function_call?.name) { const name = lastMessage.message.function_call.name; + setIsFunctionLoading(true); + try { - const message = await chatService!.executeFunction({ + let message = await chatService!.executeFunction({ name, args: lastMessage.message.function_call.arguments, messages: messagesAfterChat.slice(0, -1), signal: controller.signal, + connectorId: connectorId!, }); + let nextResponse$: Observable | undefined; + + if (isObservable(message)) { + nextResponse$ = message; + message = { content: '', data: '' }; + } + return await chat( messagesAfterChat.concat({ '@timestamp': new Date().toISOString(), @@ -198,7 +216,8 @@ export function useTimeline({ content: JSON.stringify(message.content), data: JSON.stringify(message.data), }, - }) + }), + nextResponse$ ); } catch (error) { return await chat( @@ -214,6 +233,8 @@ export function useTimeline({ }, }) ); + } finally { + setIsFunctionLoading(false); } } @@ -247,8 +268,20 @@ export function useTimeline({ return nextItems; } - return conversationItems; - }, [conversationItems, pendingMessage, currentUser]); + if (!isFunctionLoading) { + return conversationItems; + } + + return conversationItems.map((item, index) => { + if (index < conversationItems.length - 1) { + return item; + } + return { + ...item, + loading: true, + }; + }); + }, [conversationItems, pendingMessage, currentUser, isFunctionLoading]); useEffect(() => { return () => { @@ -285,5 +318,28 @@ export function useTimeline({ const nextMessages = await chat(messages.concat(message)); onChatComplete(nextMessages); }, + onActionClick: async (payload) => { + switch (payload.type) { + case ChatActionClickType.executeEsqlQuery: + const nextMessages = await chat( + messages.concat({ + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + content: '', + function_call: { + name: 'execute_query', + arguments: JSON.stringify({ + query: payload.query, + }), + trigger: MessageRole.User, + }, + }, + }) + ); + onChatComplete(nextMessages); + break; + } + }, }; } diff --git a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts index 1c109be6baaf2..bebc3bd074b20 100644 --- a/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts +++ b/x-pack/plugins/observability_ai_assistant/public/service/create_chat_service.ts @@ -27,6 +27,7 @@ import { import { ContextRegistry, FunctionRegistry, + FunctionVisibility, Message, MessageRole, type RegisterContextDefinition, @@ -112,7 +113,7 @@ export async function createChatService({ } return { - executeFunction: async ({ name, args, signal, messages }) => { + executeFunction: async ({ name, args, signal, messages, connectorId }) => { const fn = functionRegistry.get(name); if (!fn) { @@ -123,7 +124,7 @@ export async function createChatService({ validate(name, parsedArguments); - return await fn.respond({ arguments: parsedArguments, messages }, signal); + return await fn.respond({ arguments: parsedArguments, messages, connectorId }, signal); }, renderFunction: (name, args, response) => { const fn = functionRegistry.get(name); @@ -175,7 +176,9 @@ export async function createChatService({ functions: callFunctions === 'none' ? [] - : functions.map((fn) => pick(fn.options, 'name', 'description', 'parameters')), + : functions + .filter((fn) => fn.options.visibility !== FunctionVisibility.User) + .map((fn) => pick(fn.options, 'name', 'description', 'parameters')), }, }, signal: controller.signal, diff --git a/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts b/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts index 8d9cc2cf32539..4f52dfb9b9733 100644 --- a/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts +++ b/x-pack/plugins/observability_ai_assistant/public/service/get_assistant_setup_message.ts @@ -5,15 +5,19 @@ * 2.0. */ +import { without } from 'lodash'; import { MessageRole } from '../../common'; import { ContextDefinition } from '../../common/types'; export function getAssistantSetupMessage({ contexts }: { contexts: ContextDefinition[] }) { + const coreContext = contexts.find((context) => context.name === 'core')!; + + const otherContexts = without(contexts.concat(), coreContext); return { '@timestamp': new Date().toISOString(), message: { role: MessageRole.System as const, - content: contexts.map((context) => context.description).join('\n'), + content: [coreContext, ...otherContexts].map((context) => context.description).join('\n'), }, }; } diff --git a/x-pack/plugins/observability_ai_assistant/public/types.ts b/x-pack/plugins/observability_ai_assistant/public/types.ts index d1a71f6933126..5e985f5ae5f97 100644 --- a/x-pack/plugins/observability_ai_assistant/public/types.ts +++ b/x-pack/plugins/observability_ai_assistant/public/types.ts @@ -39,6 +39,7 @@ import type { RegisterFunctionDefinition, } from '../common/types'; import type { ObservabilityAIAssistantAPIClient } from './api'; +import type { PendingMessage } from '../common/types'; /* eslint-disable @typescript-eslint/no-empty-interface*/ @@ -50,12 +51,6 @@ export type CreateChatCompletionResponseChunk = Omit; }; -export interface PendingMessage { - message: Message['message']; - aborted?: boolean; - error?: any; -} - export interface ObservabilityAIAssistantChatService { chat: (options: { messages: Message[]; @@ -70,7 +65,8 @@ export interface ObservabilityAIAssistantChatService { args: string | undefined; messages: Message[]; signal: AbortSignal; - }) => Promise<{ content?: Serializable; data?: Serializable }>; + connectorId: string; + }) => Promise<{ content?: Serializable; data?: Serializable } | Observable>; renderFunction: ( name: string, args: string | undefined, @@ -118,3 +114,5 @@ export interface ObservabilityAIAssistantPluginStartDependencies { } export interface ConfigSchema {} + +export type { PendingMessage }; diff --git a/x-pack/plugins/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_ai_assistant/server/plugin.ts index 7aa6ddeeef7da..d6a256ddf6022 100644 --- a/x-pack/plugins/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_ai_assistant/server/plugin.ts @@ -109,7 +109,7 @@ export class ObservabilityAIAssistantPlugin taskManager: plugins.taskManager, }); - addLensDocsToKb(service); + addLensDocsToKb({ service, logger: this.logger.get('kb').get('lens') }); registerServerRoutes({ core, diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts index 6716da98d78a5..a250a9e8e0915 100644 --- a/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts +++ b/x-pack/plugins/observability_ai_assistant/server/routes/chat/route.ts @@ -7,6 +7,8 @@ import { notImplemented } from '@hapi/boom'; import { IncomingMessage } from 'http'; import * as t from 'io-ts'; +import { toBooleanRt } from '@kbn/io-ts-utils'; +import type { CreateChatCompletionResponse } from 'openai'; import { MessageRole } from '../../../common'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; import { messageRt } from '../runtime_types'; @@ -16,20 +18,28 @@ const chatRoute = createObservabilityAIAssistantServerRoute({ options: { tags: ['access:ai_assistant'], }, - params: t.type({ - body: t.type({ - messages: t.array(messageRt), - connectorId: t.string, - functions: t.array( + params: t.intersection([ + t.type({ + body: t.intersection([ t.type({ - name: t.string, - description: t.string, - parameters: t.any, - }) - ), + messages: t.array(messageRt), + connectorId: t.string, + functions: t.array( + t.type({ + name: t.string, + description: t.string, + parameters: t.any, + }) + ), + }), + t.partial({ + functionCall: t.string, + }), + ]), }), - }), - handler: async (resources): Promise => { + t.partial({ query: t.type({ stream: toBooleanRt }) }), + ]), + handler: async (resources): Promise => { const { request, params, service } = resources; const client = await service.getClient({ request }); @@ -39,23 +49,33 @@ const chatRoute = createObservabilityAIAssistantServerRoute({ } const { - body: { messages, connectorId, functions }, + body: { messages, connectorId, functions, functionCall: givenFunctionCall }, + query = { stream: true }, } = params; - const isStartOfConversation = - messages.some((message) => message.message.role === MessageRole.Assistant) === false; + const stream = query.stream; - const isRecallFunctionAvailable = functions.some((fn) => fn.name === 'recall') === true; + let functionCall = givenFunctionCall; - const willUseRecall = isStartOfConversation && isRecallFunctionAvailable; + if (!functionCall) { + const isStartOfConversation = + messages.some((message) => message.message.role === MessageRole.Assistant) === false; + + const isRecallFunctionAvailable = functions.some((fn) => fn.name === 'recall') === true; + + const willUseRecall = isStartOfConversation && isRecallFunctionAvailable; + + functionCall = willUseRecall ? 'recall' : undefined; + } return client.chat({ messages, connectorId, + stream, ...(functions.length ? { functions, - functionCall: willUseRecall ? 'recall' : undefined, + functionCall, } : {}), }); diff --git a/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts index d58c84a603fc1..985de114b099a 100644 --- a/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_ai_assistant/server/routes/functions/route.ts @@ -10,13 +10,13 @@ import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { omit } from 'lodash'; -import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; +import type { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; import { ALERT_STATUS, ALERT_STATUS_ACTIVE, } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; -import type { KnowledgeBaseEntry } from '../../../common/types'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; +import type { RecalledEntry } from '../../service/kb_service'; const functionElasticsearchRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/functions/elasticsearch', @@ -157,23 +157,34 @@ const functionAlertsRoute = createObservabilityAIAssistantServerRoute({ const functionRecallRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/functions/recall', params: t.type({ - body: t.type({ - queries: t.array(nonEmptyStringRt), - }), + body: t.intersection([ + t.type({ + queries: t.array(nonEmptyStringRt), + }), + t.partial({ + contexts: t.array(t.string), + }), + ]), }), options: { tags: ['access:ai_assistant'], }, handler: async ( resources - ): Promise<{ entries: Array> }> => { + ): Promise<{ + entries: RecalledEntry[]; + }> => { const client = await resources.service.getClient({ request: resources.request }); + const { + body: { queries, contexts }, + } = resources.params; + if (!client) { throw notImplemented(); } - return client.recall(resources.params.body.queries); + return client.recall({ queries, contexts }); }, }); diff --git a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts index ce3f3c39ad1a9..3a99e293cd5e2 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/client/index.ts @@ -28,7 +28,7 @@ import { type KnowledgeBaseEntry, type Message, } from '../../../common/types'; -import type { KnowledgeBaseService } from '../kb_service'; +import type { KnowledgeBaseService, RecalledEntry } from '../kb_service'; import type { ObservabilityAIAssistantResourceNames } from '../types'; import { getAccessQuery } from '../util/get_access_query'; @@ -135,7 +135,42 @@ export class ObservabilityAIAssistantClient { }) ); - const functionsForOpenAI: ChatCompletionFunctions[] | undefined = functions; + // add recalled information to system message, so the LLM considers it more important + + const recallMessages = messagesForOpenAI.filter((message) => message.name === 'recall'); + + const recalledDocuments: Map = new Map(); + + recallMessages.forEach((message) => { + const entries = message.content + ? (JSON.parse(message.content) as Array<{ id: string; text: string }>) + : []; + + const ids: string[] = []; + + entries.forEach((entry) => { + const id = entry.id; + if (!recalledDocuments.has(id)) { + recalledDocuments.set(id, entry); + } + ids.push(id); + }); + + message.content = `The following documents, present in the system message, were recalled: ${ids.join( + ', ' + )}`; + }); + + const systemMessage = messagesForOpenAI.find((message) => message.role === MessageRole.System); + + if (systemMessage && recalledDocuments.size > 0) { + systemMessage.content += `The "recall" function is not available. Do not attempt to execute it. Recalled documents: ${JSON.stringify( + Array.from(recalledDocuments.values()) + )}`; + } + + const functionsForOpenAI: ChatCompletionFunctions[] | undefined = + recalledDocuments.size > 0 ? functions?.filter((fn) => fn.name !== 'recall') : functions; const request: Omit & { model?: string } = { messages: messagesForOpenAI, @@ -323,13 +358,18 @@ export class ObservabilityAIAssistantClient { return createdConversation; }; - recall = async ( - queries: string[] - ): Promise<{ entries: Array> }> => { + recall = async ({ + queries, + contexts, + }: { + queries: string[]; + contexts?: string[]; + }): Promise<{ entries: RecalledEntry[] }> => { return this.dependencies.knowledgeBaseService.recall({ namespace: this.dependencies.namespace, user: this.dependencies.user, queries, + contexts, }); }; diff --git a/x-pack/plugins/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/index.ts index 28cc2b7293029..c116e16d27471 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/index.ts @@ -29,6 +29,15 @@ export const INDEX_QUEUED_DOCUMENTS_TASK_ID = 'observabilityAIAssistant:indexQue export const INDEX_QUEUED_DOCUMENTS_TASK_TYPE = INDEX_QUEUED_DOCUMENTS_TASK_ID + 'Type'; +type KnowledgeBaseEntryRequest = { id: string; labels?: Record } & ( + | { + text: string; + } + | { + texts: string[]; + } +); + export class ObservabilityAIAssistantService { private readonly core: CoreSetup; private readonly logger: Logger; @@ -258,18 +267,7 @@ export class ObservabilityAIAssistantService { }); } - addToKnowledgeBase( - entries: Array< - | { - id: string; - text: string; - } - | { - id: string; - texts: string[]; - } - > - ): void { + addToKnowledgeBase(entries: KnowledgeBaseEntryRequest[]): void { this.init() .then(() => { this.kbService!.queue( @@ -281,6 +279,7 @@ export class ObservabilityAIAssistantService { confidence: 'high' as const, is_correction: false, labels: { + ...entry.labels, document_id: entry.id, }, }; @@ -306,4 +305,18 @@ export class ObservabilityAIAssistantService { this.logger.error(error); }); } + + addCategoryToKnowledgeBase(categoryId: string, entries: KnowledgeBaseEntryRequest[]) { + this.addToKnowledgeBase( + entries.map((entry) => { + return { + ...entry, + labels: { + ...entry.labels, + category: categoryId, + }, + }; + }) + ); + } } diff --git a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts index f2e0b41f092c9..d70879bf46d3e 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/index.ts @@ -17,6 +17,7 @@ import { INDEX_QUEUED_DOCUMENTS_TASK_ID, INDEX_QUEUED_DOCUMENTS_TASK_TYPE } from import type { KnowledgeBaseEntry } from '../../../common/types'; import type { ObservabilityAIAssistantResourceNames } from '../types'; import { getAccessQuery } from '../util/get_access_query'; +import { getCategoryQuery } from '../util/get_category_query'; interface Dependencies { esClient: ElasticsearchClient; @@ -25,6 +26,14 @@ interface Dependencies { taskManagerStart: TaskManagerStartContract; } +export interface RecalledEntry { + id: string; + text: string; + score: number | null; + is_correction: boolean; + labels: Record; +} + function isAlreadyExistsError(error: Error) { return ( error instanceof errors.ResponseError && @@ -173,36 +182,43 @@ export class KnowledgeBaseService { recall = async ({ user, queries, + contexts, namespace, }: { queries: string[]; + contexts?: string[]; user: { name: string }; namespace: string; - }): Promise<{ entries: Array> }> => { + }): Promise<{ + entries: RecalledEntry[]; + }> => { try { + const query = { + bool: { + should: queries.map((text) => ({ + text_expansion: { + 'ml.tokens': { + model_text: text, + model_id: '.elser_model_1', + }, + } as unknown as QueryDslTextExpansionQuery, + })), + filter: [ + ...getAccessQuery({ + user, + namespace, + }), + ...getCategoryQuery({ contexts }), + ], + }, + }; + const response = await this.dependencies.esClient.search< - Pick + Pick >({ index: this.dependencies.resources.aliases.kb, - query: { - bool: { - should: queries.map((query) => ({ - text_expansion: { - 'ml.tokens': { - model_text: query, - model_id: '.elser_model_1', - }, - } as unknown as QueryDslTextExpansionQuery, - })), - filter: [ - ...getAccessQuery({ - user, - namespace, - }), - ], - }, - }, - size: 5, + query, + size: 10, _source: { includes: ['text', 'is_correction', 'labels'], }, @@ -211,7 +227,7 @@ export class KnowledgeBaseService { return { entries: response.hits.hits.map((hit) => ({ ...hit._source!, - score: hit._score, + score: hit._score!, id: hit._id, })), }; diff --git a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts index e4fdc8969c010..9baf75f6ff552 100644 --- a/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts +++ b/x-pack/plugins/observability_ai_assistant/server/service/kb_service/kb_docs/lens.ts @@ -6,10 +6,16 @@ */ import dedent from 'dedent'; +import type { Logger } from '@kbn/logging'; import type { ObservabilityAIAssistantService } from '../..'; -export function addLensDocsToKb(service: ObservabilityAIAssistantService) { - service.addToKnowledgeBase([ +export function addLensDocsToKb({ + service, +}: { + service: ObservabilityAIAssistantService; + logger: Logger; +}) { + service.addCategoryToKnowledgeBase('lens', [ { id: 'lens_formulas_how_it_works', texts: [ diff --git a/x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts b/x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts new file mode 100644 index 0000000000000..71b0a93156086 --- /dev/null +++ b/x-pack/plugins/observability_ai_assistant/server/service/util/get_category_query.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getCategoryQuery({ contexts }: { contexts?: string[] }) { + const noCategoryFilter = { + bool: { + must_not: { + exists: { + field: 'labels.category', + }, + }, + }, + }; + + if (!contexts) { + return [noCategoryFilter]; + } + + return [ + { + bool: { + should: [ + noCategoryFilter, + { + terms: { + 'labels.category': contexts, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ]; +} diff --git a/x-pack/plugins/observability_shared/public/index.ts b/x-pack/plugins/observability_shared/public/index.ts index 7e5fc02d0a0b5..a35cdce8f85e1 100644 --- a/x-pack/plugins/observability_shared/public/index.ts +++ b/x-pack/plugins/observability_shared/public/index.ts @@ -10,6 +10,7 @@ export type { ObservabilitySharedPlugin, ObservabilitySharedPluginSetup, ObservabilitySharedPluginStart, + ProfilingLocators, } from './plugin'; export const plugin = () => { return new ObservabilitySharedPlugin(); diff --git a/x-pack/plugins/observability_shared/public/plugin.ts b/x-pack/plugins/observability_shared/public/plugin.ts index 550259184dfb1..faf8990622337 100644 --- a/x-pack/plugins/observability_shared/public/plugin.ts +++ b/x-pack/plugins/observability_shared/public/plugin.ts @@ -34,6 +34,7 @@ export interface ObservabilitySharedStart { export type ObservabilitySharedPluginSetup = ReturnType; export type ObservabilitySharedPluginStart = ReturnType; +export type ProfilingLocators = ObservabilitySharedPluginSetup['locators']['profiling']; export class ObservabilitySharedPlugin implements Plugin { private readonly navigationRegistry = createNavigationRegistry(); diff --git a/x-pack/plugins/osquery/public/components/empty_state.tsx b/x-pack/plugins/osquery/public/components/empty_state.tsx index 099e9e22b1f64..18f5afe8eaf5f 100644 --- a/x-pack/plugins/osquery/public/components/empty_state.tsx +++ b/x-pack/plugins/osquery/public/components/empty_state.tsx @@ -9,7 +9,7 @@ import React, { useCallback, useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; import { EuiButton } from '@elastic/eui'; -import { KibanaPageTemplate } from '@kbn/kibana-react-plugin/public'; +import { KibanaPageTemplate } from '@kbn/shared-ux-page-kibana-template'; import { INTEGRATIONS_PLUGIN_ID } from '@kbn/fleet-plugin/common'; import { pagePathGetters } from '@kbn/fleet-plugin/public'; import { isModifiedEvent, isLeftClickEvent, useKibana } from '../common/lib/kibana'; diff --git a/x-pack/plugins/osquery/tsconfig.json b/x-pack/plugins/osquery/tsconfig.json index 6cd1086b8a850..d2344a2581df8 100644 --- a/x-pack/plugins/osquery/tsconfig.json +++ b/x-pack/plugins/osquery/tsconfig.json @@ -77,5 +77,6 @@ "@kbn/core-saved-objects-server", "@kbn/monaco", "@kbn/io-ts-utils", + "@kbn/shared-ux-page-kibana-template", ] } diff --git a/x-pack/plugins/profiling/public/index.tsx b/x-pack/plugins/profiling/public/index.tsx index ea9dc15de9927..752635a54290b 100644 --- a/x-pack/plugins/profiling/public/index.tsx +++ b/x-pack/plugins/profiling/public/index.tsx @@ -13,5 +13,3 @@ export function plugin() { } export type { ProfilingPluginSetup, ProfilingPluginStart }; - -export type ProfilingLocators = ProfilingPluginSetup['locators']; diff --git a/x-pack/plugins/profiling_data_access/server/services/status/index.ts b/x-pack/plugins/profiling_data_access/server/services/status/index.ts index 6aea8b1037bd6..31581140b12e0 100644 --- a/x-pack/plugins/profiling_data_access/server/services/status/index.ts +++ b/x-pack/plugins/profiling_data_access/server/services/status/index.ts @@ -12,7 +12,7 @@ import { getSetupState } from '../get_setup_state'; import { RegisterServicesParams } from '../register_services'; import { ProfilingSetupOptions, areResourcesSetup } from '../../../common/setup'; -interface HasSetupParams { +export interface HasSetupParams { soClient: SavedObjectsClientContract; esClient: ElasticsearchClient; spaceId?: string; diff --git a/x-pack/plugins/security_solution/public/management/cypress/common/constants.ts b/x-pack/plugins/security_solution/public/management/cypress/common/constants.ts new file mode 100644 index 0000000000000..41f08f438e3f8 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/common/constants.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EndpointSecurityRoleNames } from '../../../../scripts/endpoint/common/roles_users'; + +export type KibanaKnownUserAccounts = keyof typeof KIBANA_KNOWN_DEFAULT_ACCOUNTS; + +export type SecurityTestUser = EndpointSecurityRoleNames | KibanaKnownUserAccounts; + +/** + * List of kibana system accounts + */ +export const KIBANA_KNOWN_DEFAULT_ACCOUNTS = { + elastic: 'elastic', + elastic_serverless: 'elastic_serverless', + system_indices_superuser: 'system_indices_superuser', +} as const; diff --git a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts index 4d84dc88af9c9..a4037e32632a5 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/cypress.d.ts @@ -17,7 +17,12 @@ import type { HostPolicyResponse, LogsEndpointActionResponse, } from '../../../common/endpoint/types'; -import type { IndexEndpointHostsCyTaskOptions, HostActionResponse } from './types'; +import type { + HostActionResponse, + IndexEndpointHostsCyTaskOptions, + LoadUserAndRoleCyTaskOptions, + CreateUserAndRoleCyTaskOptions, +} from './types'; import type { DeleteIndexedFleetEndpointPoliciesResponse, IndexedFleetEndpointPolicyResponse, @@ -32,6 +37,7 @@ import type { DeletedIndexedEndpointRuleAlerts, IndexedEndpointRuleAlerts, } from '../../../common/endpoint/data_loaders/index_endpoint_rule_alerts'; +import type { LoadedRoleAndUser } from '../../../scripts/endpoint/common/role_and_user_loader'; declare global { namespace Cypress { @@ -185,6 +191,18 @@ declare global { arg: { hostname: string; path: string; password?: string }, options?: Partial ): Chainable; + + task( + name: 'loadUserAndRole', + arg: LoadUserAndRoleCyTaskOptions, + options?: Partial + ): Chainable; + + task( + name: 'createUserAndRole', + arg: CreateUserAndRoleCyTaskOptions, + options?: Partial + ): Chainable; } } } diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts index 39ebae1825365..8610ac1fc3ba5 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifact_tabs_in_policy_details.cy.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { getRoleWithArtifactReadPrivilege } from '../../fixtures/role_with_artifact_read_privilege'; import { getEndpointSecurityPolicyManager } from '../../../../../scripts/endpoint/common/roles_users/endpoint_security_policy_manager'; import { getArtifactsListTestsData } from '../../fixtures/artifacts_page'; import { visitPolicyDetailsPage } from '../../screens/policy_details'; @@ -16,27 +17,21 @@ import { yieldFirstPolicyID, } from '../../tasks/artifacts'; import { loadEndpointDataForEventFiltersIfNeeded } from '../../tasks/load_endpoint_data'; -import { - getRoleWithArtifactReadPrivilege, - login, - loginWithCustomRole, - loginWithRole, - ROLE, -} from '../../tasks/login'; +import { login, ROLE } from '../../tasks/login'; import { performUserActions } from '../../tasks/perform_user_actions'; const loginWithPrivilegeAll = () => { - loginWithRole(ROLE.endpoint_security_policy_manager); + login(ROLE.endpoint_policy_manager); }; const loginWithPrivilegeRead = (privilegePrefix: string) => { const roleWithArtifactReadPrivilege = getRoleWithArtifactReadPrivilege(privilegePrefix); - loginWithCustomRole('roleWithArtifactReadPrivilege', roleWithArtifactReadPrivilege); + login.withCustomRole({ name: 'roleWithArtifactReadPrivilege', ...roleWithArtifactReadPrivilege }); }; const loginWithPrivilegeNone = (privilegePrefix: string) => { const roleWithoutArtifactPrivilege = getRoleWithoutArtifactPrivilege(privilegePrefix); - loginWithCustomRole('roleWithoutArtifactPrivilege', roleWithoutArtifactPrivilege); + login.withCustomRole({ name: 'roleWithoutArtifactPrivilege', ...roleWithoutArtifactPrivilege }); }; const getRoleWithoutArtifactPrivilege = (privilegePrefix: string) => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts index 8f575282ad3f7..86cd86dd797b7 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/artifacts/artifacts_mocked_data.cy.ts @@ -5,13 +5,8 @@ * 2.0. */ -import { - getRoleWithArtifactReadPrivilege, - login, - loginWithCustomRole, - loginWithRole, - ROLE, -} from '../../tasks/login'; +import { getRoleWithArtifactReadPrivilege } from '../../fixtures/role_with_artifact_read_privilege'; +import { login, ROLE } from '../../tasks/login'; import { loadPage } from '../../tasks/common'; import { getArtifactsListTestsData } from '../../fixtures/artifacts_page'; @@ -20,18 +15,18 @@ import { performUserActions } from '../../tasks/perform_user_actions'; import { loadEndpointDataForEventFiltersIfNeeded } from '../../tasks/load_endpoint_data'; const loginWithWriteAccess = (url: string) => { - loginWithRole(ROLE.endpoint_security_policy_manager); + login(ROLE.endpoint_policy_manager); loadPage(url); }; const loginWithReadAccess = (privilegePrefix: string, url: string) => { const roleWithArtifactReadPrivilege = getRoleWithArtifactReadPrivilege(privilegePrefix); - loginWithCustomRole('roleWithArtifactReadPrivilege', roleWithArtifactReadPrivilege); + login.withCustomRole({ name: 'roleWithArtifactReadPrivilege', ...roleWithArtifactReadPrivilege }); loadPage(url); }; const loginWithoutAccess = (url: string) => { - loginWithRole(ROLE.t1_analyst); + login(ROLE.t1_analyst); loadPage(url); }; diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts index 84f6903c35a9b..eb5bae8624475 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/form.cy.ts @@ -16,12 +16,12 @@ import { } from '../../tasks/response_actions'; import { cleanupRule, generateRandomStringName, loadRule } from '../../tasks/api_fixtures'; import { RESPONSE_ACTION_TYPES } from '../../../../../common/api/detection_engine'; -import { loginWithRole, ROLE } from '../../tasks/login'; +import { login, ROLE } from '../../tasks/login'; describe('Form', { tags: '@ess' }, () => { describe('User with no access can not create an endpoint response action', () => { before(() => { - loginWithRole(ROLE.endpoint_response_actions_no_access); + login(ROLE.endpoint_response_actions_no_access); }); it('no endpoint response action option during rule creation', () => { @@ -36,7 +36,7 @@ describe('Form', { tags: '@ess' }, () => { const [ruleName, ruleDescription] = generateRandomStringName(2); before(() => { - loginWithRole(ROLE.endpoint_response_actions_access); + login(ROLE.endpoint_response_actions_access); }); after(() => { cleanupRule(ruleId); @@ -94,7 +94,7 @@ describe('Form', { tags: '@ess' }, () => { }); }); beforeEach(() => { - loginWithRole(ROLE.endpoint_response_actions_access); + login(ROLE.endpoint_response_actions_access); }); after(() => { cleanupRule(ruleId); @@ -146,7 +146,7 @@ describe('Form', { tags: '@ess' }, () => { const [ruleName, ruleDescription] = generateRandomStringName(2); before(() => { - loginWithRole(ROLE.endpoint_response_actions_no_access); + login(ROLE.endpoint_response_actions_no_access); }); it('response actions are disabled', () => { @@ -166,7 +166,7 @@ describe('Form', { tags: '@ess' }, () => { loadRule().then((res) => { ruleId = res.id; }); - loginWithRole(ROLE.endpoint_response_actions_no_access); + login(ROLE.endpoint_response_actions_no_access); }); after(() => { cleanupRule(ruleId); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts index edbaa90d3200c..eb6191ece0460 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/automated_response_actions/no_license.cy.ts @@ -9,7 +9,7 @@ import { disableExpandableFlyoutAdvancedSettings } from '../../tasks/common'; import { APP_ALERTS_PATH } from '../../../../../common/constants'; import { closeAllToasts } from '../../tasks/toasts'; import { fillUpNewRule } from '../../tasks/response_actions'; -import { login, loginWithRole, ROLE } from '../../tasks/login'; +import { login, ROLE } from '../../tasks/login'; import { generateRandomStringName } from '../../tasks/utils'; import type { ReturnTypeFromChainable } from '../../types'; import { indexEndpointHosts } from '../../tasks/index_endpoint_hosts'; @@ -20,7 +20,7 @@ describe('No License', { tags: '@ess', env: { ftrConfig: { license: 'basic' } } const [ruleName, ruleDescription] = generateRandomStringName(2); before(() => { - loginWithRole(ROLE.endpoint_response_actions_access); + login(ROLE.endpoint_response_actions_access); }); it('response actions are disabled', () => { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts index 58c41539361c5..32800978a968e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/endpoint_list_with_security_essentials.cy.ts @@ -7,7 +7,7 @@ import type { CyIndexEndpointHosts } from '../../tasks/index_endpoint_hosts'; import { indexEndpointHosts } from '../../tasks/index_endpoint_hosts'; -import { loginServerless } from '../../tasks/login_serverless'; +import { login } from '../../tasks/login'; import { getConsoleActionMenuItem, getUnIsolateActionMenuItem, @@ -42,7 +42,7 @@ describe( }); beforeEach(() => { - loginServerless(); + login(); visitEndpointList(); openRowActionMenu(); }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts index 7c4323b2aa689..dba7166b9a9e8 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete.cy.ts @@ -6,7 +6,7 @@ */ import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions'; -import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless'; +import { login, ROLE } from '../../../tasks/login'; import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants'; import { getNoPrivilegesPage } from '../../../screens/common'; import { getEndpointManagementPageList } from '../../../screens'; @@ -31,7 +31,7 @@ describe( let password: string; beforeEach(() => { - loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => { + login(ROLE.endpoint_operations_analyst).then((response) => { username = response.username; password = response.password; }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts index f3ae8b85a9f24..3c028b9e25040 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/complete_with_endpoint.cy.ts @@ -6,7 +6,7 @@ */ import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions'; -import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless'; +import { login, ROLE } from '../../../tasks/login'; import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants'; import { getEndpointManagementPageList, @@ -33,7 +33,7 @@ describe( let password: string; beforeEach(() => { - loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => { + login(ROLE.endpoint_operations_analyst).then((response) => { username = response.username; password = response.password; }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts index 900a5d81a9f46..fed4494722df5 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials.cy.ts @@ -6,7 +6,7 @@ */ import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions'; -import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless'; +import { login, ROLE } from '../../../tasks/login'; import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants'; import { getNoPrivilegesPage } from '../../../screens/common'; import { getEndpointManagementPageList } from '../../../screens'; @@ -33,7 +33,7 @@ describe( let password: string; beforeEach(() => { - loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => { + login(ROLE.endpoint_operations_analyst).then((response) => { username = response.username; password = response.password; }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts index 7196b73f6813a..172f850e44b7c 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/feature_access/essentials_with_endpoint.cy.ts @@ -6,7 +6,7 @@ */ import { ensureResponseActionAuthzAccess } from '../../../tasks/response_actions'; -import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless'; +import { login, ROLE } from '../../../tasks/login'; import { RESPONSE_ACTION_API_COMMANDS_NAMES } from '../../../../../../common/endpoint/service/response_actions/constants'; import { getEndpointManagementPageMap, @@ -41,7 +41,7 @@ describe( let password: string; beforeEach(() => { - loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST).then((response) => { + login(ROLE.endpoint_operations_analyst).then((response) => { username = response.username; password = response.password; }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts index faf9aba6237b3..e1516bb08e10e 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/policy_details_with_security_essentials.cy.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { loginServerless } from '../../tasks/login_serverless'; +import { login } from '../../tasks/login'; import { visitPolicyDetailsPage } from '../../screens/policy_details'; import type { IndexedFleetEndpointPolicyResponse } from '../../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; @@ -35,7 +35,7 @@ describe( }); beforeEach(() => { - loginServerless(); + login(); visitPolicyDetailsPage(loadedPolicyData.integrationPolicies[0].id); }); diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts index 879948a65c47d..534b681c2aeb4 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/complete_with_endpoint_roles.cy.ts @@ -8,7 +8,7 @@ import { pick } from 'lodash'; import type { CyIndexEndpointHosts } from '../../../tasks/index_endpoint_hosts'; import { indexEndpointHosts } from '../../../tasks/index_endpoint_hosts'; -import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless'; +import { login, ROLE } from '../../../tasks/login'; import { ensurePolicyDetailsPageAuthzAccess } from '../../../screens/policy_details'; import type { EndpointArtifactPageId } from '../../../screens'; import { @@ -63,12 +63,12 @@ describe( }); // roles `t1_analyst` and `t2_analyst` are very similar with exception of one page - (['t1_analyst', `t2_analyst`] as ServerlessUser[]).forEach((roleName) => { + (['t1_analyst', `t2_analyst`] as ROLE[]).forEach((roleName) => { describe(`for role: ${roleName}`, () => { const deniedPages = allPages.filter((page) => page.id !== 'endpointList'); beforeEach(() => { - loginServerless(roleName); + login(roleName); }); it('should have READ access to Endpoint list page', () => { @@ -124,7 +124,7 @@ describe( const deniedResponseActions = pick(consoleHelpPanelResponseActionsTestSubj, 'execute'); beforeEach(() => { - loginServerless(ServerlessUser.T3_ANALYST); + login(ROLE.t3_analyst); }); it('should have access to Endpoint list page', () => { @@ -176,7 +176,7 @@ describe( const deniedPages = allPages.filter(({ id }) => id !== 'blocklist' && id !== 'endpointList'); beforeEach(() => { - loginServerless(ServerlessUser.THREAT_INTELLIGENCE_ANALYST); + login(ROLE.threat_intelligence_analyst); }); it('should have access to Endpoint list page', () => { @@ -221,7 +221,7 @@ describe( ]; beforeEach(() => { - loginServerless(ServerlessUser.RULE_AUTHOR); + login(ROLE.rule_author); }); for (const { id, title } of artifactPagesFullAccess) { @@ -272,7 +272,7 @@ describe( const grantedAccessPages = [pageById.endpointList, pageById.policyList]; beforeEach(() => { - loginServerless(ServerlessUser.SOC_MANAGER); + login(ROLE.soc_manager); }); for (const { id, title } of artifactPagesFullAccess) { @@ -319,7 +319,7 @@ describe( const grantedAccessPages = [pageById.endpointList, pageById.policyList]; beforeEach(() => { - loginServerless(ServerlessUser.ENDPOINT_OPERATIONS_ANALYST); + login(ROLE.endpoint_operations_analyst); }); for (const { id, title } of artifactPagesFullAccess) { @@ -350,7 +350,7 @@ describe( }); }); - (['platform_engineer', 'endpoint_policy_manager'] as ServerlessUser[]).forEach((roleName) => { + (['platform_engineer', 'endpoint_policy_manager'] as ROLE[]).forEach((roleName) => { describe(`for role: ${roleName}`, () => { const artifactPagesFullAccess = [ pageById.trustedApps, @@ -361,7 +361,7 @@ describe( const grantedAccessPages = [pageById.endpointList, pageById.policyList]; beforeEach(() => { - loginServerless(roleName); + login(roleName); }); for (const { id, title } of artifactPagesFullAccess) { diff --git a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts index fec6a0f803afb..6cf3ab727980a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/e2e/serverless/roles/essentials_with_endpoint.roles.cy.ts @@ -7,7 +7,7 @@ import type { CyIndexEndpointHosts } from '../../../tasks/index_endpoint_hosts'; import { indexEndpointHosts } from '../../../tasks/index_endpoint_hosts'; -import { loginServerless, ServerlessUser } from '../../../tasks/login_serverless'; +import { login, ROLE } from '../../../tasks/login'; import type { EndpointArtifactPageId } from '../../../screens'; import { getNoPrivilegesPage, @@ -55,12 +55,12 @@ describe( }); // roles `t1_analyst` and `t2_analyst` are the same as far as endpoint access - (['t1_analyst', `t2_analyst`] as ServerlessUser[]).forEach((roleName) => { + (['t1_analyst', `t2_analyst`] as ROLE[]).forEach((roleName) => { describe(`for role: ${roleName}`, () => { const deniedPages = allPages.filter((page) => page.id !== 'endpointList'); beforeEach(() => { - loginServerless(roleName); + login(roleName); }); it('should have READ access to Endpoint list page', () => { @@ -89,7 +89,7 @@ describe( ]; beforeEach(() => { - loginServerless(ServerlessUser.T3_ANALYST); + login(ROLE.t3_analyst); }); it('should have access to Endpoint list page', () => { @@ -128,7 +128,7 @@ describe( const deniedPages = allPages.filter(({ id }) => id !== 'blocklist' && id !== 'endpointList'); beforeEach(() => { - loginServerless(ServerlessUser.THREAT_INTELLIGENCE_ANALYST); + login(ROLE.threat_intelligence_analyst); }); it('should have access to Endpoint list page', () => { @@ -163,7 +163,7 @@ describe( ]; beforeEach(() => { - loginServerless(ServerlessUser.RULE_AUTHOR); + login(ROLE.rule_author); }); for (const { id, title } of artifactPagesFullAccess) { @@ -207,7 +207,7 @@ describe( const grantedAccessPages = [pageById.endpointList, pageById.policyList]; beforeEach(() => { - loginServerless(ServerlessUser.SOC_MANAGER); + login(ROLE.soc_manager); }); for (const { id, title } of artifactPagesFullAccess) { @@ -238,11 +238,7 @@ describe( // Endpoint Operations Manager, Endpoint Policy Manager and Platform Engineer currently have the same level of access ( - [ - 'platform_engineer', - `endpoint_operations_analyst`, - 'endpoint_policy_manager', - ] as ServerlessUser[] + ['platform_engineer', `endpoint_operations_analyst`, 'endpoint_policy_manager'] as ROLE[] ).forEach((roleName) => { describe(`for role: ${roleName}`, () => { const artifactPagesFullAccess = [ @@ -253,7 +249,7 @@ describe( const grantedAccessPages = [pageById.endpointList, pageById.policyList]; beforeEach(() => { - loginServerless(roleName); + login(roleName); }); for (const { id, title } of artifactPagesFullAccess) { diff --git a/x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts b/x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts new file mode 100644 index 0000000000000..247b491f04632 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/cypress/fixtures/role_with_artifact_read_privilege.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getEndpointSecurityPolicyManager } from '../../../../scripts/endpoint/common/roles_users'; + +export const getRoleWithArtifactReadPrivilege = (privilegePrefix: string) => { + const endpointSecurityPolicyManagerRole = getEndpointSecurityPolicyManager(); + + return { + ...endpointSecurityPolicyManagerRole, + kibana: [ + { + ...endpointSecurityPolicyManagerRole.kibana[0], + feature: { + ...endpointSecurityPolicyManagerRole.kibana[0].feature, + siem: [ + ...endpointSecurityPolicyManagerRole.kibana[0].feature.siem.filter( + (privilege) => privilege !== `${privilegePrefix}all` + ), + `${privilegePrefix}read`, + ], + }, + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts index 4d8164391cb11..93614b6dbb86d 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/support/data_loaders.ts @@ -9,6 +9,14 @@ import type { CasePostRequest } from '@kbn/cases-plugin/common'; import execa from 'execa'; +import type { KbnClient } from '@kbn/test'; +import type { ToolingLog } from '@kbn/tooling-log'; +import type { KibanaKnownUserAccounts } from '../common/constants'; +import { KIBANA_KNOWN_DEFAULT_ACCOUNTS } from '../common/constants'; +import type { EndpointSecurityRoleNames } from '../../../../scripts/endpoint/common/roles_users'; +import { SECURITY_SERVERLESS_ROLE_NAMES } from '../../../../scripts/endpoint/common/roles_users'; +import type { LoadedRoleAndUser } from '../../../../scripts/endpoint/common/role_and_user_loader'; +import { EndpointSecurityTestRolesLoader } from '../../../../scripts/endpoint/common/role_and_user_loader'; import { startRuntimeServices } from '../../../../scripts/endpoint/endpoint_agent_runner/runtime'; import { runFleetServerIfNeeded } from '../../../../scripts/endpoint/endpoint_agent_runner/fleet_server'; import { @@ -22,39 +30,89 @@ import type { CreateAndEnrollEndpointHostOptions, CreateAndEnrollEndpointHostResponse, } from '../../../../scripts/endpoint/common/endpoint_host_services'; +import { + createAndEnrollEndpointHost, + destroyEndpointHost, + startEndpointHost, + stopEndpointHost, + VAGRANT_CWD, +} from '../../../../scripts/endpoint/common/endpoint_host_services'; import type { IndexedEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_endpoint_policy_response'; import { deleteIndexedEndpointPolicyResponse, indexEndpointPolicyResponse, } from '../../../../common/endpoint/data_loaders/index_endpoint_policy_response'; import type { ActionDetails, HostPolicyResponse } from '../../../../common/endpoint/types'; -import type { IndexEndpointHostsCyTaskOptions } from '../types'; import type { - IndexedEndpointRuleAlerts, + IndexEndpointHostsCyTaskOptions, + LoadUserAndRoleCyTaskOptions, + CreateUserAndRoleCyTaskOptions, +} from '../types'; +import type { DeletedIndexedEndpointRuleAlerts, + IndexedEndpointRuleAlerts, +} from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts'; +import { + deleteIndexedEndpointRuleAlerts, + indexEndpointRuleAlerts, } from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts'; import type { IndexedHostsAndAlertsResponse } from '../../../../common/endpoint/index_data'; +import { deleteIndexedHostsAndAlerts } from '../../../../common/endpoint/index_data'; import type { IndexedCase } from '../../../../common/endpoint/data_loaders/index_case'; +import { deleteIndexedCase, indexCase } from '../../../../common/endpoint/data_loaders/index_case'; import { createRuntimeServices } from '../../../../scripts/endpoint/common/stack_services'; import type { IndexedFleetEndpointPolicyResponse } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; import { - indexFleetEndpointPolicy, deleteIndexedFleetEndpointPolicies, + indexFleetEndpointPolicy, } from '../../../../common/endpoint/data_loaders/index_fleet_endpoint_policy'; -import { deleteIndexedCase, indexCase } from '../../../../common/endpoint/data_loaders/index_case'; import { cyLoadEndpointDataHandler } from './plugin_handlers/endpoint_data_loader'; -import { deleteIndexedHostsAndAlerts } from '../../../../common/endpoint/index_data'; -import { - deleteIndexedEndpointRuleAlerts, - indexEndpointRuleAlerts, -} from '../../../../common/endpoint/data_loaders/index_endpoint_rule_alerts'; -import { - startEndpointHost, - createAndEnrollEndpointHost, - destroyEndpointHost, - stopEndpointHost, - VAGRANT_CWD, -} from '../../../../scripts/endpoint/common/endpoint_host_services'; + +/** + * Test Role/User loader for cypress. Checks to see if running in serverless and handles it as appropriate + */ +class TestRoleAndUserLoader extends EndpointSecurityTestRolesLoader { + constructor( + protected readonly kbnClient: KbnClient, + protected readonly logger: ToolingLog, + private readonly isServerless: boolean + ) { + super(kbnClient, logger); + } + + async load( + name: EndpointSecurityRoleNames | KibanaKnownUserAccounts + ): Promise { + // If its a known system account, then just exit here and use the default `changeme` password + if (KIBANA_KNOWN_DEFAULT_ACCOUNTS[name as KibanaKnownUserAccounts]) { + return { + role: name, + username: name, + password: 'changeme', + }; + } + + if (this.isServerless) { + // If the username is not one that we support in serverless, then throw an error. + if (!SECURITY_SERVERLESS_ROLE_NAMES[name as keyof typeof SECURITY_SERVERLESS_ROLE_NAMES]) { + throw new Error( + `username [${name}] is not valid when running in serverless. Valid values are: ${Object.keys( + SECURITY_SERVERLESS_ROLE_NAMES + ).join(', ')}` + ); + } + + // Roles/users for serverless will be already present in the env, so just return the defaults creds + return { + role: name, + username: name, + password: 'changeme', + }; + } + + return super.load(name as EndpointSecurityRoleNames); + } +} /** * Cypress plugin for adding data loading related `task`s @@ -68,7 +126,8 @@ export const dataLoaders = ( on: Cypress.PluginEvents, config: Cypress.PluginConfigOptions ): void => { - // FIXME: investigate if we can create a `ToolingLog` that writes output to cypress and pass that to the stack services + // Env. variable is set by `cypress_serverless.config.ts` + const isServerless = config.env.IS_SERVERLESS; const stackServicesPromise = createRuntimeServices({ kibanaUrl: config.env.KIBANA_URL, @@ -81,6 +140,12 @@ export const dataLoaders = ( asSuperuser: true, }); + const roleAndUserLoaderPromise: Promise = stackServicesPromise.then( + ({ kbnClient, log }) => { + return new TestRoleAndUserLoader(kbnClient, log, isServerless); + } + ); + on('task', { indexFleetEndpointPolicy: async ({ policyName, @@ -201,6 +266,23 @@ export const dataLoaders = ( const { esClient } = await stackServicesPromise; return deleteAllEndpointData(esClient, endpointAgentIds); }, + + /** + * Loads a user/role into Kibana. Used from `login()` task. + * @param name + */ + loadUserAndRole: async ({ name }: LoadUserAndRoleCyTaskOptions): Promise => { + return (await roleAndUserLoaderPromise).load(name); + }, + + /** + * Creates a new Role/User + */ + createUserAndRole: async ({ + role, + }: CreateUserAndRoleCyTaskOptions): Promise => { + return (await roleAndUserLoaderPromise).create(role); + }, }); }; diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts index 58eba62a72f82..8ac78b508d084 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/tasks/login.ts @@ -5,242 +5,107 @@ * 2.0. */ -import * as yaml from 'js-yaml'; -import type { Role } from '@kbn/security-plugin/common'; import type { LoginState } from '@kbn/security-plugin/common/login_state'; -import { getWithResponseActionsRole } from '../../../../scripts/endpoint/common/roles_users/with_response_actions_role'; -import { getNoResponseActionsRole } from '../../../../scripts/endpoint/common/roles_users/without_response_actions_role'; -import { request } from './common'; -import { getT1Analyst } from '../../../../scripts/endpoint/common/roles_users/t1_analyst'; -import { getT2Analyst } from '../../../../scripts/endpoint/common/roles_users/t2_analyst'; -import { getHunter } from '../../../../scripts/endpoint/common/roles_users/hunter'; -import { getThreatIntelligenceAnalyst } from '../../../../scripts/endpoint/common/roles_users/threat_intelligence_analyst'; -import { getSocManager } from '../../../../scripts/endpoint/common/roles_users/soc_manager'; -import { getPlatformEngineer } from '../../../../scripts/endpoint/common/roles_users/platform_engineer'; -import { getEndpointOperationsAnalyst } from '../../../../scripts/endpoint/common/roles_users/endpoint_operations_analyst'; -import { - getEndpointSecurityPolicyManagementReadRole, - getEndpointSecurityPolicyManager, -} from '../../../../scripts/endpoint/common/roles_users/endpoint_security_policy_manager'; -import { getDetectionsEngineer } from '../../../../scripts/endpoint/common/roles_users/detections_engineer'; - -export enum ROLE { - t1_analyst = 't1Analyst', - t2_analyst = 't2Analyst', - analyst_hunter = 'hunter', - threat_intelligence_analyst = 'threatIntelligenceAnalyst', - detections_engineer = 'detectionsEngineer', - soc_manager = 'socManager', - platform_engineer = 'platformEngineer', - endpoint_operations_analyst = 'endpointOperationsAnalyst', - endpoint_security_policy_manager = 'endpointSecurityPolicyManager', - endpoint_response_actions_access = 'endpointResponseActionsAccess', - endpoint_response_actions_no_access = 'endpointResponseActionsNoAccess', - endpoint_security_policy_management_read = 'endpointSecurityPolicyManagementRead', +import type { Role } from '@kbn/security-plugin/common'; +import { ENDPOINT_SECURITY_ROLE_NAMES } from '../../../../scripts/endpoint/common/roles_users'; +import type { SecurityTestUser } from '../common/constants'; +import { COMMON_API_HEADERS, request } from './common'; + +export const ROLE = Object.freeze>({ + ...ENDPOINT_SECURITY_ROLE_NAMES, + elastic: 'elastic', + elastic_serverless: 'elastic_serverless', + system_indices_superuser: 'system_indices_superuser', +}); + +interface CyLoginTask { + (user?: SecurityTestUser): ReturnType; + + /** + * Login using any username/password + * @param username + * @param password + */ + with(username: string, password: string): ReturnType; + + /** + * Creates the provided role in kibana/ES along with a respective user (same name as role) + * and then login with this new user + * @param role + */ + withCustomRole(role: Role): ReturnType; } -export const rolesMapping: { [key in ROLE]: Omit } = { - t1Analyst: getT1Analyst(), - t2Analyst: getT2Analyst(), - hunter: getHunter(), - threatIntelligenceAnalyst: getThreatIntelligenceAnalyst(), - socManager: getSocManager(), - platformEngineer: getPlatformEngineer(), - endpointOperationsAnalyst: getEndpointOperationsAnalyst(), - endpointSecurityPolicyManager: getEndpointSecurityPolicyManager(), - detectionsEngineer: getDetectionsEngineer(), - endpointResponseActionsAccess: getWithResponseActionsRole(), - endpointResponseActionsNoAccess: getNoResponseActionsRole(), - endpointSecurityPolicyManagementRead: getEndpointSecurityPolicyManagementReadRole(), -}; -/** - * Credentials in the `kibana.dev.yml` config file will be used to authenticate - * with Kibana when credentials are not provided via environment variables - */ -const KIBANA_DEV_YML_PATH = '../../../config/kibana.dev.yml'; - -/** - * The configuration path in `kibana.dev.yml` to the username to be used when - * authenticating with Kibana. - */ -const ELASTICSEARCH_USERNAME_CONFIG_PATH = 'config.elasticsearch.username'; - -/** - * The configuration path in `kibana.dev.yml` to the password to be used when - * authenticating with Kibana. - */ -const ELASTICSEARCH_PASSWORD_CONFIG_PATH = 'config.elasticsearch.password'; - -/** - * The `CYPRESS_ELASTICSEARCH_USERNAME` environment variable specifies the - * username to be used when authenticating with Kibana - */ -const ELASTICSEARCH_USERNAME = 'ELASTICSEARCH_USERNAME'; - /** - * The `CYPRESS_ELASTICSEARCH_PASSWORD` environment variable specifies the - * username to be used when authenticating with Kibana + * Login to Kibana using API (not login page). + * By default, user will be logged in using `KIBANA_USERNAME` and `KIBANA_PASSWORD` retrieved from + * the cypress `env` + * + * @param user */ -const ELASTICSEARCH_PASSWORD = 'ELASTICSEARCH_PASSWORD'; - -const KIBANA_USERNAME = 'KIBANA_USERNAME'; -const KIBANA_PASSWORD = 'KIBANA_PASSWORD'; +export const login: CyLoginTask = ( + // FIXME:PT default user to `soc_manager` + user?: SecurityTestUser +): ReturnType => { + let username = Cypress.env('KIBANA_USERNAME'); + let password = Cypress.env('KIBANA_PASSWORD'); + + if (user) { + return cy.task('loadUserAndRole', { name: user }).then((loadedUser) => { + username = loadedUser.username; + password = loadedUser.password; + + return sendApiLoginRequest(username, password); + }); + } else { + return sendApiLoginRequest(username, password); + } +}; -export const createCustomRoleAndUser = (role: string, rolePrivileges: Omit) => { - // post the role - request({ - method: 'PUT', - url: `/api/security/role/${role}`, - body: rolePrivileges, - }); +login.with = (username: string, password: string): ReturnType => { + return sendApiLoginRequest(username, password); +}; - // post the user associated with the role to elasticsearch - request({ - method: 'POST', - url: `/internal/security/users/${role}`, - body: { - username: role, - password: Cypress.env(ELASTICSEARCH_PASSWORD), - roles: [role], - }, +login.withCustomRole = (role: Role): ReturnType => { + return cy.task('createUserAndRole', { role }).then(({ username, password }) => { + return sendApiLoginRequest(username, password); }); }; -const loginWithUsernameAndPassword = (username: string, password: string) => { +/** + * Send login via API + * @param username + * @param password + * + * @private + */ +const sendApiLoginRequest = ( + username: string, + password: string +): Cypress.Chainable<{ username: string; password: string }> => { const baseUrl = Cypress.config().baseUrl; - if (!baseUrl) { - throw Error(`Cypress config baseUrl not set!`); - } + const loginUrl = `${baseUrl}/internal/security/login`; + const headers = { ...COMMON_API_HEADERS }; + + cy.log(`Authenticating [${username}] via ${loginUrl}`); - // Programmatically authenticate without interacting with the Kibana login page. - const headers = { 'kbn-xsrf': 'cypress-creds' }; - request({ headers, url: `${baseUrl}/internal/security/login_state` }).then( - (loginState) => { + return request({ headers, url: `${baseUrl}/internal/security/login_state` }) + .then((loginState) => { const basicProvider = loginState.body.selector.providers.find( (provider) => provider.type === 'basic' ); + return request({ - url: `${baseUrl}/internal/security/login`, + url: loginUrl, method: 'POST', headers, body: { - providerType: basicProvider.type, - providerName: basicProvider.name, + providerType: basicProvider?.type, + providerName: basicProvider?.name, currentURL: '/', params: { username, password }, }, }); - } - ); -}; - -export const loginWithRole = (role: ROLE) => { - loginWithCustomRole(role, rolesMapping[role]); -}; - -export const loginWithCustomRole = (role: string, rolePrivileges: Omit) => { - createCustomRoleAndUser(role, rolePrivileges); - - cy.log(`origin: ${Cypress.config().baseUrl}`); - - loginWithUsernameAndPassword(role, Cypress.env(ELASTICSEARCH_PASSWORD)); -}; - -/** - * Authenticates with Kibana using, if specified, credentials specified by - * environment variables. The credentials in `kibana.dev.yml` will be used - * for authentication when the environment variables are unset. - * - * To speed the execution of tests, prefer this non-interactive authentication, - * which is faster than authentication via Kibana's interactive login page. - */ -export const login = (role?: ROLE) => { - if (role != null) { - loginWithRole(role); - } else if (credentialsProvidedByEnvironment()) { - loginViaEnvironmentCredentials(); - } else { - loginViaConfig(); - } -}; - -/** - * Returns `true` if the credentials used to login to Kibana are provided - * via environment variables - */ -const credentialsProvidedByEnvironment = (): boolean => - (Cypress.env(KIBANA_USERNAME) != null && Cypress.env(KIBANA_PASSWORD) != null) || - (Cypress.env(ELASTICSEARCH_USERNAME) != null && Cypress.env(ELASTICSEARCH_PASSWORD) != null); - -/** - * Authenticates with Kibana by reading credentials from the - * `CYPRESS_ELASTICSEARCH_USERNAME` and `CYPRESS_ELASTICSEARCH_PASSWORD` - * environment variables, and POSTing the username and password directly to - * Kibana's `/internal/security/login` endpoint, bypassing the login page (for speed). - */ -const loginViaEnvironmentCredentials = () => { - let username: string; - let password: string; - let usernameEnvVar: string; - let passwordEnvVar: string; - - if (Cypress.env(KIBANA_USERNAME) && Cypress.env(KIBANA_PASSWORD)) { - username = Cypress.env(KIBANA_USERNAME); - password = Cypress.env(KIBANA_PASSWORD); - usernameEnvVar = KIBANA_USERNAME; - passwordEnvVar = KIBANA_PASSWORD; - } else { - username = Cypress.env(ELASTICSEARCH_USERNAME); - password = Cypress.env(ELASTICSEARCH_PASSWORD); - usernameEnvVar = ELASTICSEARCH_USERNAME; - passwordEnvVar = ELASTICSEARCH_PASSWORD; - } - - cy.log( - `Authenticating user [${username}] retrieved via environment credentials from the \`CYPRESS_${usernameEnvVar}\` and \`CYPRESS_${passwordEnvVar}\` environment variables` - ); - - loginWithUsernameAndPassword(username, password); -}; - -/** - * Authenticates with Kibana by reading credentials from the - * `kibana.dev.yml` file and POSTing the username and password directly to - * Kibana's `/internal/security/login` endpoint, bypassing the login page (for speed). - */ -const loginViaConfig = () => { - cy.log( - `Authenticating via config credentials \`${ELASTICSEARCH_USERNAME_CONFIG_PATH}\` and \`${ELASTICSEARCH_PASSWORD_CONFIG_PATH}\` from \`${KIBANA_DEV_YML_PATH}\`` - ); - - // read the login details from `kibana.dev.yaml` - cy.readFile(KIBANA_DEV_YML_PATH).then((kibanaDevYml) => { - const config = yaml.safeLoad(kibanaDevYml); - loginWithUsernameAndPassword( - Cypress.env(ELASTICSEARCH_USERNAME), - config.elasticsearch.password - ); - }); -}; - -export const getRoleWithArtifactReadPrivilege = (privilegePrefix: string) => { - const endpointSecurityPolicyManagerRole = getEndpointSecurityPolicyManager(); - - return { - ...endpointSecurityPolicyManagerRole, - kibana: [ - { - ...endpointSecurityPolicyManagerRole.kibana[0], - feature: { - ...endpointSecurityPolicyManagerRole.kibana[0].feature, - siem: [ - ...endpointSecurityPolicyManagerRole.kibana[0].feature.siem.filter( - (privilege) => privilege !== `${privilegePrefix}all` - ), - `${privilegePrefix}read`, - ], - }, - }, - ], - }; + }) + .then(() => ({ username, password })); }; diff --git a/x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts b/x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts deleted file mode 100644 index 533a17663e16b..0000000000000 --- a/x-pack/plugins/security_solution/public/management/cypress/tasks/login_serverless.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { LoginState } from '@kbn/security-plugin/common/login_state'; -import { COMMON_API_HEADERS, request } from './common'; - -export enum ServerlessUser { - T1_ANALYST = 't1_analyst', - T2_ANALYST = 't2_analyst', - T3_ANALYST = 't3_analyst', - THREAT_INTELLIGENCE_ANALYST = 'threat_intelligence_analyst', - RULE_AUTHOR = 'rule_author', - SOC_MANAGER = 'soc_manager', - DETECTIONS_ADMIN = 'detections_admin', - PLATFORM_ENGINEER = 'platform_engineer', - ENDPOINT_OPERATIONS_ANALYST = 'endpoint_operations_analyst', - ENDPOINT_POLICY_MANAGER = 'endpoint_policy_manager', -} - -/** - * Send login via API - * @param username - * @param password - * - * @private - */ -const sendApiLoginRequest = ( - username: string, - password: string -): Cypress.Chainable<{ username: string; password: string }> => { - const baseUrl = Cypress.config().baseUrl; - const headers = { ...COMMON_API_HEADERS }; - - cy.log(`Authenticating [${username}] via ${baseUrl}`); - - return request({ headers, url: `${baseUrl}/internal/security/login_state` }) - .then((loginState) => { - const basicProvider = loginState.body.selector.providers.find( - (provider) => provider.type === 'basic' - ); - - return request({ - url: `${baseUrl}/internal/security/login`, - method: 'POST', - headers, - body: { - providerType: basicProvider?.type, - providerName: basicProvider?.name, - currentURL: '/', - params: { username, password }, - }, - }); - }) - .then(() => ({ username, password })); -}; - -interface CyLoginTask { - (user?: ServerlessUser | 'elastic'): ReturnType; - - /** - * Login using any username/password - * @param username - * @param password - */ - with(username: string, password: string): ReturnType; -} - -/** - * Login to Kibana using API (not login page). By default, user will be logged in using - * the username and password defined via `KIBANA_USERNAME` and `KIBANA_PASSWORD` cypress env - * variables. - * @param user Defaults to `soc_manager` - */ -export const loginServerless: CyLoginTask = ( - user: ServerlessUser | 'elastic' = ServerlessUser.SOC_MANAGER -): ReturnType => { - const username = Cypress.env('KIBANA_USERNAME'); - const password = Cypress.env('KIBANA_PASSWORD'); - - if (user && user !== 'elastic') { - throw new Error('Serverless usernames not yet implemented'); - - // return cy.task('loadUserAndRole', { name: user }).then((loadedUser) => { - // username = loadedUser.username; - // password = loadedUser.password; - // - // return sendApiLoginRequest(username, password); - // }); - } else { - return sendApiLoginRequest(username, password); - } -}; - -loginServerless.with = ( - username: string, - password: string -): ReturnType => { - return sendApiLoginRequest(username, password); -}; diff --git a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json index 4d7fe47ec2d23..94d50ffe7f62a 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json +++ b/x-pack/plugins/security_solution/public/management/cypress/tsconfig.json @@ -32,5 +32,6 @@ "@kbn/test", "@kbn/repo-info", "@kbn/data-views-plugin", + "@kbn/tooling-log", ] } diff --git a/x-pack/plugins/security_solution/public/management/cypress/types.ts b/x-pack/plugins/security_solution/public/management/cypress/types.ts index fecaa33a6a70a..aee97723c7d51 100644 --- a/x-pack/plugins/security_solution/public/management/cypress/types.ts +++ b/x-pack/plugins/security_solution/public/management/cypress/types.ts @@ -7,8 +7,10 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Role } from '@kbn/security-plugin/common'; import type { ActionDetails } from '../../../common/endpoint/types'; import type { CyLoadEndpointDataOptions } from './support/plugin_handlers/endpoint_data_loader'; +import type { SecurityTestUser } from './common/constants'; type PossibleChainable = | Cypress.Chainable @@ -56,3 +58,11 @@ export interface HostActionResponse { state: { state?: 'success' | 'failure' }; }; } + +export interface LoadUserAndRoleCyTaskOptions { + name: SecurityTestUser; +} + +export interface CreateUserAndRoleCyTaskOptions { + role: Role; +} diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts index 1d2b3d5f47784..4a8b2daca2af4 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/constants.ts @@ -10,3 +10,8 @@ export const HORIZONTAL_LINE = '-'.repeat(80); export const ENDPOINT_EVENTS_INDEX = 'logs-endpoint.events.process-default'; export const ENDPOINT_ALERTS_INDEX = 'logs-endpoint.alerts-default'; + +export const COMMON_API_HEADERS = Object.freeze({ + 'kbn-xsrf': 'security-solution', + 'x-elastic-internal-origin': 'security-solution', +}); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts new file mode 100644 index 0000000000000..f8c51d5255018 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/role_and_user_loader.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable max-classes-per-file */ + +import type { KbnClient } from '@kbn/test'; +import type { Role } from '@kbn/security-plugin/common'; +import type { ToolingLog } from '@kbn/tooling-log'; +import { inspect } from 'util'; +import type { AxiosError } from 'axios'; +import type { EndpointSecurityRoleDefinitions } from './roles_users'; +import { getAllEndpointSecurityRoles } from './roles_users'; +import { catchAxiosErrorFormatAndThrow } from './format_axios_error'; +import { COMMON_API_HEADERS } from './constants'; + +const ignoreHttp409Error = (error: AxiosError) => { + if (error?.response?.status === 409) { + return; + } + + throw error; +}; + +export interface LoadedRoleAndUser { + role: string; + username: string; + password: string; +} + +export interface RoleAndUserLoaderInterface = Record> { + /** + * Loads the requested Role into kibana and then creates a user by the same role name that is + * assigned to the given role + * @param name + */ + load(name: keyof R): Promise; + + /** + * Loads all roles/users + */ + loadAll(): Promise>; + + /** + * Creates a new Role in kibana along with a user (by the same name as the Role name) + * that is assigned to the given role + * @param role + */ + create(role: Role): Promise; +} + +/** + * A generic class for loading roles and creating associated user into kibana + */ +export class RoleAndUserLoader = Record> + implements RoleAndUserLoaderInterface +{ + protected readonly logPromiseError: (error: Error) => never; + + constructor( + protected readonly kbnClient: KbnClient, + protected readonly logger: ToolingLog, + protected readonly roles: R + ) { + this.logPromiseError = (error) => { + this.logger.error(inspect(error, { depth: 5 })); + throw error; + }; + } + + async load(name: keyof R): Promise { + const role = this.roles[name]; + + if (!role) { + throw new Error( + `Unknown role/user: [${String(name)}]. Valid values are: [${Object.keys(this.roles).join( + ', ' + )}]` + ); + } + + return this.create(role); + } + + async loadAll(): Promise> { + const response = {} as Record; + + for (const [name, role] of Object.entries(this.roles)) { + response[name as keyof R] = await this.create(role); + } + + return response; + } + + public async create(role: Role): Promise { + const roleName = role.name; + + await this.createRole(role); + await this.createUser(roleName, 'changeme', [roleName]); + + return { + role: roleName, + username: roleName, + password: 'changeme', + }; + } + + protected async createRole(role: Role): Promise { + const { name: roleName, ...roleDefinition } = role; + + this.logger.debug(`creating role:`, roleDefinition); + + await this.kbnClient + .request({ + method: 'PUT', + path: `/api/security/role/${roleName}`, + headers: { + ...COMMON_API_HEADERS, + }, + body: roleDefinition, + }) + .then((response) => { + this.logger.debug(`Role [${roleName}] created/updated`, response?.data); + return response; + }) + .catch(ignoreHttp409Error) + .catch(catchAxiosErrorFormatAndThrow) + .catch(this.logPromiseError); + } + + protected async createUser( + username: string, + password: string, + roles: string[] = [] + ): Promise { + const user = { + username, + password, + roles, + full_name: username, + email: '', + }; + + this.logger.debug(`creating user:`, user); + + await this.kbnClient + .request({ + method: 'POST', + path: `/internal/security/users/${username}`, + headers: { + ...COMMON_API_HEADERS, + }, + body: user, + }) + .then((response) => { + this.logger.debug(`User [${username}] created/updated`, response?.data); + return response; + }) + .catch(ignoreHttp409Error) + .catch(catchAxiosErrorFormatAndThrow) + .catch(this.logPromiseError); + } +} + +/** + * Role and user loader for Endpoint security dev/testing + */ +export class EndpointSecurityTestRolesLoader extends RoleAndUserLoader { + constructor(protected readonly kbnClient: KbnClient, protected readonly logger: ToolingLog) { + super(kbnClient, logger, getAllEndpointSecurityRoles()); + } +} diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts new file mode 100644 index 0000000000000..b035f55bf1589 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Role } from '@kbn/security-plugin/common'; +import { getRuleAuthor } from './rule_author'; +import { getT3Analyst } from './t3_analyst'; +import { getT1Analyst } from './t1_analyst'; +import { getT2Analyst } from './t2_analyst'; +import { getHunter } from './hunter'; +import { getThreatIntelligenceAnalyst } from './threat_intelligence_analyst'; +import { getSocManager } from './soc_manager'; +import { getPlatformEngineer } from './platform_engineer'; +import { getEndpointOperationsAnalyst } from './endpoint_operations_analyst'; +import { + getEndpointSecurityPolicyManagementReadRole, + getEndpointSecurityPolicyManager, +} from './endpoint_security_policy_manager'; +import { getDetectionsEngineer } from './detections_engineer'; +import { getWithResponseActionsRole } from './with_response_actions_role'; +import { getNoResponseActionsRole } from './without_response_actions_role'; + +export * from './with_response_actions_role'; +export * from './without_response_actions_role'; +export * from './t1_analyst'; +export * from './t2_analyst'; +export * from './t3_analyst'; +export * from './hunter'; +export * from './threat_intelligence_analyst'; +export * from './soc_manager'; +export * from './platform_engineer'; +export * from './endpoint_operations_analyst'; +export * from './endpoint_security_policy_manager'; +export * from './detections_engineer'; + +export type EndpointSecurityRoleNames = keyof typeof ENDPOINT_SECURITY_ROLE_NAMES; + +export type EndpointSecurityRoleDefinitions = Record; + +/** + * Security Solution set of roles that are loaded and used in serverless deployments. + * The source of these role definitions is under `project-controller` at: + * + * @see https://github.com/elastic/project-controller/blob/main/internal/project/security/config/roles.yml + * + * The role definition spreadsheet can be found here: + * + * @see https://docs.google.com/spreadsheets/d/16aGow187AunLCBFZLlbVyS81iQNuMpNxd96LOerWj4c/edit#gid=1936689222 + */ +export const SECURITY_SERVERLESS_ROLE_NAMES = Object.freeze({ + t1_analyst: 't1_analyst', + t2_analyst: 't2_analyst', + t3_analyst: 't3_analyst', + threat_intelligence_analyst: 'threat_intelligence_analyst', + rule_author: 'rule_author', + soc_manager: 'soc_manager', + detections_admin: 'detections_admin', + platform_engineer: 'platform_engineer', + endpoint_operations_analyst: 'endpoint_operations_analyst', + endpoint_policy_manager: 'endpoint_policy_manager', +}); + +export const ENDPOINT_SECURITY_ROLE_NAMES = Object.freeze({ + // -------------------------------------- + // Set of roles used in serverless + ...SECURITY_SERVERLESS_ROLE_NAMES, + + // -------------------------------------- + // Other roles used for testing + hunter: 'hunter', + endpoint_response_actions_access: 'endpoint_response_actions_access', + endpoint_response_actions_no_access: 'endpoint_response_actions_no_access', + endpoint_security_policy_management_read: 'endpoint_security_policy_management_read', +}); + +export const getAllEndpointSecurityRoles = (): EndpointSecurityRoleDefinitions => { + return { + t1_analyst: { + ...getT1Analyst(), + name: 't1_analyst', + }, + t2_analyst: { + ...getT2Analyst(), + name: 't2_analyst', + }, + t3_analyst: { + ...getT3Analyst(), + name: 't3_analyst', + }, + threat_intelligence_analyst: { + ...getThreatIntelligenceAnalyst(), + name: 'threat_intelligence_analyst', + }, + rule_author: { + ...getRuleAuthor(), + name: 'rule_author', + }, + soc_manager: { + ...getSocManager(), + name: 'soc_manager', + }, + detections_admin: { + ...getDetectionsEngineer(), + name: 'detections_admin', + }, + platform_engineer: { + ...getPlatformEngineer(), + name: 'platform_engineer', + }, + endpoint_operations_analyst: { + ...getEndpointOperationsAnalyst(), + name: 'endpoint_operations_analyst', + }, + endpoint_policy_manager: { + ...getEndpointSecurityPolicyManager(), + name: 'endpoint_policy_manager', + }, + + hunter: { + ...getHunter(), + name: 'hunter', + }, + endpoint_response_actions_access: { + ...getWithResponseActionsRole(), + name: 'endpoint_response_actions_access', + }, + endpoint_response_actions_no_access: { + ...getNoResponseActionsRole(), + name: 'endpoint_response_actions_no_access', + }, + endpoint_security_policy_management_read: { + ...getEndpointSecurityPolicyManagementReadRole(), + name: 'endpoint_security_policy_management_read', + }, + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts new file mode 100644 index 0000000000000..f957fe8947c5d --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/rule_author.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Role } from '@kbn/security-plugin/common'; +import { getNoResponseActionsRole } from './without_response_actions_role'; + +export const getRuleAuthor: () => Omit = () => { + const noResponseActionsRole = getNoResponseActionsRole(); + return { + ...noResponseActionsRole, + kibana: [ + { + ...noResponseActionsRole.kibana[0], + feature: { + ...noResponseActionsRole.kibana[0].feature, + siem: [ + 'all', + 'read_alerts', + 'crud_alerts', + 'policy_management_all', + 'endpoint_list_all', + 'trusted_applications_all', + 'event_filters_all', + 'host_isolation_exceptions_read', + 'blocklist_all', + 'actions_log_management_read', + ], + }, + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts new file mode 100644 index 0000000000000..304c4e6d744ee --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/t3_analyst.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Role } from '@kbn/security-plugin/common'; +import { getNoResponseActionsRole } from './without_response_actions_role'; + +export const getT3Analyst: () => Omit = () => { + const noResponseActionsRole = getNoResponseActionsRole(); + return { + ...noResponseActionsRole, + kibana: [ + { + ...noResponseActionsRole.kibana[0], + feature: { + ...noResponseActionsRole.kibana[0].feature, + siem: [ + 'all', + 'read_alerts', + 'crud_alerts', + 'endpoint_list_all', + 'trusted_applications_all', + 'event_filters_all', + 'host_isolation_exceptions_all', + 'blocklist_all', + 'policy_management_read', + 'host_isolation_all', + 'process_operations_all', + 'actions_log_management_all', + 'file_operations_all', + ], + }, + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index c1c38dcf8b30a..330bcf4589a74 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -14,36 +14,13 @@ import { CA_CERT_PATH } from '@kbn/dev-utils'; import { ToolingLog } from '@kbn/tooling-log'; import type { KbnClientOptions } from '@kbn/test'; import { KbnClient } from '@kbn/test'; -import type { Role } from '@kbn/security-plugin/common'; +import { EndpointSecurityTestRolesLoader } from './common/role_and_user_loader'; import { METADATA_DATASTREAM } from '../../common/endpoint/constants'; import { EndpointMetadataGenerator } from '../../common/endpoint/data_generators/endpoint_metadata_generator'; import { indexHostsAndAlerts } from '../../common/endpoint/index_data'; import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data'; import { fetchStackVersion, isServerlessKibanaFlavor } from './common/stack_services'; import { ENDPOINT_ALERTS_INDEX, ENDPOINT_EVENTS_INDEX } from './common/constants'; -import { getWithResponseActionsRole } from './common/roles_users/with_response_actions_role'; -import { getNoResponseActionsRole } from './common/roles_users/without_response_actions_role'; -import { getT1Analyst } from './common/roles_users/t1_analyst'; -import { getT2Analyst } from './common/roles_users/t2_analyst'; -import { getEndpointOperationsAnalyst } from './common/roles_users/endpoint_operations_analyst'; -import { getEndpointSecurityPolicyManager } from './common/roles_users/endpoint_security_policy_manager'; -import { getHunter } from './common/roles_users/hunter'; -import { getPlatformEngineer } from './common/roles_users/platform_engineer'; -import { getSocManager } from './common/roles_users/soc_manager'; -import { getThreatIntelligenceAnalyst } from './common/roles_users/threat_intelligence_analyst'; - -const rolesMapping: { [id: string]: Omit } = { - t1Analyst: getT1Analyst(), - t2Analyst: getT2Analyst(), - hunter: getHunter(), - threatIntelligenceAnalyst: getThreatIntelligenceAnalyst(), - socManager: getSocManager(), - platformEngineer: getPlatformEngineer(), - endpointOperationsAnalyst: getEndpointOperationsAnalyst(), - endpointSecurityPolicyManager: getEndpointSecurityPolicyManager(), - withResponseActionsRole: getWithResponseActionsRole(), - noResponseActionsRole: getNoResponseActionsRole(), -}; main(); @@ -67,31 +44,6 @@ async function deleteIndices(indices: string[], client: Client) { } } -async function addRole(kbnClient: KbnClient, role: Role): Promise { - if (!role) { - console.log('No role data given'); - return; - } - - const { name, ...permissions } = role; - const path = `/api/security/role/${name}?createOnly=true`; - - // add role if doesn't exist already - try { - console.log(`Adding ${name} role`); - await kbnClient.request({ - method: 'PUT', - path, - body: permissions, - }); - - return name; - } catch (error) { - console.log(error); - handleErr(error); - } -} - interface UserInfo { username: string; password: string; @@ -422,19 +374,7 @@ async function main() { throw new Error(`Can not use '--rbacUser' option against serverless deployment`); } - // Add roles and users with response actions kibana privileges - for (const role of Object.keys(rolesMapping)) { - const addedRole = await addRole(kbnClient, { - name: role, - ...rolesMapping[role], - }); - if (addedRole) { - logger.info(`Successfully added ${role} role`); - await addUser(client, { username: role, password: 'changeme', roles: [role] }); - } else { - logger.warning(`Failed to add role, ${role}`); - } - } + await loadRbacTestUsers(kbnClient, logger); } const seed = argv.seed || Math.random().toString(); @@ -499,3 +439,12 @@ async function main() { logger.info(`Creating and indexing documents took: ${new Date().getTime() - startTime}ms`); } + +const loadRbacTestUsers = async (kbnClient: KbnClient, logger: ToolingLog): Promise => { + const loadedRoles = await new EndpointSecurityTestRolesLoader(kbnClient, logger).loadAll(); + + logger.info(`Roles and associated users loaded. Login accounts: + ${Object.values(loadedRoles) + .map(({ username, password }) => `${username} / ${password}`) + .join('\n ')}`); +}; diff --git a/x-pack/plugins/transform/common/constants.ts b/x-pack/plugins/transform/common/constants.ts index a1ba2d8277af9..ec26333a834a7 100644 --- a/x-pack/plugins/transform/common/constants.ts +++ b/x-pack/plugins/transform/common/constants.ts @@ -43,6 +43,7 @@ export const TRANSFORM_REACT_QUERY_KEYS = { GET_TRANSFORM_NODES: 'transform.get_transform_nodes', GET_TRANSFORM_AUDIT_MESSAGES: 'transform.get_transform_audit_messages', GET_TRANSFORM_STATS: 'transform.get_transform_stats', + GET_TRANSFORMS_STATS: 'transform.get_transforms_stats', GET_TRANSFORMS: 'transform.get_transforms', GET_TRANSFORMS_PREVIEW: 'transform.get_transforms_preview', } as const; diff --git a/x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts b/x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts new file mode 100644 index 0000000000000..9cfe3ccc38ee4 --- /dev/null +++ b/x-pack/plugins/transform/common/utils/create_stats_unknown_message.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export function createNoStatsTooltipMessage({ + actionName, + count = 1, +}: { + actionName: string; + count: number; +}) { + return i18n.translate('xpack.transform.transformList.actionDisabledNoStatsTooltipMessage', { + defaultMessage: + '{actionName} is disabled because the status for {count, plural, one {this transform} other {some transforms}} is unavailable.', + values: { actionName, count }, + }); +} diff --git a/x-pack/plugins/transform/public/app/common/transform_list.ts b/x-pack/plugins/transform/public/app/common/transform_list.ts index c8ddd32f6a8ff..e588ed917958f 100644 --- a/x-pack/plugins/transform/public/app/common/transform_list.ts +++ b/x-pack/plugins/transform/public/app/common/transform_list.ts @@ -20,10 +20,24 @@ export interface TransformListRow { id: TransformId; config: TransformConfigUnion; mode?: string; // added property on client side to allow filtering by this field - stats: TransformStats; + stats?: TransformStats; alerting_rules?: TransformHealthAlertRule[]; } +export type TransformListRowWithStats = TransformListRow & { + stats: TransformStats; +}; + +export function isTransformListRowWithStats( + arg: TransformListRow +): arg is TransformListRowWithStats { + return arg.stats !== undefined; +} + +export function missingTransformStats(items: TransformListRow[]) { + return items.some((i: TransformListRow) => !isTransformListRowWithStats(i)); +} + // The single Action type is not exported as is // from EUI so we use that code to get the single // Action type from the array of actions. diff --git a/x-pack/plugins/transform/public/app/common/transform_stats.test.ts b/x-pack/plugins/transform/public/app/common/transform_stats.test.ts index 974764475eb5f..9213adba45e4d 100644 --- a/x-pack/plugins/transform/public/app/common/transform_stats.test.ts +++ b/x-pack/plugins/transform/public/app/common/transform_stats.test.ts @@ -18,7 +18,7 @@ const getRow = (statsId: string) => { // @ts-expect-error mock data does not actually match TransformListRow type ...(mockTransformListRow as TransformListRow), stats: { - ...(mockTransformStats.transforms as Array).find( + ...(mockTransformStats.transforms as Array>).find( (stats) => stats.id === statsId )!, }, @@ -47,7 +47,7 @@ describe('Transform: Transform stats.', () => { // that will be used by isCompletedBatchTransform() // followed by a call to isCompletedBatchTransform() itself // @ts-expect-error mock data is too loosely typed - const row = mockTransformListRow as TransformListRow; + const row = mockTransformListRow as Required; expect(row.stats.checkpointing.last.checkpoint === 1).toBe(true); expect(row.config.sync === undefined).toBe(true); expect(row.stats.state === TRANSFORM_STATE.STOPPED).toBe(true); diff --git a/x-pack/plugins/transform/public/app/common/transform_stats.ts b/x-pack/plugins/transform/public/app/common/transform_stats.ts index 7763e0712249e..0f7ad5ae98c3f 100644 --- a/x-pack/plugins/transform/public/app/common/transform_stats.ts +++ b/x-pack/plugins/transform/public/app/common/transform_stats.ts @@ -33,7 +33,9 @@ export function isCompletedBatchTransform(item: TransformItem) { // If `checkpoint=1`, `sync` is missing from the config and state is stopped, // then this is a completed batch transform. return ( - item.stats.checkpointing.last.checkpoint === 1 && + item.stats && + item.config && + item.stats.checkpointing?.last.checkpoint === 1 && item.config.sync === undefined && item.stats.state === TRANSFORM_STATE.STOPPED ); diff --git a/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts b/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts index d2b9d32f25853..3c6b329dd880b 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_get_transform_stats.ts @@ -35,3 +35,24 @@ export const useGetTransformStats = ( { enabled, refetchInterval } ); }; + +export const useGetTransformsStats = ({ + enabled, + refetchInterval, +}: { + enabled?: boolean; + refetchInterval?: number | false; +}) => { + const { http } = useAppDependencies(); + + return useQuery( + [TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORMS_STATS], + ({ signal }) => + http.get(addInternalBasePath(`transforms/_stats`), { + version: '1', + asSystemRequest: true, + signal, + }), + { enabled, refetchInterval } + ); +}; diff --git a/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts b/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts index f74f7a5774ded..f4afb03c4e4c0 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_get_transforms.ts @@ -11,14 +11,12 @@ import type { IHttpFetchError } from '@kbn/core-http-browser'; import { isDefined } from '@kbn/ml-is-defined'; import type { GetTransformsResponseSchema } from '../../../common/api_schemas/transforms'; -import type { GetTransformsStatsResponseSchema } from '../../../common/api_schemas/transforms_stats'; import { addInternalBasePath, DEFAULT_REFRESH_INTERVAL_MS, TRANSFORM_REACT_QUERY_KEYS, TRANSFORM_MODE, } from '../../../common/constants'; -import { isTransformStats } from '../../../common/types/transform_stats'; import { type TransformListRow } from '../common'; import { useAppDependencies } from '../app_dependencies'; @@ -55,14 +53,6 @@ export const useGetTransforms = ({ enabled }: UseGetTransformsOptions = {}) => { signal, } ); - const transformStats = await http.get( - addInternalBasePath(`transforms/_stats`), - { - version: '1', - asSystemRequest: true, - signal, - } - ); // There might be some errors with fetching certain transforms // For example, when task exists and is running but the config is deleted @@ -81,21 +71,13 @@ export const useGetTransforms = ({ enabled }: UseGetTransformsOptions = {}) => { } update.transforms = transformConfigs.transforms.reduce((reducedtableRows, config) => { - const stats = transformStats.transforms.find((d) => config.id === d.id); - - // A newly created transform might not have corresponding stats yet. - // If that's the case we just skip the transform and don't add it to the transform list yet. - if (!isTransformStats(stats)) { - return reducedtableRows; - } - - // Table with expandable rows requires `id` on the outer most level + // Table with expandable rows requires `id` on the outermost level reducedtableRows.push({ id: config.id, config, mode: typeof config.sync !== 'undefined' ? TRANSFORM_MODE.CONTINUOUS : TRANSFORM_MODE.BATCH, - stats, + stats: undefined, alerting_rules: config.alerting_rules, }); return reducedtableRows; diff --git a/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts b/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts index 651886ba76f7b..3098946157305 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_refresh_transform_list.ts @@ -15,6 +15,7 @@ export const useRefreshTransformList = () => { return () => { queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORM_NODES]); queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORMS]); + queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORMS_STATS]); queryClient.invalidateQueries([TRANSFORM_REACT_QUERY_KEYS.GET_TRANSFORM_AUDIT_MESSAGES]); }; }; diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx index 37daeeb75e138..bec2407ba631c 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.test.tsx @@ -16,9 +16,11 @@ jest.mock('../../../../app_dependencies'); describe('Transform: Transform List Actions ', () => { test('Minimal initialization', () => { const props: DeleteActionNameProps = { + items: [], canDeleteTransform: true, disabled: false, isBulkAction: false, + forceDisable: false, }; const { container } = render(); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx index 93e346e6c7a1d..7f68645cb2f18 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/delete_action_name.tsx @@ -11,10 +11,12 @@ import { i18n } from '@kbn/i18n'; import { EuiToolTip } from '@elastic/eui'; -import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants'; -import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message'; - +import { missingTransformStats } from '../../../../common/transform_list'; +import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message'; +import { TransformCapabilities } from '../../../../../../common/types/capabilities'; import { TransformListRow } from '../../../../common'; +import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message'; +import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants'; export const deleteActionNameText = i18n.translate( 'xpack.transform.transformList.deleteActionNameText', @@ -24,45 +26,67 @@ export const deleteActionNameText = i18n.translate( ); const transformCanNotBeDeleted = (i: TransformListRow) => + i.stats && !([TRANSFORM_STATE.STOPPED, TRANSFORM_STATE.FAILED] as TransformState[]).includes(i.stats.state); export const isDeleteActionDisabled = (items: TransformListRow[], forceDisable: boolean) => { const disabled = items.some(transformCanNotBeDeleted); - return forceDisable === true || disabled; + + return forceDisable === true || disabled || missingTransformStats(items); }; export interface DeleteActionNameProps { + items: TransformListRow[]; canDeleteTransform: boolean; disabled: boolean; isBulkAction: boolean; + forceDisable: boolean; } +export const getDeleteActionDisabledMessage = ({ + items, + canDeleteTransform, + forceDisable, +}: { + items: TransformListRow[]; + canDeleteTransform: TransformCapabilities['canDeleteTransform']; + forceDisable: boolean; +}) => { + const isBulkAction = items.length > 1; + + if (missingTransformStats(items)) { + return createNoStatsTooltipMessage({ + actionName: deleteActionNameText, + count: items.length, + }); + } + + if (!canDeleteTransform) { + return createCapabilityFailureMessage('canDeleteTransform'); + } + + const disabled = items.some(transformCanNotBeDeleted); + + if (disabled) { + return isBulkAction === true + ? i18n.translate('xpack.transform.transformList.deleteBulkActionDisabledToolTipContent', { + defaultMessage: 'One or more selected transforms must be stopped in order to be deleted.', + }) + : i18n.translate('xpack.transform.transformList.deleteActionDisabledToolTipContent', { + defaultMessage: 'Stop the transform in order to delete it.', + }); + } +}; + export const DeleteActionName: FC = ({ + items, canDeleteTransform, disabled, isBulkAction, + forceDisable, }) => { - const bulkDeleteButtonDisabledText = i18n.translate( - 'xpack.transform.transformList.deleteBulkActionDisabledToolTipContent', - { - defaultMessage: 'One or more selected transforms must be stopped in order to be deleted.', - } - ); - const deleteButtonDisabledText = i18n.translate( - 'xpack.transform.transformList.deleteActionDisabledToolTipContent', - { - defaultMessage: 'Stop the transform in order to delete it.', - } - ); - - if (disabled || !canDeleteTransform) { - let content; - if (disabled) { - content = isBulkAction ? bulkDeleteButtonDisabledText : deleteButtonDisabledText; - } else { - content = createCapabilityFailureMessage('canDeleteTransform'); - } - + const content = getDeleteActionDisabledMessage({ items, canDeleteTransform, forceDisable }); + if (content) { return ( <>{deleteActionNameText} diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx index 5996e271604e3..1942a9a9dc8b7 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_delete/use_delete_action.tsx @@ -7,9 +7,10 @@ import React, { useMemo, useState } from 'react'; +import { isTransformListRowWithStats } from '../../../../common/transform_list'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; -import { TransformListAction, TransformListRow } from '../../../../common'; +import type { TransformListAction, TransformListRow } from '../../../../common'; import { useDeleteIndexAndTargetIndex, useDeleteTransforms, @@ -33,7 +34,7 @@ export const useDeleteAction = (forceDisable: boolean) => { const isBulkAction = items.length > 1; const shouldForceDelete = useMemo( - () => items.some((i: TransformListRow) => i.stats.state === TRANSFORM_STATE.FAILED), + () => items.some((i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.FAILED), [items] ); @@ -59,10 +60,10 @@ export const useDeleteAction = (forceDisable: boolean) => { // else, force delete only when the item user picks has failed const forceDelete = isBulkAction ? shouldForceDelete - : items[0] && items[0] && items[0].stats.state === TRANSFORM_STATE.FAILED; + : items[0] && items[0] && items[0].stats?.state === TRANSFORM_STATE.FAILED; deleteTransforms({ - transformsInfo: items.map((i) => ({ + transformsInfo: items.filter(isTransformListRowWithStats).map((i) => ({ id: i.config.id, state: i.stats.state, })), @@ -87,11 +88,15 @@ export const useDeleteAction = (forceDisable: boolean) => { canDeleteTransform, disabled: isDeleteActionDisabled([item], forceDisable), isBulkAction: false, + items: [item], + forceDisable, }} /> ), enabled: (item: TransformListRow) => - !isDeleteActionDisabled([item], forceDisable) && canDeleteTransform, + isTransformListRowWithStats(item) && + !isDeleteActionDisabled([item], forceDisable) && + canDeleteTransform, description: deleteActionNameText, icon: 'trash', type: 'icon', diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx index 4fb0f9e655208..05d804cc9147b 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.test.tsx @@ -11,16 +11,16 @@ import { __IntlProvider as IntlProvider } from '@kbn/i18n-react'; import { render, waitFor, screen } from '@testing-library/react'; -import { TransformListRow } from '../../../../common'; import { isDiscoverActionDisabled, DiscoverActionName } from './discover_action_name'; import transformListRow from '../../../../common/__mocks__/transform_list_row.json'; +import { TransformListRowWithStats } from '../../../../common/transform_list'; jest.mock('../../../../../shared_imports'); jest.mock('../../../../app_dependencies'); // @ts-expect-error mock data is too loosely typed -const item: TransformListRow = transformListRow; +const item: TransformListRowWithStats = transformListRow; describe('Transform: Transform List Actions isDiscoverActionDisabled()', () => { it('should be disabled when more than one item is passed in', () => { diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx index c3d359e648c28..b164dbee25fd2 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_discover/discover_action_name.tsx @@ -33,7 +33,7 @@ export const isDiscoverActionDisabled = ( const item = items[0]; // Disable discover action if it's a batch transform and was never started - const stoppedTransform = item.stats.state === TRANSFORM_STATE.STOPPED; + const stoppedTransform = item.stats?.state === TRANSFORM_STATE.STOPPED; const transformProgress = getTransformProgress(item); const isBatchTransform = typeof item.config.sync === 'undefined'; const transformNeverStarted = @@ -52,7 +52,7 @@ export const DiscoverActionName: FC = ({ dataViewExists const item = items[0]; // Disable discover action if it's a batch transform and was never started - const stoppedTransform = item.stats.state === TRANSFORM_STATE.STOPPED; + const stoppedTransform = item.stats?.state === TRANSFORM_STATE.STOPPED; const transformProgress = getTransformProgress(item); const isBatchTransform = typeof item.config.sync === 'undefined'; const transformNeverStarted = diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx index ee883a3e7e77b..c0abb4fa5e51c 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/reauthorize_action_name.tsx @@ -49,7 +49,6 @@ export const ReauthorizeActionName: FC = ({ }) => { const { canStartStopTransform } = useTransformCapabilities(); - // Disable start for batch transforms which have completed. const someNeedsReauthorization = items.some(needsReauthorization); const actionIsDisabled = isReauthorizeActionDisabled( diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx index 67e618765e42e..3d216af23db8a 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reauthorize/use_reauthorize_action.tsx @@ -7,6 +7,7 @@ import React, { useMemo, useState } from 'react'; +import { isTransformListRowWithStats } from '../../../../common/transform_list'; import { sortTransformsToReauthorize } from './sort_transforms_to_reauthorize'; import { needsReauthorization } from '../../../../common/reauthorization_utils'; import { useReauthorizeTransforms } from '../../../../hooks/use_reauthorize_transform'; @@ -54,6 +55,7 @@ export const useReauthorizeAction = (forceDisable: boolean, transformNodes: numb ), available: (item: TransformListRow) => needsReauthorization(item), enabled: (item: TransformListRow) => + isTransformListRowWithStats(item) && !isReauthorizeActionDisabled([item], canStartStopTransform, transformNodes), description: reauthorizeActionNameText, icon: 'alert', diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx index 6d5f56d3e7297..0f19701fb0914 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/reset_action_name.tsx @@ -11,6 +11,9 @@ import { EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message'; +import { missingTransformStats } from '../../../../common/transform_list'; +import { TransformCapabilities } from '../../../../../../common/types/capabilities'; import { TransformState, TRANSFORM_STATE } from '../../../../../../common/constants'; import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message'; @@ -24,45 +27,74 @@ export const resetActionNameText = i18n.translate( ); const transformCanNotBeReseted = (i: TransformListRow) => + i.stats && !([TRANSFORM_STATE.STOPPED, TRANSFORM_STATE.FAILED] as TransformState[]).includes(i.stats.state); export const isResetActionDisabled = (items: TransformListRow[], forceDisable: boolean) => { const disabled = items.some(transformCanNotBeReseted); - return forceDisable === true || disabled; + + return forceDisable === true || disabled || missingTransformStats(items); }; +export const getResetActionDisabledMessage = ({ + items, + canResetTransform, + forceDisable, +}: { + items: TransformListRow[]; + canResetTransform: TransformCapabilities['canResetTransform']; + forceDisable: boolean; +}) => { + const isBulkAction = items.length > 1; + + if (missingTransformStats(items)) { + return createNoStatsTooltipMessage({ + actionName: resetActionNameText, + count: items.length, + }); + } + + if (!canResetTransform) { + return createCapabilityFailureMessage('canResetTransform'); + } + + if (isResetActionDisabled(items, forceDisable)) { + const bulkResetButtonDisabledText = i18n.translate( + 'xpack.transform.transformList.resetBulkActionDisabledToolTipContent', + { + defaultMessage: 'One or more selected transforms must be stopped to be reset.', + } + ); + const resetButtonDisabledText = i18n.translate( + 'xpack.transform.transformList.resetActionDisabledToolTipContent', + { + defaultMessage: 'Stop the transform in order to reset it.', + } + ); + + return isBulkAction ? bulkResetButtonDisabledText : resetButtonDisabledText; + } +}; export interface ResetActionNameProps { + items: TransformListRow[]; + canResetTransform: boolean; disabled: boolean; isBulkAction: boolean; } export const ResetActionName: FC = ({ + items, canResetTransform, disabled, - isBulkAction, }) => { - const bulkResetButtonDisabledText = i18n.translate( - 'xpack.transform.transformList.resetBulkActionDisabledToolTipContent', - { - defaultMessage: 'One or more selected transforms must be stopped to be reset.', - } - ); - const resetButtonDisabledText = i18n.translate( - 'xpack.transform.transformList.resetActionDisabledToolTipContent', - { - defaultMessage: 'Stop the transform in order to reset it.', - } - ); - - if (disabled || !canResetTransform) { - let content; - if (disabled) { - content = isBulkAction ? bulkResetButtonDisabledText : resetButtonDisabledText; - } else { - content = createCapabilityFailureMessage('canResetTransform'); - } + const content = getResetActionDisabledMessage({ + items, + canResetTransform, + forceDisable: disabled, + }); + if (content) { return ( <>{resetActionNameText} diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx index 23a399fdb9086..b00ea69d7e4fd 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_reset/use_reset_action.tsx @@ -7,12 +7,21 @@ import React, { useMemo, useState } from 'react'; +import { + isTransformListRowWithStats, + TransformListRowWithStats, +} from '../../../../common/transform_list'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { TransformListAction, TransformListRow } from '../../../../common'; import { useTransformCapabilities, useResetTransforms } from '../../../../hooks'; -import { resetActionNameText, isResetActionDisabled, ResetActionName } from './reset_action_name'; +import { + resetActionNameText, + isResetActionDisabled, + ResetActionName, + getResetActionDisabledMessage, +} from './reset_action_name'; export type ResetAction = ReturnType; export const useResetAction = (forceDisable: boolean) => { @@ -24,7 +33,7 @@ export const useResetAction = (forceDisable: boolean) => { const [items, setItems] = useState([]); const shouldForceReset = useMemo( - () => items.some((i: TransformListRow) => i.stats.state === TRANSFORM_STATE.FAILED), + () => items.some((i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.FAILED), [items] ); @@ -34,10 +43,12 @@ export const useResetAction = (forceDisable: boolean) => { setModalVisible(false); resetTransforms({ - transformsInfo: items.map((i) => ({ - id: i.config.id, - state: i.stats.state, - })), + transformsInfo: items + .filter(isTransformListRowWithStats) + .map((i) => ({ + id: i.config.id, + state: i.stats.state, + })), }); }; @@ -56,11 +67,13 @@ export const useResetAction = (forceDisable: boolean) => { canResetTransform, disabled: isResetActionDisabled([item], forceDisable), isBulkAction: false, + items: [item], }} /> ), enabled: (item: TransformListRow) => - !isResetActionDisabled([item], forceDisable) && canResetTransform, + getResetActionDisabledMessage({ items: [item], canResetTransform, forceDisable }) === + undefined, description: resetActionNameText, icon: 'refresh', type: 'icon', diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx index 0c3be1cdad70b..5b4ff6908afdf 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/schedule_now_action_name.tsx @@ -11,6 +11,8 @@ import { EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { missingTransformStats } from '../../../../common/transform_list'; +import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message'; import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message'; import { useTransformCapabilities } from '../../../../hooks'; @@ -35,7 +37,8 @@ export const isScheduleNowActionDisabled = ( !canScheduleNowTransform || completedBatchTransform || items.length === 0 || - transformNodes === 0 + transformNodes === 0 || + missingTransformStats(items) ); }; @@ -92,6 +95,11 @@ export const ScheduleNowActionName: FC = ({ content = createCapabilityFailureMessage('canScheduleNowTransform'); } else if (completedBatchTransform) { content = completedBatchTransformMessage; + } else if (missingTransformStats(items)) { + content = createNoStatsTooltipMessage({ + actionName: scheduleNowActionNameText, + count: items.length, + }); } } diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx index a13d3da89f677..e5d330eb9ba28 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_schedule_now/use_schedule_now_action.tsx @@ -7,6 +7,7 @@ import React, { useMemo } from 'react'; +import { isTransformListRowWithStats } from '../../../../common/transform_list'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { useTransformCapabilities } from '../../../../hooks'; @@ -33,8 +34,10 @@ export const useScheduleNowAction = (forceDisable: boolean, transformNodes: numb transformNodes={transformNodes} /> ), - available: (item: TransformListRow) => item.stats.state === TRANSFORM_STATE.STARTED, + available: (item: TransformListRow) => + isTransformListRowWithStats(item) ? item.stats.state === TRANSFORM_STATE.STARTED : true, enabled: (item: TransformListRow) => + isTransformListRowWithStats(item) && !isScheduleNowActionDisabled([item], canScheduleNowTransform, transformNodes), description: scheduleNowActionNameText, icon: 'play', diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx index c50c83d25edc5..22ddbf1d12c3d 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/start_action_name.tsx @@ -9,6 +9,8 @@ import React, { type FC } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiToolTip } from '@elastic/eui'; +import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message'; +import { missingTransformStats } from '../../../../common/transform_list'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message'; @@ -31,7 +33,7 @@ export const isStartActionDisabled = ( const completedBatchTransform = items.some((i: TransformListRow) => isCompletedBatchTransform(i)); // Disable start action if one of the transforms is already started or trying to restart will throw error const startedTransform = items.some( - (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STARTED + (i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.STARTED ); return ( @@ -39,7 +41,8 @@ export const isStartActionDisabled = ( completedBatchTransform || startedTransform || items.length === 0 || - transformNodes === 0 + transformNodes === 0 || + missingTransformStats(items) ); }; @@ -58,9 +61,9 @@ export const StartActionName: FC = ({ // Disable start for batch transforms which have completed. const completedBatchTransform = items.some((i: TransformListRow) => isCompletedBatchTransform(i)); - // Disable start action if one of the transforms is already started or trying to restart will throw error + // Disable if one of the transforms is already started or trying to restart will throw error const startedTransform = items.some( - (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STARTED + (i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.STARTED ); let startedTransformMessage; @@ -107,6 +110,11 @@ export const StartActionName: FC = ({ content = completedBatchTransformMessage; } else if (startedTransform) { content = startedTransformMessage; + } else if (missingTransformStats(items)) { + content = createNoStatsTooltipMessage({ + actionName: startActionNameText, + count: items.length, + }); } } diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx index 168174730b706..3724e53cf69b8 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_start/use_start_action.tsx @@ -7,6 +7,7 @@ import React, { useMemo, useState } from 'react'; +import { isTransformListRowWithStats } from '../../../../common/transform_list'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { TransformListAction, TransformListRow } from '../../../../common'; @@ -46,7 +47,8 @@ export const useStartAction = (forceDisable: boolean, transformNodes: number) => transformNodes={transformNodes} /> ), - available: (item: TransformListRow) => item.stats.state === TRANSFORM_STATE.STOPPED, + available: (item: TransformListRow) => + isTransformListRowWithStats(item) ? item.stats.state === TRANSFORM_STATE.STOPPED : true, enabled: (item: TransformListRow) => !isStartActionDisabled([item], canStartStopTransform, transformNodes), description: startActionNameText, diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx index e5bc1425cdd99..b2a079a425273 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/stop_action_name.tsx @@ -9,6 +9,12 @@ import React, { type FC } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiToolTip } from '@elastic/eui'; +import { TransformCapabilities } from '../../../../../../common/types/capabilities'; +import { + isTransformListRowWithStats, + missingTransformStats, +} from '../../../../common/transform_list'; +import { createNoStatsTooltipMessage } from '../../../../../../common/utils/create_stats_unknown_message'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { createCapabilityFailureMessage } from '../../../../../../common/utils/create_capability_failure_message'; @@ -22,6 +28,46 @@ export const stopActionNameText = i18n.translate( } ); +export const getStopActionDisabledMessage = ({ + items, + capabilities, +}: { + items: TransformListRow[]; + capabilities: TransformCapabilities; +}) => { + const isBulkAction = items.length > 1; + + const { canStartStopTransform } = capabilities; + + if (missingTransformStats(items)) { + return createNoStatsTooltipMessage({ + actionName: stopActionNameText, + count: items.length, + }); + } + + // Disable stop action if one of the transforms is stopped already + const stoppedTransform = items.some( + (i: TransformListRow) => + isTransformListRowWithStats(i) && i.stats.state === TRANSFORM_STATE.STOPPED + ); + + if (!canStartStopTransform) { + return createCapabilityFailureMessage('canStartStopTransform'); + } + + if (stoppedTransform) { + return isBulkAction === true + ? i18n.translate('xpack.transform.transformList.stoppedTransformBulkToolTip', { + defaultMessage: 'One or more transforms are already stopped.', + }) + : i18n.translate('xpack.transform.transformList.stoppedTransformToolTip', { + defaultMessage: '{transformId} is already stopped.', + values: { transformId: items[0] && items[0].config.id }, + }); + } +}; + export const isStopActionDisabled = ( items: TransformListRow[], canStartStopTransform: boolean, @@ -29,10 +75,15 @@ export const isStopActionDisabled = ( ) => { // Disable stop action if one of the transforms is stopped already const stoppedTransform = items.some( - (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STOPPED + (i: TransformListRow) => i.stats?.state === TRANSFORM_STATE.STOPPED ); - return forceDisable === true || !canStartStopTransform || stoppedTransform === true; + return ( + forceDisable === true || + !canStartStopTransform || + stoppedTransform === true || + missingTransformStats(items) + ); }; export interface StopActionNameProps { @@ -40,42 +91,15 @@ export interface StopActionNameProps { forceDisable?: boolean; } export const StopActionName: FC = ({ items, forceDisable }) => { - const isBulkAction = items.length > 1; - const { canStartStopTransform } = useTransformCapabilities(); - - // Disable stop action if one of the transforms is stopped already - const stoppedTransform = items.some( - (i: TransformListRow) => i.stats.state === TRANSFORM_STATE.STOPPED - ); - - let stoppedTransformMessage; - if (isBulkAction === true) { - stoppedTransformMessage = i18n.translate( - 'xpack.transform.transformList.stoppedTransformBulkToolTip', - { - defaultMessage: 'One or more transforms are already stopped.', - } - ); - } else { - stoppedTransformMessage = i18n.translate( - 'xpack.transform.transformList.stoppedTransformToolTip', - { - defaultMessage: '{transformId} is already stopped.', - values: { transformId: items[0] && items[0].config.id }, - } - ); - } - - if (!canStartStopTransform || stoppedTransform) { + const capabilities = useTransformCapabilities(); + // Disable transforms if stats does not exist + const stoppedTransformMessage = getStopActionDisabledMessage({ + items, + capabilities, + }); + if (forceDisable || stoppedTransformMessage) { return ( - + <>{stopActionNameText} ); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx index e410704341177..36ec5139d14ee 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/action_stop/use_stop_action.tsx @@ -6,6 +6,7 @@ */ import React, { useCallback, useMemo, useState } from 'react'; +import { isTransformListRowWithStats } from '../../../../common/transform_list'; import { TRANSFORM_STATE } from '../../../../../../common/constants'; import { TransformListAction, TransformListRow } from '../../../../common'; import { useTransformCapabilities, useStopTransforms } from '../../../../hooks'; @@ -30,13 +31,16 @@ export const useStopAction = (forceDisable: boolean) => { const stopAndCloseModal = useCallback( (transformSelection: TransformListRow[]) => { setModalVisible(false); - stopTransforms(transformSelection.map((t) => ({ id: t.id, state: t.stats.state }))); + stopTransforms( + transformSelection.map((t) => ({ id: t.id, state: t.stats ? t.stats.state : 'waiting' })) + ); }, [stopTransforms] ); const clickHandler = useCallback( - (i: TransformListRow) => stopTransforms([{ id: i.id, state: i.stats.state }]), + (t: TransformListRow) => + stopTransforms([{ id: t.id, state: t.stats ? t.stats.state : 'waiting' }]), [stopTransforms] ); @@ -45,8 +49,10 @@ export const useStopAction = (forceDisable: boolean) => { name: (item: TransformListRow) => ( ), - available: (item: TransformListRow) => item.stats.state !== TRANSFORM_STATE.STOPPED, + available: (item: TransformListRow) => + isTransformListRowWithStats(item) ? item.stats.state !== TRANSFORM_STATE.STOPPED : true, enabled: (item: TransformListRow) => + isTransformListRowWithStats(item) && !isStopActionDisabled([item], canStartStopTransform, forceDisable), description: stopActionNameText, icon: 'stop', diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx index f8b55dbbeda3c..370d7c2da05d5 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.test.tsx @@ -38,7 +38,7 @@ describe('Transform: Transform List ', () => { render( - + ); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx index f4766da492f67..c43e3d096f32b 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row.tsx @@ -9,14 +9,23 @@ import React, { FC, useMemo } from 'react'; import { css } from '@emotion/react'; import moment from 'moment-timezone'; -import { EuiButtonEmpty, EuiTabbedContent } from '@elastic/eui'; +import { + EuiButtonEmpty, + EuiLoadingSpinner, + EuiTabbedContent, + EuiFlexGroup, + useEuiTheme, + EuiCallOut, + EuiFlexItem, +} from '@elastic/eui'; -import { Optional } from '@kbn/utility-types'; import { i18n } from '@kbn/i18n'; import { formatHumanReadableDateTimeSeconds } from '@kbn/ml-date-utils'; import { stringHash } from '@kbn/ml-string-hash'; import { isDefined } from '@kbn/ml-is-defined'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { isTransformListRowWithStats } from '../../../../common/transform_list'; import { useIsServerless } from '../../../../serverless_context'; import { TransformHealthAlertRule } from '../../../../../../common/types/alerting'; @@ -42,39 +51,66 @@ type Item = SectionItem; interface Props { item: TransformListRow; onAlertEdit: (alertRule: TransformHealthAlertRule) => void; + transformsStatsLoading: boolean; } -type StateValues = Optional; +const NoStatsFallbackTabContent = ({ + transformsStatsLoading, +}: { + transformsStatsLoading: boolean; +}) => { + const { euiTheme } = useEuiTheme(); -export const ExpandedRow: FC = ({ item, onAlertEdit }) => { - const hideNodeInfo = useIsServerless(); + const content = transformsStatsLoading ? ( + + ) : ( + + + } + /> + + ); + return ( + + {content} + + ); +}; - const stateValues: StateValues = { ...item.stats }; - delete stateValues.stats; - delete stateValues.checkpointing; +export const ExpandedRow: FC = ({ item, onAlertEdit, transformsStatsLoading }) => { + const hideNodeInfo = useIsServerless(); const stateItems: Item[] = []; - stateItems.push( - { - title: 'ID', - description: item.id, - }, - { + stateItems.push({ + title: 'ID', + description: item.id, + }); + if (isTransformListRowWithStats(item)) { + stateItems.push({ title: 'state', description: item.stats.state, - } - ); - if (!hideNodeInfo && item.stats.node !== undefined) { - stateItems.push({ - title: 'node.name', - description: item.stats.node.name, - }); - } - if (item.stats.health !== undefined) { - stateItems.push({ - title: 'health', - description: , }); + + if (!hideNodeInfo && item.stats.node !== undefined) { + stateItems.push({ + title: 'node.name', + description: item.stats.node.name, + }); + } + if (item.stats.health !== undefined) { + stateItems.push({ + title: 'health', + description: , + }); + } } const state: SectionConfig = { @@ -137,69 +173,71 @@ export const ExpandedRow: FC = ({ item, onAlertEdit }) => { }; const checkpointingItems: Item[] = []; - if (item.stats.checkpointing.changes_last_detected_at !== undefined) { - checkpointingItems.push({ - title: 'changes_last_detected_at', - description: formatHumanReadableDateTimeSeconds( - item.stats.checkpointing.changes_last_detected_at - ), - }); - } - - if (item.stats.checkpointing.last !== undefined) { - checkpointingItems.push({ - title: 'last.checkpoint', - description: item.stats.checkpointing.last.checkpoint, - }); - if (item.stats.checkpointing.last.timestamp_millis !== undefined) { + if (isTransformListRowWithStats(item)) { + if (item.stats.checkpointing.changes_last_detected_at !== undefined) { checkpointingItems.push({ - title: 'last.timestamp', + title: 'changes_last_detected_at', description: formatHumanReadableDateTimeSeconds( - item.stats.checkpointing.last.timestamp_millis + item.stats.checkpointing.changes_last_detected_at ), }); + } + + if (item.stats.checkpointing.last !== undefined) { checkpointingItems.push({ - title: 'last.timestamp_millis', - description: item.stats.checkpointing.last.timestamp_millis, + title: 'last.checkpoint', + description: item.stats.checkpointing.last.checkpoint, }); + if (item.stats.checkpointing.last.timestamp_millis !== undefined) { + checkpointingItems.push({ + title: 'last.timestamp', + description: formatHumanReadableDateTimeSeconds( + item.stats.checkpointing.last.timestamp_millis + ), + }); + checkpointingItems.push({ + title: 'last.timestamp_millis', + description: item.stats.checkpointing.last.timestamp_millis, + }); + } } - } - - if (item.stats.checkpointing.last_search_time !== undefined) { - checkpointingItems.push({ - title: 'last_search_time', - description: formatHumanReadableDateTimeSeconds(item.stats.checkpointing.last_search_time), - }); - } - if (item.stats.checkpointing.next !== undefined) { - checkpointingItems.push({ - title: 'next.checkpoint', - description: item.stats.checkpointing.next.checkpoint, - }); - if (item.stats.checkpointing.next.checkpoint_progress !== undefined) { + if (item.stats.checkpointing.last_search_time !== undefined) { checkpointingItems.push({ - title: 'next.checkpoint_progress.total_docs', - description: item.stats.checkpointing.next.checkpoint_progress.total_docs, + title: 'last_search_time', + description: formatHumanReadableDateTimeSeconds(item.stats.checkpointing.last_search_time), }); + } + + if (item.stats.checkpointing.next !== undefined) { checkpointingItems.push({ - title: 'next.checkpoint_progress.docs_remaining', - description: item.stats.checkpointing.next.checkpoint_progress.docs_remaining, + title: 'next.checkpoint', + description: item.stats.checkpointing.next.checkpoint, }); + if (item.stats.checkpointing.next.checkpoint_progress !== undefined) { + checkpointingItems.push({ + title: 'next.checkpoint_progress.total_docs', + description: item.stats.checkpointing.next.checkpoint_progress.total_docs, + }); + checkpointingItems.push({ + title: 'next.checkpoint_progress.docs_remaining', + description: item.stats.checkpointing.next.checkpoint_progress.docs_remaining, + }); + checkpointingItems.push({ + title: 'next.checkpoint_progress.percent_complete', + description: item.stats.checkpointing.next.checkpoint_progress.percent_complete, + }); + } + } + + if (item.stats.checkpointing.operations_behind !== undefined) { checkpointingItems.push({ - title: 'next.checkpoint_progress.percent_complete', - description: item.stats.checkpointing.next.checkpoint_progress.percent_complete, + title: 'operations_behind', + description: item.stats.checkpointing.operations_behind, }); } } - if (item.stats.checkpointing.operations_behind !== undefined) { - checkpointingItems.push({ - title: 'operations_behind', - description: item.stats.checkpointing.operations_behind, - }); - } - const alertRuleItems: Item[] | undefined = item.alerting_rules?.map((rule) => { return { title: ( @@ -236,9 +274,11 @@ export const ExpandedRow: FC = ({ item, onAlertEdit }) => { const stats: SectionConfig = { title: 'Stats', - items: Object.entries(item.stats.stats).map((s) => { - return { title: s[0].toString(), description: getItemDescription(s[1]) }; - }), + items: item.stats + ? Object.entries(item.stats.stats).map((s) => { + return { title: s[0].toString(), description: getItemDescription(s[1]) }; + }) + : [], position: 'left', }; @@ -275,8 +315,10 @@ export const ExpandedRow: FC = ({ item, onAlertEdit }) => { defaultMessage: 'Stats', } ), - content: ( + content: item.stats ? ( + ) : ( + ), }, { @@ -285,7 +327,7 @@ export const ExpandedRow: FC = ({ item, onAlertEdit }) => { name: 'JSON', content: , }, - ...(item.stats.health + ...(item.stats?.health ? [ { id: `transform-health-tab-${tabId}`, diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx index 0e8ab94dc2086..3a088d5a9b273 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/expanded_row_health_pane.tsx @@ -12,14 +12,15 @@ import { formatDate, EuiPanel, EuiSpacer, EuiInMemoryTable } from '@elastic/eui' import { i18n } from '@kbn/i18n'; import { TIME_FORMAT } from '../../../../../../common/constants'; -import type { TransformHealthIssue } from '../../../../../../common/types/transform_stats'; - -import { TransformListRow } from '../../../../common'; +import type { + TransformHealthIssue, + TransformStats, +} from '../../../../../../common/types/transform_stats'; import { TransformHealthColoredDot } from './transform_health_colored_dot'; interface ExpandedRowHealthPaneProps { - health: TransformListRow['stats']['health']; + health: TransformStats['health']; } export const ExpandedRowHealthPane: FC = ({ health }) => { diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx index fc4bf0e24d14d..e3f98beea9e7e 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.test.tsx @@ -39,6 +39,7 @@ describe('Transform: Transform List ', () => { transformNodes={1} transforms={[]} transformsLoading={false} + transformsStatsLoading={false} /> diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx index 2e0106afc0e92..a55253a08b5eb 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_list.tsx @@ -70,12 +70,19 @@ type ItemIdToExpandedRowMap = Record; function getItemIdToExpandedRowMap( itemIds: TransformId[], transforms: TransformListRow[], - onAlertEdit: (alertRule: TransformHealthAlertRule) => void + onAlertEdit: (alertRule: TransformHealthAlertRule) => void, + transformsStatsLoading: boolean ): ItemIdToExpandedRowMap { return itemIds.reduce((m: ItemIdToExpandedRowMap, transformId: TransformId) => { const item = transforms.find((transform) => transform.config.id === transformId); if (item !== undefined) { - m[transformId] = ; + m[transformId] = ( + + ); } return m; }, {} as ItemIdToExpandedRowMap); @@ -87,6 +94,7 @@ interface TransformListProps { transformNodes: number; transforms: TransformListRow[]; transformsLoading: boolean; + transformsStatsLoading: boolean; } export const TransformList: FC = ({ @@ -95,6 +103,7 @@ export const TransformList: FC = ({ transformNodes, transforms, transformsLoading, + transformsStatsLoading, }) => { const refreshTransformList = useRefreshTransformList(); const { setEditAlertRule } = useAlertRuleFlyout(); @@ -126,7 +135,8 @@ export const TransformList: FC = ({ expandedRowItemIds, setExpandedRowItemIds, transformNodes, - transformSelection + transformSelection, + transformsStatsLoading ); const searchError = query?.error ? query?.error.message : undefined; @@ -166,7 +176,8 @@ export const TransformList: FC = ({ const itemIdToExpandedRowMap = getItemIdToExpandedRowMap( expandedRowItemIds, transforms, - setEditAlertRule + setEditAlertRule, + transformsStatsLoading ); const bulkActionMenuItems = [ @@ -235,6 +246,7 @@ export const TransformList: FC = ({ canResetTransform={capabilities.canResetTransform} disabled={isResetActionDisabled(transformSelection, false)} isBulkAction={true} + items={transformSelection} />
    , @@ -247,6 +259,8 @@ export const TransformList: FC = ({ canDeleteTransform={capabilities.canDeleteTransform} disabled={isDeleteActionDisabled(transformSelection, false)} isBulkAction={true} + items={transformSelection} + forceDisable={false} />
    , diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx index 5a2df943e9a54..637f32558cb41 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transform_search_bar_filters.tsx @@ -106,9 +106,12 @@ export const filterTransforms = (transforms: TransformListRow[], clauses: Clause // filter other clauses, i.e. the mode and status filters if (c.type !== 'is' && Array.isArray(c.value)) { // the status value is an array of string(s) e.g. ['failed', 'stopped'] - ts = transforms.filter((transform) => (c.value as Value[]).includes(transform.stats.state)); + ts = transforms.filter( + (transform) => transform.stats && (c.value as Value[]).includes(transform.stats.state) + ); } else { ts = transforms.filter((transform) => { + if (!transform.stats) return false; if (c.type === 'field' && c.field === 'health') { return transform.stats.health?.status === c.value; } diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx index 79f6321c2419f..ea85f705e46ef 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/transforms_stats_bar.tsx @@ -94,10 +94,12 @@ function createTranformStats( transformStats.batch.value++; } - if (transform.stats.state === TRANSFORM_STATE.FAILED) { - failedTransforms++; - } else if (transform.stats.state === TRANSFORM_STATE.STARTED) { - startedTransforms++; + if (transform.stats) { + if (transform.stats.state === TRANSFORM_STATE.FAILED) { + failedTransforms++; + } else if (transform.stats.state === TRANSFORM_STATE.STARTED) { + startedTransforms++; + } } }); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx index 87f550e433bc2..e49df35b42a55 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.test.tsx @@ -20,7 +20,7 @@ describe('Transform: Job List Columns', () => { const wrapper: FC = ({ children }) => ( {children} ); - const { result, waitForNextUpdate } = renderHook(() => useColumns([], () => {}, 1, []), { + const { result, waitForNextUpdate } = renderHook(() => useColumns([], () => {}, 1, [], false), { wrapper, }); diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx index 2ae8edf30a1de..fb578ec06aa3c 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/transform_list/use_columns.tsx @@ -22,6 +22,7 @@ import { EuiToolTip, RIGHT_ALIGNMENT, EuiIcon, + EuiLoadingSpinner, } from '@elastic/eui'; import { useTransformCapabilities } from '../../../../hooks'; @@ -46,12 +47,20 @@ const TRANSFORM_INSUFFICIENT_PERMISSIONS_MSG = i18n.translate( defaultMessage: 'This transform was created with insufficient permissions.', } ); + +const StatsUnknown = () => ( + + + +); export const useColumns = ( expandedRowItemIds: TransformId[], setExpandedRowItemIds: React.Dispatch>, transformNodes: number, - transformSelection: TransformListRow[] + transformSelection: TransformListRow[], + transformsStatsLoading: boolean ) => { + const NoStatsFallbackComponent = transformsStatsLoading ? EuiLoadingSpinner : StatsUnknown; const { canStartStopTransform } = useTransformCapabilities(); const { actions, modals } = useActions({ @@ -239,10 +248,14 @@ export const useColumns = ( { name: i18n.translate('xpack.transform.status', { defaultMessage: 'Status' }), 'data-test-subj': 'transformListColumnStatus', - sortable: (item: TransformListRow) => item.stats.state, + sortable: (item: TransformListRow) => item.stats?.state, truncateText: true, render(item: TransformListRow) { - return ; + return item.stats ? ( + + ) : ( + + ); }, width: '100px', }, @@ -271,6 +284,7 @@ export const useColumns = ( if (progress === undefined && isBatchTransform === true) { return null; } + if (!item.stats) return ; return ( @@ -292,7 +306,7 @@ export const useColumns = ( )} - {!isBatchTransform && ( + {!isBatchTransform && item.stats && ( <> {/* If not stopped, failed or waiting show the animated progress bar */} @@ -321,10 +335,14 @@ export const useColumns = ( { name: i18n.translate('xpack.transform.health', { defaultMessage: 'Health' }), 'data-test-subj': 'transformListColumnHealth', - sortable: (item: TransformListRow) => item.stats.health.status, + sortable: (item: TransformListRow) => item.stats?.health.status, truncateText: true, render(item: TransformListRow) { - return ; + return item.stats ? ( + + ) : ( + + ); }, width: '100px', }, diff --git a/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx b/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx index cce3128b3f3d0..858954976cf82 100644 --- a/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx +++ b/x-pack/plugins/transform/public/app/sections/transform_management/transform_management_section.tsx @@ -21,6 +21,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { IHttpFetchError } from '@kbn/core-http-browser'; +import type { TransformListRow } from '../../common'; +import { isTransformStats } from '../../../../common/types/transform_stats'; +import { useGetTransformsStats } from '../../hooks/use_get_transform_stats'; import { useIsServerless } from '../../serverless_context'; import { needsReauthorization } from '../../common/reauthorization_utils'; import { TRANSFORM_STATE } from '../../../../common/constants'; @@ -85,13 +88,37 @@ export const TransformManagement: FC = () => { const { isInitialLoading: transformsInitialLoading, - isLoading: transformsLoading, + isLoading: transformsWithoutStatsLoading, error: transformsErrorMessage, - data: { transforms, transformIdsWithoutConfig }, + data: { transforms: transformsWithoutStats, transformIdsWithoutConfig }, } = useGetTransforms({ enabled: !transformNodesInitialLoading && (transformNodes > 0 || hideNodeInfo), }); + const { + isLoading: transformsStatsLoading, + error: transformsStatsErrorMessage, + data: transformsStats, + } = useGetTransformsStats({ + enabled: !transformNodesInitialLoading && (transformNodes > 0 || hideNodeInfo), + }); + + const transforms: TransformListRow[] = useMemo(() => { + if (!transformsStats) return transformsWithoutStats; + + return transformsWithoutStats.map((t) => { + const stats = transformsStats.transforms.find((d) => t.config.id === d.id); + + // A newly created transform might not have corresponding stats yet. + // If that's the case we just skip the transform and don't add it to the transform list yet. + if (!isTransformStats(stats)) { + return t; + } + + return { ...t, stats }; + }); + }, [transformsStats, transformsWithoutStats]); + const isInitialLoading = transformNodesInitialLoading || transformsInitialLoading; const { canStartStopTransform } = useTransformCapabilities(); @@ -219,6 +246,17 @@ export const TransformManagement: FC = () => { errorMessage={transformsErrorMessage} /> )} + {transformsStatsErrorMessage !== null ? ( + + } + errorMessage={transformsStatsErrorMessage} + /> + ) : null} @@ -272,11 +310,12 @@ export const TransformManagement: FC = () => { ) : null} {(transformNodes > 0 || transforms.length > 0) && ( )} diff --git a/x-pack/plugins/transform/tsconfig.json b/x-pack/plugins/transform/tsconfig.json index 52e0747cc1fb9..7b41f101c15c1 100644 --- a/x-pack/plugins/transform/tsconfig.json +++ b/x-pack/plugins/transform/tsconfig.json @@ -40,7 +40,6 @@ "@kbn/std", "@kbn/es-query", "@kbn/ml-agg-utils", - "@kbn/utility-types", "@kbn/ml-string-hash", "@kbn/ui-theme", "@kbn/field-types", diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 150f72792a861..0e167e1ce60d3 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -3212,7 +3212,7 @@ "guidedOnboardingPackage.gettingStarted.cards.progressLabel": "{numberCompleteSteps} étape(s) terminée(s) sur {numberSteps}", "guidedOnboardingPackage.gettingStarted.cards.siemSecurity.title": "Détecter les menaces dans {lineBreak} mes données avec SIEM", "guidedOnboardingPackage.gettingStarted.cards.completeLabel": "Guide terminé", - "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "Créer une expérience de recherche sémantique", + "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "Créer une expérience de recherche basée sur l'IA", "guidedOnboardingPackage.gettingStarted.cards.hostsObservability.title": "Monitorer mes indicateurs d'hôte", "guidedOnboardingPackage.gettingStarted.cards.kubernetesObservability.title": "Monitorer les clusters Kubernetes", "guidedOnboardingPackage.gettingStarted.cards.logsObservability.title": "Collecter et analyser mes logs", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 41096d8c2e990..f0e47690da767 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -3227,7 +3227,7 @@ "guidedOnboardingPackage.gettingStarted.cards.progressLabel": "{numberSteps}ステップ中{numberCompleteSteps}ステップ完了", "guidedOnboardingPackage.gettingStarted.cards.siemSecurity.title": "SIEMで{lineBreak}データの脅威を検出", "guidedOnboardingPackage.gettingStarted.cards.completeLabel": "ガイド完了", - "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "セマンティック検索エクスペリエンスを構築", + "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "AI を活用した検索エクスペリエンスを構築する", "guidedOnboardingPackage.gettingStarted.cards.hostsObservability.title": "ホストメトリックを監視", "guidedOnboardingPackage.gettingStarted.cards.kubernetesObservability.title": "Kubernetesクラスターの監視", "guidedOnboardingPackage.gettingStarted.cards.logsObservability.title": "ログを収集して分析", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index b17bf98b8ad3a..17beacb3184e2 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -3226,7 +3226,7 @@ "guidedOnboardingPackage.gettingStarted.cards.progressLabel": "已完成 {numberCompleteSteps} 个（共 {numberSteps} 个）步骤", "guidedOnboardingPackage.gettingStarted.cards.siemSecurity.title": "通过 SIEM{lineBreak}在我的数据中检测威胁", "guidedOnboardingPackage.gettingStarted.cards.completeLabel": "指南完成", - "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "构建语义搜索体验", + "guidedOnboardingPackage.gettingStarted.cards.aiSearch.title": "打造人工智能驱动的搜索体验", "guidedOnboardingPackage.gettingStarted.cards.hostsObservability.title": "监测我的主机指标", "guidedOnboardingPackage.gettingStarted.cards.kubernetesObservability.title": "监测 Kubernetes 集群", "guidedOnboardingPackage.gettingStarted.cards.logsObservability.title": "收集并分析我的日志", diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts index ddc90ec10e9b6..02a41b6de7afa 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts @@ -26,8 +26,7 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const esTestIndexTool = new ESTestIndexTool(es, retry); const supertestWithoutAuth = getService('supertestWithoutAuth'); - // FLAKY: https://github.com/elastic/kibana/issues/140973 - describe.skip('telemetry', () => { + describe('telemetry', () => { const objectRemover = new ObjectRemover(supertest); const alwaysFiringRuleId: { [key: string]: string } = {}; diff --git a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts index 94b023757bf19..ab0e0302e163d 100644 --- a/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts +++ b/x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts @@ -55,14 +55,20 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider refresh: true, }), ]), - add: async (findingsMock: T[]) => { + add: async < + T extends { + '@timestamp'?: string; + } + >( + findingsMock: T[] + ) => { await Promise.all([ ...findingsMock.map((finding) => es.index({ index: FINDINGS_INDEX, body: { ...finding, - '@timestamp': new Date().toISOString(), + '@timestamp': finding['@timestamp'] ?? new Date().toISOString(), }, refresh: true, }) @@ -72,7 +78,7 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider index: FINDINGS_LATEST_INDEX, body: { ...finding, - '@timestamp': new Date().toISOString(), + '@timestamp': finding['@timestamp'] ?? new Date().toISOString(), }, refresh: true, }) @@ -81,6 +87,20 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider }, }; + const detectionRuleApi = { + remove: async () => { + await supertest + .post('/api/detection_engine/rules/_bulk_action?dry_run=false') + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') + .send({ + action: 'delete', + query: '', + }) + .expect(200); + }, + }; + const distributionBar = { filterBy: async (type: 'passed' | 'failed') => testSubjects.click(type === 'failed' ? 'distribution_bar_failed' : 'distribution_bar_passed'), @@ -203,6 +223,12 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider await nonStaleElement.click(); } }, + + async openFlyoutAt(rowIndex: number) { + const table = await this.getElement(); + const flyoutButton = await table.findAllByTestSubject('findings_table_expand_column'); + await flyoutButton[rowIndex].click(); + }, }); const navigateToLatestFindingsPage = async () => { @@ -247,6 +273,41 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider const notInstalledVulnerabilities = createNotInstalledObject('cnvm-integration-not-installed'); const notInstalledCSP = createNotInstalledObject('cloud_posture_page_package_not_installed'); + const createFlyoutObject = (tableTestSubject: string) => ({ + async getElement() { + return await testSubjects.find(tableTestSubject); + }, + async clickTakeActionButton() { + const element = await this.getElement(); + const button = await element.findByCssSelector('[data-test-subj="csp:take_action"] button'); + await button.click(); + return button; + }, + async clickTakeActionCreateRuleButton() { + await this.clickTakeActionButton(); + const button = await testSubjects.find('csp:create_rule'); + await button.click(); + return button; + }, + async getVisibleText(testSubj: string) { + const element = await this.getElement(); + return await (await element.findByTestSubject(testSubj)).getVisibleText(); + }, + }); + + const misconfigurationsFlyout = createFlyoutObject('findings_flyout'); + + const toastMessage = async (testSubj = 'csp:toast-success') => ({ + async getElement() { + return await testSubjects.find(testSubj); + }, + async clickToastMessageLink(linkTestSubj = 'csp:toast-success-link') { + const element = await this.getElement(); + const link = await element.findByTestSubject(linkTestSubj); + await link.click(); + }, + }); + return { navigateToLatestFindingsPage, navigateToVulnerabilities, @@ -259,5 +320,8 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider index, waitForPluginInitialized, distributionBar, + misconfigurationsFlyout, + toastMessage, + detectionRuleApi, }; } diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts b/x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts new file mode 100644 index 0000000000000..53f0b765165ef --- /dev/null +++ b/x-pack/test/cloud_security_posture_functional/pages/findings_alerts.ts @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import Chance from 'chance'; +import type { FtrProviderContext } from '../ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getPageObjects, getService }: FtrProviderContext) { + const testSubjects = getService('testSubjects'); + const retry = getService('retry'); + const pageObjects = getPageObjects(['common', 'findings', 'header']); + const chance = new Chance(); + + // We need to use a dataset for the tests to run + const data = [ + { + resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' }, + result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' }, + rule: { + tags: ['CIS', 'CIS K8S'], + rationale: 'rationale steps for rule 1.1', + references: '1. https://elastic.co/rules/1.1', + name: 'Upper case rule name', + section: 'Upper case section', + benchmark: { + rule_number: '1.1', + id: 'cis_k8s', + posture_type: 'kspm', + name: 'CIS Kubernetes V1.23', + version: 'v1.0.0', + remediation: 'remediation guide', + }, + type: 'process', + }, + cluster_id: 'Upper case cluster id', + }, + { + '@timestamp': '2023-09-10T14:01:00.000Z', + resource: { id: chance.guid(), name: `Pod`, sub_type: 'Upper case sub type' }, + result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' }, + rule: { + tags: ['CIS', 'CIS K8S'], + rationale: 'rationale steps', + references: '1. https://elastic.co', + name: 'lower case rule name', + section: 'Another upper case section', + benchmark: { + rule_number: '1.2', + id: 'cis_k8s', + posture_type: 'kspm', + name: 'CIS Kubernetes V1.23', + version: 'v1.0.0', + remediation: 'remediation guide', + }, + type: 'process', + }, + cluster_id: 'Another Upper case cluster id', + }, + { + '@timestamp': '2023-09-10T14:02:00.000Z', + resource: { id: chance.guid(), name: `process`, sub_type: 'another lower case type' }, + result: { evaluation: 'passed' }, + rule: { + tags: ['CIS', 'CIS K8S'], + rationale: 'rationale steps', + references: '1. https://elastic.co', + name: 'Another upper case rule name', + section: 'lower case section', + benchmark: { + rule_number: '1.3', + id: 'cis_k8s', + posture_type: 'kspm', + name: 'CIS Kubernetes V1.23', + version: 'v1.0.0', + remediation: 'remediation guide', + }, + type: 'process', + }, + cluster_id: 'lower case cluster id', + }, + { + '@timestamp': '2023-09-10T14:03:00.000Z', + resource: { id: chance.guid(), name: `process`, sub_type: 'Upper case type again' }, + result: { evaluation: 'failed' }, + rule: { + tags: ['CIS', 'CIS K8S'], + rationale: 'rationale steps', + references: '1. https://elastic.co', + name: 'some lower case rule name', + section: 'another lower case section', + benchmark: { + rule_number: '1.4', + id: 'cis_k8s', + posture_type: 'kspm', + name: 'CIS Kubernetes V1.23', + version: 'v1.0.0', + remediation: 'remediation guide', + }, + type: 'process', + }, + cluster_id: 'another lower case cluster id', + }, + ]; + + const ruleName1 = data[0].rule.name; + + describe('Findings Page - Alerts', function () { + this.tags(['cloud_security_posture_findings_alerts']); + let findings: typeof pageObjects.findings; + let latestFindingsTable: typeof findings.latestFindingsTable; + let misconfigurationsFlyout: typeof findings.misconfigurationsFlyout; + + before(async () => { + findings = pageObjects.findings; + latestFindingsTable = findings.latestFindingsTable; + misconfigurationsFlyout = findings.misconfigurationsFlyout; + // Before we start any test we must wait for cloud_security_posture plugin to complete its initialization + await findings.waitForPluginInitialized(); + // Prepare mocked findings + await findings.index.remove(); + await findings.index.add(data); + }); + + after(async () => { + await findings.index.remove(); + await findings.detectionRuleApi.remove(); + }); + + beforeEach(async () => { + await findings.detectionRuleApi.remove(); + await findings.navigateToLatestFindingsPage(); + await retry.waitFor( + 'Findings table to be loaded', + async () => (await latestFindingsTable.getRowsCount()) === data.length + ); + pageObjects.header.waitUntilLoadingHasFinished(); + }); + + describe('Create detection rule', () => { + it('Creates a detection rule from the Take Action button and navigates to rule page', async () => { + await latestFindingsTable.openFlyoutAt(0); + await misconfigurationsFlyout.clickTakeActionCreateRuleButton(); + + expect( + await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-alert-count') + ).to.be('0 alerts'); + + expect( + await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-detection-rule-count') + ).to.be('1 detection rule'); + + const toastMessage = await (await findings.toastMessage()).getElement(); + expect(toastMessage).to.be.ok(); + + const toastMessageTitle = await toastMessage.findByTestSubject('csp:toast-success-title'); + expect(await toastMessageTitle.getVisibleText()).to.be(ruleName1); + + await (await findings.toastMessage()).clickToastMessageLink(); + + const rulePageTitle = await testSubjects.find('header-page-title'); + expect(await rulePageTitle.getVisibleText()).to.be(ruleName1); + }); + it('Creates a detection rule from the Alerts section and navigates to rule page', async () => { + await latestFindingsTable.openFlyoutAt(0); + const flyout = await misconfigurationsFlyout.getElement(); + + await ( + await flyout.findByTestSubject('csp:findings-flyout-create-detection-rule-link') + ).click(); + + expect( + await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-alert-count') + ).to.be('0 alerts'); + + expect( + await misconfigurationsFlyout.getVisibleText('csp:findings-flyout-detection-rule-count') + ).to.be('1 detection rule'); + + const toastMessage = await (await findings.toastMessage()).getElement(); + expect(toastMessage).to.be.ok(); + + const toastMessageTitle = await toastMessage.findByTestSubject('csp:toast-success-title'); + expect(await toastMessageTitle.getVisibleText()).to.be(ruleName1); + + await (await findings.toastMessage()).clickToastMessageLink(); + + const rulePageTitle = await testSubjects.find('header-page-title'); + expect(await rulePageTitle.getVisibleText()).to.be(ruleName1); + }); + }); + describe('Rule details', () => { + it('The rule page contains the expected matching data', async () => { + await latestFindingsTable.openFlyoutAt(0); + await misconfigurationsFlyout.clickTakeActionCreateRuleButton(); + + await (await findings.toastMessage()).clickToastMessageLink(); + + const rulePageDescription = await testSubjects.find( + 'stepAboutRuleDetailsToggleDescriptionText' + ); + expect(await rulePageDescription.getVisibleText()).to.be(data[0].rule.rationale); + + const severity = await testSubjects.find('severity'); + expect(await severity.getVisibleText()).to.be('Low'); + + const referenceUrls = await testSubjects.find('urlsDescriptionReferenceLinkItem'); + expect(await referenceUrls.getVisibleText()).to.contain('https://elastic.co/rules/1.1'); + }); + }); + describe('Navigation', () => { + it('Clicking on count of Rules should navigate to the rules page with benchmark tags as a filter', async () => { + await latestFindingsTable.openFlyoutAt(0); + await misconfigurationsFlyout.clickTakeActionCreateRuleButton(); + const flyout = await misconfigurationsFlyout.getElement(); + await (await flyout.findByTestSubject('csp:findings-flyout-detection-rule-count')).click(); + + expect(await (await testSubjects.find('ruleName')).getVisibleText()).to.be(ruleName1); + }); + it('Clicking on count of Alerts should navigate to the alerts page', async () => { + await latestFindingsTable.openFlyoutAt(0); + await misconfigurationsFlyout.clickTakeActionCreateRuleButton(); + const flyout = await misconfigurationsFlyout.getElement(); + await (await flyout.findByTestSubject('csp:findings-flyout-alert-count')).click(); + + expect(await (await testSubjects.find('header-page-title')).getVisibleText()).to.be( + 'Alerts' + ); + }); + }); + }); +} diff --git a/x-pack/test/cloud_security_posture_functional/pages/index.ts b/x-pack/test/cloud_security_posture_functional/pages/index.ts index 1d30a3b27fda9..81e905ddaca35 100644 --- a/x-pack/test/cloud_security_posture_functional/pages/index.ts +++ b/x-pack/test/cloud_security_posture_functional/pages/index.ts @@ -12,6 +12,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('Cloud Security Posture', function () { loadTestFile(require.resolve('./findings_onboarding')); loadTestFile(require.resolve('./findings')); + loadTestFile(require.resolve('./findings_alerts')); loadTestFile(require.resolve('./compliance_dashboard')); }); } diff --git a/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts b/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts index 5046eb8c9c556..583cc02908025 100644 --- a/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts +++ b/x-pack/test/profiling_api_integration/tests/has_setup.spec.ts @@ -23,9 +23,9 @@ export default function featureControlsTests({ getService }: FtrProviderContext) const logger = getService('log'); const es = getService('es'); - registry.when('Profiling status check', { config: 'cloud' }, () => { - // Failing: See https://github.com/elastic/kibana/issues/167076 - describe.skip('Profiling is not set up and no data is loaded', () => { + // Failing: See https://github.com/elastic/kibana/issues/167076 + registry.when.skip('Profiling status check', { config: 'cloud' }, () => { + describe('Profiling is not set up and no data is loaded', () => { describe('Admin user', () => { let statusCheck: ProfilingStatus; before(async () => { diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts index afedab5dbb094..c48fa80baad46 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/investigations/timelines/discover/search_filter.cy.ts @@ -38,7 +38,8 @@ const INITIAL_START_DATE = 'Jan 18, 2021 @ 20:33:29.186'; const INITIAL_END_DATE = 'Jan 19, 2024 @ 20:33:29.186'; const NEW_START_DATE = 'Jan 18, 2023 @ 20:33:29.186'; -describe( +// Failing: See https://github.com/elastic/kibana/issues/167186 +describe.skip( 'Basic discover search and filter operations', { env: { ftrConfig: { enableExperimental: ['discoverInTimeline'] } }, diff --git a/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts b/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts index c935ab3f15f83..30d99d112e640 100644 --- a/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts +++ b/x-pack/test_serverless/functional/test_suites/search/dashboards/import_dashboard.ts @@ -27,8 +27,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'svlCommonPage', ]); - // Failing: See https://github.com/elastic/kibana/issues/166573 - describe.skip('Importing an existing dashboard', () => { + describe('Importing an existing dashboard', () => { before(async () => { await PageObjects.svlCommonPage.login(); await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/logstash_functional');