diff --git a/docs/user/alerting/create-and-manage-rules.asciidoc b/docs/user/alerting/create-and-manage-rules.asciidoc index b796128b95bc..7e35cd232566 100644 --- a/docs/user/alerting/create-and-manage-rules.asciidoc +++ b/docs/user/alerting/create-and-manage-rules.asciidoc @@ -166,7 +166,8 @@ responses: Click the rule name to access a rule details page: [role="screenshot"] -image::images/rule-details-alerts-active.png[Rule details page with three alerts] +image::images/rule-details-alerts-active.png[Rule details page with multiple alerts] +// NOTE: This is an autogenerated screenshot. Do not edit it directly. In this example, the rule detects when a site serves more than a threshold number of bytes in a 24 hour period. Four sites are above the threshold. These are called alerts - occurrences of the condition being detected - and the alert name, status, time of detection, and duration of the condition are shown in this view. Alerts come and go from the list depending on whether the rule conditions are met. @@ -182,4 +183,4 @@ You can suppress future actions for a specific alert by turning on the *Mute* to [role="screenshot"] image::images/rule-details-disabling.png[Use the disable toggle to turn off rule checks and clear alerts tracked] - +// NOTE: This is an autogenerated screenshot. Do not edit it directly. diff --git a/docs/user/alerting/images/individual-enable-disable.png b/docs/user/alerting/images/individual-enable-disable.png index 60de6079befb..dfac27dec39e 100644 Binary files a/docs/user/alerting/images/individual-enable-disable.png and b/docs/user/alerting/images/individual-enable-disable.png differ diff --git a/docs/user/alerting/images/rule-details-alerts-active.png b/docs/user/alerting/images/rule-details-alerts-active.png index 978ee180223b..deb2feff7993 100644 Binary files a/docs/user/alerting/images/rule-details-alerts-active.png and b/docs/user/alerting/images/rule-details-alerts-active.png differ diff --git a/docs/user/alerting/images/rule-details-disabling.png b/docs/user/alerting/images/rule-details-disabling.png index bee157ad2856..ad74410ab14e 100644 Binary files a/docs/user/alerting/images/rule-details-disabling.png and b/docs/user/alerting/images/rule-details-disabling.png differ diff --git a/docs/user/alerting/images/rules-ui.png b/docs/user/alerting/images/rules-ui.png index ba7b4db071fe..250f46392a45 100644 Binary files a/docs/user/alerting/images/rules-ui.png and b/docs/user/alerting/images/rules-ui.png differ diff --git a/docs/user/alerting/images/snooze-panel.png b/docs/user/alerting/images/snooze-panel.png index a65bfa6bf2e6..080c661ccccd 100644 Binary files a/docs/user/alerting/images/snooze-panel.png and b/docs/user/alerting/images/snooze-panel.png differ diff --git a/package.json b/package.json index ee6a2bd39713..b6b1ae184578 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "@dnd-kit/utilities": "^2.0.0", "@elastic/apm-rum": "^5.12.0", "@elastic/apm-rum-react": "^1.4.2", - "@elastic/charts": "53.1.0", + "@elastic/charts": "54.0.0", "@elastic/datemath": "5.0.3", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.6.0-canary.3", "@elastic/ems-client": "8.4.0", @@ -1302,7 +1302,7 @@ "apidoc-markdown": "^7.2.4", "argsplit": "^1.0.5", "autoprefixer": "^10.4.7", - "axe-core": "^4.0.2", + "axe-core": "^4.6.1", "babel-jest": "^29.3.1", "babel-loader": "^8.2.5", "babel-plugin-add-module-exports": "^1.0.4", diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap index d86194ac7dba..37aeb88feefa 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap @@ -20,6 +20,21 @@ Array [ test - Elastic , + ,
`; + +exports[`SkipToMainContent renders 1`] = ` + +`; diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx index 227a94a208ca..5449ee34bb66 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx @@ -46,7 +46,7 @@ import { HeaderActionMenu } from './header_action_menu'; import { HeaderExtension } from './header_extension'; import { HeaderTopBanner } from './header_top_banner'; import { HeaderMenuButton } from './header_menu_button'; -import { ScreenReaderRouteAnnouncements } from './screen_reader_a11y'; +import { ScreenReaderRouteAnnouncements, SkipToMainContent } from './screen_reader_a11y'; export interface HeaderProps { kibanaVersion: string; @@ -114,6 +114,7 @@ export function Header({ customBranding$={customBranding$} appId$={application.currentAppId$} /> +
diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx index 3dfc25c93c25..b468eefd51d7 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx @@ -9,8 +9,8 @@ import React from 'react'; import { BehaviorSubject } from 'rxjs'; import { mountWithIntl } from '@kbn/test-jest-helpers'; -import { ScreenReaderRouteAnnouncements } from './screen_reader_a11y'; -import { mount } from 'enzyme'; +import { ScreenReaderRouteAnnouncements, SkipToMainContent } from './screen_reader_a11y'; +import { mount, render } from 'enzyme'; describe('ScreenReaderRouteAnnouncements', () => { it('renders', () => { @@ -67,3 +67,10 @@ describe('ScreenReaderRouteAnnouncements', () => { ).toBeTruthy(); }); }); + +describe('SkipToMainContent', () => { + it('renders', () => { + const component = render(); + expect(component).toMatchSnapshot(); + }); +}); diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx index 811da5234141..f879e896297d 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx @@ -8,7 +8,8 @@ import React, { FC, useState, useEffect } from 'react'; import useObservable from 'react-use/lib/useObservable'; -import { EuiScreenReaderLive } from '@elastic/eui'; +import { EuiScreenReaderLive, EuiSkipLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import type { InternalApplicationStart } from '@kbn/core-application-browser-internal'; import type { HeaderProps } from './header'; @@ -56,3 +57,25 @@ export const ScreenReaderRouteAnnouncements: FC<{ ); }; + +const fallbackContentQueries = [ + 'main', // Ideal target for all plugins using KibanaPageTemplate + '[role="main"]', // Fallback for plugins using deprecated EuiPageContent + '.kbnAppWrapper', // Last-ditch fallback for all plugins regardless of page template +]; + +export const SkipToMainContent = () => { + return ( + + {i18n.translate('core.ui.skipToMainButton', { + defaultMessage: 'Skip to main content', + })} + + ); +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index f93602bd9935..bfe48e43491a 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -15,6 +15,7 @@ export { type IndexMapping, type IndexMappingMeta, type SavedObjectsTypeMappingDefinitions, + type IndexMappingMigrationStateMeta, } from './src/mappings'; export { SavedObjectsSerializer } from './src/serialization'; export { SavedObjectsTypeValidator } from './src/validation'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts index b7869bd12337..7b2bb933fab3 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts @@ -7,4 +7,9 @@ */ export { getTypes, getProperty, getRootProperties, getRootPropertiesObjects } from './lib'; -export type { SavedObjectsTypeMappingDefinitions, IndexMappingMeta, IndexMapping } from './types'; +export type { + SavedObjectsTypeMappingDefinitions, + IndexMappingMeta, + IndexMapping, + IndexMappingMigrationStateMeta, +} from './types'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index 0267f2ce27c1..10faa1b03d31 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -77,4 +77,19 @@ export interface IndexMappingMeta { * @remark: Only defined for indices using the zdt migration algorithm. */ docVersions?: { [k: string]: number }; + /** + * Info about the current state of the migration. + * Should only be present if a migration is in progress or was interrupted. + * + * @remark: Only defined for indices using the zdt migration algorithm. + */ + migrationState?: IndexMappingMigrationStateMeta; +} + +/** @internal */ +export interface IndexMappingMigrationStateMeta { + /** + * Indicates that the algorithm is currently converting the documents. + */ + convertingDocuments: boolean; } diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts index 8e7816a12fb5..01fc57d46462 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -10,16 +10,21 @@ import type { IndexMapping, IndexMappingMeta } from '../mappings'; import type { ModelVersionMap } from './version_map'; import { assertValidModelVersion } from './conversion'; +export interface GetModelVersionsFromMappingsOpts { + mappings: IndexMapping; + source: 'mappingVersions' | 'docVersions'; + /** if specified, will filter the types with the provided list */ + knownTypes?: string[]; +} + /** * Build the version map from the specified source of the provided mappings. */ export const getModelVersionsFromMappings = ({ mappings, source, -}: { - mappings: IndexMapping; - source: 'mappingVersions' | 'docVersions'; -}): ModelVersionMap | undefined => { + knownTypes, +}: GetModelVersionsFromMappingsOpts): ModelVersionMap | undefined => { if (!mappings._meta) { return undefined; } @@ -27,25 +32,35 @@ export const getModelVersionsFromMappings = ({ return getModelVersionsFromMappingMeta({ meta: mappings._meta, source, + knownTypes, }); }; +export interface GetModelVersionsFromMappingMetaOpts { + meta: IndexMappingMeta; + source: 'mappingVersions' | 'docVersions'; + /** if specified, will filter the types with the provided list */ + knownTypes?: string[]; +} + /** * Build the version map from the specified source of the provided mappings meta. */ export const getModelVersionsFromMappingMeta = ({ meta, source, -}: { - meta: IndexMappingMeta; - source: 'mappingVersions' | 'docVersions'; -}): ModelVersionMap | undefined => { + knownTypes, +}: GetModelVersionsFromMappingMetaOpts): ModelVersionMap | undefined => { const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; if (!indexVersions) { return undefined; } + const typeSet = knownTypes ? new Set(knownTypes) : undefined; + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { - map[type] = assertValidModelVersion(rawVersion); + if (!typeSet || typeSet.has(type)) { + map[type] = assertValidModelVersion(rawVersion); + } return map; }, {}); }; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts index c4d48d66a6a0..6d1c0e3df4ce 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts @@ -32,6 +32,9 @@ const migrationSchema = schema.object({ pollInterval: schema.number({ defaultValue: 1_500 }), skip: schema.boolean({ defaultValue: false }), retryAttempts: schema.number({ defaultValue: 15 }), + zdt: schema.object({ + metaPickupSyncDelaySec: schema.number({ min: 1, defaultValue: 120 }), + }), }); export type SavedObjectsMigrationConfigType = TypeOf; @@ -60,6 +63,7 @@ export const savedObjectsConfig: ServiceConfigDescriptor path: 'savedObjects', schema: soSchema, }; + export class SavedObjectConfig { public maxImportPayloadBytes: number; public maxImportExportSize: number; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts index 5dfdb05a0bca..8b8e0f830261 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts @@ -7,3 +7,5 @@ */ export const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; + +export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts new file mode 100644 index 000000000000..1181eb992be3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { BulkOperation } from '../model/create_batches'; + +export const redactBulkOperationBatches = ( + bulkOperationBatches: BulkOperation[][] +): BulkOperationContainer[][] => { + return bulkOperationBatches.map((batch) => + batch.map((operation) => (Array.isArray(operation) ? operation[0] : operation)) + ); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts index 22c58ae62adc..c83cab1c1d0b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts @@ -15,8 +15,8 @@ import type { SavedObjectSanitizedDoc, SavedObjectsRawDoc, SavedObjectUnsanitizedDoc, + ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import { SavedObjectsSerializer } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrateAndConvertFn } from '../document_migrator/document_migrator'; import { TransformSavedObjectDocumentError } from '.'; @@ -65,7 +65,7 @@ export class CorruptSavedObjectError extends Error { * @returns {SavedObjectsRawDoc[]} */ export async function migrateRawDocs( - serializer: SavedObjectsSerializer, + serializer: ISavedObjectsSerializer, migrateDoc: MigrateAndConvertFn, rawDocs: SavedObjectsRawDoc[] ): Promise { @@ -86,7 +86,7 @@ export async function migrateRawDocs( } interface MigrateRawDocsSafelyDeps { - serializer: SavedObjectsSerializer; + serializer: ISavedObjectsSerializer; migrateDoc: MigrateAndConvertFn; rawDocs: SavedObjectsRawDoc[]; } @@ -181,7 +181,7 @@ function transformNonBlocking( async function migrateMapToRawDoc( migrateMethod: MigrateFn, savedObject: SavedObjectSanitizedDoc, - serializer: SavedObjectsSerializer + serializer: ISavedObjectsSerializer ): Promise { return [...(await migrateMethod(savedObject))].map((attrs) => serializer.savedObjectToRaw({ @@ -201,7 +201,7 @@ async function migrateMapToRawDoc( function convertToRawAddMigrationVersion( rawDoc: SavedObjectsRawDoc, options: { namespaceTreatment: 'lax' }, - serializer: SavedObjectsSerializer + serializer: ISavedObjectsSerializer ): SavedObjectSanitizedDoc { const savedObject = serializer.rawToSavedObject(rawDoc, options); if (!savedObject.migrationVersion && !savedObject.typeMigrationVersion) { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index f5a462ebceda..e6855a1256b5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -288,6 +288,9 @@ const mockOptions = () => { scrollDuration: '10m', skip: false, retryAttempts: 20, + zdt: { + metaPickupSyncDelaySec: 120, + }, }, client: mockedClient, docLinks: docLinksServiceMock.createSetupContract(), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts index 48a3bad0d096..d03b4f7378da 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts @@ -51,6 +51,9 @@ describe('migrationsStateActionMachine', () => { scrollDuration: '0s', skip: false, retryAttempts: 5, + zdt: { + metaPickupSyncDelaySec: 120, + }, }, typeRegistry, docLinks, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts index 1b5caf3c4e75..92b0054bd47c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts @@ -15,12 +15,11 @@ import { getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; import { logActionResponse, logStateTransition } from './common/utils/logs'; import { type Model, type Next, stateActionMachine } from './state_action_machine'; import { cleanup } from './migrations_state_machine_cleanup'; import type { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './state'; -import type { BulkOperation } from './model/create_batches'; +import { redactBulkOperationBatches } from './common/redact_state'; /** * A specialized migrations-specific state-action machine that: @@ -159,11 +158,3 @@ export async function migrationStateActionMachine({ } } } - -const redactBulkOperationBatches = ( - bulkOperationBatches: BulkOperation[][] -): BulkOperationContainer[][] => { - return bulkOperationBatches.map((batch) => - batch.map((operation) => (Array.isArray(operation) ? operation[0] : operation)) - ); -}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts index ec19f834d9ce..008d074b2cd6 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts @@ -14,6 +14,7 @@ import type { TransformErrorObjects } from '../core'; export type BulkIndexOperationTuple = [BulkOperationContainer, SavedObjectsRawDocSource]; export type BulkOperation = BulkIndexOperationTuple | BulkOperationContainer; +export type BulkOperationBatch = BulkOperation[]; export interface CreateBatchesParams { documents: SavedObjectsRawDoc[]; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 2f78d8745e55..23ddee504326 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -46,9 +46,10 @@ import { } from './helpers'; import { createBatches } from './create_batches'; import type { MigrationLog } from '../types'; -import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../common/constants'; - -export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; +import { + CLUSTER_SHARD_LIMIT_EXCEEDED_REASON, + FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, +} from '../common/constants'; export const model = (currentState: State, resW: ResponseType): State => { // The action response `resW` is weakly typed, the type includes all action diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index bb135c115ce9..a3db45a3748c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -6,17 +6,7 @@ * Side Public License, v 1. */ -import type { - IncompatibleClusterRoutingAllocation, - RetryableEsClientError, - WaitForTaskCompletionTimeout, - IndexNotYellowTimeout, - IndexNotGreenTimeout, - ClusterShardLimitExceeded, - IndexNotFound, - AliasNotFound, - IncompatibleMappingException, -} from '../../actions'; +import type { ActionErrorTypeMap as BaseActionErrorTypeMap } from '../../actions'; export { initAction as init, @@ -25,7 +15,16 @@ export { updateAliases, updateMappings, updateAndPickupMappings, + cleanupUnknownAndExcluded, + waitForDeleteByQueryTask, waitForPickupUpdatedMappingsTask, + refreshIndex, + openPit, + readWithPit, + closePit, + transformDocs, + bulkOverwriteTransformedDocuments, + noop, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -33,17 +32,11 @@ export { type IndexNotFound, } from '../../actions'; -export interface ActionErrorTypeMap { - wait_for_task_completion_timeout: WaitForTaskCompletionTimeout; - incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation; - retryable_es_client_error: RetryableEsClientError; - index_not_found_exception: IndexNotFound; - index_not_green_timeout: IndexNotGreenTimeout; - index_not_yellow_timeout: IndexNotYellowTimeout; - cluster_shard_limit_exceeded: ClusterShardLimitExceeded; - alias_not_found_exception: AliasNotFound; - incompatible_mapping_exception: IncompatibleMappingException; -} +export { updateIndexMeta, type UpdateIndexMetaParams } from './update_index_meta'; +export { waitForDelay, type WaitForDelayParams } from './wait_for_delay'; + +// alias in case we need to extend it with zdt specific actions/errors +export type ActionErrorTypeMap = BaseActionErrorTypeMap; /** Type guard for narrowing the type of a left */ export function isTypeof( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts new file mode 100644 index 000000000000..8f209365a285 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const updateMappingsMock = jest.fn(); + +jest.doMock('../../actions/update_mappings', () => { + const actual = jest.requireActual('../../actions/update_mappings'); + return { + ...actual, + updateMappings: updateMappingsMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts new file mode 100644 index 000000000000..6ed55ccb49eb --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { updateMappingsMock } from './update_index_meta.test.mocks'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { updateIndexMeta } from './update_index_meta'; + +describe('updateIndexMeta', () => { + it('calls updateMappings with the correct parameters', () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + const index = '.kibana_1'; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + }, + }; + + updateIndexMeta({ client, index, meta }); + + expect(updateMappingsMock).toHaveBeenCalledTimes(1); + expect(updateMappingsMock).toHaveBeenCalledWith({ + client, + index, + mappings: { + properties: {}, + _meta: meta, + }, + }); + }); + + it('returns the response from updateMappings', () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + const index = '.kibana_1'; + const meta: IndexMappingMeta = {}; + + const expected = Symbol(); + updateMappingsMock.mockReturnValue(expected); + + const actual = updateIndexMeta({ client, index, meta }); + + expect(actual).toBe(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts new file mode 100644 index 000000000000..195e282dce5a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { updateMappings } from '../../actions'; + +export interface UpdateIndexMetaParams { + client: ElasticsearchClient; + index: string; + meta: IndexMappingMeta; +} + +export const updateIndexMeta = ({ + client, + index, + meta, +}: UpdateIndexMetaParams): ReturnType => { + return updateMappings({ + client, + index, + mappings: { + properties: {}, + _meta: meta, + }, + }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts new file mode 100644 index 000000000000..fbfc144bd815 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { waitForDelay } from './wait_for_delay'; + +const nextTick = () => new Promise((resolve) => resolve()); +const aFewTicks = () => nextTick().then(nextTick).then(nextTick); + +describe('waitForDelay', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('resolves after the specified amount of time', async () => { + const handler = jest.fn(); + + waitForDelay({ delayInSec: 5 })().then(handler); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(5000); + await aFewTicks(); + + expect(handler).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts new file mode 100644 index 000000000000..302a702331de --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import * as TaskEither from 'fp-ts/lib/TaskEither'; + +export interface WaitForDelayParams { + delayInSec: number; +} + +export const waitForDelay = ({ + delayInSec, +}: WaitForDelayParams): TaskEither.TaskEither => { + return () => { + return delay(delayInSec) + .then(() => Either.right('wait_succeeded' as const)) + .catch((err) => { + // will never happen + throw err; + }); + }; +}; + +const delay = (delayInSec: number) => + new Promise((resolve) => setTimeout(resolve, delayInSec * 1000)); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index 7a660ea47044..c31d4bc799b3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -21,12 +21,15 @@ export const createContext = ({ types, docLinks, migrationConfig, + documentMigrator, elasticsearchClient, indexPrefix, typeRegistry, serializer, }: CreateContextOps): MigratorContext => { return { + migrationConfig, + documentMigrator, kibanaVersion, indexPrefix, types, @@ -37,5 +40,6 @@ export const createContext = ({ maxRetryAttempts: migrationConfig.retryAttempts, migrationDocLinks: docLinks.links.kibanaUpgradeSavedObjects, deletedTypes: REMOVED_TYPES, + discardCorruptObjects: Boolean(migrationConfig.discardCorruptObjects), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 5b6d4b2fe27e..95ca7282daf5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -11,13 +11,19 @@ import type { ISavedObjectTypeRegistry, ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import type { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { + ModelVersionMap, + SavedObjectsMigrationConfigType, +} from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; +import { VersionedTransformer } from '../../document_migrator'; /** * The set of static, precomputed values and services used by the ZDT migration */ export interface MigratorContext { + /** The migration configuration */ + readonly migrationConfig: SavedObjectsMigrationConfigType; /** The current Kibana version */ readonly kibanaVersion: string; /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ @@ -34,8 +40,12 @@ export interface MigratorContext { readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects']; /** SO serializer to use for migration */ readonly serializer: ISavedObjectsSerializer; + /** The doc migrator to use */ + readonly documentMigrator: VersionedTransformer; /** The SO type registry to use for the migration */ readonly typeRegistry: ISavedObjectTypeRegistry; /** List of types that are no longer registered */ readonly deletedTypes: string[]; + /** If true, corrupted objects will be discarded instead of failing the migration */ + readonly discardCorruptObjects: boolean; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts index 8982a1a9c6c7..cb6a30ce50c6 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts @@ -12,11 +12,17 @@ import { getErrorMessage, getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; +import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; import { logStateTransition, logActionResponse } from '../common/utils'; import { type Next, stateActionMachine } from '../state_action_machine'; import { cleanup } from '../migrations_state_machine_cleanup'; -import type { State } from './state'; +import type { + State, + OutdatedDocumentsSearchTransformState, + OutdatedDocumentsSearchBulkIndexState, +} from './state'; import type { MigratorContext } from './context'; +import { redactBulkOperationBatches } from '../common/redact_state'; /** * A specialized migrations-specific state-action machine that: @@ -60,23 +66,12 @@ export async function migrationStateActionMachine({ // the _id's of documents const redactedNewState = { ...newState, - /* TODO: commented until we have model stages that process outdated docs. (attrs not on model atm) - ...{ - outdatedDocuments: ( - (newState as ReindexSourceToTempTransform).outdatedDocuments ?? [] - ).map( - (doc) => - ({ - _id: doc._id, - } as SavedObjectsRawDoc) - ), - }, - ...{ - transformedDocBatches: ( - (newState as ReindexSourceToTempIndexBulk).transformedDocBatches ?? [] - ).map((batches) => batches.map((doc) => ({ _id: doc._id }))) as [SavedObjectsRawDoc[]], - }, - */ + outdatedDocuments: ( + (newState as OutdatedDocumentsSearchTransformState).outdatedDocuments ?? [] + ).map((doc) => ({ _id: doc._id } as SavedObjectsRawDoc)), + bulkOperationBatches: redactBulkOperationBatches( + (newState as OutdatedDocumentsSearchBulkIndexState).bulkOperationBatches ?? [[]] + ), }; const now = Date.now(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts index c0316b954e5f..bef92e198674 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts @@ -6,13 +6,13 @@ * Side Public License, v 1. */ -import { StageMocks } from './model.test.mocks'; +import './model.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { createContextMock, MockedMigratorContext } from '../test_helpers'; import type { RetryableEsClientError } from '../../actions'; import type { State, BaseState, FatalState, AllActionStates } from '../state'; import type { StateActionResponse } from './types'; -import { model } from './model'; +import { model, modelStageMap } from './model'; describe('model', () => { let context: MockedMigratorContext; @@ -128,16 +128,7 @@ describe('model', () => { }, }); - const stageMapping: Record = { - INIT: StageMocks.init, - CREATE_TARGET_INDEX: StageMocks.createTargetIndex, - UPDATE_INDEX_MAPPINGS: StageMocks.updateIndexMappings, - UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: StageMocks.updateIndexMappingsWaitForTask, - UPDATE_MAPPING_MODEL_VERSIONS: StageMocks.updateMappingModelVersion, - UPDATE_ALIASES: StageMocks.updateAliases, - }; - - Object.entries(stageMapping).forEach(([stage, handler]) => { + Object.entries(modelStageMap).forEach(([stage, handler]) => { test(`dispatch ${stage} state`, () => { const state = createStubState(stage as AllActionStates); const res = createStubResponse(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 62971c3a614a..483e97e3a7f8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -12,9 +12,44 @@ import type { ResponseType } from '../next'; import { delayRetryState, resetRetryState } from '../../model/retry_state'; import { throwBadControlState } from '../../model/helpers'; import { isTypeof } from '../actions'; -import { MigratorContext } from '../context'; +import type { MigratorContext } from '../context'; +import type { ModelStage } from './types'; import * as Stages from './stages'; -import { StateActionResponse } from './types'; + +type ModelStageMap = { + [K in AllActionStates]: ModelStage; +}; + +type AnyModelStageHandler = ( + state: State, + response: Either.Either, + ctx: MigratorContext +) => State; + +export const modelStageMap: ModelStageMap = { + INIT: Stages.init, + CREATE_TARGET_INDEX: Stages.createTargetIndex, + UPDATE_ALIASES: Stages.updateAliases, + UPDATE_INDEX_MAPPINGS: Stages.updateIndexMappings, + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: Stages.updateIndexMappingsWaitForTask, + UPDATE_MAPPING_MODEL_VERSIONS: Stages.updateMappingModelVersion, + INDEX_STATE_UPDATE_DONE: Stages.indexStateUpdateDone, + DOCUMENTS_UPDATE_INIT: Stages.documentsUpdateInit, + SET_DOC_MIGRATION_STARTED: Stages.setDocMigrationStarted, + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: Stages.setDocMigrationStartedWaitForInstances, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: Stages.cleanupUnknownAndExcludedDocs, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: Stages.cleanupUnknownAndExcludedDocsWaitForTask, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: Stages.cleanupUnknownAndExcludedDocsRefresh, + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: Stages.outdatedDocumentsSearchOpenPit, + OUTDATED_DOCUMENTS_SEARCH_READ: Stages.outdatedDocumentsSearchRead, + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: Stages.outdatedDocumentsSearchTransform, + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: Stages.outdatedDocumentsSearchBulkIndex, + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: Stages.outdatedDocumentsSearchClosePit, + OUTDATED_DOCUMENTS_SEARCH_REFRESH: Stages.outdatedDocumentsSearchRefresh, + UPDATE_DOCUMENT_MODEL_VERSIONS: Stages.updateDocumentModelVersion, + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: + Stages.updateDocumentModelVersionWaitForInstances, +}; export const model = ( current: State, @@ -29,44 +64,14 @@ export const model = ( current = resetRetryState(current); } - switch (current.controlState) { - case 'INIT': - return Stages.init(current, response as StateActionResponse<'INIT'>, context); - case 'CREATE_TARGET_INDEX': - return Stages.createTargetIndex( - current, - response as StateActionResponse<'CREATE_TARGET_INDEX'>, - context - ); - case 'UPDATE_ALIASES': - return Stages.updateAliases( - current, - response as StateActionResponse<'UPDATE_ALIASES'>, - context - ); - case 'UPDATE_INDEX_MAPPINGS': - return Stages.updateIndexMappings( - current, - response as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, - context - ); - case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': - return Stages.updateIndexMappingsWaitForTask( - current, - response as StateActionResponse<'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'>, - context - ); - case 'UPDATE_MAPPING_MODEL_VERSIONS': - return Stages.updateMappingModelVersion( - current, - response as StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'>, - context - ); - case 'DONE': - case 'FATAL': - // The state-action machine will never call the model in the terminating states - return throwBadControlState(current as never); - default: - return throwBadControlState(current); + if (current.controlState === 'DONE' || current.controlState === 'FATAL') { + return throwBadControlState(current as never); + } + + const stageHandler = modelStageMap[current.controlState] as AnyModelStageHandler; + if (!stageHandler) { + return throwBadControlState(current as never); } + + return stageHandler(current, response, context); }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts new file mode 100644 index 000000000000..29730ab1ec78 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocs } from './cleanup_unknown_and_excluded_docs'; + +describe('Stage: cleanupUnknownAndExcludedDocs', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'> = Either.right({ + type: 'cleanup_started', + taskId: '42', + errorsByType: {}, + unknownDocs: [], + }); + + const newState = cleanupUnknownAndExcludedDocs(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: '42', + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> FATAL when unsuccessful', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'> = Either.left({ + type: 'unknown_docs_found', + unknownDocs: [], + }); + + const newState = cleanupUnknownAndExcludedDocs(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts new file mode 100644 index 000000000000..6aca449a73de --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { extractUnknownDocFailureReason } from '../../../model/extract_errors'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocs: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + return { + ...state, + controlState: 'FATAL', + reason: extractUnknownDocFailureReason( + context.migrationDocLinks.resolveMigrationFailures, + res.left.unknownDocs + ), + }; + } + + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: res.right.taskId, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts new file mode 100644 index 000000000000..10d7ecc6a384 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsRefreshState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocsRefresh } from './cleanup_unknown_and_excluded_docs_refresh'; + +describe('Stage: cleanupUnknownAndExcludedDocsRefresh', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsRefreshState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT when successful', () => { + const state = createState(); + const res = Either.right({ + refreshed: true, + }) as StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH'>; + + const newState = cleanupUnknownAndExcludedDocsRefresh(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts new file mode 100644 index 000000000000..32c4dc5bf663 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocsRefresh: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts new file mode 100644 index 000000000000..e6deae559077 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsWaitForTaskState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocsWaitForTask } from './cleanup_unknown_and_excluded_docs_wait_for_task'; + +describe('Stage: cleanupUnknownAndExcludedDocsWaitForTask', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsWaitForTaskState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: '42', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK in case of wait_for_task_completion_timeout', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'wait_for_task_completion_timeout', + message: 'woups', + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + retryCount: 1, + retryDelay: expect.any(Number), + logs: expect.any(Array), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS in case of cleanup_failed', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'cleanup_failed', + failures: [], + versionConflicts: 42, + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + hasDeletedDocs: true, + retryCount: 1, + retryDelay: expect.any(Number), + logs: expect.any(Array), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> FATAL in case of cleanup_failed when exceeding retry count', () => { + const state = createState({ + retryCount: context.maxRetryAttempts + 1, + }); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'cleanup_failed', + failures: [], + versionConflicts: 42, + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH when successful and docs were deleted', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = + Either.right({ + type: 'cleanup_successful', + deleted: 9000, + }); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT when successful and no docs were deleted', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = + Either.right({ + type: 'cleanup_successful', + deleted: 0, + }); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts new file mode 100644 index 000000000000..83baaccea992 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocsWaitForTask: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + | 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH' + | 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' + | 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' + | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + if (isTypeof(res.left, 'wait_for_task_completion_timeout')) { + // After waiting for the specified timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticsearch task succeeds or fails. + return delayRetryState(state, res.left.message, Number.MAX_SAFE_INTEGER); + } else { + if (state.retryCount < context.maxRetryAttempts) { + const retryCount = state.retryCount + 1; + const retryDelay = 1500 + 1000 * Math.random(); + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + hasDeletedDocs: true, + retryCount, + retryDelay, + logs: [ + ...state.logs, + { + level: 'warning', + message: `Errors occurred whilst deleting unwanted documents. Retrying attempt ${retryCount}.`, + }, + ], + }; + } else { + const failures = res.left.failures.length; + const versionConflicts = res.left.versionConflicts ?? 0; + let reason = `Migration failed because it was unable to delete unwanted documents from the ${state.currentIndex} system index (${failures} failures and ${versionConflicts} conflicts)`; + if (failures) { + reason += `:\n` + res.left.failures.map((failure: string) => `- ${failure}\n`).join(''); + } + return { + ...state, + controlState: 'FATAL', + reason, + }; + } + } + } + + const mustRefresh = + state.hasDeletedDocs || typeof res.right.deleted === 'undefined' || res.right.deleted > 0; + + if (mustRefresh) { + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + }; + } else { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts index cd15594d32b2..4aaee33a8f97 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts @@ -84,7 +84,7 @@ describe('Stage: createTargetIndex', () => { }); }); - it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful', () => { + it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful and alias actions are not empty', () => { const state = createState(); const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.right('create_index_succeeded'); @@ -101,6 +101,27 @@ describe('Stage: createTargetIndex', () => { currentIndexMeta: state.indexMappings._meta, aliases: [], aliasActions, + newIndexCreation: true, + }); + }); + + it('CREATE_TARGET_INDEX -> INDEX_STATE_UPDATE_DONE when successful and alias actions are empty', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + getAliasActionsMock.mockReturnValue([]); + + const newState = createTargetIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'INDEX_STATE_UPDATE_DONE', + previousMappings: state.indexMappings, + currentIndexMeta: state.indexMappings._meta, + aliases: [], + aliasActions: [], + newIndexCreation: true, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts index bb0697a70cf5..ab0f42182267 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts @@ -15,11 +15,10 @@ import { isTypeof } from '../../actions'; import { getAliasActions } from '../../utils'; import type { ModelStage } from '../types'; -export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASES' | 'FATAL'> = ( - state, - res, - context -) => { +export const createTargetIndex: ModelStage< + 'CREATE_TARGET_INDEX', + 'UPDATE_ALIASES' | 'INDEX_STATE_UPDATE_DONE' | 'FATAL' +> = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'index_not_green_timeout')) { @@ -48,10 +47,11 @@ export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASE return { ...state, - controlState: 'UPDATE_ALIASES', + controlState: aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', previousMappings: state.indexMappings, currentIndexMeta, aliases: [], aliasActions, + newIndexCreation: true, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts new file mode 100644 index 000000000000..a16bed57fec6 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getOutdatedDocumentsQueryMock = jest.fn(); +export const createDocumentTransformFnMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getOutdatedDocumentsQuery: getOutdatedDocumentsQueryMock, + createDocumentTransformFn: createDocumentTransformFnMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts new file mode 100644 index 000000000000..abf8a689ad62 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + getOutdatedDocumentsQueryMock, + createDocumentTransformFnMock, +} from './documents_update_init.test.mocks'; +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { DocumentsUpdateInitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { documentsUpdateInit } from './documents_update_init'; +import { createType } from '../../test_helpers'; + +describe('Stage: documentsUpdateInit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): DocumentsUpdateInitState => ({ + ...createPostInitState(), + controlState: 'DOCUMENTS_UPDATE_INIT', + ...parts, + }); + + beforeEach(() => { + getOutdatedDocumentsQueryMock.mockReset(); + createDocumentTransformFnMock.mockReset(); + + context = createContextMock(); + context.typeRegistry.registerType(createType({ name: 'foo' })); + context.typeRegistry.registerType(createType({ name: 'bar' })); + }); + + it('calls getOutdatedDocumentsQuery with the correct parameters', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + documentsUpdateInit(state, res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, context); + + expect(getOutdatedDocumentsQueryMock).toHaveBeenCalledTimes(1); + expect(getOutdatedDocumentsQueryMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + }); + }); + + it('calls createDocumentTransformFn with the correct parameters', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + documentsUpdateInit(state, res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, context); + + expect(createDocumentTransformFnMock).toHaveBeenCalledTimes(1); + expect(createDocumentTransformFnMock).toHaveBeenCalledWith({ + serializer: context.serializer, + documentMigrator: context.documentMigrator, + }); + }); + + it('DOCUMENTS_UPDATE_INIT -> SET_DOC_MIGRATION_STARTED when successful', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + const newState = documentsUpdateInit( + state, + res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, + context + ); + expect(newState.controlState).toEqual('SET_DOC_MIGRATION_STARTED'); + }); + + it('updates the state with the expected properties', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + const transformRawDocs = jest.fn(); + createDocumentTransformFnMock.mockReturnValue(transformRawDocs); + + const outdatedDocumentsQuery = Symbol(); + getOutdatedDocumentsQueryMock.mockReturnValue(outdatedDocumentsQuery); + + const newState = documentsUpdateInit( + state, + res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED', + transformRawDocs, + outdatedDocumentsQuery, + excludeFromUpgradeFilterHooks: expect.any(Object), + excludeOnUpgradeQuery: expect.any(Object), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts new file mode 100644 index 000000000000..a4f94e3cbe7b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { excludeUnusedTypesQuery } from '../../../core'; +import type { ModelStage } from '../types'; +import { getOutdatedDocumentsQuery, createDocumentTransformFn } from '../../utils'; + +export const documentsUpdateInit: ModelStage< + 'DOCUMENTS_UPDATE_INIT', + 'SET_DOC_MIGRATION_STARTED' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const excludeFilterHooks = Object.fromEntries( + context.types + .map((name) => context.typeRegistry.getType(name)!) + .filter((type) => !!type.excludeOnUpgrade) + .map((type) => [type.name, type.excludeOnUpgrade!]) + ); + + const types = context.types.map((type) => context.typeRegistry.getType(type)!); + const outdatedDocumentsQuery = getOutdatedDocumentsQuery({ types }); + + const transformRawDocs = createDocumentTransformFn({ + serializer: context.serializer, + documentMigrator: context.documentMigrator, + }); + + return { + ...state, + excludeOnUpgradeQuery: excludeUnusedTypesQuery, + excludeFromUpgradeFilterHooks: excludeFilterHooks, + outdatedDocumentsQuery, + transformRawDocs, + controlState: 'SET_DOC_MIGRATION_STARTED', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index 0322f92eb35a..532f15b5417e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -12,3 +12,18 @@ export { updateAliases } from './update_aliases'; export { updateIndexMappings } from './update_index_mappings'; export { updateIndexMappingsWaitForTask } from './update_index_mappings_wait_for_task'; export { updateMappingModelVersion } from './update_mapping_model_version'; +export { indexStateUpdateDone } from './index_state_update_done'; +export { documentsUpdateInit } from './documents_update_init'; +export { setDocMigrationStarted } from './set_doc_migration_started'; +export { setDocMigrationStartedWaitForInstances } from './set_doc_migration_started_wait_for_instances'; +export { cleanupUnknownAndExcludedDocs } from './cleanup_unknown_and_excluded_docs'; +export { cleanupUnknownAndExcludedDocsWaitForTask } from './cleanup_unknown_and_excluded_docs_wait_for_task'; +export { cleanupUnknownAndExcludedDocsRefresh } from './cleanup_unknown_and_excluded_docs_refresh'; +export { outdatedDocumentsSearchOpenPit } from './outdated_documents_search_open_pit'; +export { outdatedDocumentsSearchRead } from './outdated_documents_search_read'; +export { outdatedDocumentsSearchTransform } from './outdated_documents_search_transform'; +export { outdatedDocumentsSearchBulkIndex } from './outdated_documents_search_bulk_index'; +export { outdatedDocumentsSearchClosePit } from './outdated_documents_search_close_pit'; +export { outdatedDocumentsSearchRefresh } from './outdated_documents_search_refresh'; +export { updateDocumentModelVersion } from './update_document_model_version'; +export { updateDocumentModelVersionWaitForInstances } from './update_document_model_version_wait_for_instances'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts new file mode 100644 index 000000000000..2b523887109b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { IndexStateUpdateDoneState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { indexStateUpdateDone } from './index_state_update_done'; + +describe('Stage: indexStateUpdateDone', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): IndexStateUpdateDoneState => ({ + ...createPostDocInitState(), + controlState: 'INDEX_STATE_UPDATE_DONE', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT when successful and newIndexCreation is false', () => { + const state = createState({ + newIndexCreation: false, + }); + const res = Either.right('noop') as StateActionResponse<'INDEX_STATE_UPDATE_DONE'>; + + const newState = indexStateUpdateDone(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DOCUMENTS_UPDATE_INIT', + }); + }); + + it('INDEX_STATE_UPDATE_DONE -> DONE when successful and newIndexCreation is true', () => { + const state = createState({ + newIndexCreation: true, + }); + const res = Either.right('noop') as StateActionResponse<'INDEX_STATE_UPDATE_DONE'>; + + const newState = indexStateUpdateDone(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts new file mode 100644 index 000000000000..2ca41886c8b6 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const indexStateUpdateDone: ModelStage< + 'INDEX_STATE_UPDATE_DONE', + 'DOCUMENTS_UPDATE_INIT' | 'DONE' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + if (state.newIndexCreation) { + // we created the index, so we can safely skip the whole document migration + // and go directly to DONE + return { + ...state, + controlState: 'DONE', + }; + } else { + return { + ...state, + controlState: 'DOCUMENTS_UPDATE_INIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts index 8f773fe95117..89c513fa2a66 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -10,6 +10,7 @@ export const getCurrentIndexMock = jest.fn(); export const checkVersionCompatibilityMock = jest.fn(); export const buildIndexMappingsMock = jest.fn(); export const generateAdditiveMappingDiffMock = jest.fn(); +export const getAliasActionsMock = jest.fn(); jest.doMock('../../utils', () => { const realModule = jest.requireActual('../../utils'); @@ -19,5 +20,6 @@ jest.doMock('../../utils', () => { checkVersionCompatibility: checkVersionCompatibilityMock, buildIndexMappings: buildIndexMappingsMock, generateAdditiveMappingDiff: generateAdditiveMappingDiffMock, + getAliasActions: getAliasActionsMock, }; }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index ea6f4424404e..d8c176af4be0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -11,6 +11,7 @@ import { checkVersionCompatibilityMock, buildIndexMappingsMock, generateAdditiveMappingDiffMock, + getAliasActionsMock, } from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { FetchIndexResponse } from '../../../actions'; @@ -49,6 +50,7 @@ describe('Stage: init', () => { status: 'equal', }); generateAdditiveMappingDiffMock.mockReset().mockReturnValue({}); + getAliasActionsMock.mockReset().mockReturnValue([]); context = createContextMock({ indexPrefix: '.kibana', types: ['foo', 'bar'] }); context.typeRegistry.registerType({ @@ -65,7 +67,7 @@ describe('Stage: init', () => { }); }); - it('loops to INIT when cluster routing allocation is incompatible', () => { + it('INIT -> INIT when cluster routing allocation is incompatible', () => { const state = createState(); const res: StateActionResponse<'INIT'> = Either.left({ type: 'incompatible_cluster_routing_allocation', @@ -124,7 +126,7 @@ describe('Stage: init', () => { }); }); - it('forwards to CREATE_TARGET_INDEX', () => { + it('INIT -> CREATE_TARGET_INDEX', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -164,7 +166,7 @@ describe('Stage: init', () => { }); }); - it('forwards to UPDATE_INDEX_MAPPINGS', () => { + it('INIT -> UPDATE_INDEX_MAPPINGS', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -182,6 +184,7 @@ describe('Stage: init', () => { currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, additiveMappingChanges: { someToken: {} }, + newIndexCreation: false, }) ); }); @@ -203,7 +206,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `equal`', () => { - it('forwards to UPDATE_ALIASES', () => { + it('INIT -> UPDATE_ALIASES if alias actions are not empty', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -211,6 +214,7 @@ describe('Stage: init', () => { checkVersionCompatibilityMock.mockReturnValue({ status: 'equal', }); + getAliasActionsMock.mockReturnValue([{ add: { index: '.kibana_1', alias: '.kibana' } }]); const newState = init(state, res, context); @@ -219,6 +223,29 @@ describe('Stage: init', () => { controlState: 'UPDATE_ALIASES', currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, + newIndexCreation: false, + }) + ); + }); + + it('INIT -> INDEX_STATE_UPDATE_DONE if alias actions are empty', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + getAliasActionsMock.mockReturnValue([]); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'INDEX_STATE_UPDATE_DONE', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + newIndexCreation: false, }) ); }); @@ -240,7 +267,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `lesser`', () => { - it('forwards to FATAL', () => { + it('INIT -> FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -276,7 +303,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `conflict`', () => { - it('forwards to FATAL', () => { + it('INIT -> FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index da138730364f..19538cc1a4a8 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -23,7 +23,11 @@ import type { ModelStage } from '../types'; export const init: ModelStage< 'INIT', - 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_ALIASES' | 'FATAL' + | 'CREATE_TARGET_INDEX' + | 'UPDATE_INDEX_MAPPINGS' + | 'UPDATE_ALIASES' + | 'INDEX_STATE_UPDATE_DONE' + | 'FATAL' > = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; @@ -78,6 +82,15 @@ export const init: ModelStage< // cloning as we may be mutating it in later stages. const currentIndexMeta = cloneDeep(currentMappings._meta!); + const commonState = { + logs, + currentIndex, + currentIndexMeta, + aliases, + aliasActions, + previousMappings: currentMappings, + }; + switch (versionCheck.status) { // app version is greater than the index mapping version. // scenario of an upgrade: we need to update the mappings @@ -90,13 +103,9 @@ export const init: ModelStage< return { ...state, controlState: 'UPDATE_INDEX_MAPPINGS', - logs, - currentIndex, - currentIndexMeta, - aliases, - aliasActions, - previousMappings: currentMappings, + ...commonState, additiveMappingChanges, + newIndexCreation: false, }; // app version and index mapping version are the same. // either application upgrade without model change, or a simple reboot on the same version. @@ -104,13 +113,9 @@ export const init: ModelStage< case 'equal': return { ...state, - controlState: 'UPDATE_ALIASES', - logs, - currentIndex, - currentIndexMeta, - aliases, - aliasActions, - previousMappings: currentMappings, + controlState: aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', + ...commonState, + newIndexCreation: false, }; // app version is lower than the index mapping version. // likely a rollback scenario - unsupported for the initial implementation diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts new file mode 100644 index 000000000000..323a8ba64687 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchBulkIndexState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchBulkIndex } from './outdated_documents_search_bulk_index'; + +describe('Stage: outdatedDocumentsSearchBulkIndex', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchBulkIndexState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + bulkOperationBatches: [], + currentBatch: 0, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX when there are remaining batches', () => { + const state = createState({ + currentBatch: 0, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res = Either.right( + 'bulk_index_succeeded' + ) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'>; + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: 1, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_READ when there are no remaining batches', () => { + const state = createState({ + currentBatch: 1, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'> = + Either.right('bulk_index_succeeded'); + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: true, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> FATAL in case of request_entity_too_large_exception', () => { + const state = createState({ + currentBatch: 1, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'> = Either.left({ + type: 'request_entity_too_large_exception', + }); + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts new file mode 100644 index 000000000000..4d6bba19c515 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { FATAL_REASON_REQUEST_ENTITY_TOO_LARGE } from '../../../common/constants'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchBulkIndex: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + 'OUTDATED_DOCUMENTS_SEARCH_READ' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + if (isTypeof(res.left, 'request_entity_too_large_exception')) { + return { + ...state, + controlState: 'FATAL', + reason: FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, + }; + } else if ( + isTypeof(res.left, 'target_index_had_write_block') || + isTypeof(res.left, 'index_not_found_exception') + ) { + // we fail on these errors since the target index will never get + // deleted and should only have a write block if a newer version of + // Kibana started an upgrade + throwBadResponse(state, res.left as never); + } else { + throwBadResponse(state, res.left); + } + } + + if (state.currentBatch + 1 < state.bulkOperationBatches.length) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: state.currentBatch + 1, + }; + } + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: true, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts new file mode 100644 index 000000000000..a5db9fd119a8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchClosePitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchClosePit } from './outdated_documents_search_close_pit'; + +describe('Stage: outdatedDocumentsSearchClosePit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchClosePitState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_SEARCH_REFRESH when documents were transformed', () => { + const state = createState({ + hasTransformedDocs: true, + }); + const res = Either.right({}) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'>; + + const newState = outdatedDocumentsSearchClosePit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> UPDATE_DOCUMENT_MODEL_VERSIONS when no documents were transformed', () => { + const state = createState({ + hasTransformedDocs: false, + }); + const res = Either.right({}) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'>; + + const newState = outdatedDocumentsSearchClosePit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts new file mode 100644 index 000000000000..a467a4a7d90f --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchClosePit: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH' | 'UPDATE_DOCUMENT_MODEL_VERSIONS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const { hasTransformedDocs } = state; + if (hasTransformedDocs) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + }; + } else { + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts new file mode 100644 index 000000000000..90417d66f1a3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchOpenPitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchOpenPit } from './outdated_documents_search_open_pit'; + +describe('Stage: outdatedDocumentsSearchOpenPit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchOpenPitState => ({ + ...createPostDocInitState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ when successful', () => { + const state = createState(); + const res = Either.right({ + pitId: '42', + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'>; + + const newState = outdatedDocumentsSearchOpenPit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + pitId: '42', + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + progress: { + processed: undefined, + total: undefined, + }, + hasTransformedDocs: false, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts new file mode 100644 index 000000000000..f7a76e57779c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { createInitialProgress } from '../../../model/progress'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchOpenPit: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + 'OUTDATED_DOCUMENTS_SEARCH_READ' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const pitId = res.right.pitId; + + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + pitId, + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + progress: createInitialProgress(), + hasTransformedDocs: false, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts new file mode 100644 index 000000000000..de312c966dbd --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + createSavedObjectRawDoc, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchReadState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchRead } from './outdated_documents_search_read'; + +describe('Stage: outdatedDocumentsSearchRead', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchReadState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_TRANSFORM when outdated documents are found', () => { + const state = createState({ + progress: { + total: 300, + processed: 0, + }, + }); + const outdatedDocuments = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + const res = Either.right({ + outdatedDocuments, + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments, + lastHitSortValue: [12, 24], + logs: expect.any(Array), + progress: { + total: 9000, + processed: 0, + }, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when no more outdated documents', () => { + const state = createState({ + progress: { + total: 300, + processed: 0, + }, + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL when corrupt ids are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + corruptDocumentIds: ['foo_1', 'bar_2'], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL when transform errors are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + transformErrors: [{ rawId: 'foo_1', err: new Error('woups') }], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + //// + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when corrupt ids are are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: true, + }); + const state = createState({ + corruptDocumentIds: ['foo_1', 'bar_2'], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when transform errors are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: true, + }); + const state = createState({ + transformErrors: [{ rawId: 'foo_1', err: new Error('woups') }], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts new file mode 100644 index 000000000000..d3e8b2b5b901 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; +import { logProgress, setProgressTotal } from '../../../model/progress'; +import { + extractDiscardedCorruptDocs, + extractTransformFailuresReason, +} from '../../../model/extract_errors'; + +export const outdatedDocumentsSearchRead: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_READ', + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM' | 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + let logs = state.logs; + + if (res.right.outdatedDocuments.length > 0) { + // search returned outdated documents, so we process them + const progress = setProgressTotal(state.progress, res.right.totalHits); + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments: res.right.outdatedDocuments, + lastHitSortValue: res.right.lastHitSortValue, + logs: logProgress(state.logs, progress), + progress, + }; + } else { + // no more outdated documents , we need to move on + if (state.corruptDocumentIds.length > 0 || state.transformErrors.length > 0) { + if (!context.discardCorruptObjects) { + const transformFailureReason = extractTransformFailuresReason( + context.migrationDocLinks.resolveMigrationFailures, + state.corruptDocumentIds, + state.transformErrors + ); + return { + ...state, + controlState: 'FATAL', + reason: transformFailureReason, + }; + } + + // at this point, users have configured kibana to discard corrupt objects + // thus, we can ignore corrupt documents and transform errors and proceed with the migration + logs = [ + ...state.logs, + { + level: 'warning', + message: extractDiscardedCorruptDocs(state.corruptDocumentIds, state.transformErrors), + }, + ]; + } + + // If there are no more results we have transformed all outdated + // documents and we didn't encounter any corrupt documents or transformation errors + // and can proceed to the next step + return { + ...state, + logs, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts new file mode 100644 index 000000000000..9584d97b2ace --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchRefreshState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchRefresh } from './outdated_documents_search_refresh'; + +describe('Stage: outdatedDocumentsSearchRefresh', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchRefreshState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_REFRESH -> UPDATE_DOCUMENT_MODEL_VERSIONS when successful', () => { + const state = createState({}); + const res = Either.right({ + refreshed: true, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_REFRESH'>; + + const newState = outdatedDocumentsSearchRefresh(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts new file mode 100644 index 000000000000..0816174c5b58 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchRefresh: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + 'UPDATE_DOCUMENT_MODEL_VERSIONS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts new file mode 100644 index 000000000000..6c08e3467821 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + createSavedObjectRawDoc, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchTransformState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchTransform } from './outdated_documents_search_transform'; + +describe('Stage: outdatedDocumentsSearchTransform', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchTransformState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments: [], + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX when outdated documents were converted', () => { + const state = createState({ + progress: { + processed: 0, + total: 100, + }, + outdatedDocuments: [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ], + }); + const processedDocs = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + const res = Either.right({ + processedDocs, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'>; + + const newState = outdatedDocumentsSearchTransform(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: 0, + hasTransformedDocs: true, + bulkOperationBatches: expect.any(Array), + progress: { + processed: 2, + total: 100, + }, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ in case of documents_transform_failed when discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + progress: { + processed: 0, + total: 100, + }, + outdatedDocuments: [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ], + corruptDocumentIds: ['init_1'], + }); + const processedDocs = [ + createSavedObjectRawDoc({ _id: '3' }), + createSavedObjectRawDoc({ _id: '4' }), + ]; + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'> = Either.left({ + type: 'documents_transform_failed', + processedDocs, + corruptDocumentIds: ['foo_1', 'bar_2'], + transformErrors: [], + }); + + const newState = outdatedDocumentsSearchTransform(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: ['init_1', 'foo_1', 'bar_2'], + transformErrors: [], + hasTransformedDocs: false, + progress: { + processed: 2, + total: 100, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts new file mode 100644 index 000000000000..b174a57096e1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; +import { incrementProcessedProgress } from '../../../model/progress'; +import { fatalReasonDocumentExceedsMaxBatchSizeBytes } from '../../../model/extract_errors'; +import { createBatches } from '../../../model/create_batches'; +import { isTypeof } from '../../actions'; + +export const outdatedDocumentsSearchTransform: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX' | 'OUTDATED_DOCUMENTS_SEARCH_READ' | 'FATAL' +> = (state, res, context) => { + // Increment the processed documents, no matter what the results are. + // Otherwise the progress might look off when there are errors. + const progress = incrementProcessedProgress(state.progress, state.outdatedDocuments.length); + const discardCorruptObjects = context.discardCorruptObjects; + if ( + Either.isRight(res) || + (isTypeof(res.left, 'documents_transform_failed') && discardCorruptObjects) + ) { + // we might have some transformation errors, but user has chosen to discard them + if ( + (state.corruptDocumentIds.length === 0 && state.transformErrors.length === 0) || + discardCorruptObjects + ) { + const documents = Either.isRight(res) ? res.right.processedDocs : res.left.processedDocs; + + let corruptDocumentIds = state.corruptDocumentIds; + let transformErrors = state.transformErrors; + + if (Either.isLeft(res)) { + corruptDocumentIds = [...state.corruptDocumentIds, ...res.left.corruptDocumentIds]; + transformErrors = [...state.transformErrors, ...res.left.transformErrors]; + } + + const batches = createBatches({ + documents, + corruptDocumentIds, + transformErrors, + maxBatchSizeBytes: context.migrationConfig.maxBatchSizeBytes.getValueInBytes(), + }); + if (Either.isRight(batches)) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + bulkOperationBatches: batches.right, + currentBatch: 0, + hasTransformedDocs: true, + progress, + }; + } else { + return { + ...state, + controlState: 'FATAL', + reason: fatalReasonDocumentExceedsMaxBatchSizeBytes({ + _id: batches.left.documentId, + docSizeBytes: batches.left.docSizeBytes, + maxBatchSizeBytes: batches.left.maxBatchSizeBytes, + }), + }; + } + } else { + // We have seen corrupt documents and/or transformation errors + // skip indexing and go straight to reading and transforming more docs + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + progress, + }; + } + } else { + if (isTypeof(res.left, 'documents_transform_failed')) { + // continue to build up any more transformation errors before failing the migration. + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [...state.corruptDocumentIds, ...res.left.corruptDocumentIds], + transformErrors: [...state.transformErrors, ...res.left.transformErrors], + hasTransformedDocs: false, + progress, + }; + } else { + throwBadResponse(state, res as never); + } + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts new file mode 100644 index 000000000000..fca84ab858a2 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { SetDocMigrationStartedState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { setDocMigrationStarted } from './set_doc_migration_started'; + +describe('Stage: setDocMigrationStarted', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): SetDocMigrationStartedState => ({ + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('SET_DOC_MIGRATION_STARTED -> SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES when successful', () => { + const state = createState(); + const res: StateActionResponse<'SET_DOC_MIGRATION_STARTED'> = Either.right( + 'update_mappings_succeeded' as const + ); + + const newState = setDocMigrationStarted(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + currentIndexMeta: { + ...state.currentIndexMeta, + migrationState: { + ...state.currentIndexMeta.migrationState, + convertingDocuments: true, + }, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts new file mode 100644 index 000000000000..f1c24dbba386 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { setMetaDocMigrationStarted } from '../../utils'; +import type { ModelStage } from '../types'; + +export const setDocMigrationStarted: ModelStage< + 'SET_DOC_MIGRATION_STARTED', + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + currentIndexMeta: setMetaDocMigrationStarted({ + meta: state.currentIndexMeta, + }), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts new file mode 100644 index 000000000000..530430d79586 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { SetDocMigrationStartedWaitForInstancesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { setDocMigrationStartedWaitForInstances } from './set_doc_migration_started_wait_for_instances'; + +describe('Stage: setDocMigrationStartedWaitForInstances', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): SetDocMigrationStartedWaitForInstancesState => ({ + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS when successful', () => { + const state = createState(); + const res = Either.right( + 'wait_succeeded' as const + ) as StateActionResponse<'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'>; + + const newState = setDocMigrationStartedWaitForInstances(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts new file mode 100644 index 000000000000..7053f3149579 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const setDocMigrationStartedWaitForInstances: ModelStage< + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts index 4fac3d02db04..9849e6b46403 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts @@ -60,7 +60,7 @@ describe('Stage: updateAliases', () => { }); }); - it('UPDATE_ALIASES -> DONE if successful', () => { + it('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE if successful', () => { const state = createState(); const res: StateActionResponse<'UPDATE_ALIASES'> = Either.right('update_aliases_succeeded'); @@ -68,7 +68,7 @@ describe('Stage: updateAliases', () => { expect(newState).toEqual({ ...state, - controlState: 'DONE', + controlState: 'INDEX_STATE_UPDATE_DONE', }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts index 4d91eb116871..5d7f0914d0d5 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts @@ -11,7 +11,7 @@ import { throwBadResponse } from '../../../model/helpers'; import { isTypeof } from '../../actions'; import type { ModelStage } from '../types'; -export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( +export const updateAliases: ModelStage<'UPDATE_ALIASES', 'INDEX_STATE_UPDATE_DONE' | 'FATAL'> = ( state, res, context @@ -41,6 +41,6 @@ export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( return { ...state, - controlState: 'DONE', + controlState: 'INDEX_STATE_UPDATE_DONE', }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts new file mode 100644 index 000000000000..c3d3fd67422b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateDocumentModelVersionsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateDocumentModelVersion } from './update_document_model_version'; + +describe('Stage: updateDocumentModelVersion', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateDocumentModelVersionsState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_DOCUMENT_MODEL_VERSIONS -> UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES when successful', () => { + const state = createState({}); + const res = Either.right( + 'update_mappings_succeeded' + ) as StateActionResponse<'UPDATE_DOCUMENT_MODEL_VERSIONS'>; + + const newState = updateDocumentModelVersion(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + currentIndexMeta: expect.any(Object), + }); + }); + + it('updates state.currentIndexMeta when successful', () => { + const state = createState({ + currentIndexMeta: { + mappingVersions: { foo: 1, bar: 2 }, + docVersions: { foo: 0, bar: 0 }, + migrationState: { + convertingDocuments: true, + }, + }, + }); + const res = Either.right( + 'update_mappings_succeeded' + ) as StateActionResponse<'UPDATE_DOCUMENT_MODEL_VERSIONS'>; + + const newState = updateDocumentModelVersion(state, res, context); + + expect(newState.currentIndexMeta).toEqual({ + mappingVersions: { foo: 1, bar: 2 }, + docVersions: { foo: 1, bar: 2 }, + migrationState: { + convertingDocuments: false, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts new file mode 100644 index 000000000000..ea3ea142cbd1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { setMetaDocMigrationComplete } from '../../utils'; +import type { ModelStage } from '../types'; + +export const updateDocumentModelVersion: ModelStage< + 'UPDATE_DOCUMENT_MODEL_VERSIONS', + 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + currentIndexMeta: setMetaDocMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts new file mode 100644 index 000000000000..b2ea37919418 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateDocumentModelVersionsWaitForInstancesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateDocumentModelVersionWaitForInstances } from './update_document_model_version_wait_for_instances'; + +describe('Stage: updateDocumentModelVersionWaitForInstances', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateDocumentModelVersionsWaitForInstancesState => ({ + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES -> DONE when successful', () => { + const state = createState(); + const res = Either.right( + 'wait_succeeded' as const + ) as StateActionResponse<'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'>; + + const newState = updateDocumentModelVersionWaitForInstances(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts new file mode 100644 index 000000000000..82e3ce7c10cd --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateDocumentModelVersionWaitForInstances: ModelStage< + 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + 'DONE' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts index 9856cb0c5a1e..c47ef54030b3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts @@ -33,11 +33,5 @@ export const updateIndexMappingsWaitForTask: ModelStage< return { ...state, controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', - currentIndexMeta: { - ...state.currentIndexMeta, - mappingVersions: { - ...context.typeModelVersions, - }, - }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts index 971482d3262b..b21ec69a531f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -12,20 +12,19 @@ import { createPostInitState, type MockedMigratorContext, } from '../../test_helpers'; -import type { ResponseType } from '../../next'; -import type { UpdateIndexMappingsState } from '../../state'; +import type { UpdateMappingModelVersionState } from '../../state'; import type { StateActionResponse } from '../types'; -import { updateIndexMappings } from './update_index_mappings'; +import { updateMappingModelVersion } from './update_mapping_model_version'; +import { setMetaMappingMigrationComplete } from '../../utils'; -describe('Stage: updateIndexMappings', () => { +describe('Stage: updateMappingModelVersion', () => { let context: MockedMigratorContext; const createState = ( - parts: Partial = {} - ): UpdateIndexMappingsState => ({ + parts: Partial = {} + ): UpdateMappingModelVersionState => ({ ...createPostInitState(), - controlState: 'UPDATE_INDEX_MAPPINGS', - additiveMappingChanges: {}, + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', ...parts, }); @@ -33,21 +32,50 @@ describe('Stage: updateIndexMappings', () => { context = createContextMock(); }); - it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { - const state = createState(); - const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ - taskId: '42', + it('updates state.currentIndexMeta', () => { + const state = createState({}); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' + ); + + const newState = updateMappingModelVersion(state, res, context); + expect(newState.currentIndexMeta).toEqual( + setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }) + ); + }); + + it('UPDATE_MAPPING_MODEL_VERSIONS -> UPDATE_ALIASES when at least one aliasActions', () => { + const state = createState({ + aliasActions: [{ add: { alias: '.kibana', index: '.kibana_1' } }], }); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' + ); - const newState = updateIndexMappings( - state, - res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, - context + const newState = updateMappingModelVersion(state, res, context); + expect(newState).toEqual({ + ...state, + currentIndexMeta: expect.any(Object), + controlState: 'UPDATE_ALIASES', + }); + }); + + it('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE when no aliasActions', () => { + const state = createState({ + aliasActions: [], + }); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' ); + + const newState = updateMappingModelVersion(state, res, context); expect(newState).toEqual({ ...state, - controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', - updateTargetMappingsTaskId: '42', + currentIndexMeta: expect.any(Object), + controlState: 'INDEX_STATE_UPDATE_DONE', }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts index 8b4df56fc83a..946c4a4ab1ef 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts @@ -9,10 +9,11 @@ import * as Either from 'fp-ts/lib/Either'; import { throwBadResponse } from '../../../model/helpers'; import type { ModelStage } from '../types'; +import { setMetaMappingMigrationComplete } from '../../utils'; export const updateMappingModelVersion: ModelStage< 'UPDATE_MAPPING_MODEL_VERSIONS', - 'DONE' | 'FATAL' + 'UPDATE_ALIASES' | 'INDEX_STATE_UPDATE_DONE' > = (state, res, context) => { if (Either.isLeft(res)) { throwBadResponse(state, res as never); @@ -20,6 +21,10 @@ export const updateMappingModelVersion: ModelStage< return { ...state, - controlState: 'DONE', + controlState: state.aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', + currentIndexMeta: setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts new file mode 100644 index 000000000000..fc274bcfd121 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const setMetaMappingMigrationCompleteMock = jest.fn(); +export const setMetaDocMigrationCompleteMock = jest.fn(); +export const setMetaDocMigrationStartedMock = jest.fn(); + +jest.doMock('./utils', () => { + const actual = jest.requireActual('./utils'); + return { + ...actual, + setMetaDocMigrationStarted: setMetaDocMigrationStartedMock, + setMetaMappingMigrationComplete: setMetaMappingMigrationCompleteMock, + setMetaDocMigrationComplete: setMetaDocMigrationCompleteMock, + }; +}); + +const realActions = jest.requireActual('./actions'); + +export const ActionMocks = Object.keys(realActions).reduce((mocks, key) => { + mocks[key] = jest.fn().mockImplementation((state: unknown) => state); + return mocks; +}, {} as Record>); + +jest.doMock('./actions', () => ActionMocks); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts new file mode 100644 index 000000000000..d9135fff65a3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + ActionMocks, + setMetaDocMigrationStartedMock, + setMetaDocMigrationCompleteMock, + setMetaMappingMigrationCompleteMock, +} from './next.test.mocks'; +import { nextActionMap, type ActionMap } from './next'; +import { + createContextMock, + type MockedMigratorContext, + createPostDocInitState, +} from './test_helpers'; +import type { + SetDocMigrationStartedState, + UpdateMappingModelVersionState, + UpdateDocumentModelVersionsState, +} from './state'; + +describe('actions', () => { + let context: MockedMigratorContext; + let actionMap: ActionMap; + + beforeEach(() => { + jest.clearAllMocks(); + + context = createContextMock(); + actionMap = nextActionMap(context); + }); + + describe('SET_DOC_MIGRATION_STARTED', () => { + it('calls setMetaDocMigrationStarted with the correct parameters', () => { + const state: SetDocMigrationStartedState = { + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + }; + const action = actionMap.SET_DOC_MIGRATION_STARTED; + + action(state); + + expect(setMetaDocMigrationStartedMock).toHaveBeenCalledTimes(1); + expect(setMetaDocMigrationStartedMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: SetDocMigrationStartedState = { + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + }; + const action = actionMap.SET_DOC_MIGRATION_STARTED; + + const someMeta = { some: 'meta' }; + setMetaDocMigrationStartedMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); + + describe('UPDATE_MAPPING_MODEL_VERSIONS', () => { + it('calls setMetaMappingMigrationComplete with the correct parameters', () => { + const state: UpdateMappingModelVersionState = { + ...createPostDocInitState(), + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_MAPPING_MODEL_VERSIONS; + + action(state); + + expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledTimes(1); + expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: UpdateMappingModelVersionState = { + ...createPostDocInitState(), + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_MAPPING_MODEL_VERSIONS; + + const someMeta = { some: 'meta' }; + setMetaMappingMigrationCompleteMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); + + describe('UPDATE_DOCUMENT_MODEL_VERSIONS', () => { + it('calls setMetaDocMigrationComplete with the correct parameters', () => { + const state: UpdateDocumentModelVersionsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_DOCUMENT_MODEL_VERSIONS; + + action(state); + + expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledTimes(1); + expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: UpdateDocumentModelVersionsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_DOCUMENT_MODEL_VERSIONS; + + const someMeta = { some: 'meta' }; + setMetaDocMigrationCompleteMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index a85e9bfde6b5..cb3e1b5b5ad2 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -15,10 +15,30 @@ import type { UpdateIndexMappingsWaitForTaskState, UpdateMappingModelVersionState, UpdateAliasesState, + CleanupUnknownAndExcludedDocsState, + CleanupUnknownAndExcludedDocsWaitForTaskState, + DocumentsUpdateInitState, + IndexStateUpdateDoneState, + OutdatedDocumentsSearchBulkIndexState, + OutdatedDocumentsSearchClosePitState, + OutdatedDocumentsSearchOpenPitState, + OutdatedDocumentsSearchReadState, + OutdatedDocumentsSearchTransformState, + CleanupUnknownAndExcludedDocsRefreshState, + SetDocMigrationStartedState, + SetDocMigrationStartedWaitForInstancesState, + OutdatedDocumentsSearchRefreshState, + UpdateDocumentModelVersionsState, + UpdateDocumentModelVersionsWaitForInstancesState, } from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; import { createDelayFn } from '../common/utils'; +import { + setMetaMappingMigrationComplete, + setMetaDocMigrationComplete, + setMetaDocMigrationStarted, +} from './utils'; export type ActionMap = ReturnType; @@ -59,19 +79,108 @@ export const nextActionMap = (context: MigratorContext) => { timeout: '60s', }), UPDATE_MAPPING_MODEL_VERSIONS: (state: UpdateMappingModelVersionState) => - Actions.updateMappings({ + Actions.updateIndexMeta({ client, index: state.currentIndex, - mappings: { - properties: {}, - _meta: state.currentIndexMeta, - }, + meta: setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), }), UPDATE_ALIASES: (state: UpdateAliasesState) => Actions.updateAliases({ client, aliasActions: state.aliasActions, }), + INDEX_STATE_UPDATE_DONE: (state: IndexStateUpdateDoneState) => () => Actions.noop(), + DOCUMENTS_UPDATE_INIT: (state: DocumentsUpdateInitState) => () => Actions.noop(), + SET_DOC_MIGRATION_STARTED: (state: SetDocMigrationStartedState) => + Actions.updateIndexMeta({ + client, + index: state.currentIndex, + meta: setMetaDocMigrationStarted({ + meta: state.currentIndexMeta, + }), + }), + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: ( + state: SetDocMigrationStartedWaitForInstancesState + ) => + Actions.waitForDelay({ + delayInSec: context.migrationConfig.zdt.metaPickupSyncDelaySec, + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: (state: CleanupUnknownAndExcludedDocsState) => + Actions.cleanupUnknownAndExcluded({ + client, + indexName: state.currentIndex, + discardUnknownDocs: true, + excludeOnUpgradeQuery: state.excludeOnUpgradeQuery, + excludeFromUpgradeFilterHooks: state.excludeFromUpgradeFilterHooks, + knownTypes: context.types, + removedTypes: context.deletedTypes, + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: ( + state: CleanupUnknownAndExcludedDocsWaitForTaskState + ) => + Actions.waitForDeleteByQueryTask({ + client, + taskId: state.deleteTaskId, + timeout: '120s', + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: (state: CleanupUnknownAndExcludedDocsRefreshState) => + Actions.refreshIndex({ + client, + index: state.currentIndex, + }), + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: (state: OutdatedDocumentsSearchOpenPitState) => + Actions.openPit({ + client, + index: state.currentIndex, + }), + OUTDATED_DOCUMENTS_SEARCH_READ: (state: OutdatedDocumentsSearchReadState) => + Actions.readWithPit({ + client, + pitId: state.pitId, + searchAfter: state.lastHitSortValue, + batchSize: context.migrationConfig.batchSize, + query: state.outdatedDocumentsQuery, + }), + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: (state: OutdatedDocumentsSearchTransformState) => + Actions.transformDocs({ + outdatedDocuments: state.outdatedDocuments, + transformRawDocs: state.transformRawDocs, + }), + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: (state: OutdatedDocumentsSearchBulkIndexState) => + Actions.bulkOverwriteTransformedDocuments({ + client, + index: state.currentIndex, + operations: state.bulkOperationBatches[state.currentBatch], + refresh: false, + }), + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: (state: OutdatedDocumentsSearchClosePitState) => + Actions.closePit({ + client, + pitId: state.pitId, + }), + OUTDATED_DOCUMENTS_SEARCH_REFRESH: (state: OutdatedDocumentsSearchRefreshState) => + Actions.refreshIndex({ + client, + index: state.currentIndex, + }), + UPDATE_DOCUMENT_MODEL_VERSIONS: (state: UpdateDocumentModelVersionsState) => + Actions.updateIndexMeta({ + client, + index: state.currentIndex, + meta: setMetaDocMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), + }), + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: ( + state: UpdateDocumentModelVersionsWaitForInstancesState + ) => + Actions.waitForDelay({ + delayInSec: context.migrationConfig.zdt.metaPickupSyncDelaySec, + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index 0f7d28507bb4..45d958720a91 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -21,5 +21,20 @@ export type { AllControlStates, StateFromActionState, StateFromControlState, + IndexStateUpdateDoneState, + DocumentsUpdateInitState, + SetDocMigrationStartedState, + SetDocMigrationStartedWaitForInstancesState, + CleanupUnknownAndExcludedDocsState, + CleanupUnknownAndExcludedDocsWaitForTaskState, + CleanupUnknownAndExcludedDocsRefreshState, + OutdatedDocumentsSearchOpenPitState, + OutdatedDocumentsSearchReadState, + OutdatedDocumentsSearchTransformState, + OutdatedDocumentsSearchBulkIndexState, + OutdatedDocumentsSearchClosePitState, + OutdatedDocumentsSearchRefreshState, + UpdateDocumentModelVersionsState, + UpdateDocumentModelVersionsWaitForInstancesState, } from './types'; export { createInitialState } from './create_initial_state'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index d43c6e49dd5e..123df0455c9b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -6,11 +6,18 @@ * Side Public License, v 1. */ -import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { + SavedObjectsRawDoc, + SavedObjectsMappingProperties, + SavedObjectTypeExcludeFromUpgradeFilterHook, +} from '@kbn/core-saved-objects-server'; import type { IndexMapping, IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; -import type { MigrationLog } from '../../types'; +import type { MigrationLog, Progress, TransformRawDocs } from '../../types'; import type { ControlState } from '../../state_action_machine'; +import type { BulkOperationBatch } from '../../model/create_batches'; import type { AliasAction } from '../../actions'; +import { TransformErrorObjects } from '../../core'; export interface BaseState extends ControlState { readonly retryCount: number; @@ -23,6 +30,9 @@ export interface InitState extends BaseState { readonly controlState: 'INIT'; } +/** + * Common state properties available after the `INIT` stage + */ export interface PostInitState extends BaseState { /** * The index we're currently migrating. @@ -46,6 +56,35 @@ export interface PostInitState extends BaseState { * All operations updating this field will update in the state accordingly. */ readonly currentIndexMeta: IndexMappingMeta; + /** + * When true, will fully skip document migration after the INDEX_STATE_UPDATE_DONE stage. + * Used when 'upgrading' a fresh cluster (via CREATE_TARGET_INDEX), as we create + * the index with the correct meta and because we're sure we don't need to migrate documents + * in that case. + */ + readonly newIndexCreation: boolean; +} + +/** + * Common state properties available after the `DOCUMENTS_UPDATE_INIT` stage + */ +export interface PostDocInitState extends PostInitState { + readonly excludeOnUpgradeQuery: QueryDslQueryContainer; + readonly excludeFromUpgradeFilterHooks: Record< + string, + SavedObjectTypeExcludeFromUpgradeFilterHook + >; + readonly outdatedDocumentsQuery: QueryDslQueryContainer; + readonly transformRawDocs: TransformRawDocs; +} + +export interface OutdatedDocumentsSearchState extends PostDocInitState { + readonly pitId: string; + readonly lastHitSortValue: number[] | undefined; + readonly corruptDocumentIds: string[]; + readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; + readonly hasTransformedDocs: boolean; } export interface CreateTargetIndexState extends BaseState { @@ -72,6 +111,72 @@ export interface UpdateAliasesState extends PostInitState { readonly controlState: 'UPDATE_ALIASES'; } +export interface IndexStateUpdateDoneState extends PostInitState { + readonly controlState: 'INDEX_STATE_UPDATE_DONE'; +} + +export interface DocumentsUpdateInitState extends PostInitState { + readonly controlState: 'DOCUMENTS_UPDATE_INIT'; +} + +export interface SetDocMigrationStartedState extends PostDocInitState { + readonly controlState: 'SET_DOC_MIGRATION_STARTED'; +} + +export interface SetDocMigrationStartedWaitForInstancesState extends PostDocInitState { + readonly controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'; +} + +export interface CleanupUnknownAndExcludedDocsState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'; + readonly hasDeletedDocs?: boolean; +} + +export interface CleanupUnknownAndExcludedDocsWaitForTaskState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'; + readonly deleteTaskId: string; + readonly hasDeletedDocs?: boolean; +} + +export interface CleanupUnknownAndExcludedDocsRefreshState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH'; +} + +export interface OutdatedDocumentsSearchOpenPitState extends PostDocInitState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'; +} + +export interface OutdatedDocumentsSearchReadState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ'; +} + +export interface OutdatedDocumentsSearchTransformState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'; + readonly outdatedDocuments: SavedObjectsRawDoc[]; +} + +export interface OutdatedDocumentsSearchBulkIndexState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'; + readonly bulkOperationBatches: BulkOperationBatch[]; + readonly currentBatch: number; +} + +export interface OutdatedDocumentsSearchClosePitState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'; +} + +export interface OutdatedDocumentsSearchRefreshState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH'; +} + +export interface UpdateDocumentModelVersionsState extends PostDocInitState { + readonly controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS'; +} + +export interface UpdateDocumentModelVersionsWaitForInstancesState extends PostInitState { + readonly controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES'; +} + /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; @@ -92,7 +197,22 @@ export type State = | UpdateIndexMappingsState | UpdateIndexMappingsWaitForTaskState | UpdateMappingModelVersionState - | UpdateAliasesState; + | UpdateAliasesState + | IndexStateUpdateDoneState + | DocumentsUpdateInitState + | SetDocMigrationStartedState + | SetDocMigrationStartedWaitForInstancesState + | CleanupUnknownAndExcludedDocsState + | CleanupUnknownAndExcludedDocsWaitForTaskState + | CleanupUnknownAndExcludedDocsRefreshState + | OutdatedDocumentsSearchOpenPitState + | OutdatedDocumentsSearchReadState + | OutdatedDocumentsSearchTransformState + | OutdatedDocumentsSearchBulkIndexState + | OutdatedDocumentsSearchClosePitState + | UpdateDocumentModelVersionsState + | UpdateDocumentModelVersionsWaitForInstancesState + | OutdatedDocumentsSearchRefreshState; export type AllControlStates = State['controlState']; @@ -110,6 +230,21 @@ export interface ControlStateMap { UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; UPDATE_MAPPING_MODEL_VERSIONS: UpdateMappingModelVersionState; UPDATE_ALIASES: UpdateAliasesState; + INDEX_STATE_UPDATE_DONE: IndexStateUpdateDoneState; + DOCUMENTS_UPDATE_INIT: DocumentsUpdateInitState; + SET_DOC_MIGRATION_STARTED: SetDocMigrationStartedState; + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: SetDocMigrationStartedWaitForInstancesState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: CleanupUnknownAndExcludedDocsState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: CleanupUnknownAndExcludedDocsWaitForTaskState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: CleanupUnknownAndExcludedDocsRefreshState; + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: OutdatedDocumentsSearchOpenPitState; + OUTDATED_DOCUMENTS_SEARCH_READ: OutdatedDocumentsSearchReadState; + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: OutdatedDocumentsSearchTransformState; + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: OutdatedDocumentsSearchBulkIndexState; + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: OutdatedDocumentsSearchClosePitState; + OUTDATED_DOCUMENTS_SEARCH_REFRESH: OutdatedDocumentsSearchRefreshState; + UPDATE_DOCUMENT_MODEL_VERSIONS: UpdateDocumentModelVersionsState; + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: UpdateDocumentModelVersionsWaitForInstancesState; } /** diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index faf9f9c89c9f..0dafc36108b9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { ByteSizeValue } from '@kbn/config-schema'; import { ElasticsearchClientMock, elasticsearchClientMock, @@ -14,6 +15,7 @@ import { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-int import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import type { MigratorContext } from '../context'; +import { createDocumentMigrator } from './document_migrator'; export type MockedMigratorContext = Omit & { elasticsearchClient: ElasticsearchClientMock; @@ -33,12 +35,26 @@ export const createContextMock = ( foo: 1, bar: 2, }, + documentMigrator: createDocumentMigrator(), + migrationConfig: { + algorithm: 'zdt', + batchSize: 1000, + maxBatchSizeBytes: new ByteSizeValue(1e8), + pollInterval: 0, + scrollDuration: '0s', + skip: false, + retryAttempts: 5, + zdt: { + metaPickupSyncDelaySec: 120, + }, + }, elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(), maxRetryAttempts: 15, migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects, typeRegistry, serializer: serializerMock.create(), deletedTypes: ['deleted-type'], + discardCorruptObjects: false, ...parts, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts new file mode 100644 index 000000000000..524da0595205 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { VersionedTransformer } from '../../document_migrator'; + +export const createDocumentMigrator = (): jest.Mocked => { + return { + migrationVersion: {}, + migrate: jest.fn().mockImplementation((doc: unknown) => doc), + migrateAndConvert: jest.fn().mockImplementation((doc: unknown) => [doc]), + prepareMigrations: jest.fn(), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts index 5658828fc2e0..8b79eef0069f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts @@ -7,5 +7,11 @@ */ export { createContextMock, type MockedMigratorContext } from './context'; -export { createPostInitState } from './state'; +export { + createPostInitState, + createPostDocInitState, + createOutdatedDocumentSearchState, +} from './state'; export { createType } from './saved_object_type'; +export { createDocumentMigrator } from './document_migrator'; +export { createSavedObjectRawDoc } from './saved_object'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts new file mode 100644 index 000000000000..9b5520a8508d --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; + +export const createSavedObjectRawDoc = ( + parts: Partial +): SavedObjectsRawDoc => ({ + _id: '42', + _source: { + type: 'some-type', + }, + ...parts, +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts index bd95881abbba..b91f2482326c 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { PostInitState } from '../state/types'; +import { PostInitState, PostDocInitState, OutdatedDocumentsSearchState } from '../state/types'; export const createPostInitState = (): PostInitState => ({ controlState: 'INIT', @@ -18,4 +18,26 @@ export const createPostInitState = (): PostInitState => ({ aliasActions: [], previousMappings: { properties: {} }, currentIndexMeta: {}, + newIndexCreation: false, +}); + +export const createPostDocInitState = (): PostDocInitState => ({ + ...createPostInitState(), + excludeOnUpgradeQuery: { bool: {} }, + excludeFromUpgradeFilterHooks: {}, + outdatedDocumentsQuery: { bool: {} }, + transformRawDocs: jest.fn(), +}); + +export const createOutdatedDocumentSearchState = (): OutdatedDocumentsSearchState => ({ + ...createPostDocInitState(), + pitId: '42', + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: false, + progress: { + processed: undefined, + total: undefined, + }, }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts index f4536cf1c75b..35354001f680 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts @@ -78,6 +78,9 @@ describe('buildIndexMeta', () => { bar: 1, dolly: 3, }, + migrationState: { + convertingDocuments: false, + }, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts index 6221221ab993..a75ebd4dbdc1 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -53,5 +53,8 @@ export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta return { mappingVersions: modelVersions, docVersions: modelVersions, + migrationState: { + convertingDocuments: false, + }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts index 6ad12656229f..8430f1f89842 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts @@ -62,6 +62,7 @@ describe('checkVersionCompatibility', () => { expect(getModelVersionsFromMappingsMock).toHaveBeenCalledWith({ mappings, source: 'mappingVersions', + knownTypes: ['foo', 'bar'], }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts index 4499ce419d34..c231645fb799 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -29,7 +29,11 @@ export const checkVersionCompatibility = ({ deletedTypes, }: CheckVersionCompatibilityOpts): CompareModelVersionResult => { const appVersions = getModelVersionMapForTypes(types); - const indexVersions = getModelVersionsFromMappings({ mappings, source }); + const indexVersions = getModelVersionsFromMappings({ + mappings, + source, + knownTypes: types.map((type) => type.name), + }); if (!indexVersions) { throw new Error(`Cannot check version: ${source} not present in the mapping meta`); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts index f23b1e84a87e..400e01e99979 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -36,7 +36,11 @@ export const generateAdditiveMappingDiff = ({ deletedTypes, }: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { const typeVersions = getModelVersionMapForTypes(types); - const mappingVersion = getModelVersionsFromMappingMeta({ meta, source: 'mappingVersions' }); + const mappingVersion = getModelVersionsFromMappingMeta({ + meta, + source: 'mappingVersions', + knownTypes: types.map((type) => type.name), + }); if (!mappingVersion) { // should never occur given we checked previously in the flow but better safe than sorry. throw new Error( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index ebc22e623f60..76c66a9fc9bd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -12,3 +12,10 @@ export { checkVersionCompatibility } from './check_version_compatibility'; export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; export { getAliasActions } from './get_alias_actions'; export { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; +export { getOutdatedDocumentsQuery } from './outdated_documents_query'; +export { createDocumentTransformFn } from './transform_raw_docs'; +export { + setMetaMappingMigrationComplete, + setMetaDocMigrationStarted, + setMetaDocMigrationComplete, +} from './update_index_meta'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts new file mode 100644 index 000000000000..f39016b7d86a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { getOutdatedDocumentsQuery } from './outdated_documents_query'; +import { createType } from '../test_helpers/saved_object_type'; + +const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +describe('getOutdatedDocumentsQuery', () => { + it('generates the correct query', () => { + const fooType = createType({ + name: 'foo', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + }, + }); + const barType = createType({ + name: 'bar', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + 3: dummyModelVersion, + }, + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.foo": "10.2.0", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + Object { + "term": Object { + "typeMigrationVersion": "10.2.0", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.bar": "10.3.0", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + Object { + "term": Object { + "typeMigrationVersion": "10.3.0", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + } + `); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts new file mode 100644 index 000000000000..e15a2e447b55 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + getModelVersionMapForTypes, + modelVersionToVirtualVersion, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface GetOutdatedDocumentsQueryOps { + types: SavedObjectsType[]; +} + +export const getOutdatedDocumentsQuery = ({ + types, +}: GetOutdatedDocumentsQueryOps): QueryDslQueryContainer => { + // Note: in theory, we could check the difference of model version with the index's + // and narrow the search filter only on the type that have different versions. + // however, it feels safer to just search for all outdated document, just in case. + const modelVersions = getModelVersionMapForTypes(types); + return { + bool: { + should: types.map((type) => { + const virtualVersion = modelVersionToVirtualVersion(modelVersions[type.name]); + return { + bool: { + must: [ + { term: { type: type.name } }, + { + bool: { + should: [ + { + bool: { + must: { exists: { field: 'migrationVersion' } }, + must_not: { term: { [`migrationVersion.${type.name}`]: virtualVersion } }, + }, + }, + { + bool: { + must_not: [ + { exists: { field: 'migrationVersion' } }, + { term: { typeMigrationVersion: virtualVersion } }, + ], + }, + }, + ], + }, + }, + ], + }, + }; + }), + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts new file mode 100644 index 000000000000..ab6e3f0c5773 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const migrateRawDocsSafelyMock = jest.fn(); + +jest.doMock('../../core/migrate_raw_docs', () => { + const actual = jest.requireActual('../../core/migrate_raw_docs'); + return { + ...actual, + migrateRawDocsSafely: migrateRawDocsSafelyMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts new file mode 100644 index 000000000000..91a1f9983f33 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { migrateRawDocsSafelyMock } from './transform_raw_docs.test.mocks'; +import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; +import { createDocumentMigrator, createSavedObjectRawDoc } from '../test_helpers'; +import { createDocumentTransformFn } from './transform_raw_docs'; + +describe('createDocumentTransformFn', () => { + let serializer: ReturnType; + let documentMigrator: ReturnType; + + beforeEach(() => { + migrateRawDocsSafelyMock.mockReset(); + serializer = serializerMock.create(); + documentMigrator = createDocumentMigrator(); + }); + + it('returns a function calling migrateRawDocsSafely', () => { + const transformFn = createDocumentTransformFn({ + serializer, + documentMigrator, + }); + + expect(migrateRawDocsSafelyMock).not.toHaveBeenCalled(); + + const documents = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + transformFn(documents); + + expect(migrateRawDocsSafelyMock).toHaveBeenCalledTimes(1); + expect(migrateRawDocsSafelyMock).toHaveBeenCalledWith({ + rawDocs: documents, + serializer, + migrateDoc: documentMigrator.migrateAndConvert, + }); + }); + + it('forward the return from migrateRawDocsSafely', () => { + const transformFn = createDocumentTransformFn({ + serializer, + documentMigrator, + }); + + const documents = [createSavedObjectRawDoc({ _id: '1' })]; + + const expected = Symbol(); + migrateRawDocsSafelyMock.mockReturnValue(expected); + + const result = transformFn(documents); + + expect(result).toBe(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts new file mode 100644 index 000000000000..1af1cee9d3b8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ISavedObjectsSerializer, SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; +import { VersionedTransformer } from '../../document_migrator'; +import { TransformRawDocs } from '../../types'; +import { migrateRawDocsSafely } from '../../core/migrate_raw_docs'; + +export interface CreateDocumentTransformFnOpts { + serializer: ISavedObjectsSerializer; + documentMigrator: VersionedTransformer; +} + +export const createDocumentTransformFn = ({ + documentMigrator, + serializer, +}: CreateDocumentTransformFnOpts): TransformRawDocs => { + return (documents: SavedObjectsRawDoc[]) => + migrateRawDocsSafely({ + rawDocs: documents, + migrateDoc: documentMigrator.migrateAndConvert, + serializer, + }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts new file mode 100644 index 000000000000..4298c6b02707 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndexMappingMeta, + ModelVersionMap, +} from '@kbn/core-saved-objects-base-server-internal'; +import { + setMetaDocMigrationStarted, + setMetaDocMigrationComplete, + setMetaMappingMigrationComplete, +} from './update_index_meta'; + +const getDefaultMeta = (): IndexMappingMeta => ({ + mappingVersions: { + foo: 1, + bar: 1, + }, + docVersions: { + foo: 1, + bar: 1, + }, + migrationState: { + convertingDocuments: false, + }, +}); + +describe('setMetaMappingMigrationComplete', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = getDefaultMeta(); + const versions: ModelVersionMap = { foo: 3, bar: 2 }; + + const updated = setMetaMappingMigrationComplete({ meta, versions }); + + expect(updated).toEqual({ + ...meta, + mappingVersions: versions, + }); + }); +}); + +describe('setMetaDocMigrationStarted', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = getDefaultMeta(); + + const updated = setMetaDocMigrationStarted({ meta }); + + expect(updated).toEqual({ + ...meta, + migrationState: { + convertingDocuments: true, + }, + }); + }); +}); + +describe('setMetaDocMigrationComplete', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = { + ...getDefaultMeta(), + migrationState: { + convertingDocuments: true, + }, + }; + const versions: ModelVersionMap = { foo: 3, bar: 2 }; + + const updated = setMetaDocMigrationComplete({ meta, versions }); + + expect(updated).toEqual({ + ...meta, + docVersions: versions, + migrationState: { + convertingDocuments: false, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts new file mode 100644 index 000000000000..73d693d4a1f1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndexMappingMeta, + ModelVersionMap, +} from '@kbn/core-saved-objects-base-server-internal'; + +export const setMetaMappingMigrationComplete = ({ + meta, + versions, +}: { + meta: IndexMappingMeta; + versions: ModelVersionMap; +}): IndexMappingMeta => { + return { + ...meta, + mappingVersions: { + ...versions, + }, + }; +}; + +export const setMetaDocMigrationStarted = ({ + meta, +}: { + meta: IndexMappingMeta; +}): IndexMappingMeta => { + return { + ...meta, + migrationState: { + convertingDocuments: true, + }, + }; +}; + +export const setMetaDocMigrationComplete = ({ + meta, + versions, +}: { + meta: IndexMappingMeta; + versions: ModelVersionMap; +}): IndexMappingMeta => { + return { + ...meta, + docVersions: { + ...versions, + }, + migrationState: { + convertingDocuments: false, + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts index 3094a1ccb3e0..44e8c348778e 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts @@ -37,10 +37,7 @@ export interface SavedObjectsModelVersion { * * @public */ -export interface SavedObjectsModelExpansionChange< - PreviousAttributes = unknown, - NewAttributes = unknown -> { +export interface SavedObjectsModelExpansionChange { /** * The type of {@link SavedObjectsModelChange | change}, used to identify them internally. */ diff --git a/packages/kbn-axe-config/index.ts b/packages/kbn-axe-config/index.ts index df175dbf0817..7f7340873372 100644 --- a/packages/kbn-axe-config/index.ts +++ b/packages/kbn-axe-config/index.ts @@ -54,5 +54,8 @@ export const AXE_OPTIONS = { bypass: { enabled: false, // disabled because it's too flaky }, + 'nested-interactive': { + enabled: false, // tracker here - https://github.com/elastic/kibana/issues/152494 disabled because we have too many failures on interactive controls + }, }, }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts b/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts new file mode 100644 index 000000000000..2065dcf27576 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MatcherFunction } from 'expect'; +import { LogRecord } from '@kbn/logging'; + +const toContainLogEntry: MatcherFunction<[entry: string]> = (actual, entry) => { + if (!Array.isArray(actual)) { + throw new Error('actual must be an array'); + } + const logEntries = actual as LogRecord[]; + if (logEntries.find((item) => item.message.includes(entry))) { + return { + pass: true, + message: () => `Entry "${entry}" found in log file`, + }; + } else { + return { + pass: false, + message: () => `Entry "${entry}" not found in log file`, + }; + } +}; + +expect.extend({ + toContainLogEntry, +}); + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toContainLogEntry(entry: string): R; + } + } +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts index 0a84b9bc4b7e..610981bab56a 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts @@ -8,9 +8,12 @@ import { Env } from '@kbn/config'; import { getDocLinksMeta, getDocLinks } from '@kbn/doc-links'; +import { LogRecord } from '@kbn/logging'; import { REPO_ROOT } from '@kbn/repo-info'; import { getEnvOptions } from '@kbn/config-mocks'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; export const getDocVersion = () => { const env = Env.createDefault(REPO_ROOT, getEnvOptions()); @@ -33,3 +36,11 @@ export const createType = (parts: Partial): SavedObjectsType = mappings: { properties: {} }, ...parts, }); + +export const parseLogFile = async (filePath: string): Promise => { + const logFileContent = await fs.readFile(filePath, 'utf-8'); + return logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts new file mode 100644 index 000000000000..3cf5e499eaa6 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { createType } from '../test_utils'; +import { type KibanaMigratorTestKitParams } from '../kibana_migrator_test_kit'; + +export const getBaseMigratorParams = (): KibanaMigratorTestKitParams => ({ + kibanaIndex: '.kibana', + kibanaVersion: '8.8.0', + settings: { + migrations: { + algorithm: 'zdt', + zdt: { + metaPickupSyncDelaySec: 5, + }, + }, + }, +}); + +export const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +export const getFooType = () => { + return createType({ + name: 'foo', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + '2': dummyModelVersion, + }, + }); +}; + +export const getBarType = () => { + return createType({ + name: 'bar', + mappings: { + properties: { + aKeyword: { type: 'keyword' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getSampleAType = () => { + return createType({ + name: 'sample_a', + mappings: { + properties: { + keyword: { type: 'keyword' }, + boolean: { type: 'boolean' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getSampleBType = () => { + return createType({ + name: 'sample_b', + mappings: { + properties: { + text: { type: 'text' }, + text2: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getDeletedType = () => { + return createType({ + // we cant' easily introduce a deleted type, so we're using an existing one + name: 'server', + mappings: { + properties: { + text: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getExcludedType = () => { + return createType({ + // we cant' easily introduce a deleted type, so we're using an existing one + name: 'excluded', + mappings: { + properties: { + value: { type: 'integer' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + excludeOnUpgrade: () => { + return { + bool: { + must: [{ term: { type: 'excluded' } }, { range: { 'excluded.value': { lte: 1 } } }], + }, + }; + }, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts deleted file mode 100644 index 9bfac3ac5fd4..000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; -import { createType } from '../test_utils'; - -export const dummyModelVersion: SavedObjectsModelVersion = { - modelChange: { - type: 'expansion', - }, -}; - -export const getFooType = () => { - return createType({ - name: 'foo', - mappings: { - properties: { - someField: { type: 'text' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - '2': dummyModelVersion, - }, - }); -}; - -export const getBarType = () => { - return createType({ - name: 'bar', - mappings: { - properties: { - aKeyword: { type: 'keyword' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - }, - }); -}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts new file mode 100644 index 000000000000..418adb0a7894 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); + +describe('ZDT upgrades - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { + text: `i am number ${number}`, + text2: `some static text`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + // typeA -> we add a new field and bump the model version by one with a migration + + typeA.mappings.properties = { + ...typeA.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + someAddedField: `${doc.attributes.keyword}-mig`, + }, + }, + }; + }, + down: jest.fn(), + }, + addedMappings: { + someAddedField: { type: 'keyword' }, + }, + }, + }, + }; + + // typeB -> we add two new model version with migrations + + typeB.modelVersions = { + ...typeB.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + text2: `${doc.attributes.text2} - mig2`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + '3': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + text2: `${doc.attributes.text2} - mig3`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + }; + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, typeB], + }); + + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + sample_a: typeA.mappings, + sample_b: typeB.mappings, + }) + ); + + expect(mappingMeta.docVersions).toEqual({ + sample_a: 2, + sample_b: 3, + }); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ type: 'sample_a' }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ type: 'sample_b' }); + + expect(sampleADocs).toHaveLength(5); + expect(sampleBDocs).toHaveLength(5); + + const sampleAData = sortBy(sampleADocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleAData).toEqual([ + { + id: 'a-0', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_0', someAddedField: 'a_0-mig' }, + }, + { + id: 'a-1', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_1', someAddedField: 'a_1-mig' }, + }, + { + id: 'a-2', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_2', someAddedField: 'a_2-mig' }, + }, + { + id: 'a-3', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_3', someAddedField: 'a_3-mig' }, + }, + { + id: 'a-4', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_4', someAddedField: 'a_4-mig' }, + }, + ]); + + const sampleBData = sortBy(sampleBDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleBData).toEqual([ + { + id: 'b-0', + type: 'sample_b', + attributes: { text: 'i am number 0', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-1', + type: 'sample_b', + attributes: { text: 'i am number 1', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-2', + type: 'sample_b', + attributes: { text: 'i am number 2', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-3', + type: 'sample_b', + attributes: { text: 'i am number 3', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-4', + type: 'sample_b', + attributes: { text: 'i am number 4', text2: 'some static text - mig2 - mig3' }, + }, + ]); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('Starting to process 10 documents'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts new file mode 100644 index 000000000000..4f1b0a4bfe46 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'conversion_failures.test.log'); + +describe('ZDT upgrades - encountering conversion failures', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + beforeEach(async () => { + await fs.unlink(logFilePath).catch(() => {}); + }); + + describe('when discardCorruptObjects is true', () => { + it('completes the migration and discard the documents', async () => { + const { runMigrations, savedObjectsRepository } = await prepareScenario({ + discardCorruptObjects: true, + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('-> DONE'); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ + type: 'sample_a', + }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ + type: 'sample_b', + }); + + expect(sampleADocs).toHaveLength(0); + expect(sampleBDocs.map((doc) => doc.id).sort()).toEqual(['b-1', 'b-2', 'b-3', 'b-4']); + }); + }); + + describe('when discardCorruptObjects is false', () => { + it('fails the migration with an explicit message and keep the documents', async () => { + const { runMigrations, savedObjectsRepository } = await prepareScenario({ + discardCorruptObjects: false, + }); + + try { + await runMigrations(); + fail('migration should have failed'); + } catch (err) { + const errorMessage = err.message; + expect(errorMessage).toMatch('6 transformation errors were encountered'); + expect(errorMessage).toMatch('error from a-0'); + expect(errorMessage).toMatch('error from a-1'); + expect(errorMessage).toMatch('error from a-2'); + expect(errorMessage).toMatch('error from a-3'); + expect(errorMessage).toMatch('error from a-4'); + expect(errorMessage).toMatch('error from b-0'); + } + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL'); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ + type: 'sample_a', + }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ + type: 'sample_b', + }); + + expect(sampleADocs).toHaveLength(5); + expect(sampleBDocs).toHaveLength(5); + }); + }); + + const prepareScenario = async ({ discardCorruptObjects }: { discardCorruptObjects: boolean }) => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + // typeA -> migration failing all the documents + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + throw new Error(`error from ${doc.id}`); + }, + down: jest.fn(), + }, + }, + }, + }; + + // typeB -> migration failing the first doc + typeB.modelVersions = { + ...typeB.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + if (doc.id === 'b-0') { + throw new Error(`error from ${doc.id}`); + } + return { document: doc }; + }, + down: jest.fn(), + }, + }, + }, + }; + + const baseParams = getBaseMigratorParams(); + if (discardCorruptObjects) { + baseParams!.settings!.migrations!.discardCorruptObjects = '8.7.0'; + } + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...baseParams, + logFilePath, + types: [typeA, typeB], + }); + + return { runMigrations, client, savedObjectsRepository }; + }; + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + + try { + await client.indices.delete({ index: '.kibana_1' }); + } catch (e) { + /* index wasn't created, that's fine */ + } + + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { + text: `i am number ${number}`, + text2: `some static text`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts index e3f45bb561cf..760b0f113a8f 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -8,12 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; +import '../jest_matchers'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType } from './base_types.fixtures'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'create_index.test.log'); @@ -47,15 +46,9 @@ describe('ZDT upgrades - running on a fresh cluster', () => { const barType = getBarType(); const { runMigrations, client } = await getKibanaMigratorTestKit({ - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', + ...getBaseMigratorParams(), logFilePath, types: [fooType, barType], - settings: { - migrations: { - algorithm: 'zdt', - }, - }, }); const result = await runMigrations(); @@ -77,7 +70,7 @@ describe('ZDT upgrades - running on a fresh cluster', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappings.properties).toEqual( expect.objectContaining({ @@ -95,21 +88,17 @@ describe('ZDT upgrades - running on a fresh cluster', () => { foo: 2, bar: 1, }, + migrationState: expect.objectContaining({ + convertingDocuments: false, + }), }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; + const records = await parseLogFile(logFilePath); - expectLogsContains('INIT -> CREATE_TARGET_INDEX'); - expectLogsContains('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); - expectLogsContains('UPDATE_ALIASES -> DONE'); - expectLogsContains('Migration completed'); + expect(records).toContainLogEntry('INIT -> CREATE_TARGET_INDEX'); + expect(records).toContainLogEntry('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); + expect(records).toContainLogEntry('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DONE'); + expect(records).toContainLogEntry('Migration completed'); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts new file mode 100644 index 000000000000..b321a400684b --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { + getBaseMigratorParams, + getDeletedType, + getExcludedType, + getFooType, + getBarType, + dummyModelVersion, +} from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'document_cleanup.test.log'); + +describe('ZDT upgrades - document cleanup', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getFooType(), getBarType(), getDeletedType(), getExcludedType()], + }); + await runMigrations(); + + const fooObjs = range(5).map((number) => ({ + id: `foo-${number}`, + type: 'foo', + attributes: { + someField: `foo_${number}`, + }, + })); + + const barObjs = range(5).map((number) => ({ + id: `bar-${number}`, + type: 'bar', + attributes: { + aKeyword: `bar_${number}`, + }, + })); + + const deletedObjs = range(5).map((number) => ({ + id: `server-${number}`, + type: 'server', + attributes: { + text: `some text`, + }, + })); + + const excludedObjs = range(5).map((number) => ({ + id: `excluded-${number}`, + type: 'excluded', + attributes: { + value: number, + }, + })); + + await savedObjectsRepository.bulkCreate([ + ...fooObjs, + ...barObjs, + ...deletedObjs, + ...excludedObjs, + ]); + }; + + it('deletes the documents', async () => { + await createBaseline(); + + const fooType = getFooType(); + const excludedType = getExcludedType(); + + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, excludedType], + }); + + await runMigrations(); + + const indexContent = await client.search<{ type: string }>({ index: '.kibana_1', size: 100 }); + + // normal type + expect(countResultsByType(indexContent, 'foo')).toEqual(5); + // unknown type + expect(countResultsByType(indexContent, 'bar')).toEqual(0); + // deleted type + expect(countResultsByType(indexContent, 'server')).toEqual(0); + // excludeOnUpgrade type + expect(countResultsByType(indexContent, 'excluded')).toEqual(3); + }); +}); + +const countResultsByType = ( + indexContents: SearchResponse<{ type: string }>, + type: string +): number => { + return indexContents.hits.hits.filter((result) => result._source?.type === type).length; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts index eadd24bf447a..1674bf747f7b 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts @@ -8,15 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - getKibanaMigratorTestKit, - type KibanaMigratorTestKitParams, -} from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log'); @@ -35,15 +31,7 @@ describe('ZDT upgrades - mapping model version conflict', () => { return await startES(); }; - const baseMigratorParams: KibanaMigratorTestKitParams = { - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', - settings: { - migrations: { - algorithm: 'zdt', - }, - }, - }; + const baseMigratorParams = getBaseMigratorParams(); beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); @@ -115,25 +103,16 @@ describe('ZDT upgrades - mapping model version conflict', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappingMeta.mappingVersions).toEqual({ foo: 2, bar: 2, }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; + const records = await parseLogFile(logFilePath); - // - expectLogsContains('Mappings model version check result: conflict'); - expectLogsContains('INIT -> FATAL'); + expect(records).toContainLogEntry('Mappings model version check result: conflict'); + expect(records).toContainLogEntry('INIT -> FATAL'); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts new file mode 100644 index 000000000000..2be7c9396569 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'rerun_same_version.test.log'); + +describe('ZDT upgrades - rerun migration on same version', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('should perform a no-op upgrade', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, barType], + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT'); + expect(records).toContainLogEntry('-> DONE'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts new file mode 100644 index 000000000000..2fbbe7e09026 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { + getBaseMigratorParams, + getSampleAType, + getSampleBType, + dummyModelVersion, +} from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'standard_workflow.test.log'); + +describe('ZDT upgrades - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { keyword: `a_${number}`, boolean: true }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { text: `i am number ${number}`, text2: `some static text` }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; + + it('follows the expected stages and transitions', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': dummyModelVersion, + }; + + typeB.modelVersions = { + ...typeB.modelVersions, + '2': dummyModelVersion, + }; + + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, typeB], + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT'); + expect(records).toContainLogEntry('DOCUMENTS_UPDATE_INIT -> SET_DOC_MIGRATION_STARTED'); + expect(records).toContainLogEntry( + 'SET_DOC_MIGRATION_STARTED -> SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES' + ); + expect(records).toContainLogEntry( + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' + ); + expect(records).toContainLogEntry( + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_TRANSFORM' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_READ' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_SEARCH_REFRESH' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH -> UPDATE_DOCUMENT_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry( + 'UPDATE_DOCUMENT_MODEL_VERSIONS -> UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES' + ); + expect(records).toContainLogEntry('UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES -> DONE'); + + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts index 5d27c32c1ee6..9829da66a965 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -8,15 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - getKibanaMigratorTestKit, - type KibanaMigratorTestKitParams, -} from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); @@ -35,16 +31,6 @@ describe('ZDT upgrades - basic mapping update', () => { return await startES(); }; - const baseMigratorParams: KibanaMigratorTestKitParams = { - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', - settings: { - migrations: { - algorithm: 'zdt', - }, - }, - }; - beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); esServer = await startElasticsearch(); @@ -59,7 +45,7 @@ describe('ZDT upgrades - basic mapping update', () => { const fooType = getFooType(); const barType = getBarType(); const { runMigrations } = await getKibanaMigratorTestKit({ - ...baseMigratorParams, + ...getBaseMigratorParams(), types: [fooType, barType], }); await runMigrations(); @@ -91,7 +77,7 @@ describe('ZDT upgrades - basic mapping update', () => { }; const { runMigrations, client } = await getKibanaMigratorTestKit({ - ...baseMigratorParams, + ...getBaseMigratorParams(), logFilePath, types: [fooType, barType], }); @@ -115,7 +101,7 @@ describe('ZDT upgrades - basic mapping update', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappings.properties).toEqual( expect.objectContaining({ @@ -124,32 +110,21 @@ describe('ZDT upgrades - basic mapping update', () => { }) ); - expect(mappingMeta).toEqual({ - // doc migration not implemented yet - docVersions are not bumped. - docVersions: { - foo: 2, - bar: 1, - }, - mappingVersions: { - foo: 3, - bar: 2, - }, + expect(mappingMeta.mappingVersions).toEqual({ + foo: 3, + bar: 2, }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; + const records = await parseLogFile(logFilePath); - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; - - expectLogsContains('INIT -> UPDATE_INDEX_MAPPINGS'); - expectLogsContains('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'); - expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS'); - expectLogsContains('UPDATE_MAPPING_MODEL_VERSIONS -> DONE'); - expectLogsContains('Migration completed'); + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('Migration completed'); }); }); diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_distinct_series.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_distinct_series.ts index b3efd7f7c045..520ce1356977 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_distinct_series.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_distinct_series.ts @@ -8,29 +8,46 @@ import { DatatableRow } from '@kbn/expressions-plugin/public'; import { BucketColumns } from '../../common/types'; +/** + * All the available categories of a datatable. + */ export interface DistinctSeries { - allSeries: string[]; - parentSeries: string[]; + /** + * An array of unique category/bucket available on all the categorical/bucket columns. + * It could be `string` or `RangeKey` but is typed as unknown for now due to the loose nature of the DatatableRow type + */ + allSeries: unknown[]; + /** + * An array of unique category/bucket available on the first column of a datatable. + * It could be `string` or `RangeKey` but is typed as unknown for now due to the loose nature of the DatatableRow type + */ + parentSeries: unknown[]; } +/** + * This method returns all the categories available in a datatable. + * Here, categorical values are described as `bucket`, following the Elasticsearch bucket aggregation naming. + * It describes as `parentSeries` all the categories available on the `first` available column. + * It describes as `allSeries` each unique category/bucket available on all the categorical/bucket columns. + * The output order depends on the original datatable configuration. + */ export const getDistinctSeries = ( rows: DatatableRow[], - buckets: Array> + bucketColumns: Array> ): DistinctSeries => { - const parentBucketId = buckets[0].id; - const parentSeries: string[] = []; - const allSeries: string[] = []; - buckets.forEach(({ id }) => { + const parentBucketId = bucketColumns[0].id; + // using unknown here because there the DatatableRow is just a plain Record + // At least we can prevent some issues, see https://github.com/elastic/kibana/issues/153437 + const parentSeries: Set = new Set(); + const allSeries: Set = new Set(); + bucketColumns.forEach(({ id }) => { if (!id) return; rows.forEach((row) => { - const name = row[id]; - if (!allSeries.includes(name)) { - allSeries.push(name); - } - if (id === parentBucketId && !parentSeries.includes(row[parentBucketId])) { - parentSeries.push(row[parentBucketId]); + allSeries.add(row[id]); + if (id === parentBucketId) { + parentSeries.add(row[parentBucketId]); } }); }); - return { allSeries, parentSeries }; + return { allSeries: [...allSeries], parentSeries: [...parentSeries] }; }; diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_legend_actions.tsx b/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_legend_actions.tsx index 9852040060e7..b14bf88edf1f 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_legend_actions.tsx +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/get_legend_actions.tsx @@ -54,6 +54,7 @@ export const getLegendActions = ( visData, columnIndex, formatter.deserialize, + // FIXME key could be a RangeKey see https://github.com/elastic/kibana/issues/153437 pieSeries.key ); diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts index 06fa1d9aae83..c6fffce60ba0 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts @@ -9,8 +9,7 @@ import type { PaletteOutput, PaletteDefinition } from '@kbn/coloring'; import { chartPluginMock } from '@kbn/charts-plugin/public/mocks'; import { Datatable } from '@kbn/expressions-plugin/common'; -import { byDataColorPaletteMap } from './get_color'; -import { ShapeTreeNode } from '@elastic/charts'; +import { byDataColorPaletteMap, SimplifiedArrayNode } from './get_color'; import type { SeriesLayer } from '@kbn/coloring'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks'; @@ -19,6 +18,7 @@ import { getColor } from './get_color'; import { createMockVisData, createMockBucketColumns, createMockPieParams } from '../../mocks'; import { generateFormatters } from '../formatters'; import { ChartTypes } from '../../../common/types'; +import { getDistinctSeries } from '..'; describe('#byDataColorPaletteMap', () => { let datatable: Datatable; @@ -110,6 +110,8 @@ describe('getColor', () => { } const defaultFormatter = jest.fn((...args) => fieldFormatsMock.deserialize(...args)); const formatters = generateFormatters(visData, defaultFormatter); + const distinctSeries = getDistinctSeries(visData.rows, buckets); + const dataLength = { columnsLength: buckets.length, rowsLength: visData.rows.length }; dataMock.fieldFormats = { deserialize: jest.fn(() => ({ @@ -145,24 +147,29 @@ describe('getColor', () => { }; }; it('should return the correct color based on the parent sortIndex', () => { - const d = { - dataName: 'ES-Air', + const d: SimplifiedArrayNode = { depth: 1, sortIndex: 0, parent: { - children: [['ES-Air'], ['Kibana Airlines']], + children: [ + ['ES-Air', undefined], + ['Kibana Airlines', undefined], + ], depth: 0, sortIndex: 0, }, - } as unknown as ShapeTreeNode; + children: [], + }; + const color = getColor( ChartTypes.PIE, + 'ES-Air', d, 0, false, {}, - buckets, - visData.rows, + distinctSeries, + dataLength, visParams, getPaletteRegistry(), { getColor: () => undefined }, @@ -176,24 +183,28 @@ describe('getColor', () => { }); it('slices with the same label should have the same color for small multiples', () => { - const d = { - dataName: 'ES-Air', + const d: SimplifiedArrayNode = { depth: 1, sortIndex: 0, parent: { - children: [['ES-Air'], ['Kibana Airlines']], + children: [ + ['ES-Air', undefined], + ['Kibana Airlines', undefined], + ], depth: 0, sortIndex: 0, }, - } as unknown as ShapeTreeNode; + children: [], + }; const color = getColor( ChartTypes.PIE, + 'ES-Air', d, 0, true, {}, - buckets, - visData.rows, + distinctSeries, + dataLength, visParams, getPaletteRegistry(), { getColor: () => undefined }, @@ -206,24 +217,28 @@ describe('getColor', () => { expect(color).toEqual('color3'); }); it('returns the overwriteColor if exists', () => { - const d = { - dataName: 'ES-Air', + const d: SimplifiedArrayNode = { depth: 1, sortIndex: 0, parent: { - children: [['ES-Air'], ['Kibana Airlines']], + children: [ + ['ES-Air', undefined], + ['Kibana Airlines', undefined], + ], depth: 0, sortIndex: 0, }, - } as unknown as ShapeTreeNode; + children: [], + }; const color = getColor( ChartTypes.PIE, + 'ES-Air', d, 0, true, { 'ES-Air': '#000028' }, - buckets, - visData.rows, + distinctSeries, + dataLength, visParams, getPaletteRegistry(), { getColor: () => undefined }, @@ -237,11 +252,7 @@ describe('getColor', () => { }); it('returns the overwriteColor for older visualizations with formatted values', () => { - const d = { - dataName: { - gte: 1000, - lt: 2000, - }, + const d: SimplifiedArrayNode = { depth: 1, sortIndex: 0, parent: { @@ -250,19 +261,22 @@ describe('getColor', () => { { gte: 1000, lt: 2000, - }, + }.toString(), + undefined, ], [ { gte: 2000, lt: 3000, - }, + }.toString(), + undefined, ], ], depth: 0, sortIndex: 0, }, - } as unknown as ShapeTreeNode; + children: [], + }; const visParamsNew = { ...visParams, distinctColors: true, @@ -278,12 +292,17 @@ describe('getColor', () => { }; const color = getColor( ChartTypes.PIE, + // There is the unhandled situation that the categoricalName passed is not a plain string but a RangeKey + // In this case, the internal code, thankfully, requires the stringified version of it and/or the formatted one + // handling also this badly configured type + // FIXME when getColor could handle both strings and RangeKey + { gte: 1000, lt: 2000 } as unknown as string, d, 0, true, { '≥ 1000 and < 2000': '#3F6833' }, - buckets, - visData.rows, + distinctSeries, + dataLength, visParamsNew, getPaletteRegistry(), { getColor: () => undefined }, @@ -297,30 +316,34 @@ describe('getColor', () => { }); it('should only pass the second layer for mosaic', () => { - const d = { - dataName: 'Second level 1', + const d: SimplifiedArrayNode = { depth: 2, sortIndex: 0, parent: { - children: [['Second level 1'], ['Second level 2']], + children: [ + ['Second level 1', undefined], + ['Second level 2', undefined], + ], depth: 1, sortIndex: 0, parent: { - children: [['First level']], + children: [['First level', undefined]], depth: 0, sortIndex: 0, }, }, - } as unknown as ShapeTreeNode; + children: [], + }; const registry = getPaletteRegistry(); getColor( ChartTypes.MOSAIC, + 'Second level 1', d, 1, true, {}, - buckets, - visData.rows, + distinctSeries, + dataLength, visParams, registry, undefined, @@ -338,30 +361,34 @@ describe('getColor', () => { }); it('should only pass the first layer for treemap', () => { - const d = { - dataName: 'Second level 1', + const d: SimplifiedArrayNode = { depth: 2, sortIndex: 0, parent: { - children: [['Second level 1'], ['Second level 2']], + children: [ + ['Second level 1', undefined], + ['Second level 2', undefined], + ], depth: 1, sortIndex: 0, parent: { - children: [['First level']], + children: [['First level', undefined]], depth: 0, sortIndex: 0, }, }, - } as unknown as ShapeTreeNode; + children: [], + }; const registry = getPaletteRegistry(); getColor( ChartTypes.TREEMAP, + 'Second level 1', d, 1, true, {}, - buckets, - visData.rows, + distinctSeries, + dataLength, visParams, registry, undefined, diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts index 16c49f1c3188..2f93b9bc8d37 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts @@ -5,15 +5,15 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ -import { ShapeTreeNode } from '@elastic/charts'; +import { ArrayNode } from '@elastic/charts'; import { isEqual } from 'lodash'; import type { PaletteRegistry, SeriesLayer, PaletteOutput, PaletteDefinition } from '@kbn/coloring'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { FieldFormat } from '@kbn/field-formats-plugin/common'; import { lightenColor } from '@kbn/charts-plugin/public'; -import type { Datatable, DatatableRow } from '@kbn/expressions-plugin/public'; +import type { Datatable } from '@kbn/expressions-plugin/public'; import { BucketColumns, ChartTypes, PartitionVisParams } from '../../../common/types'; -import { DistinctSeries, getDistinctSeries } from '../get_distinct_series'; +import { DistinctSeries } from '../get_distinct_series'; import { getNodeLabel } from './get_node_labels'; const isTreemapOrMosaicChart = (shape: ChartTypes) => @@ -59,39 +59,34 @@ export const byDataColorPaletteMap = ( }; const getDistinctColor = ( - d: ShapeTreeNode, + categoricalKey: string, isSplitChart: boolean, overwriteColors: { [key: string]: string } = {}, visParams: PartitionVisParams, palettes: PaletteRegistry | null, syncColors: boolean, { parentSeries, allSeries }: DistinctSeries, - name: string + formattedCategoricalKey: string ) => { - let overwriteColor; + // TODO move away from Record to a Map to avoid issues with reserved JS keywords + if (overwriteColors.hasOwnProperty(categoricalKey)) { + return overwriteColors[categoricalKey]; + } // this is for supporting old visualizations (created by vislib plugin) // it seems that there for some aggs, the uiState saved from vislib is - // different than the es-charts handle it - if (overwriteColors.hasOwnProperty(name)) { - overwriteColor = overwriteColors[name]; - } - - if (Object.keys(overwriteColors).includes(d.dataName.toString())) { - overwriteColor = overwriteColors[d.dataName]; - } - - if (overwriteColor) { - return overwriteColor; + // different from how es-charts handles it + if (overwriteColors.hasOwnProperty(formattedCategoricalKey)) { + return overwriteColors[formattedCategoricalKey]; } - const index = allSeries.findIndex((dataName) => isEqual(dataName, d.dataName)); - const isSplitParentLayer = isSplitChart && parentSeries.includes(d.dataName); + const index = allSeries.findIndex((d) => isEqual(d, categoricalKey)); + const isSplitParentLayer = isSplitChart && parentSeries.includes(categoricalKey); return palettes?.get(visParams.palette.name).getCategoricalColor( [ { - name: d.dataName, + name: categoricalKey, rankAtDepth: isSplitParentLayer - ? parentSeries.findIndex((dataName) => dataName === d.dataName) + ? parentSeries.findIndex((d) => d === categoricalKey) : index > -1 ? index : 0, @@ -108,29 +103,57 @@ const getDistinctColor = ( ); }; +/** + * This interface is introduced to simplify the used logic on testing an ArrayNode outside elastic-charts. + * The SimplifiedArrayNode structure resembles the hierarchical configuration of an ArrayNode, + * by presenting only the necessary fields used by the functions in this file. + * The main difference is in the parent node, that to simplify this infinite tree structure we configured it as optional + * so that in test I don't need to add a type assertion on an undefined parent as in elastic-charts. + * The children are slight different in implementation and they accept `unknown` as key + * due to the situation described in https://github.com/elastic/kibana/issues/153437 + */ +export interface SimplifiedArrayNode { + depth: ArrayNode['depth']; + sortIndex: ArrayNode['depth']; + parent?: SimplifiedArrayNode; + children: Array<[unknown, SimplifiedArrayNode | undefined]>; +} + +/** + * This method returns the path of each hierarchical layer encountered from the given node + * (a node of a hierarchical tree, currently a partition tree) up to the root of the hierarchy tree. + * The resulting array only shows, for each parent, the name of the node, its child index within the parent branch + * (called rankInDepth) and the total number of children of the parent. + * + */ const createSeriesLayers = ( - d: ShapeTreeNode, + arrayNode: SimplifiedArrayNode, parentSeries: DistinctSeries['parentSeries'], isSplitChart: boolean, formatters: Record, formatter: FieldFormatsStart, column: Partial -) => { +): SeriesLayer[] => { const seriesLayers: SeriesLayer[] = []; - let tempParent: typeof d | typeof d['parent'] = d; + let tempParent: typeof arrayNode | typeof arrayNode['parent'] = arrayNode; while (tempParent.parent && tempParent.depth > 0) { - const seriesName = String(tempParent.parent.children[tempParent.sortIndex][0]); + const nodeKey = tempParent.parent.children[tempParent.sortIndex][0]; + const seriesName = String(nodeKey); + /** + * FIXME this is a bad implementation: The `parentSeries` is an array of both `string` and `RangeKey` even if its type + * is marked as `string[]` in `DistinctSeries`. Here instead we are checking if a stringified `RangeKey` is included into this array that + * is conceptually wrong. + * see https://github.com/elastic/kibana/issues/153437 + */ const isSplitParentLayer = isSplitChart && parentSeries.includes(seriesName); - const formattedName = getNodeLabel( - tempParent.parent.children[tempParent.sortIndex][0], - column, - formatters, - formatter.deserialize - ); + const formattedName = getNodeLabel(nodeKey, column, formatters, formatter.deserialize); seriesLayers.unshift({ + // by construction and types `formattedName` should be always be a string, but I leave this Nullish Coalescing + // because I don't trust much our formatting functions name: formattedName ?? seriesName, rankAtDepth: isSplitParentLayer - ? parentSeries.findIndex((name) => name === seriesName) + ? // FIXME as described above this will not work correctly if the `nodeKey` is a `RangeKey` + parentSeries.findIndex((name) => name === seriesName) : tempParent.sortIndex, totalSeriesAtDepth: isSplitParentLayer ? parentSeries.length @@ -163,12 +186,14 @@ const overrideColors = ( export const getColor = ( chartType: ChartTypes, - d: ShapeTreeNode, + // FIXME this could be both a string or a RangeKey see https://github.com/elastic/kibana/issues/153437 + categoricalKey: string, // could be RangeKey + arrayNode: SimplifiedArrayNode, layerIndex: number, isSplitChart: boolean, overwriteColors: { [key: string]: string } = {}, - columns: Array>, - rows: DatatableRow[], + distinctSeries: DistinctSeries, + { columnsLength, rowsLength }: { columnsLength: number; rowsLength: number }, visParams: PartitionVisParams, palettes: PaletteRegistry | null, byDataPalette: ReturnType | undefined, @@ -178,23 +203,18 @@ export const getColor = ( column: Partial, formatters: Record ) => { - const distinctSeries = getDistinctSeries(rows, columns); - const { parentSeries } = distinctSeries; - const dataName = d.dataName; - // Mind the difference here: the contrast computation for the text ignores the alpha/opacity - // therefore change it for dask mode + // therefore change it for dark mode const defaultColor = isDarkMode ? 'rgba(0,0,0,0)' : 'rgba(255,255,255,0)'; - let name = ''; - if (column.format) { - name = formatter.deserialize(column.format).convert(dataName) ?? ''; - } + const name = column.format + ? formatter.deserialize(column.format).convert(categoricalKey) ?? '' + : ''; if (visParams.distinctColors) { return ( getDistinctColor( - d, + categoricalKey, isSplitChart, overwriteColors, visParams, @@ -207,8 +227,8 @@ export const getColor = ( } const seriesLayers = createSeriesLayers( - d, - parentSeries, + arrayNode, + distinctSeries.parentSeries, isSplitChart, formatters, formatter, @@ -218,7 +238,7 @@ export const getColor = ( const overriddenColor = overrideColors(seriesLayers, overwriteColors, name); if (overriddenColor) { // this is necessary for supporting some old visualizations that defined their own colors (created by vislib plugin) - return lightenColor(overriddenColor, seriesLayers.length, columns.length); + return lightenColor(overriddenColor, seriesLayers.length, columnsLength); } if (chartType === ChartTypes.MOSAIC && byDataPalette && seriesLayers[1]) { @@ -226,7 +246,7 @@ export const getColor = ( } if (isTreemapOrMosaicChart(chartType)) { - if (layerIndex < columns.length - 1) { + if (layerIndex < columnsLength - 1) { return defaultColor; } // for treemap use the top layer for coloring, for mosaic use the second layer @@ -243,8 +263,8 @@ export const getColor = ( seriesLayers, { behindText: visParams.labels.show || isTreemapOrMosaicChart(chartType), - maxDepth: columns.length, - totalSeries: rows.length, + maxDepth: columnsLength, + totalSeries: rowsLength, syncColors, }, visParams.palette?.params ?? { colors: [] } diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts index 93a9cc2b6c7d..19782f0b9f60 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts @@ -11,6 +11,7 @@ import type { PaletteRegistry } from '@kbn/coloring'; import { FieldFormat } from '@kbn/field-formats-plugin/common'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { Datatable, DatatableRow } from '@kbn/expressions-plugin/public'; +import { getDistinctSeries } from '..'; import { BucketColumns, ChartTypes, PartitionVisParams } from '../../../common/types'; import { sortPredicateByType, sortPredicateSaveSourceOrder } from './sort_predicate'; import { byDataColorPaletteMap, getColor } from './get_color'; @@ -53,6 +54,9 @@ export const getLayers = ( } const sortPredicateForType = sortPredicateByType(chartType, visParams, visData, columns); + + const distinctSeries = getDistinctSeries(rows, columns); + return columns.map((col, layerIndex) => { return { groupByRollup: (d: Datum) => (col.id ? d[col.id] ?? EMPTY_SLICE : col.name), @@ -66,15 +70,16 @@ export const getLayers = ( ? sortPredicateSaveSourceOrder() : sortPredicateForType, shape: { - fillColor: (d) => + fillColor: (key, sortIndex, node) => getColor( chartType, - d, + key, + node, layerIndex, isSplitChart, overwriteColors, - columns, - rows, + distinctSeries, + { columnsLength: columns.length, rowsLength: rows.length }, visParams, palettes, byDataPalette, diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_node_labels.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_node_labels.ts index 70236d53a3e5..a8a7eb8cb4f6 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_node_labels.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_node_labels.ts @@ -15,7 +15,7 @@ export const getNodeLabel = ( column: Partial, formatters: Record, defaultFormatFactory: FormatFactory -) => { +): string => { const formatter = getAvailableFormatter(column, formatters, defaultFormatFactory); if (formatter) { return formatter.convert(nodeName) ?? ''; diff --git a/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.test.ts b/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.test.ts index 51f1e06f278a..4bae40076e58 100644 --- a/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.test.ts +++ b/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.test.ts @@ -13,6 +13,8 @@ import { encode } from 'cbor-x'; import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; import { ContentStream, ContentStreamEncoding, ContentStreamParameters } from './content_stream'; import type { GetResponse } from '@elastic/elasticsearch/lib/api/types'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { FileDocument } from '../../../../file_client/file_metadata_client/adapters/es_index'; describe('ContentStream', () => { let client: ReturnType; @@ -30,8 +32,9 @@ describe('ContentStream', () => { encoding: 'base64' as ContentStreamEncoding, size: 1, } as ContentStreamParameters, + indexIsAlias = false, } = {}) => { - return new ContentStream(client, id, index, logger, params); + return new ContentStream(client, id, index, logger, params, indexIsAlias); }; beforeEach(() => { @@ -43,124 +46,193 @@ describe('ContentStream', () => { }); describe('read', () => { - beforeEach(() => { - stream = getContentStream({ params: { size: 1 } }); - }); + describe('with `indexIsAlias` set to `true`', () => { + let searchResponse: estypes.SearchResponse>; + + beforeEach(() => { + searchResponse = { + took: 3, + timed_out: false, + _shards: { + total: 2, + successful: 2, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 0, + hits: [ + { + _index: 'foo', + _id: '123', + _score: 1.0, + }, + ], + }, + }; + + client.search.mockResolvedValue(searchResponse); + }); - it('should perform a search using index and the document id', async () => { - await new Promise((resolve) => stream.once('data', resolve)); + it('should use es.search() to find chunk index', async () => { + stream = getContentStream({ params: { size: 1 }, indexIsAlias: true }); + const data = await new Promise((resolve) => stream.once('data', resolve)); + + expect(client.search).toHaveBeenCalledWith({ + body: { + _source: false, + query: { + term: { + _id: 'something.0', + }, + }, + size: 1, + }, + index: 'somewhere', + }); + expect(data).toEqual(Buffer.from('some content')); + }); - expect(client.get).toHaveBeenCalledTimes(1); + it('should throw if chunk is not found', async () => { + searchResponse.hits.hits = []; + stream = getContentStream({ params: { size: 1 }, indexIsAlias: true }); - const [[request]] = client.get.mock.calls; - expect(request).toHaveProperty('index', 'somewhere'); - expect(request).toHaveProperty('id', 'something.0'); - }); + const readPromise = new Promise((resolve, reject) => { + stream.once('data', resolve); + stream.once('error', reject); + }); - it('should read the document contents', async () => { - const data = await new Promise((resolve) => stream.once('data', resolve)); - expect(data).toEqual(Buffer.from('some content')); + await expect(readPromise).rejects.toHaveProperty( + 'message', + 'Unable to determine index for file chunk id [something.0] in index (alias) [somewhere]' + ); + }); }); - it('should be an empty stream on empty response', async () => { - client.get.mockResponseOnce(toReadable()); - const onData = jest.fn(); + describe('with `indexIsAlias` set to `false`', () => { + beforeEach(() => { + stream = getContentStream({ params: { size: 1 } }); + }); - stream.on('data', onData); - await new Promise((resolve) => stream.once('end', resolve)); + it('should perform a search using index and the document id', async () => { + await new Promise((resolve) => stream.once('data', resolve)); - expect(onData).not.toHaveBeenCalled(); - }); + expect(client.get).toHaveBeenCalledTimes(1); - it('should emit an error event', async () => { - client.get.mockRejectedValueOnce('some error'); + const [[request]] = client.get.mock.calls; + expect(request).toHaveProperty('index', 'somewhere'); + expect(request).toHaveProperty('id', 'something.0'); + }); - stream.read(); - const error = await new Promise((resolve) => stream.once('error', resolve)); + it('should read the document contents', async () => { + const data = await new Promise((resolve) => stream.once('data', resolve)); + expect(data).toEqual(Buffer.from('some content')); + }); - expect(error).toBe('some error'); - }); + it('should be an empty stream on empty response', async () => { + client.get.mockResponseOnce(toReadable()); + const onData = jest.fn(); - it('should decode base64 encoded content', async () => { - client.get.mockResponseOnce( - toReadable(set({ found: true }, '_source.data', Buffer.from('encoded content'))) - ); - const data = await new Promise((resolve) => stream.once('data', resolve)); + stream.on('data', onData); + await new Promise((resolve) => stream.once('end', resolve)); - expect(data).toEqual(Buffer.from('encoded content')); - }); + expect(onData).not.toHaveBeenCalled(); + }); + + it('should emit an error event', async () => { + client.get.mockRejectedValueOnce('some error'); - it('should compound content from multiple chunks', async () => { - const [one, two, three] = ['12', '34', '56'].map(Buffer.from); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', three))); + stream.read(); + const error = await new Promise((resolve) => stream.once('error', resolve)); - stream = getContentStream({ - params: { size: 6 }, + expect(error).toBe('some error'); }); - let data = ''; - for await (const chunk of stream) { - data += chunk; - } + it('should decode base64 encoded content', async () => { + client.get.mockResponseOnce( + toReadable(set({ found: true }, '_source.data', Buffer.from('encoded content'))) + ); + const data = await new Promise((resolve) => stream.once('data', resolve)); - expect(data).toEqual('123456'); - expect(client.get).toHaveBeenCalledTimes(3); + expect(data).toEqual(Buffer.from('encoded content')); + }); - const [[request1], [request2], [request3]] = client.get.mock.calls; + it('should compound content from multiple chunks', async () => { + const [one, two, three] = ['12', '34', '56'].map(Buffer.from); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', three))); - expect(request1).toHaveProperty('index', 'somewhere'); - expect(request1).toHaveProperty('id', 'something.0'); - expect(request2).toHaveProperty('index', 'somewhere'); - expect(request2).toHaveProperty('id', 'something.1'); - expect(request3).toHaveProperty('index', 'somewhere'); - expect(request3).toHaveProperty('id', 'something.2'); - }); + stream = getContentStream({ + params: { size: 6 }, + }); - it('should stop reading on empty chunk', async () => { - const [one, two, three] = ['12', '34', ''].map(Buffer.from); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', three))); - stream = getContentStream({ params: { size: 12 } }); - let data = ''; - for await (const chunk of stream) { - data += chunk; - } - - expect(data).toEqual('1234'); - expect(client.get).toHaveBeenCalledTimes(3); - }); + let data = ''; + for await (const chunk of stream) { + data += chunk; + } - it('should read while chunks are present when there is no size', async () => { - const [one, two] = ['12', '34'].map(Buffer.from); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); - client.get.mockResponseOnce(toReadable({ found: true })); - stream = getContentStream({ params: { size: undefined } }); - let data = ''; - for await (const chunk of stream) { - data += chunk; - } - - expect(data).toEqual('1234'); - expect(client.get).toHaveBeenCalledTimes(3); - }); + expect(data).toEqual('123456'); + expect(client.get).toHaveBeenCalledTimes(3); - it('should decode every chunk separately', async () => { - const [one, two, three, four] = ['12', '34', '56', ''].map(Buffer.from); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', three))); - client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', four))); - stream = getContentStream({ params: { size: 12 } }); - let data = ''; - for await (const chunk of stream) { - data += chunk; - } - - expect(data).toEqual('123456'); + const [[request1], [request2], [request3]] = client.get.mock.calls; + + expect(request1).toHaveProperty('index', 'somewhere'); + expect(request1).toHaveProperty('id', 'something.0'); + expect(request2).toHaveProperty('index', 'somewhere'); + expect(request2).toHaveProperty('id', 'something.1'); + expect(request3).toHaveProperty('index', 'somewhere'); + expect(request3).toHaveProperty('id', 'something.2'); + }); + + it('should stop reading on empty chunk', async () => { + const [one, two, three] = ['12', '34', ''].map(Buffer.from); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', three))); + stream = getContentStream({ params: { size: 12 } }); + let data = ''; + for await (const chunk of stream) { + data += chunk; + } + + expect(data).toEqual('1234'); + expect(client.get).toHaveBeenCalledTimes(3); + }); + + it('should read while chunks are present when there is no size', async () => { + const [one, two] = ['12', '34'].map(Buffer.from); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); + client.get.mockResponseOnce(toReadable({ found: true })); + stream = getContentStream({ params: { size: undefined } }); + let data = ''; + for await (const chunk of stream) { + data += chunk; + } + + expect(data).toEqual('1234'); + expect(client.get).toHaveBeenCalledTimes(3); + }); + + it('should decode every chunk separately', async () => { + const [one, two, three, four] = ['12', '34', '56', ''].map(Buffer.from); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', one))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', two))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', three))); + client.get.mockResponseOnce(toReadable(set({ found: true }, '_source.data', four))); + stream = getContentStream({ params: { size: 12 } }); + let data = ''; + for await (const chunk of stream) { + data += chunk; + } + + expect(data).toEqual('123456'); + }); }); }); diff --git a/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.ts b/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.ts index 99e29383f020..98aebda3c773 100644 --- a/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.ts +++ b/src/plugins/files/server/blob_storage_service/adapters/es/content_stream/content_stream.ts @@ -101,12 +101,15 @@ export class ContentStream extends Duplex { }, }); - const docIndex = chunkDocMeta.hits.hits[0]._index; + const docIndex = chunkDocMeta.hits.hits?.[0]?._index; if (!docIndex) { - throw new Error( + const err = new Error( `Unable to determine index for file chunk id [${id}] in index (alias) [${this.index}]` ); + + this.logger.error(err); + throw err; } return docIndex; diff --git a/src/plugins/files/server/file_client/create_es_file_client.test.ts b/src/plugins/files/server/file_client/create_es_file_client.test.ts new file mode 100644 index 000000000000..68589b334c8e --- /dev/null +++ b/src/plugins/files/server/file_client/create_es_file_client.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + type ElasticsearchClientMock, + elasticsearchServiceMock, + loggingSystemMock, +} from '@kbn/core/server/mocks'; +import { MockedLogger } from '@kbn/logging-mocks'; +import { createEsFileClient } from './create_es_file_client'; +import { FileClient } from './types'; +import { ElasticsearchBlobStorageClient } from '../blob_storage_service'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { FileDocument } from './file_metadata_client/adapters/es_index'; + +describe('When initializing file client via createESFileClient()', () => { + let esClient: ElasticsearchClientMock; + let logger: MockedLogger; + + beforeEach(() => { + ElasticsearchBlobStorageClient.configureConcurrentUpload(Infinity); + esClient = elasticsearchServiceMock.createElasticsearchClient(); + logger = loggingSystemMock.createLogger(); + }); + + describe('and `indexIsAlias` argument is used', () => { + let fileClient: FileClient; + let searchResponse: estypes.SearchResponse>; + + beforeEach(() => { + searchResponse = { + took: 3, + timed_out: false, + _shards: { + total: 2, + successful: 2, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 0, + hits: [ + { + _index: 'foo', + _id: '123', + _score: 1.0, + _source: { + file: { + name: 'foo.txt', + Status: 'READY', + created: '2023-03-27T20:45:31.490Z', + Updated: '2023-03-27T20:45:31.490Z', + FileKind: '', + }, + }, + }, + ], + }, + }; + + esClient.search.mockResolvedValue(searchResponse); + fileClient = createEsFileClient({ + logger, + metadataIndex: 'file-meta', + blobStorageIndex: 'file-data', + elasticsearchClient: esClient, + indexIsAlias: true, + }); + }); + + it('should use es.search() to retrieve file metadata', async () => { + await fileClient.get({ id: '123' }); + expect(esClient.search).toHaveBeenCalledWith({ + body: { + query: { + term: { + _id: '123', + }, + }, + size: 1, + }, + index: 'file-meta', + }); + }); + + it('should throw an error if file is not found', async () => { + (searchResponse.hits.total as estypes.SearchTotalHits).value = 0; + searchResponse.hits.hits = []; + await expect(fileClient.get({ id: '123 ' })).rejects.toHaveProperty( + 'message', + 'File not found' + ); + }); + }); +}); diff --git a/src/plugins/files/server/file_client/file_metadata_client/adapters/es_index.ts b/src/plugins/files/server/file_client/file_metadata_client/adapters/es_index.ts index 37f3988c0cc7..d139c7cf6330 100644 --- a/src/plugins/files/server/file_client/file_metadata_client/adapters/es_index.ts +++ b/src/plugins/files/server/file_client/file_metadata_client/adapters/es_index.ts @@ -12,7 +12,6 @@ import { Logger } from '@kbn/core/server'; import { toElasticsearchQuery } from '@kbn/es-query'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { MappingProperty, SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; -import { fetchDoc } from '../../utils'; import type { FilesMetrics, FileMetadata, Pagination } from '../../../../common'; import type { FindFileArgs } from '../../../file_service'; import type { @@ -36,7 +35,7 @@ const fileMappings: MappingProperty = { }, }; -interface FileDocument { +export interface FileDocument { file: FileMetadata; } @@ -82,11 +81,36 @@ export class EsIndexFilesMetadataClient implements FileMetadataClie } async get({ id }: GetArg): Promise> { - const { _source: doc } = - (await fetchDoc>(this.esClient, this.index, id, this.indexIsAlias)) ?? {}; + const { esClient, index, indexIsAlias } = this; + let doc: FileDocument | undefined; + + if (indexIsAlias) { + doc = ( + await esClient.search>({ + index, + body: { + size: 1, + query: { + term: { + _id: id, + }, + }, + }, + }) + ).hits.hits?.[0]?._source; + } else { + doc = ( + await esClient.get>({ + index, + id, + }) + )._source; + } if (!doc) { - this.logger.error(`File with id "${id}" not found`); + this.logger.error( + `File with id "${id}" not found in index ${indexIsAlias ? 'alias ' : ''}"${index}"` + ); throw new Error('File not found'); } diff --git a/src/plugins/files/server/file_client/utils.ts b/src/plugins/files/server/file_client/utils.ts index 9c7d07312f63..88e0901680f0 100644 --- a/src/plugins/files/server/file_client/utils.ts +++ b/src/plugins/files/server/file_client/utils.ts @@ -6,8 +6,6 @@ * Side Public License, v 1. */ -import { GetResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { FileMetadata } from '../../common'; export function createDefaultFileAttributes(): Pick< @@ -21,31 +19,3 @@ export function createDefaultFileAttributes(): Pick< Updated: dateString, }; } - -export const fetchDoc = async ( - esClient: ElasticsearchClient, - index: string, - docId: string, - indexIsAlias: boolean = false -): Promise | undefined> => { - if (indexIsAlias) { - const fileDocSearchResult = await esClient.search({ - index, - body: { - size: 1, - query: { - term: { - _id: docId, - }, - }, - }, - }); - - return fileDocSearchResult.hits.hits[0] as GetResponse; - } - - return esClient.get({ - index, - id: docId, - }); -}; diff --git a/src/plugins/files/tsconfig.json b/src/plugins/files/tsconfig.json index 8a14fd5ef06b..ffeed3f22d46 100644 --- a/src/plugins/files/tsconfig.json +++ b/src/plugins/files/tsconfig.json @@ -28,6 +28,7 @@ "@kbn/core-logging-server-mocks", "@kbn/ecs", "@kbn/safer-lodash-set", + "@kbn/logging-mocks", ], "exclude": [ "target/**/*", diff --git a/test/accessibility/apps/console.ts b/test/accessibility/apps/console.ts index e619b4bbf553..5bfc40952ffc 100644 --- a/test/accessibility/apps/console.ts +++ b/test/accessibility/apps/console.ts @@ -12,7 +12,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['common', 'console']); const a11y = getService('a11y'); - describe('Dev tools console', () => { + // https://github.com/elastic/kibana/issues/148538 + describe.skip('Dev tools console', () => { before(async () => { await PageObjects.common.navigateToApp('console'); }); diff --git a/test/accessibility/apps/dashboard.ts b/test/accessibility/apps/dashboard.ts index 3db8b99c577a..b334563167fc 100644 --- a/test/accessibility/apps/dashboard.ts +++ b/test/accessibility/apps/dashboard.ts @@ -109,12 +109,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await a11y.testAppSnapshot(); }); - it('Test full screen', async () => { + // https://github.com/elastic/kibana/issues/153597 + it.skip('Test full screen', async () => { await PageObjects.dashboard.clickFullScreenMode(); await a11y.testAppSnapshot(); }); - it('Exit out of full screen mode', async () => { + // https://github.com/elastic/kibana/issues/153597 + it.skip('Exit out of full screen mode', async () => { await PageObjects.dashboard.exitFullScreenMode(); await a11y.testAppSnapshot(); }); diff --git a/test/accessibility/apps/discover.ts b/test/accessibility/apps/discover.ts index b2fa7946817e..df59456ee548 100644 --- a/test/accessibility/apps/discover.ts +++ b/test/accessibility/apps/discover.ts @@ -172,13 +172,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.click('unifiedHistogramChartOptionsToggle'); }); - it('a11y test for data grid sort panel', async () => { + // https://github.com/elastic/kibana/issues/148567 + it.skip('a11y test for data grid sort panel', async () => { await testSubjects.click('dataGridColumnSortingButton'); await a11y.testAppSnapshot(); await browser.pressKeys(browser.keys.ESCAPE); }); - - it('a11y test for setting row height for display panel', async () => { + // https://github.com/elastic/kibana/issues/148567 + it.skip('a11y test for setting row height for display panel', async () => { await testSubjects.click('dataGridDisplaySelectorPopover'); await a11y.testAppSnapshot(); await browser.pressKeys(browser.keys.ESCAPE); diff --git a/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.ts b/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.ts index 63d2106c8cbb..4ab85f87e01d 100644 --- a/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.ts +++ b/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.ts @@ -36,7 +36,7 @@ export const getSummaryData = ( { categoryId: 'ecs-compliant', mappings: partitionedFieldMetadata.ecsCompliant.length }, ]; -export const getFillColor = (categoryId: CategoryId): string => { +export const getFillColor = (categoryId: CategoryId | string): string => { switch (categoryId) { case 'incompatible': return euiThemeVars.euiColorDanger; diff --git a/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/ecs_summary_donut_chart/index.tsx b/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/ecs_summary_donut_chart/index.tsx index dc25d12031b8..b09a04b131cc 100644 --- a/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/ecs_summary_donut_chart/index.tsx +++ b/x-pack/packages/kbn-ecs-data-quality-dashboard/impl/data_quality/ecs_summary_donut_chart/index.tsx @@ -16,6 +16,7 @@ import type { Theme, WordCloudElementEvent, XYChartElementEvent, + PartitionLayer, } from '@elastic/charts'; import { Chart, Partition, PartitionLayout, Settings } from '@elastic/charts'; import { @@ -101,12 +102,12 @@ const EcsSummaryDonutChartComponent: React.FC = ({ const valueAccessor = useCallback((d: Datum) => d.mappings as number, []); const valueFormatter = useCallback((d: number) => `${d}`, []); const layers = useMemo( - () => [ + (): PartitionLayer[] => [ { groupByRollup: (d: Datum) => d.categoryId, nodeLabel: (d: Datum) => getNodeLabel(d), shape: { - fillColor: (d: Datum) => getFillColor(d.dataName), + fillColor: getFillColor, }, }, ], diff --git a/x-pack/plugins/apm/public/components/app/storage_explorer/services_table/storage_details_per_service.tsx b/x-pack/plugins/apm/public/components/app/storage_explorer/services_table/storage_details_per_service.tsx index 0e47d657da30..dffd3559d16e 100644 --- a/x-pack/plugins/apm/public/components/app/storage_explorer/services_table/storage_details_per_service.tsx +++ b/x-pack/plugins/apm/public/components/app/storage_explorer/services_table/storage_details_per_service.tsx @@ -219,7 +219,8 @@ export function StorageDetailsPerService({ { groupByRollup: (d: Datum) => d.processorEventLabel, shape: { - fillColor: (d) => groupedPalette[d.sortIndex], + fillColor: (dataName, sortIndex) => + groupedPalette[sortIndex], }, }, ]} diff --git a/x-pack/plugins/apm/public/components/shared/charts/treemap_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/treemap_chart/index.tsx index 1c201ffaee19..80f0980bbadd 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/treemap_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/treemap_chart/index.tsx @@ -47,7 +47,7 @@ export function TreemapChart({ { groupByRollup: (d: Datum) => d.label, shape: { - fillColor: ({ sortIndex }: { sortIndex: number }) => + fillColor: (dataName, sortIndex) => colorPalette[Math.floor(sortIndex % 10)], }, fillLabel: { diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 04e31f9e0b64..37453e1364f8 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { PostureTypes } from './types'; + export const STATUS_ROUTE_PATH = '/internal/cloud_security_posture/status'; export const STATS_ROUTE_PATH = '/internal/cloud_security_posture/stats/{policy_template}'; export const BENCHMARKS_ROUTE_PATH = '/internal/cloud_security_posture/benchmarks'; @@ -85,3 +87,10 @@ export const SUPPORTED_CLOUDBEAT_INPUTS = [ CLOUDBEAT_VULN_MGMT_GCP, CLOUDBEAT_VULN_MGMT_AZURE, ] as const; + +export const POSTURE_TYPES: { [x: string]: PostureTypes } = { + [KSPM_POLICY_TEMPLATE]: KSPM_POLICY_TEMPLATE, + [CSPM_POLICY_TEMPLATE]: CSPM_POLICY_TEMPLATE, + [VULN_MGMT_POLICY_TEMPLATE]: VULN_MGMT_POLICY_TEMPLATE, + [POSTURE_TYPE_ALL]: POSTURE_TYPE_ALL, +} as const; diff --git a/x-pack/plugins/cloud_security_posture/common/types.ts b/x-pack/plugins/cloud_security_posture/common/types.ts index 83fd9a4b276c..3552a663866e 100644 --- a/x-pack/plugins/cloud_security_posture/common/types.ts +++ b/x-pack/plugins/cloud_security_posture/common/types.ts @@ -12,7 +12,7 @@ import { CspRuleTemplateMetadata } from './schemas/csp_rule_template_metadata'; export type Evaluation = 'passed' | 'failed' | 'NA'; -export type PostureTypes = 'cspm' | 'kspm' | 'all'; +export type PostureTypes = 'cspm' | 'kspm' | 'vuln_mgmt' | 'all'; /** number between 1-100 */ export type Score = number; @@ -85,6 +85,7 @@ export interface BaseCspSetupStatus { latestPackageVersion: string; cspm: BaseCspSetupBothPolicy; kspm: BaseCspSetupBothPolicy; + vuln_mgmt: BaseCspSetupBothPolicy; isPluginInitialized: boolean; } diff --git a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts index c8d876a7281d..47ab871dc9d2 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts @@ -6,13 +6,13 @@ */ import { ElasticsearchClient, type Logger } from '@kbn/core/server'; -import { IndexStatus } from '../../common/types'; +import { IndexStatus, PostureTypes } from '../../common/types'; export const checkIndexStatus = async ( esClient: ElasticsearchClient, index: string, logger: Logger, - postureType: 'cspm' | 'kspm' | 'all' = 'all' + postureType: PostureTypes = 'all' ): Promise => { const query = postureType === 'all' diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts index 7f1345ced245..5cf11f8d1f98 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts @@ -6,10 +6,10 @@ */ import { calculateCspStatusCode } from './status'; -import { CSPM_POLICY_TEMPLATE } from '../../../common/constants'; +import { CSPM_POLICY_TEMPLATE, VULN_MGMT_POLICY_TEMPLATE } from '../../../common/constants'; -describe('calculateCspStatusCode test', () => { - it('Verify status when there are no permission', async () => { +describe('calculateCspStatusCode for cspm', () => { + it('Verify status when there are no permission for cspm', async () => { const statusCode = calculateCspStatusCode( CSPM_POLICY_TEMPLATE, { @@ -145,3 +145,141 @@ describe('calculateCspStatusCode test', () => { expect(statusCode).toMatch('indexing'); }); }); + +describe('calculateCspStatusCode for vul_mgmt', () => { + it('Verify status when there are no permission for vul_mgmt', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'unprivileged', + findings: 'unprivileged', + score: 'unprivileged', + }, + 1, + 1, + 1, + ['cspm'] + ); + + expect(statusCode).toMatch('unprivileged'); + }); + + it('Verify status when there are no vul_mgmt findings, no healthy agents and no installed policy templates', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 0, + 0, + 0, + [] + ); + + expect(statusCode).toMatch('not-installed'); + }); + + it('Verify status when there are vul_mgmt findings and installed policies but no healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 0, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('not-deployed'); + }); + + it('Verify status when there are vul_mgmt findings ,installed policies and healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'not-empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 1, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('indexed'); + }); + + it('Verify status when there are no vul_mgmt findings ,installed policies and no healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 0, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('not-deployed'); + }); + + it('Verify status when there are installed policies, healthy agents and no vul_mgmt findings', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 1, + 9, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('waiting_for_results'); + }); + + it('Verify status when there are installed policies, healthy agents and no vul_mgmt findings and been more than 10 minutes', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 1, + 11, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('index-timeout'); + }); + + it('Verify status when there are installed policies, healthy agents past vul_mgmt findings but no recent findings', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 1, + 0, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('indexing'); + }); +}); diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts index 23578194422e..e643741bc432 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts @@ -17,10 +17,14 @@ import { LATEST_FINDINGS_INDEX_DEFAULT_NS, FINDINGS_INDEX_PATTERN, BENCHMARK_SCORE_INDEX_DEFAULT_NS, + VULNERABILITIES_INDEX_PATTERN, KSPM_POLICY_TEMPLATE, CSPM_POLICY_TEMPLATE, + POSTURE_TYPES, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + VULN_MGMT_POLICY_TEMPLATE, } from '../../../common/constants'; -import type { CspApiRequestHandlerContext, CspRouter } from '../../types'; +import type { CspApiRequestHandlerContext, CspRouter, StatusResponseInfo } from '../../types'; import type { CspSetupStatus, CspStatusCode, @@ -72,7 +76,7 @@ export const calculateCspStatusCode = ( indicesStatus: { findingsLatest: IndexStatus; findings: IndexStatus; - score: IndexStatus; + score?: IndexStatus; }, installedCspPackagePolicies: number, healthyAgents: number, @@ -80,8 +84,7 @@ export const calculateCspStatusCode = ( installedPolicyTemplates: string[] ): CspStatusCode => { // We check privileges only for the relevant indices for our pages to appear - const postureTypeCheck = - postureType === CSPM_POLICY_TEMPLATE ? CSPM_POLICY_TEMPLATE : KSPM_POLICY_TEMPLATE; + const postureTypeCheck: PostureTypes = POSTURE_TYPES[postureType]; if (indicesStatus.findingsLatest === 'unprivileged' || indicesStatus.score === 'unprivileged') return 'unprivileged'; if (!installedPolicyTemplates.includes(postureTypeCheck)) return 'not-installed'; @@ -133,10 +136,13 @@ const getCspStatus = async ({ findingsLatestIndexStatusKspm, findingsIndexStatusKspm, scoreIndexStatusKspm, + vulnerabilitiesLatestIndexStatus, + vulnerabilitiesIndexStatus, installation, latestCspPackage, installedPackagePoliciesKspm, installedPackagePoliciesCspm, + installedPackagePoliciesVulnMgmt, installedPolicyTemplates, ] = await Promise.all([ checkIndexStatus(esClient.asCurrentUser, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger), @@ -151,8 +157,22 @@ const getCspStatus = async ({ checkIndexStatus(esClient.asCurrentUser, FINDINGS_INDEX_PATTERN, logger, 'kspm'), checkIndexStatus(esClient.asCurrentUser, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'kspm'), + checkIndexStatus( + esClient.asCurrentUser, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + logger, + VULN_MGMT_POLICY_TEMPLATE + ), + checkIndexStatus( + esClient.asCurrentUser, + VULNERABILITIES_INDEX_PATTERN, + logger, + VULN_MGMT_POLICY_TEMPLATE + ), + packageService.asInternalUser.getInstallation(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), packageService.asInternalUser.fetchFindLatestPackage(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), + getCspPackagePolicies( soClient, packagePolicyService, @@ -171,9 +191,17 @@ const getCspStatus = async ({ }, CSPM_POLICY_TEMPLATE ), + getCspPackagePolicies( + soClient, + packagePolicyService, + CLOUD_SECURITY_POSTURE_PACKAGE_NAME, + { + per_page: 10000, + }, + VULN_MGMT_POLICY_TEMPLATE + ), getInstalledPolicyTemplates(packagePolicyService, soClient), ]); - const healthyAgentsKspm = await getHealthyAgents( soClient, installedPackagePoliciesKspm.items, @@ -189,8 +217,18 @@ const getCspStatus = async ({ agentService, logger ); + + const healthyAgentsVulMgmt = await getHealthyAgents( + soClient, + installedPackagePoliciesVulnMgmt.items, + agentPolicyService, + agentService, + logger + ); const installedPackagePoliciesTotalKspm = installedPackagePoliciesKspm.total; const installedPackagePoliciesTotalCspm = installedPackagePoliciesCspm.total; + const installedPackagePoliciesTotalVulnMgmt = installedPackagePoliciesVulnMgmt.total; + const latestCspPackageVersion = latestCspPackage.version; const MIN_DATE = 0; @@ -207,6 +245,10 @@ const getCspStatus = async ({ index: BENCHMARK_SCORE_INDEX_DEFAULT_NS, status: scoreIndexStatus, }, + { + index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + status: vulnerabilitiesLatestIndexStatus, + }, ]; const statusCspm = calculateCspStatusCode( @@ -235,38 +277,38 @@ const getCspStatus = async ({ installedPolicyTemplates ); - if ((statusCspm && statusKspm) === 'not-installed') - return { - cspm: { - status: statusCspm, - healthyAgents: healthyAgentsCspm, - installedPackagePolicies: installedPackagePoliciesTotalCspm, - }, - kspm: { - status: statusKspm, - healthyAgents: healthyAgentsKspm, - installedPackagePolicies: installedPackagePoliciesTotalKspm, - }, - indicesDetails, - latestPackageVersion: latestCspPackageVersion, - isPluginInitialized: isPluginInitialized(), - }; - - const response = { - cspm: { - status: statusCspm, - healthyAgents: healthyAgentsCspm, - installedPackagePolicies: installedPackagePoliciesTotalCspm, - }, - kspm: { - status: statusKspm, - healthyAgents: healthyAgentsKspm, - installedPackagePolicies: installedPackagePoliciesTotalKspm, + const statusVulnMgmt = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: vulnerabilitiesLatestIndexStatus, + findings: vulnerabilitiesIndexStatus, }, + installedPackagePoliciesTotalVulnMgmt, + healthyAgentsVulMgmt, + calculateDiffFromNowInMinutes(installation?.install_started_at || MIN_DATE), + installedPolicyTemplates + ); + + const statusResponseInfo = getStatusResponse({ + statusCspm, + statusKspm, + statusVulnMgmt, + healthyAgentsCspm, + healthyAgentsKspm, + healthyAgentsVulMgmt, + installedPackagePoliciesTotalKspm, + installedPackagePoliciesTotalCspm, + installedPackagePoliciesTotalVulnMgmt, indicesDetails, - latestPackageVersion: latestCspPackageVersion, - installedPackageVersion: installation?.install_version, + latestCspPackageVersion, isPluginInitialized: isPluginInitialized(), + }); + + if ((statusCspm && statusKspm && statusVulnMgmt) === 'not-installed') return statusResponseInfo; + + const response = { + ...statusResponseInfo, + installedPackageVersion: installation?.install_version, }; assertResponse(response, logger); @@ -316,3 +358,40 @@ export const defineGetCspStatusRoute = (router: CspRouter): void => } } ); + +const getStatusResponse = (statusResponseInfo: StatusResponseInfo) => { + const { + statusCspm, + statusKspm, + statusVulnMgmt, + healthyAgentsCspm, + healthyAgentsKspm, + healthyAgentsVulMgmt, + installedPackagePoliciesTotalKspm, + installedPackagePoliciesTotalCspm, + installedPackagePoliciesTotalVulnMgmt, + indicesDetails, + latestCspPackageVersion, + isPluginInitialized, + }: StatusResponseInfo = statusResponseInfo; + return { + [CSPM_POLICY_TEMPLATE]: { + status: statusCspm, + healthyAgents: healthyAgentsCspm, + installedPackagePolicies: installedPackagePoliciesTotalCspm, + }, + [KSPM_POLICY_TEMPLATE]: { + status: statusKspm, + healthyAgents: healthyAgentsKspm, + installedPackagePolicies: installedPackagePoliciesTotalKspm, + }, + [VULN_MGMT_POLICY_TEMPLATE]: { + status: statusVulnMgmt, + healthyAgents: healthyAgentsVulMgmt, + installedPackagePolicies: installedPackagePoliciesTotalVulnMgmt, + }, + indicesDetails, + isPluginInitialized, + latestPackageVersion: latestCspPackageVersion, + }; +}; diff --git a/x-pack/plugins/cloud_security_posture/server/types.ts b/x-pack/plugins/cloud_security_posture/server/types.ts index 503a529f4a68..1b3c4fd84034 100644 --- a/x-pack/plugins/cloud_security_posture/server/types.ts +++ b/x-pack/plugins/cloud_security_posture/server/types.ts @@ -34,6 +34,7 @@ import type { import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import type { FleetStartContract, FleetRequestHandlerContext } from '@kbn/fleet-plugin/server'; import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; +import { CspStatusCode, IndexDetails } from '../common/types'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface CspServerPluginSetup {} @@ -97,3 +98,18 @@ export type CspRequestHandler< * @internal */ export type CspRouter = IRouter; + +export interface StatusResponseInfo { + statusCspm: CspStatusCode; + statusKspm: CspStatusCode; + statusVulnMgmt: CspStatusCode; + healthyAgentsCspm: number; + healthyAgentsKspm: number; + healthyAgentsVulMgmt: number; + installedPackagePoliciesTotalKspm: number; + installedPackagePoliciesTotalCspm: number; + installedPackagePoliciesTotalVulnMgmt: number; + indicesDetails: IndexDetails[]; + latestCspPackageVersion: string; + isPluginInitialized: boolean; +} diff --git a/x-pack/plugins/enterprise_search/common/types/connectors.ts b/x-pack/plugins/enterprise_search/common/types/connectors.ts index 5869d19ae42d..be05831daa85 100644 --- a/x-pack/plugins/enterprise_search/common/types/connectors.ts +++ b/x-pack/plugins/enterprise_search/common/types/connectors.ts @@ -8,7 +8,7 @@ export interface KeyValuePair { label: string; order?: number | null; - value: string | null; + value: string | number | boolean | null; } export type ConnectorConfiguration = Record & { diff --git a/x-pack/plugins/enterprise_search/common/types/engines.ts b/x-pack/plugins/enterprise_search/common/types/engines.ts index 4aa6cf9368c0..7a484094a46d 100644 --- a/x-pack/plugins/enterprise_search/common/types/engines.ts +++ b/x-pack/plugins/enterprise_search/common/types/engines.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { HealthStatus, FieldCapsResponse } from '@elastic/elasticsearch/lib/api/types'; +import { FieldCapsResponse, HealthStatus } from '@elastic/elasticsearch/lib/api/types'; export interface EnterpriseSearchEnginesResponse { count: number; - params: { q?: string; from: number; size: number }; + params: { from: number; q?: string; size: number }; results: EnterpriseSearchEngine[]; } @@ -21,8 +21,8 @@ export interface EnterpriseSearchEngine { export interface EnterpriseSearchEngineDetails { indices: EnterpriseSearchEngineIndex[]; - updated_at_millis: number; name: string; + updated_at_millis: number; } export interface EnterpriseSearchEngineIndex { @@ -33,14 +33,10 @@ export interface EnterpriseSearchEngineIndex { export interface EnterpriseSearchEngineFieldCapabilities { field_capabilities: FieldCapsResponse; - fields?: SchemaField[]; + fields: SchemaField[]; name: string; updated_at_millis: number; } -export interface EnterpriseSearchSchemaField { - field_name: string; - field_type: string[]; -} export interface EnterpriseSearchEngineUpsertResponse { result: string; @@ -51,7 +47,6 @@ export interface SchemaFieldIndex { } export interface SchemaField { - fields: SchemaField[]; indices: SchemaFieldIndex[]; name: string; type: string; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/update_connector_configuration_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/update_connector_configuration_api_logic.ts index 267bd325895b..72510ea9fc1d 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/update_connector_configuration_api_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/connector/update_connector_configuration_api_logic.ts @@ -12,7 +12,7 @@ import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_lo import { HttpLogic } from '../../../shared/http'; export interface PostConnectorConfigurationArgs { - configuration: Record; + configuration: Record; connectorId: string; indexName: string; } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts index 588f0a61c7fb..a3ad08c77584 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_overview_logic.test.ts @@ -8,7 +8,7 @@ import { LogicMounter } from '../../../__mocks__/kea_logic'; import { Status } from '../../../../../common/types/api'; -import { EnterpriseSearchEngineIndex } from '../../../../../common/types/engines'; +import { EnterpriseSearchEngineIndex, SchemaField } from '../../../../../common/types/engines'; import { EngineOverviewLogic, @@ -311,6 +311,50 @@ describe('EngineOverviewLogic', () => { }, indices: ['index-001', 'index-002'], }, + fields: [ + { + indices: [ + { + name: 'index-001', + type: 'integer', + }, + { + name: 'index-002', + type: 'integer', + }, + ], + name: 'age', + type: 'integer', + }, + { + indices: [ + { + name: 'index-001', + type: 'keyword', + }, + { + name: 'index-002', + type: 'keyword', + }, + ], + name: 'color', + type: 'keyword', + }, + { + indices: [ + { + name: 'index-001', + type: 'text', + }, + { + name: 'index-002', + type: 'text', + }, + ], + name: 'name', + type: 'text', + }, + ] as SchemaField[], name: 'engine-001', updated_at_millis: 2202018295, }; @@ -389,6 +433,120 @@ describe('EngineOverviewLogic', () => { }, indices: ['index-001', 'index-002'], }, + fields: [ + { + indices: [ + { + name: 'index-001', + type: 'integer', + }, + { + name: 'index-002', + type: 'integer', + }, + ], + name: '_doc_count', + type: 'integer', + }, + { + indices: [ + { + name: 'index-001', + type: '_id', + }, + { + name: 'index-002', + type: '_id', + }, + ], + name: '_id', + type: '_id', + }, + { + indices: [ + { + name: 'index-001', + type: '_index', + }, + { + name: 'index-002', + type: '_index', + }, + ], + name: '_index', + type: '_index', + }, + { + indices: [ + { + name: 'index-001', + type: '_source', + }, + { + name: 'index-002', + type: '_source', + }, + ], + name: '_source', + type: '_source', + }, + { + indices: [ + { + name: 'index-001', + type: '_version', + }, + { + name: 'index-002', + type: '_version', + }, + ], + name: '_version', + type: '_version', + }, + { + indices: [ + { + name: 'index-001', + type: 'integer', + }, + { + name: 'index-002', + type: 'integer', + }, + ], + name: 'age', + type: 'integer', + }, + { + indices: [ + { + name: 'index-001', + type: 'keyword', + }, + { + name: 'index-002', + type: 'keyword', + }, + ], + name: 'color', + type: 'keyword', + }, + { + indices: [ + { + name: 'index-001', + type: 'text', + }, + { + name: 'index-002', + type: 'text', + }, + ], + name: 'name', + type: 'text', + }, + ] as SchemaField[], name: 'foo-engine', updated_at_millis: 2202018295, }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx index b64c144f78dc..a5ec5aa7a055 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_schema.tsx @@ -5,43 +5,278 @@ * 2.0. */ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import { useActions, useValues } from 'kea'; -import { EuiBasicTable, EuiBasicTableColumn } from '@elastic/eui'; +import { + EuiBadge, + EuiBasicTable, + EuiBasicTableColumn, + EuiButtonEmpty, + EuiCallOut, + EuiFlexGroup, + EuiIcon, + EuiLink, + EuiPanel, + EuiText, +} from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; -import { EnterpriseSearchSchemaField } from '../../../../../common/types/engines'; -import { EngineViewTabs } from '../../routes'; +import { FieldIcon } from '@kbn/react-field'; + +import { SchemaField } from '../../../../../common/types/engines'; +import { docLinks } from '../../../shared/doc_links'; +import { generateEncodedPath } from '../../../shared/encode_path_params'; +import { KibanaLogic } from '../../../shared/kibana'; +import { EuiLinkTo } from '../../../shared/react_router_helpers'; +import { EngineViewTabs, SEARCH_INDEX_TAB_PATH } from '../../routes'; import { EnterpriseSearchEnginesPageTemplate } from '../layout/engines_page_template'; import { EngineIndicesLogic } from './engine_indices_logic'; import { EngineViewLogic } from './engine_view_logic'; +const SchemaFieldDetails: React.FC<{ schemaField: SchemaField }> = ({ schemaField }) => { + const { navigateToUrl } = useValues(KibanaLogic); + const notInAllIndices = schemaField.indices.some((i) => i.type === 'unmapped'); + + const columns: Array> = [ + { + field: 'name', + name: i18n.translate( + 'xpack.enterpriseSearch.content.engine.schema.fieldIndices.index.columnTitle', + { + defaultMessage: 'Parent index', + } + ), + render: (name: string) => ( + + {name} + + ), + }, + { + field: 'type', + name: i18n.translate( + 'xpack.enterpriseSearch.content.engine.schema.fieldIndices.type.columnTitle', + { + defaultMessage: 'Field mapped as', + } + ), + render: (name: string) => { + if (name === 'unmapped') + return ( + + + + ); + return name; + }, + }, + { + actions: [ + { + description: 'View index mappings', + icon: 'eye', + name: 'View index', + onClick: (item: SchemaField['indices'][0]) => { + navigateToUrl( + generateEncodedPath(SEARCH_INDEX_TAB_PATH, { + indexName: item.name, + tabId: 'index_mappings', + }) + ); + }, + type: 'icon', + }, + ], + }, + ]; + + return ( + + + {notInAllIndices && ( + + } + > + +

+ {' '} + + + +

+ + + )} + + + + ); +}; + export const EngineSchema: React.FC = () => { const { engineName } = useValues(EngineIndicesLogic); const { isLoadingEngineSchema, schemaFields } = useValues(EngineViewLogic); const { fetchEngineSchema } = useActions(EngineViewLogic); - const columns: Array> = [ + const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState>( + {} + ); + + useEffect(() => { + fetchEngineSchema({ engineName }); + }, [engineName]); + + const toggleDetails = (schemaField: SchemaField) => { + const newItemIdToExpandedRowMap = { ...itemIdToExpandedRowMap }; + if (itemIdToExpandedRowMap[schemaField.name]) { + delete newItemIdToExpandedRowMap[schemaField.name]; + } else { + newItemIdToExpandedRowMap[schemaField.name] = ( + + ); + } + + setItemIdToExpandedRowMap(newItemIdToExpandedRowMap); + }; + + const columns: Array> = [ + { + render: ({ type }: SchemaField) => { + if (type !== 'conflict') return null; + return ; + }, + width: '2%', + }, { - field: 'field_name', name: i18n.translate('xpack.enterpriseSearch.content.engine.schema.field_name.columnTitle', { defaultMessage: 'Field name', }), + render: ({ name, type }: SchemaField) => ( + + {name.includes('.') && } + +

{name}

+ + + ), + width: '43%', }, { - field: 'field_type', name: i18n.translate('xpack.enterpriseSearch.content.engine.schema.field_type.columnTitle', { - defaultMessage: 'Field Type', + defaultMessage: 'Field type', }), + render: ({ type }: SchemaField) => { + if (type === 'conflict') { + return ( + + + + + + ); + } + + return ( + + + +

{type}

+ + + ); + }, + width: '30%', + }, + { + name: i18n.translate( + 'xpack.enterpriseSearch.content.engine.schema.field_indices.columnTitle', + { + defaultMessage: 'In all indices?', + } + ), + render: ({ indices }: SchemaField) => { + const inAllIndices = indices.every((i) => i.type !== 'unmapped'); + return inAllIndices ? ( + +

+ +

+ + ) : ( + + + + ); + }, + width: '15%', + }, + { + render: (schemaField: SchemaField) => { + const { name, type, indices } = schemaField; + if (type === 'conflict' || indices.some((i) => i.type === 'unmapped')) { + const icon = itemIdToExpandedRowMap[name] ? 'arrowUp' : 'arrowDown'; + return ( + + { + toggleDetails(schemaField); + }} + > + + + + ); + } + return null; + }, + width: '10%', }, ]; - useEffect(() => { - fetchEngineSchema({ engineName }); - }, [engineName]); return ( { engineName={engineName} > <> - + ); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_logic.ts index 172eb16f5921..5b82dd3f7640 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/engine/engine_view_logic.ts @@ -8,7 +8,7 @@ import { kea, MakeLogicType } from 'kea'; import { Status } from '../../../../../common/types/api'; -import { EnterpriseSearchSchemaField } from '../../../../../common/types/engines'; +import { SchemaField } from '../../../../../common/types/engines'; import { KibanaLogic } from '../../../shared/kibana'; @@ -43,7 +43,7 @@ export interface EngineViewValues { isDeleteModalVisible: boolean; isLoadingEngine: boolean; isLoadingEngineSchema: boolean; - schemaFields: EnterpriseSearchSchemaField[]; + schemaFields: SchemaField[]; } export const EngineViewLogic = kea>({ @@ -103,15 +103,7 @@ export const EngineViewLogic = kea [selectors.engineSchemaData], - (data) => - Object.entries(data?.field_capabilities?.fields ?? {}) - .map(([name]) => - Object.entries({ - field_name: name, - field_type: [...Object.keys(data?.field_capabilities?.fields[name])], - }) - ) - .map((fields) => Object.fromEntries(fields)), + (data: EngineViewValues['engineSchemaData']) => data?.fields || [], ], }), }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts index 5abebb3c1bee..ecb134b48c37 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/connector/connector_configuration_logic.ts @@ -155,7 +155,10 @@ export const ConnectorConfigurationLogic = kea< ) .filter(isNotNullish) .reduce( - (prev: Record, { key, value }) => ({ ...prev, [key]: value }), + (prev: Record, { key, value }) => ({ + ...prev, + [key]: value, + }), {} ), connectorId: values.index.connector.id, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx index 7d0622e14b94..f478a3347b0a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.test.tsx @@ -319,7 +319,7 @@ describe('AddInferencePipelineFlyout', () => { ); expect(wrapper.find(EuiButtonEmpty)).toHaveLength(2); - const cancelBtn = wrapper.find(EuiButtonEmpty).at(1); + const cancelBtn = wrapper.find(EuiButtonEmpty).at(0); expect(cancelBtn.prop('children')).toBe('Cancel'); cancelBtn.prop('onClick')!({} as any); expect(onClose).toHaveBeenCalledTimes(1); @@ -336,7 +336,7 @@ describe('AddInferencePipelineFlyout', () => { ); expect(wrapper.find(EuiButtonEmpty)).toHaveLength(2); - const cancelBtn = wrapper.find(EuiButtonEmpty).at(1); + const cancelBtn = wrapper.find(EuiButtonEmpty).at(0); expect(cancelBtn.prop('children')).toBe('Cancel'); cancelBtn.prop('onClick')!({} as any); expect(onClose).toHaveBeenCalledTimes(1); @@ -353,7 +353,7 @@ describe('AddInferencePipelineFlyout', () => { ); expect(wrapper.find(EuiButtonEmpty)).toHaveLength(2); - const cancelBtn = wrapper.find(EuiButtonEmpty).at(1); + const cancelBtn = wrapper.find(EuiButtonEmpty).at(0); expect(cancelBtn.prop('children')).toBe('Cancel'); cancelBtn.prop('onClick')!({} as any); expect(onClose).toHaveBeenCalledTimes(1); @@ -370,7 +370,7 @@ describe('AddInferencePipelineFlyout', () => { ); expect(wrapper.find(EuiButtonEmpty)).toHaveLength(2); - const backBtn = wrapper.find(EuiButtonEmpty).at(0); + const backBtn = wrapper.find(EuiButtonEmpty).at(1); expect(backBtn.prop('children')).toBe('Back'); backBtn.prop('onClick')!({} as any); expect(actions.setAddInferencePipelineStep).toHaveBeenCalledWith( @@ -389,7 +389,7 @@ describe('AddInferencePipelineFlyout', () => { ); expect(wrapper.find(EuiButtonEmpty)).toHaveLength(2); - const backBtn = wrapper.find(EuiButtonEmpty).at(0); + const backBtn = wrapper.find(EuiButtonEmpty).at(1); expect(backBtn.prop('children')).toBe('Back'); backBtn.prop('onClick')!({} as any); expect(actions.setAddInferencePipelineStep).toHaveBeenCalledWith( @@ -408,7 +408,7 @@ describe('AddInferencePipelineFlyout', () => { ); expect(wrapper.find(EuiButtonEmpty)).toHaveLength(2); - const backBtn = wrapper.find(EuiButtonEmpty).at(0); + const backBtn = wrapper.find(EuiButtonEmpty).at(1); expect(backBtn.prop('children')).toBe('Back'); backBtn.prop('onClick')!({} as any); expect(actions.setAddInferencePipelineStep).toHaveBeenCalledWith( diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.tsx index 4a8aa61a2197..05e9b886487e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/add_inference_pipeline_flyout.tsx @@ -127,6 +127,7 @@ export const AddInferencePipelineContent = ({ onClose }: AddInferencePipelineFly )} + {step === AddInferencePipelineSteps.Configuration && } {step === AddInferencePipelineSteps.Fields && } {step === AddInferencePipelineSteps.Test && } @@ -255,6 +256,15 @@ export const AddInferencePipelineFooter: React.FC< } return ( + + + {CANCEL_BUTTON_LABEL} + + + {previousStep !== undefined ? ( ) : null} - - - - {CANCEL_BUTTON_LABEL} - - {nextStep !== undefined ? ( { return ( <> - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.title', - { defaultMessage: 'Select field mappings' } - )} -

- - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.description', - { - defaultMessage: - 'Choose fields to be enhanced from your existing documents or manually enter in fields you anticipate using.', - } - )} -

- - - - - - + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.title', + { defaultMessage: 'Select field mappings' } + )} +

+ + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.description', { - defaultMessage: 'Source field', + defaultMessage: + 'Choose fields to be enhanced from your existing documents or manually enter in fields you anticipate using.', } )} - error={emptySourceFields && } - isInvalid={emptySourceFields} - > - + + + + + + + + + ({ - text: field, - value: field, - })) ?? []), - ]} - onChange={(e) => - setInferencePipelineConfiguration({ - ...configuration, - sourceField: e.target.value, - }) - } - /> - - - - - ) - } - error={formErrors.destinationField} - isInvalid={formErrors.destinationField !== undefined} - fullWidth - > - - setInferencePipelineConfiguration({ - ...configuration, - destinationField: e.target.value, - }) + defaultMessage: 'Source field', + } + )} + error={emptySourceFields && } + isInvalid={emptySourceFields} + > + ({ + text: field, + value: field, + })) ?? []), + ]} + onChange={(e) => + setInferencePipelineConfiguration({ + ...configuration, + sourceField: e.target.value, + }) + } + /> + + + + + ) } + error={formErrors.destinationField} + isInvalid={formErrors.destinationField !== undefined} fullWidth - /> - - - - - + > + + setInferencePipelineConfiguration({ + ...configuration, + destinationField: e.target.value, + }) + } + fullWidth + /> + + + + + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx index 75692ab749db..86581f5f2436 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/configure_pipeline.tsx @@ -22,6 +22,8 @@ import { EuiSpacer, EuiTitle, EuiText, + EuiPanel, + EuiHorizontalRule, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -45,11 +47,11 @@ const CHOOSE_EXISTING_LABEL = i18n.translate( ); const CHOOSE_NEW_LABEL = i18n.translate( 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.newLabel', - { defaultMessage: 'New' } + { defaultMessage: 'New pipeline' } ); const CHOOSE_PIPELINE_LABEL = i18n.translate( 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.existingPipeline.existingLabel', - { defaultMessage: 'Existing' } + { defaultMessage: 'Existing pipeline' } ); export const ConfigurePipeline: React.FC = () => { @@ -102,177 +104,220 @@ export const ConfigurePipeline: React.FC = () => { return ( <> - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title', - { defaultMessage: 'Add a new pipeline' } - )} -

- - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description', - { - defaultMessage: - "Once created, this pipeline will be added as a processor on your Enterprise Search Ingestion Pipeline. You'll also be able to use this pipeline elsewhere in your Elastic deployment.", - } - )} -

- - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.docsLink', - { - defaultMessage: 'Learn more about using ML models in Enterprise Search', - } - )} - - - - - - - + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.title', + { defaultMessage: 'Create or select a pipeline' } )} - > - - setInferencePipelineConfiguration({ - ...EMPTY_PIPELINE_CONFIGURATION, - existingPipeline: e.target.value === 'true', - }) +

+ + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.description', + { + defaultMessage: + 'Build or reuse a child pipeline that will be used as a processor in your main pipeline.', } - value={configuration.existingPipeline?.toString() ?? ''} - /> - - - - {configuration.existingPipeline === true ? ( + )} +

+

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionUsePipelines', + { + defaultMessage: + 'Pipelines you create will be saved to be used elsewhere in your Elastic deployment.', + } + )} +

+ + + + + - 0 ? pipelineName : PIPELINE_SELECT_PLACEHOLDER_VALUE + options={[ + { + disabled: true, + text: CHOOSE_EXISTING_LABEL, + value: '', + }, + { + text: CHOOSE_NEW_LABEL, + value: 'false', + }, + { + disabled: + !existingInferencePipelines || existingInferencePipelines.length === 0, + text: CHOOSE_PIPELINE_LABEL, + value: 'true', + }, + ]} + onChange={(e) => + setInferencePipelineConfiguration({ + ...EMPTY_PIPELINE_CONFIGURATION, + existingPipeline: e.target.value === 'true', + }) } - options={pipelineOptions} - onChange={(value) => selectExistingPipeline(value)} + value={configuration.existingPipeline?.toString() ?? ''} /> - ) : ( + {configuration.existingPipeline === true ? ( + + 0 ? pipelineName : PIPELINE_SELECT_PLACEHOLDER_VALUE + } + options={pipelineOptions} + onChange={(value) => selectExistingPipeline(value)} + /> + + ) : ( + + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText', + { + defaultMessage: + 'Pipeline names are unique within a deployment and can only contain letters, numbers, underscores, and hyphens. This will create a pipeline named {pipelineName}.', + values: { + pipelineName: `ml-inference-${ + pipelineName.length > 0 ? pipelineName : '' + }`, + }, + } + )} + + ) + } + error={nameError && formErrors.pipelineName} + isInvalid={nameError} + > + + setInferencePipelineConfiguration({ + ...configuration, + pipelineName: e.target.value, + }) + } + /> + + )} + + + + + + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.titleSelectTrainedModel', + { defaultMessage: 'Select a trained ML Model' } + )} +

+ + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.descriptionDeployTrainedModel', + { + defaultMessage: + 'To perform natural language processing tasks in your cluster, you must deploy an appropriate trained model.', + } + )} +

+ + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.docsLink', + { + defaultMessage: + 'Learn more about importing and using ML models in Enterprise Search', + } + )} + + + + + + - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.configure.name.helpText', - { - defaultMessage: - 'Pipeline names are unique within a deployment and can only contain letters, numbers, underscores, and hyphens. This will create a pipeline named {pipelineName}.', - values: { - pipelineName: `ml-inference-${ - pipelineName.length > 0 ? pipelineName : '' - }`, - }, - } - )} - - ) - } - error={nameError && formErrors.pipelineName} - isInvalid={nameError} + fullWidth > - + hasDividers + disabled={inputsDisabled} + itemLayoutAlign="top" + onChange={(value) => setInferencePipelineConfiguration({ ...configuration, - pipelineName: e.target.value, + inferenceConfig: undefined, + modelID: value, }) } + options={modelOptions} + valueOfSelected={modelID === '' ? MODEL_SELECT_PLACEHOLDER_VALUE : modelID} /> - )} - - - - - - setInferencePipelineConfiguration({ - ...configuration, - inferenceConfig: undefined, - modelID: value, - }) - } - options={modelOptions} - valueOfSelected={modelID === '' ? MODEL_SELECT_PLACEHOLDER_VALUE : modelID} - /> - - - + + + + + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/review_pipeline.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/review_pipeline.tsx index 080a277fe326..4b3dc62a5135 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/review_pipeline.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/review_pipeline.tsx @@ -25,39 +25,42 @@ import { MLInferenceLogic } from './ml_inference_logic'; export const ReviewPipeline: React.FC = () => { const { mlInferencePipeline } = useValues(MLInferenceLogic); return ( - - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.title', - { - defaultMessage: 'Pipeline configuration', - } - )} -

- - - - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.description', - { - defaultMessage: - "This pipeline will be created and injected as a processor into your default pipeline for this index. You'll be able to use this new pipeline independently as well.", - } - )} -

- - - - - - {JSON.stringify(mlInferencePipeline ?? {}, null, 2)} - - - + <> + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.title', + { + defaultMessage: 'Review your pipeline configuration', + } + )} +

+ + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.review.description', + { + defaultMessage: + "This pipeline will be created and injected as a processor into your default pipeline for this index. You'll be able to use this new pipeline independently as well.", + } + )} +

+ + + + + + + + {JSON.stringify(mlInferencePipeline ?? {}, null, 2)} + + + + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx index 130ca9fe1b81..effbe28a86a0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/ml_inference/test_pipeline.tsx @@ -11,7 +11,6 @@ import { useValues, useActions } from 'kea'; import { EuiButton, - EuiCode, EuiCodeBlock, EuiFieldText, EuiFlexGroup, @@ -22,10 +21,10 @@ import { EuiTitle, EuiText, useIsWithinMaxBreakpoint, + EuiPanel, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; import { CodeEditor } from '@kbn/kibana-react-plugin/public'; import { TestPipelineLogic } from './test_pipeline_logic'; @@ -52,158 +51,188 @@ export const TestPipeline: React.FC = () => { const inputRef = useRef(); return ( - - - -

- {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.title', - { defaultMessage: 'Review pipeline results' } - )} -

- - - - - -

- + <> + + + +

{i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.optionalCallout', - { defaultMessage: 'This step is optional.' } + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.title', + { defaultMessage: 'Test your pipeline results' } )} - -   - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.description', - { - defaultMessage: - 'You can simulate your pipeline results by passing an array of documents.', - } - )} -
- {`[{"_index":"index","_id":"id","_source":{"${sourceField}":"bar"}}]`} - ), - }} - /> -

- - - - - - { - inputRef.current = ref; - }} - isInvalid={showGetDocumentErrors} - isLoading={isGetDocumentsLoading} - onKeyDown={(e) => { - if (e.key === 'Enter' && inputRef.current?.value.trim().length !== 0) { - makeGetDocumentRequest({ - documentId: inputRef.current?.value.trim() ?? '', - indexName, - }); - } - }} - /> - - - - - - - -
+
+ + + + + +

+ {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.subtitle.documents', - { defaultMessage: 'Documents' } + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.optionalCallout', + { defaultMessage: 'This is an optional step.' } )} -

- + +   + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.description', + { + defaultMessage: + 'Use this tool to run a simulation of your pipeline in order to confirm that it produces your anticipated results.', + } + )} +

+ + + + + + + + + + + { + inputRef.current = ref; + }} + isInvalid={showGetDocumentErrors} + isLoading={isGetDocumentsLoading} + onKeyDown={(e) => { + if (e.key === 'Enter' && inputRef.current?.value.trim().length !== 0) { + makeGetDocumentRequest({ + documentId: inputRef.current?.value.trim() ?? '', + indexName, + }); + } + }} + /> + + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.or', + { defaultMessage: 'Or' } + )} +

+ + + + +

+ + {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.useJsonFormat', + { + defaultMessage: 'Use this JSON format to add your own array of documents', + } + )} + +

+ + + {`[{"_index":"index","_id":"id","_source":{"${sourceField}":"bar"}}]`} + + + + + + + + + +
+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.subtitle.documents', + { defaultMessage: 'Raw document' } + )} +
+ + + + +
+ {i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.subtitle.result', + { defaultMessage: 'Result' } + )} +
+ + + - -
+ + {(EuiResizablePanel, EuiResizableButton) => ( + <> + + { + setPipelineSimulateBody(value); + }} + /> + + + + + + + {simulatePipelineErrors.length > 0 + ? JSON.stringify(simulatePipelineErrors, null, 2) + : simulatePipelineResult + ? JSON.stringify(simulatePipelineResult, null, 2) + : '{}'} + + + + )} + + + + +
+ {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.subtitle.result', - { defaultMessage: 'Result' } + 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.runButton', + { defaultMessage: 'Simulate Pipeline' } )} -
- + + - - - - {(EuiResizablePanel, EuiResizableButton) => ( - <> - - { - setPipelineSimulateBody(value); - }} - /> - - - - - - - {simulatePipelineErrors.length > 0 - ? JSON.stringify(simulatePipelineErrors, null, 2) - : JSON.stringify(simulatePipelineResult || '', null, 2)} - - - - )} - - - - -
- - {i18n.translate( - 'xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.runButton', - { defaultMessage: 'Simulate Pipeline' } - )} - -
- - + + ); }; diff --git a/x-pack/plugins/enterprise_search/server/integrations.ts b/x-pack/plugins/enterprise_search/server/integrations.ts index 94b12b35d412..1ae3de167d3f 100644 --- a/x-pack/plugins/enterprise_search/server/integrations.ts +++ b/x-pack/plugins/enterprise_search/server/integrations.ts @@ -10,6 +10,8 @@ import type { IntegrationCategory } from '@kbn/custom-integrations-plugin/common import type { CustomIntegrationsPluginSetup } from '@kbn/custom-integrations-plugin/server'; import { i18n } from '@kbn/i18n'; +import { ConfigType } from '.'; + interface WorkplaceSearchIntegration { id: string; title: string; @@ -297,65 +299,70 @@ const workplaceSearchIntegrations: WorkplaceSearchIntegration[] = [ ]; export const registerEnterpriseSearchIntegrations = ( + config: ConfigType, http: HttpServiceSetup, customIntegrations: CustomIntegrationsPluginSetup ) => { - workplaceSearchIntegrations.forEach((integration) => { + if (config.canDeployEntSearch) { + workplaceSearchIntegrations.forEach((integration) => { + customIntegrations.registerCustomIntegration({ + uiInternalPath: `/app/enterprise_search/workplace_search/sources/add/${integration.id}`, + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + `/plugins/enterpriseSearch/assets/source_icons/${integration.id}.svg` + ), + }, + ], + isBeta: false, + shipper: 'enterprise_search', + ...integration, + }); + }); + customIntegrations.registerCustomIntegration({ - uiInternalPath: `/app/enterprise_search/workplace_search/sources/add/${integration.id}`, + id: 'app_search_json', + title: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonName', { + defaultMessage: 'JSON', + }), + description: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonDescription', { + defaultMessage: 'Search over your JSON data with App Search.', + }), + categories: ['upload_file'], + uiInternalPath: '/app/enterprise_search/app_search/engines/new?method=json', icons: [ { - type: 'svg', - src: http.basePath.prepend( - `/plugins/enterpriseSearch/assets/source_icons/${integration.id}.svg` - ), + type: 'eui', + src: 'logoAppSearch', }, ], - isBeta: false, shipper: 'enterprise_search', - ...integration, + isBeta: false, }); - }); + } - customIntegrations.registerCustomIntegration({ - id: 'app_search_json', - title: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonName', { - defaultMessage: 'JSON', - }), - description: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonDescription', { - defaultMessage: 'Search over your JSON data with App Search.', - }), - categories: ['upload_file'], - uiInternalPath: '/app/enterprise_search/app_search/engines/new?method=json', - icons: [ - { - type: 'eui', - src: 'logoAppSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); - - customIntegrations.registerCustomIntegration({ - id: 'web_crawler', - title: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerName', { - defaultMessage: 'Web crawler', - }), - description: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerDescription', { - defaultMessage: 'Add search to your website with the Enterprise Search web crawler.', - }), - categories: ['enterprise_search', 'website_search', 'web', 'elastic_stack'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=crawler', - icons: [ - { - type: 'eui', - src: 'logoEnterpriseSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + if (config.hasWebCrawler) { + customIntegrations.registerCustomIntegration({ + id: 'web_crawler', + title: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerName', { + defaultMessage: 'Web crawler', + }), + description: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerDescription', { + defaultMessage: 'Add search to your website with the Enterprise Search web crawler.', + }), + categories: ['enterprise_search', 'website_search', 'web', 'elastic_stack'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=crawler', + icons: [ + { + type: 'eui', + src: 'logoEnterpriseSearch', + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); + } customIntegrations.registerCustomIntegration({ id: 'api', @@ -377,265 +384,282 @@ export const registerEnterpriseSearchIntegrations = ( isBeta: false, }); - customIntegrations.registerCustomIntegration({ - id: 'build_a_connector', - title: i18n.translate('xpack.enterpriseSearch.integrations.buildAConnectorName', { - defaultMessage: 'Build a connector', - }), - description: i18n.translate('xpack.enterpriseSearch.integrations.buildAConnectorDescription', { - defaultMessage: 'Search over data stored on custom data sources with Enterprise Search.', - }), - categories: ['enterprise_search', 'custom', 'elastic_stack'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'eui', - src: 'logoEnterpriseSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + if (config.hasNativeConnectors) { + customIntegrations.registerCustomIntegration({ + id: 'native_connector', + title: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorName', + { + defaultMessage: 'Use a connector', + } + ), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorDescription', + { + defaultMessage: + 'Search over your data sources with a native Enterprise Search connector.', + } + ), + categories: ['elastic_stack', 'enterprise_search'], + uiInternalPath: + '/app/enterprise_search/content/search_indices/new_index?method=native_connector', + icons: [ + { + type: 'eui', + src: 'logoEnterpriseSearch', + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'native_connector', - title: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorName', - { - defaultMessage: 'Use a connector', - } - ), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorDescription', - { - defaultMessage: 'Search over your data sources with a native Enterprise Search connector.', - } - ), - categories: ['elastic_stack', 'enterprise_search'], - uiInternalPath: - '/app/enterprise_search/content/search_indices/new_index?method=native_connector', - icons: [ - { - type: 'eui', - src: 'logoEnterpriseSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'mongodb', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBName', { + defaultMessage: 'MongoDB', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBDescription', + { + defaultMessage: 'Search over your MongoDB content with Enterprise Search.', + } + ), + categories: ['datastore', 'enterprise_search'], + uiInternalPath: + '/app/enterprise_search/content/search_indices/new_index?method=native_connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mongodb.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'mongodb', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBName', { - defaultMessage: 'MongoDB', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBDescription', - { - defaultMessage: 'Search over your MongoDB content with Enterprise Search.', - } - ), - categories: ['datastore', 'enterprise_search'], - uiInternalPath: - '/app/enterprise_search/content/search_indices/new_index?method=native_connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mongodb.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'mysql', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mysqlName', { + defaultMessage: 'MySQL', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.mysqlDescription', + { + defaultMessage: 'Search over your MySQL content with Enterprise Search.', + } + ), + categories: ['datastore', 'enterprise_search'], + uiInternalPath: + '/app/enterprise_search/content/search_indices/new_index?method=native_connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mysql.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); + } - customIntegrations.registerCustomIntegration({ - id: 'mysql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mysqlName', { - defaultMessage: 'MySQL', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.mysqlDescription', - { - defaultMessage: 'Search over your MySQL content with Enterprise Search.', - } - ), - categories: ['datastore', 'enterprise_search'], - uiInternalPath: - '/app/enterprise_search/content/search_indices/new_index?method=native_connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mysql.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + if (config.hasConnectors) { + customIntegrations.registerCustomIntegration({ + id: 'build_a_connector', + title: i18n.translate('xpack.enterpriseSearch.integrations.buildAConnectorName', { + defaultMessage: 'Build a connector', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.integrations.buildAConnectorDescription', + { + defaultMessage: 'Search over data stored on custom data sources with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'custom', 'elastic_stack'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'eui', + src: 'logoEnterpriseSearch', + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'postgresql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.postgresqlName', { - defaultMessage: 'PostgreSQL', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.postgreSQLDescription', - { - defaultMessage: 'Search over your content on PostgreSQL with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/postgresql.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'postgresql', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.postgresqlName', { + defaultMessage: 'PostgreSQL', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.postgreSQLDescription', + { + defaultMessage: 'Search over your content on PostgreSQL with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/postgresql.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'oracle', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.oracleName', { - defaultMessage: 'Oracle', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.oracleDescription', - { - defaultMessage: 'Search over your content on Oracle with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/oracle.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'oracle', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.oracleName', { + defaultMessage: 'Oracle', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.oracleDescription', + { + defaultMessage: 'Search over your content on Oracle with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/oracle.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'ms_sql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.msSqlName', { - defaultMessage: 'Microsoft SQL', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.msSqlDescription', - { - defaultMessage: 'Search over your content on Microsoft SQL Server with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'custom', 'elastic_stack', 'datastore'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend( - '/plugins/enterpriseSearch/assets/source_icons/microsoft_sql.svg' - ), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'ms_sql', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.msSqlName', { + defaultMessage: 'Microsoft SQL', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.msSqlDescription', + { + defaultMessage: + 'Search over your content on Microsoft SQL Server with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'custom', 'elastic_stack', 'datastore'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/microsoft_sql.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'ms_sql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.networkDriveName', { - defaultMessage: 'Network Drive', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.netowkrDriveDescription', - { - defaultMessage: 'Search over your Network Drive content with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'custom', 'elastic_stack', 'file_storage'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend( - '/plugins/enterpriseSearch/assets/source_icons/network_drive.svg' - ), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'ms_sql', + title: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.networkDriveName', + { + defaultMessage: 'Network Drive', + } + ), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.netowkrDriveDescription', + { + defaultMessage: 'Search over your Network Drive content with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'custom', 'elastic_stack', 'file_storage'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/network_drive.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'amazon_s3', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.s3', { - defaultMessage: 'Amazon S3', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.s3Description', - { - defaultMessage: 'Search over your content on Amazon S3 with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'datastore', 'elastic_stack'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/amazon_s3.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'amazon_s3', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.s3', { + defaultMessage: 'Amazon S3', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.s3Description', + { + defaultMessage: 'Search over your content on Amazon S3 with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'datastore', 'elastic_stack'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/amazon_s3.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'google_cloud_storage', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.googleCloud', { - defaultMessage: 'Google Cloud Storage', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.googleCloudDescription', - { - defaultMessage: 'Search over your content on Google Cloud Storage with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend( - '/plugins/enterpriseSearch/assets/source_icons/google_cloud.svg' - ), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'google_cloud_storage', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.googleCloud', { + defaultMessage: 'Google Cloud Storage', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.googleCloudDescription', + { + defaultMessage: + 'Search over your content on Google Cloud Storage with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/google_cloud.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'azure_blob_storage', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.azureBlob', { - defaultMessage: 'Azure Blob Storage', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.azureBlobDescription', - { - defaultMessage: 'Search over your content on Azure Blob Storage with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/azure_blob.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'azure_blob_storage', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.azureBlob', { + defaultMessage: 'Azure Blob Storage', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.azureBlobDescription', + { + defaultMessage: 'Search over your content on Azure Blob Storage with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/azure_blob.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); + } }; diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/update_connector_configuration.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/update_connector_configuration.ts index c19fcb2f0129..cd36ccc7786c 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/update_connector_configuration.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/update_connector_configuration.ts @@ -23,7 +23,7 @@ import { fetchConnectorById } from './fetch_connectors'; export const updateConnectorConfiguration = async ( client: IScopedClusterClient, connectorId: string, - configuration: Record + configuration: Record ) => { const connectorResult = await fetchConnectorById(client, connectorId); const connector = connectorResult?.value; diff --git a/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.test.ts b/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.test.ts index ba3bfe3b96a2..e0c82b00ecf6 100644 --- a/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.test.ts @@ -8,7 +8,7 @@ import { FieldCapsResponse } from '@elastic/elasticsearch/lib/api/types'; import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; -import { EnterpriseSearchEngineDetails, SchemaField } from '../../../common/types/engines'; +import { EnterpriseSearchEngine, SchemaField } from '../../../common/types/engines'; import { fetchEngineFieldCapabilities, parseFieldsCapabilities } from './field_capabilities'; @@ -19,8 +19,8 @@ describe('engines field_capabilities', () => { }, asInternalUser: {}, }; - const mockEngine: EnterpriseSearchEngineDetails = { - indices: [], + const mockEngine: EnterpriseSearchEngine = { + indices: ['index-001'], name: 'unit-test-engine', updated_at_millis: 2202018295, }; @@ -51,7 +51,6 @@ describe('engines field_capabilities', () => { field_capabilities: fieldCapsResponse, fields: [ { - fields: [], indices: [ { name: 'index-001', @@ -70,7 +69,7 @@ describe('engines field_capabilities', () => { expect(mockClient.asCurrentUser.fieldCaps).toHaveBeenCalledWith({ fields: '*', include_unmapped: true, - index: 'search-engine-unit-test-engine', + index: ['index-001'], }); }); }); @@ -100,7 +99,6 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [], indices: [ { name: 'index-001', @@ -111,7 +109,6 @@ describe('engines field_capabilities', () => { type: 'text', }, { - fields: [], indices: [ { name: 'index-001', @@ -148,19 +145,6 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ - { - fields: [], - indices: [ - { - name: 'index-001', - type: 'keyword', - }, - ], - name: 'keyword', - type: 'keyword', - }, - ], indices: [ { name: 'index-001', @@ -170,6 +154,16 @@ describe('engines field_capabilities', () => { name: 'body', type: 'text', }, + { + indices: [ + { + name: 'index-001', + type: 'keyword', + }, + ], + name: 'body.keyword', + type: 'keyword', + }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); }); @@ -205,38 +199,34 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ + indices: [ { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - ], - name: 'first', - type: 'text', + name: 'index-001', + type: 'object', }, + ], + name: 'name', + type: 'object', + }, + { + indices: [ { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - ], - name: 'last', + name: 'index-001', type: 'text', }, ], + name: 'name.first', + type: 'text', + }, + { indices: [ { name: 'index-001', - type: 'object', + type: 'text', }, ], - name: 'name', - type: 'object', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -273,38 +263,34 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ + indices: [ { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - ], - name: 'first', - type: 'text', + name: 'index-001', + type: 'nested', }, + ], + name: 'name', + type: 'nested', + }, + { + indices: [ { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - ], - name: 'last', + name: 'index-001', type: 'text', }, ], + name: 'name.first', + type: 'text', + }, + { indices: [ { name: 'index-001', - type: 'nested', + type: 'text', }, ], - name: 'name', - type: 'nested', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -333,7 +319,6 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [], indices: [ { name: 'index-001', @@ -406,50 +391,46 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ + indices: [ { - fields: [], - indices: [ - { - name: 'index-002', - type: 'text', - }, - { - name: 'index-001', - type: 'unmapped', - }, - ], - name: 'first', + name: 'index-001', type: 'text', }, { - fields: [], - indices: [ - { - name: 'index-002', - type: 'text', - }, - { - name: 'index-001', - type: 'unmapped', - }, - ], - name: 'last', - type: 'text', + name: 'index-002', + type: 'object', }, ], + name: 'name', + type: 'conflict', + }, + { indices: [ + { + name: 'index-001', + type: 'unmapped', + }, { name: 'index-002', - type: 'object', + type: 'text', }, + ], + name: 'name.first', + type: 'text', + }, + { + indices: [ { name: 'index-001', + type: 'unmapped', + }, + { + name: 'index-002', type: 'text', }, ], - name: 'name', - type: 'conflict', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -518,63 +499,63 @@ describe('engines field_capabilities', () => { const expectedFields: SchemaField[] = [ { - fields: [ + indices: [ { - fields: [], - indices: [ - { - name: 'index-002', - type: 'text', - }, - { - name: 'index-003', - type: 'text', - }, - { - name: 'index-001', - type: 'unmapped', - }, - ], - name: 'first', + name: 'index-001', type: 'text', }, { - fields: [], - indices: [ - { - name: 'index-002', - type: 'text', - }, - { - name: 'index-001', - type: 'unmapped', - }, - ], - name: 'last', - type: 'text', + name: 'index-002', + type: 'object', }, - ], - indices: [ { name: 'index-003', type: 'keyword', }, + ], + name: 'name', + type: 'conflict', + }, + { + indices: [ + { + name: 'index-001', + type: 'unmapped', + }, { name: 'index-002', - type: 'object', + type: 'text', + }, + { + name: 'index-003', + type: 'text', }, + ], + name: 'name.first', + type: 'text', + }, + { + indices: [ { name: 'index-001', + type: 'unmapped', + }, + { + name: 'index-002', type: 'text', }, + { + name: 'index-003', + type: 'unmapped', + }, ], - name: 'name', - type: 'conflict', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); }); - it('handles conflicts & unmapped fields together', () => { + it('handles conflicts & unmapped fields together', () => { const fieldCapabilities: FieldCapsResponse = { fields: { body: { @@ -653,12 +634,7 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [], indices: [ - { - name: 'index-003', - type: 'text', - }, { name: 'index-001', type: 'unmapped', @@ -667,58 +643,58 @@ describe('engines field_capabilities', () => { name: 'index-002', type: 'unmapped', }, + { + name: 'index-003', + type: 'text', + }, ], name: 'body', type: 'text', }, { - fields: [ + indices: [ { - fields: [], - indices: [ - { - name: 'index-002', - type: 'text', - }, - { - name: 'index-001', - type: 'unmapped', - }, - { - name: 'index-003', - type: 'unmapped', - }, - ], - name: 'first', + name: 'index-001', type: 'text', }, { - fields: [], - indices: [ - { - name: 'index-002', - type: 'text', - }, - { - name: 'index-001', - type: 'unmapped', - }, - { - name: 'index-003', - type: 'unmapped', - }, - ], - name: 'last', - type: 'text', + name: 'index-002', + type: 'object', + }, + { + name: 'index-003', + type: 'unmapped', }, ], + name: 'name', + type: 'conflict', + }, + { indices: [ + { + name: 'index-001', + type: 'unmapped', + }, { name: 'index-002', - type: 'object', + type: 'text', }, + { + name: 'index-003', + type: 'unmapped', + }, + ], + name: 'name.first', + type: 'text', + }, + { + indices: [ { name: 'index-001', + type: 'unmapped', + }, + { + name: 'index-002', type: 'text', }, { @@ -726,8 +702,8 @@ describe('engines field_capabilities', () => { type: 'unmapped', }, ], - name: 'name', - type: 'conflict', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -772,50 +748,46 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ + indices: [ + { + name: 'index-001', + type: 'object', + }, + { + name: 'index-002', + type: 'object', + }, + ], + name: 'name', + type: 'object', + }, + { + indices: [ { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - { - name: 'index-002', - type: 'text', - }, - ], - name: 'first', + name: 'index-001', type: 'text', }, { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - { - name: 'index-002', - type: 'unmapped', - }, - ], - name: 'last', + name: 'index-002', type: 'text', }, ], + name: 'name.first', + type: 'text', + }, + { indices: [ { name: 'index-001', - type: 'object', + type: 'text', }, { name: 'index-002', - type: 'object', + type: 'unmapped', }, ], - name: 'name', - type: 'object', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -860,50 +832,46 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ + indices: [ + { + name: 'index-001', + type: 'nested', + }, { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - { - name: 'index-002', - type: 'text', - }, - ], - name: 'first', + name: 'index-002', + type: 'nested', + }, + ], + name: 'name', + type: 'nested', + }, + { + indices: [ + { + name: 'index-001', type: 'text', }, { - fields: [], - indices: [ - { - name: 'index-001', - type: 'text', - }, - { - name: 'index-002', - type: 'unmapped', - }, - ], - name: 'last', + name: 'index-002', type: 'text', }, ], + name: 'name.first', + type: 'text', + }, + { indices: [ { name: 'index-001', - type: 'nested', + type: 'text', }, { name: 'index-002', - type: 'nested', + type: 'unmapped', }, ], - name: 'name', - type: 'nested', + name: 'name.last', + type: 'text', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -940,23 +908,6 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ - { - fields: [], - indices: [ - { - name: 'index-001', - type: 'keyword', - }, - { - name: 'index-002', - type: 'unmapped', - }, - ], - name: 'keyword', - type: 'keyword', - }, - ], indices: [ { name: 'index-001', @@ -970,6 +921,20 @@ describe('engines field_capabilities', () => { name: 'body', type: 'text', }, + { + indices: [ + { + name: 'index-001', + type: 'keyword', + }, + { + name: 'index-002', + type: 'unmapped', + }, + ], + name: 'body.keyword', + type: 'keyword', + }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); }); @@ -1005,23 +970,6 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ - { - fields: [], - indices: [ - { - name: 'index-002', - type: 'number', - }, - { - name: 'index-001', - type: 'text', - }, - ], - name: 'id', - type: 'conflict', - }, - ], indices: [ { name: 'index-001', @@ -1033,7 +981,21 @@ describe('engines field_capabilities', () => { }, ], name: 'order', - type: 'object', // Should this be 'conflict' too? + type: 'object', + }, + { + indices: [ + { + name: 'index-001', + type: 'text', + }, + { + name: 'index-002', + type: 'number', + }, + ], + name: 'order.id', + type: 'conflict', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); @@ -1070,23 +1032,6 @@ describe('engines field_capabilities', () => { }; const expectedFields: SchemaField[] = [ { - fields: [ - { - fields: [], - indices: [ - { - name: 'index-002', - type: 'number', - }, - { - name: 'index-001', - type: 'text', - }, - ], - name: 'id', - type: 'conflict', - }, - ], indices: [ { name: 'index-001', @@ -1098,7 +1043,21 @@ describe('engines field_capabilities', () => { }, ], name: 'order', - type: 'nested', // Should this be 'conflict' too? + type: 'nested', + }, + { + indices: [ + { + name: 'index-001', + type: 'text', + }, + { + name: 'index-002', + type: 'number', + }, + ], + name: 'order.id', + type: 'conflict', }, ]; expect(parseFieldsCapabilities(fieldCapabilities)).toEqual(expectedFields); diff --git a/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.ts b/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.ts index a9d1025123b5..9d7082ffbbd8 100644 --- a/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.ts +++ b/x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.ts @@ -5,24 +5,24 @@ * 2.0. */ -import { FieldCapsResponse, FieldCapsFieldCapability } from '@elastic/elasticsearch/lib/api/types'; +import { FieldCapsResponse } from '@elastic/elasticsearch/lib/api/types'; import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; import { - EnterpriseSearchEngineDetails, + EnterpriseSearchEngine, EnterpriseSearchEngineFieldCapabilities, SchemaField, } from '../../../common/types/engines'; export const fetchEngineFieldCapabilities = async ( client: IScopedClusterClient, - engine: EnterpriseSearchEngineDetails + engine: EnterpriseSearchEngine ): Promise => { - const { name, updated_at_millis } = engine; + const { name, updated_at_millis, indices } = engine; const fieldCapabilities = await client.asCurrentUser.fieldCaps({ fields: '*', include_unmapped: true, - index: getEngineIndexAliasName(name), + index: indices, }); const fields = parseFieldsCapabilities(fieldCapabilities); return { @@ -33,46 +33,52 @@ export const fetchEngineFieldCapabilities = async ( }; }; -const ensureIndices = (indices: string | string[] | undefined): string[] => { +const ensureIndices = (indices: string[] | string | undefined): string[] => { if (!indices) return []; return Array.isArray(indices) ? indices : [indices]; }; -export const parseFieldsCapabilities = ( - fieldCapsResponse: FieldCapsResponse, - prefix: string = '' -): SchemaField[] => { +export const parseFieldsCapabilities = (fieldCapsResponse: FieldCapsResponse): SchemaField[] => { const { fields, indices: indexOrIndices } = fieldCapsResponse; - const inThisPass: Array<[string, Record]> = Object.entries( - fields - ) - .filter(([key]) => key.startsWith(prefix)) - .map(([key, value]) => [key.replace(prefix, ''), value]); + const indices = ensureIndices(indexOrIndices); - const atThisLevel = inThisPass.filter(([key]) => !key.includes('.')); + return Object.entries(fields) + .map(([fieldName, typesObject]) => { + const typeValues = Object.values(typesObject); + const type = calculateType(Object.keys(typesObject)); - return atThisLevel.map(([name, value]) => { - const type = calculateType(Object.keys(value)); - let indices = Object.values(value).flatMap((fieldCaps) => { - return ensureIndices(fieldCaps.indices).map((index) => ({ - name: index, - type: fieldCaps.type, - })); - }); + const indicesToType = typeValues.reduce( + (acc: Record, { type: indexType, indices: typeIndexOrIndices }) => { + const typeIndices = ensureIndices(typeIndexOrIndices); + typeIndices.forEach((index) => { + acc[index] = indexType; + }); + return acc; + }, + {} + ); - indices = - indices.length === 0 - ? ensureIndices(indexOrIndices).map((index) => ({ name: index, type })) - : indices; + const fieldIndices = + Object.keys(indicesToType).length > 0 + ? indices.map((index) => { + const indexType = indicesToType[index] || 'unmapped'; + return { + name: index, + type: indexType, + }; + }) + : indices.map((index) => ({ + name: index, + type, + })); - const subFields = parseFieldsCapabilities(fieldCapsResponse, `${prefix}${name}.`); - return { - fields: subFields, - indices, - name, - type, - }; - }); + return { + indices: fieldIndices, + name: fieldName, + type, + }; + }) + .sort((a, b) => a.name.localeCompare(b.name)) as SchemaField[]; }; const calculateType = (types: string[]): string => { @@ -87,6 +93,3 @@ const calculateType = (types: string[]): string => { // Otherwise there is a conflict return 'conflict'; }; - -// Note: This will likely need to be modified when engines move to es module -const getEngineIndexAliasName = (engineName: string): string => `search-engine-${engineName}`; diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts index 14a46bdcb659..729258fe4e97 100644 --- a/x-pack/plugins/enterprise_search/server/plugin.ts +++ b/x-pack/plugins/enterprise_search/server/plugin.ts @@ -136,7 +136,7 @@ export class EnterpriseSearchPlugin implements Plugin { ]; if (customIntegrations) { - registerEnterpriseSearchIntegrations(http, customIntegrations); + registerEnterpriseSearchIntegrations(config, http, customIntegrations); } /* diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts index 249a722fa57d..24fc921e6e58 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts @@ -99,7 +99,10 @@ export function registerConnectorRoutes({ router, log }: RouteDependencies) { { path: '/internal/enterprise_search/connectors/{connectorId}/configuration', validate: { - body: schema.recordOf(schema.string(), schema.string()), + body: schema.recordOf( + schema.string(), + schema.oneOf([schema.string(), schema.number(), schema.boolean()]) + ), params: schema.object({ connectorId: schema.string(), }), diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.test.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.test.ts index 14c5082c366f..f674df90e368 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.test.ts @@ -7,10 +7,6 @@ import { mockDependencies, MockRouter } from '../../__mocks__'; -jest.mock('../../utils/fetch_enterprise_search', () => ({ - ...jest.requireActual('../../utils/fetch_enterprise_search'), - fetchEnterpriseSearch: jest.fn(), -})); jest.mock('../../lib/engines/field_capabilities', () => ({ fetchEngineFieldCapabilities: jest.fn(), })); @@ -18,7 +14,6 @@ jest.mock('../../lib/engines/field_capabilities', () => ({ import { RequestHandlerContext } from '@kbn/core/server'; import { fetchEngineFieldCapabilities } from '../../lib/engines/field_capabilities'; -import { fetchEnterpriseSearch } from '../../utils/fetch_enterprise_search'; import { registerEnginesRoutes } from './engines'; @@ -171,20 +166,20 @@ describe('engines routes', () => { })); await mockRouter.callRoute({ + body: { + indices: ['test-indices-1'], + }, params: { engine_name: 'engine-name', }, query: { create: true }, - body: { - indices: ['test-indices-1'], - }, }); expect(mockClient.asCurrentUser.transport.request).toHaveBeenCalledWith({ - method: 'PUT', - path: '/_application/search_application/engine-name', body: { indices: ['test-indices-1'], }, + method: 'PUT', + path: '/_application/search_application/engine-name', querystring: { create: true }, }); const mock = jest.fn(); @@ -202,19 +197,19 @@ describe('engines routes', () => { })); await mockRouter.callRoute({ - params: { - engine_name: 'engine-name', - }, body: { indices: ['test-indices-1'], }, + params: { + engine_name: 'engine-name', + }, }); expect(mockClient.asCurrentUser.transport.request).toHaveBeenCalledWith({ - method: 'PUT', - path: '/_application/search_application/engine-name', body: { indices: ['test-indices-1'], }, + method: 'PUT', + path: '/_application/search_application/engine-name', querystring: {}, }); const mock = jest.fn(); @@ -229,10 +224,8 @@ describe('engines routes', () => { it('validates correctly with engine_name', () => { const request = { + body: { indices: ['search-unit-test'] }, params: { engine_name: 'some-engine' }, - body: { - indices: ['search-unit-test'], - }, }; mockRouter.shouldValidate(request); @@ -246,10 +239,10 @@ describe('engines routes', () => { it('fails validation without indices', () => { const request = { - params: { engine_name: 'some-engine' }, body: { name: 'some-engine', }, + params: { engine_name: 'some-engine' }, }; mockRouter.shouldThrow(request); @@ -354,9 +347,9 @@ describe('engines routes', () => { }, }); expect(mockClient.asCurrentUser.transport.request).toHaveBeenCalledWith({ + body: {}, method: 'POST', path: '/engine-name/_search', - body: {}, }); expect(mockRouter.response.ok).toHaveBeenCalledWith({ body: { @@ -368,8 +361,8 @@ describe('engines routes', () => { it('validates correctly with engine_name and pagination', () => { const request = { body: { - query: 'test-query', fields: ['test-field-1', 'test-field-2'], + query: 'test-query', }, params: { engine_name: 'some-engine', @@ -393,12 +386,12 @@ describe('engines routes', () => { it('validation with query and without fields', () => { const request = { - params: { - engine_name: 'my-test-engine', - }, body: { - query: 'sample-query', fields: [], + query: 'sample-query', + }, + params: { + engine_name: 'my-test-engine', }, }; mockRouter.shouldValidate(request); @@ -414,7 +407,7 @@ describe('engines routes', () => { describe('GET /internal/enterprise_search/engines/{engine_name}/field_capabilities', () => { let mockRouter: MockRouter; const mockClient = { - asCurrentUser: {}, + asCurrentUser: { transport: { request: jest.fn() } }, }; const mockCore = { elasticsearch: { client: mockClient }, @@ -451,26 +444,35 @@ describe('engines routes', () => { name: 'unit-test', }; - (fetchEnterpriseSearch as jest.Mock).mockResolvedValueOnce(engineResult); + (mockClient.asCurrentUser.transport.request as jest.Mock).mockResolvedValueOnce(engineResult); (fetchEngineFieldCapabilities as jest.Mock).mockResolvedValueOnce(fieldCapabilitiesResult); await mockRouter.callRoute({ params: { engine_name: 'unit-test' }, }); - expect(fetchEnterpriseSearch).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - '/api/engines/unit-test' - ); + expect(mockClient.asCurrentUser.transport.request).toHaveBeenCalledWith({ + method: 'GET', + path: '/_application/search_application/unit-test', + }); expect(fetchEngineFieldCapabilities).toHaveBeenCalledWith(mockClient, engineResult); expect(mockRouter.response.ok).toHaveBeenCalledWith({ body: fieldCapabilitiesResult, headers: { 'content-type': 'application/json' }, }); }); - it('returns 404 when fetch engine is undefined', async () => { - (fetchEnterpriseSearch as jest.Mock).mockResolvedValueOnce(undefined); + it('returns 404 when fetch engine throws a not found exception', async () => { + (mockClient.asCurrentUser.transport.request as jest.Mock).mockRejectedValueOnce({ + meta: { + body: { + error: { + type: 'resource_not_found_exception', + }, + }, + statusCode: 404, + }, + name: 'ResponseError', + }); await mockRouter.callRoute({ params: { engine_name: 'unit-test' }, }); @@ -485,29 +487,15 @@ describe('engines routes', () => { statusCode: 404, }); }); - it('returns 404 when fetch engine is returns 404', async () => { - (fetchEnterpriseSearch as jest.Mock).mockResolvedValueOnce({ - responseStatus: 404, - responseStatusText: 'NOT_FOUND', - }); - await mockRouter.callRoute({ - params: { engine_name: 'unit-test' }, - }); - - expect(mockRouter.response.customError).toHaveBeenCalledWith({ + it('returns error when fetch engine returns an unknown error', async () => { + (mockClient.asCurrentUser.transport.request as jest.Mock).mockRejectedValueOnce({ body: { attributes: { - error_code: 'engine_not_found', + error_code: 'unknown_error', }, - message: 'Could not find engine', + message: 'Unknown error', }, - statusCode: 404, - }); - }); - it('returns error when fetch engine returns an error', async () => { - (fetchEnterpriseSearch as jest.Mock).mockResolvedValueOnce({ - responseStatus: 500, - responseStatusText: 'INTERNAL_SERVER_ERROR', + statusCode: 500, }); await mockRouter.callRoute({ params: { engine_name: 'unit-test' }, @@ -518,9 +506,9 @@ describe('engines routes', () => { attributes: { error_code: 'uncaught_exception', }, - message: 'Error fetching engine', + message: 'Enterprise Search encountered an error. Check Kibana Server logs for details.', }, - statusCode: 500, + statusCode: 502, }); }); }); diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts index b422d968c5d2..2bd7fd079bac 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/engines.ts @@ -8,7 +8,7 @@ import { SearchResponse, AcknowledgedResponseBase } from '@elastic/elasticsearch import { schema } from '@kbn/config-schema'; import { - EnterpriseSearchEngineDetails, + EnterpriseSearchEngine, EnterpriseSearchEnginesResponse, EnterpriseSearchEngineUpsertResponse, } from '../../../common/types/engines'; @@ -20,9 +20,9 @@ import { RouteDependencies } from '../../plugin'; import { createError } from '../../utils/create_error'; import { elasticsearchErrorHandler } from '../../utils/elasticsearch_error_handler'; -import { fetchEnterpriseSearch, isResponseError } from '../../utils/fetch_enterprise_search'; +import { isNotFoundException } from '../../utils/identify_exceptions'; -export function registerEnginesRoutes({ config, log, router }: RouteDependencies) { +export function registerEnginesRoutes({ log, router }: RouteDependencies) { router.get( { path: '/internal/enterprise_search/engines', @@ -59,7 +59,7 @@ export function registerEnginesRoutes({ config, log, router }: RouteDependencies }, elasticsearchErrorHandler(log, async (context, request, response) => { const { client } = (await context.core).elasticsearch; - const engine = await client.asCurrentUser.transport.request({ + const engine = await client.asCurrentUser.transport.request({ method: 'GET', path: `/_application/search_application/${request.params.engine_name}`, }); @@ -75,21 +75,21 @@ export function registerEnginesRoutes({ config, log, router }: RouteDependencies indices: schema.arrayOf(schema.string()), name: schema.maybe(schema.string()), }), - query: schema.object({ - create: schema.maybe(schema.boolean()), - }), params: schema.object({ engine_name: schema.string(), }), + query: schema.object({ + create: schema.maybe(schema.boolean()), + }), }, }, elasticsearchErrorHandler(log, async (context, request, response) => { const { client } = (await context.core).elasticsearch; const engine = await client.asCurrentUser.transport.request({ + body: { indices: request.body.indices }, method: 'PUT', path: `/_application/search_application/${request.params.engine_name}`, - body: { indices: request.body.indices }, querystring: request.query, }); return response.ok({ body: engine }); @@ -130,9 +130,9 @@ export function registerEnginesRoutes({ config, log, router }: RouteDependencies elasticsearchErrorHandler(log, async (context, request, response) => { const { client } = (await context.core).elasticsearch; const engines = await client.asCurrentUser.transport.request({ + body: {}, method: 'POST', path: `/${request.params.engine_name}/_search`, - body: {}, }); return response.ok({ body: engines }); }) @@ -168,37 +168,31 @@ export function registerEnginesRoutes({ config, log, router }: RouteDependencies validate: { params: schema.object({ engine_name: schema.string() }) }, }, elasticsearchErrorHandler(log, async (context, request, response) => { - const engineName = decodeURIComponent(request.params.engine_name); - const { client } = (await context.core).elasticsearch; - - const engine = await fetchEnterpriseSearch( - config, - request, - `/api/engines/${engineName}` - ); + try { + const engineName = decodeURIComponent(request.params.engine_name); + const { client } = (await context.core).elasticsearch; - if (!engine || (isResponseError(engine) && engine.responseStatus === 404)) { - return createError({ - errorCode: ErrorCode.ENGINE_NOT_FOUND, - message: 'Could not find engine', - response, - statusCode: 404, + const engine = await client.asCurrentUser.transport.request({ + method: 'GET', + path: `/_application/search_application/${engineName}`, }); - } - if (isResponseError(engine)) { - return createError({ - errorCode: ErrorCode.UNCAUGHT_EXCEPTION, - message: 'Error fetching engine', - response, - statusCode: engine.responseStatus, + + const data = await fetchEngineFieldCapabilities(client, engine); + return response.ok({ + body: data, + headers: { 'content-type': 'application/json' }, }); + } catch (e) { + if (isNotFoundException(e)) { + return createError({ + errorCode: ErrorCode.ENGINE_NOT_FOUND, + message: 'Could not find engine', + response, + statusCode: 404, + }); + } + throw e; } - - const data = await fetchEngineFieldCapabilities(client, engine); - return response.ok({ - body: data, - headers: { 'content-type': 'application/json' }, - }); }) ); } diff --git a/x-pack/plugins/enterprise_search/server/utils/fetch_enterprise_search.test.ts b/x-pack/plugins/enterprise_search/server/utils/fetch_enterprise_search.test.ts deleted file mode 100644 index 20fe7b57350e..000000000000 --- a/x-pack/plugins/enterprise_search/server/utils/fetch_enterprise_search.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import '../__mocks__/http_agent.mock'; - -jest.mock('node-fetch'); -import fetch from 'node-fetch'; - -import { KibanaRequest } from '@kbn/core/server'; - -import { ConfigType } from '..'; - -import { fetchEnterpriseSearch, isResponseError } from './fetch_enterprise_search'; - -describe('fetchEnterpriseSearch', () => { - const mockConfig = { - accessCheckTimeout: 200, - accessCheckTimeoutWarning: 100, - host: 'http://localhost:3002', - }; - const mockRequest = { - headers: { authorization: '==someAuth' }, - }; - - beforeEach(() => { - jest.clearAllMocks(); - }); - - it('returns json fetch response', async () => { - const response = { foo: 'bar' }; - (fetch as unknown as jest.Mock).mockResolvedValueOnce({ - json: jest.fn().mockResolvedValueOnce(response), - ok: true, - }); - await expect( - fetchEnterpriseSearch(mockConfig as ConfigType, mockRequest as KibanaRequest, '/api/v1/test') - ).resolves.toBe(response); - }); - it('calls expected endpoint', async () => { - (fetch as unknown as jest.Mock).mockResolvedValueOnce({ - json: jest.fn().mockResolvedValueOnce({}), - ok: true, - }); - await fetchEnterpriseSearch( - mockConfig as ConfigType, - mockRequest as KibanaRequest, - '/api/v1/test' - ); - - expect(fetch).toHaveBeenCalledTimes(1); - expect(fetch).toHaveBeenCalledWith('http://localhost:3002/api/v1/test', expect.anything()); - }); - it('uses request auth header & config custom headers', async () => { - (fetch as unknown as jest.Mock).mockResolvedValueOnce({ - json: jest.fn().mockResolvedValueOnce({}), - ok: true, - }); - const config = { - ...mockConfig, - customHeaders: { - foo: 'bar', - }, - }; - await fetchEnterpriseSearch( - config as unknown as ConfigType, - mockRequest as KibanaRequest, - '/api/v1/test' - ); - - expect(fetch).toHaveBeenCalledTimes(1); - expect(fetch).toHaveBeenCalledWith(expect.anything(), { - agent: expect.anything(), - headers: { - Authorization: mockRequest.headers.authorization, - foo: 'bar', - }, - }); - }); - it('returns undefined when config.host is unavailable', async () => { - await expect( - fetchEnterpriseSearch( - { host: '' } as ConfigType, - mockRequest as KibanaRequest, - '/api/v1/test' - ) - ).resolves.toBeUndefined(); - }); -}); - -describe('isResponseError', () => { - it('returns true for ResponseError object', () => { - expect(isResponseError({ responseStatus: 404, responseStatusText: 'NOT_FOUND' })).toBe(true); - }); - it('returns false for null/undefined', () => { - expect(isResponseError(null)).toBe(false); - expect(isResponseError(undefined)).toBe(false); - }); - it('returns false for object without expected keys', () => { - expect(isResponseError({})).toBe(false); - expect(isResponseError({ responseStatusText: 'NOT_FOUND' })).toBe(false); - expect(isResponseError({ responseStatus: 404 })).toBe(false); - expect(isResponseError([])).toBe(false); - }); - it('returns false for non-object', () => { - expect(isResponseError(100)).toBe(false); - expect(isResponseError('test')).toBe(false); - }); -}); diff --git a/x-pack/plugins/enterprise_search/server/utils/fetch_enterprise_search.ts b/x-pack/plugins/enterprise_search/server/utils/fetch_enterprise_search.ts deleted file mode 100644 index b1a2146a86ab..000000000000 --- a/x-pack/plugins/enterprise_search/server/utils/fetch_enterprise_search.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import fetch, { RequestInit } from 'node-fetch'; - -import { KibanaRequest } from '@kbn/core/server'; - -import { ConfigType } from '..'; - -import { entSearchHttpAgent } from '../lib/enterprise_search_http_agent'; - -export interface ResponseError { - responseStatus: number; - responseStatusText: string; -} - -export function isResponseError(resp: unknown): resp is ResponseError { - if (typeof resp !== 'object') return false; - if (resp === null) return false; - if ('responseStatus' in resp && 'responseStatusText' in resp) return true; - return false; -} - -export async function fetchEnterpriseSearch( - config: ConfigType, - request: KibanaRequest, - endpoint: string -): Promise { - if (!config.host) return undefined; - - const enterpriseSearchUrl = encodeURI(`${config.host}${endpoint}`); - const options: RequestInit = { - agent: entSearchHttpAgent.getHttpAgent(), - headers: { - Authorization: request.headers.authorization as string, - ...config.customHeaders, - }, - }; - - const response = await fetch(enterpriseSearchUrl, options); - - if (!response.ok) { - return { - responseStatus: response.status, - responseStatusText: response.statusText, - }; - } - - return await response.json(); -} diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index 0b7ac287c0bc..8f595e7b5078 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -54,5 +54,6 @@ "@kbn/es-query", "@kbn/datemath", "@kbn/expressions-plugin", + "@kbn/react-field", ] } diff --git a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts index 996e556ca170..dd809258ef0c 100644 --- a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts @@ -7,6 +7,7 @@ /* eslint-disable @typescript-eslint/consistent-type-definitions */ +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/data-plugin/common'; import type { Filter } from '@kbn/es-query'; import type { TimeRange } from '@kbn/es-query'; @@ -33,6 +34,7 @@ export type DataFilters = { zoom: number; isReadOnly: boolean; joinKeyFilter?: Filter; + executionContext: KibanaExecutionContext; }; export type VectorSourceRequestMeta = DataFilters & { diff --git a/x-pack/plugins/maps/common/execution_context.test.ts b/x-pack/plugins/maps/common/execution_context.test.ts deleted file mode 100644 index 25e3813a7e8f..000000000000 --- a/x-pack/plugins/maps/common/execution_context.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { makeExecutionContext } from './execution_context'; - -describe('makeExecutionContext', () => { - test('returns basic fields if nothing is provided', () => { - const context = makeExecutionContext({}); - expect(context).toStrictEqual({ - name: 'maps', - type: 'application', - }); - }); - - test('merges in context', () => { - const context = makeExecutionContext({ id: '123' }); - expect(context).toStrictEqual({ - name: 'maps', - type: 'application', - id: '123', - }); - }); - - test('omits undefined values', () => { - const context = makeExecutionContext({ id: '123', description: undefined }); - expect(context).toStrictEqual({ - name: 'maps', - type: 'application', - id: '123', - }); - }); -}); diff --git a/x-pack/plugins/maps/common/execution_context.ts b/x-pack/plugins/maps/common/execution_context.ts deleted file mode 100644 index f62f1da85f99..000000000000 --- a/x-pack/plugins/maps/common/execution_context.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isUndefined, omitBy } from 'lodash'; -import type { KibanaExecutionContext } from '@kbn/core/public'; -import { APP_ID } from './constants'; - -export function makeExecutionContext(context: { - id?: string; - url?: string; - description?: string; -}): KibanaExecutionContext { - return omitBy( - { - name: APP_ID, - type: 'application', - ...context, - }, - isUndefined - ); -} diff --git a/x-pack/plugins/maps/public/actions/map_action_constants.ts b/x-pack/plugins/maps/public/actions/map_action_constants.ts index aabca948ec80..483615e44c1d 100644 --- a/x-pack/plugins/maps/public/actions/map_action_constants.ts +++ b/x-pack/plugins/maps/public/actions/map_action_constants.ts @@ -39,6 +39,7 @@ export const REMOVE_TRACKED_LAYER_STATE = 'REMOVE_TRACKED_LAYER_STATE'; export const SET_OPEN_TOOLTIPS = 'SET_OPEN_TOOLTIPS'; export const UPDATE_DRAW_STATE = 'UPDATE_DRAW_STATE'; export const UPDATE_EDIT_STATE = 'UPDATE_EDIT_STATE'; +export const SET_EXECUTION_CONTEXT = 'SET_EXECUTION_CONTEXT'; export const SET_MAP_INIT_ERROR = 'SET_MAP_INIT_ERROR'; export const SET_WAITING_FOR_READY_HIDDEN_LAYERS = 'SET_WAITING_FOR_READY_HIDDEN_LAYERS'; export const SET_MAP_SETTINGS = 'SET_MAP_SETTINGS'; diff --git a/x-pack/plugins/maps/public/actions/map_actions.ts b/x-pack/plugins/maps/public/actions/map_actions.ts index e51a65235916..7ae6a5690040 100644 --- a/x-pack/plugins/maps/public/actions/map_actions.ts +++ b/x-pack/plugins/maps/public/actions/map_actions.ts @@ -11,6 +11,7 @@ import { AnyAction, Dispatch } from 'redux'; import { ThunkDispatch } from 'redux-thunk'; import turfBboxPolygon from '@turf/bbox-polygon'; import turfBooleanContains from '@turf/boolean-contains'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Filter } from '@kbn/es-query'; import type { Query, TimeRange } from '@kbn/es-query'; import { Geometry, Position } from 'geojson'; @@ -44,6 +45,7 @@ import { MAP_READY, ROLLBACK_MAP_SETTINGS, SET_EMBEDDABLE_SEARCH_CONTEXT, + SET_EXECUTION_CONTEXT, SET_GOTO, SET_MAP_INIT_ERROR, SET_MAP_SETTINGS, @@ -357,6 +359,13 @@ export function setEmbeddableSearchContext({ }; } +export function setExecutionContext(executionContext: KibanaExecutionContext) { + return { + type: SET_EXECUTION_CONTEXT, + executionContext, + }; +} + export function updateDrawState(drawState: DrawState | null) { return (dispatch: Dispatch) => { if (drawState !== null) { diff --git a/x-pack/plugins/maps/public/classes/joins/inner_join.ts b/x-pack/plugins/maps/public/classes/joins/inner_join.ts index b3a55ea55dec..a754650cdef8 100644 --- a/x-pack/plugins/maps/public/classes/joins/inner_join.ts +++ b/x-pack/plugins/maps/public/classes/joins/inner_join.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/es-query'; import { Feature, GeoJsonProperties } from 'geojson'; import { ESTermSource } from '../sources/es_term_source'; @@ -137,8 +138,11 @@ export class InnerJoin { return this._descriptor; } - async getTooltipProperties(properties: GeoJsonProperties) { - return await this.getRightJoinSource().getTooltipProperties(properties); + async getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ) { + return await this.getRightJoinSource().getTooltipProperties(properties, executionContext); } getIndexPatternIds() { diff --git a/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts b/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts index ef4741873679..1160a8a1aa76 100644 --- a/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts +++ b/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts @@ -34,6 +34,7 @@ export class MockSyncContext implements DataRequestContext { }, zoom: 0, isReadOnly: false, + executionContext: {}, ...dataFilters, }; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts index d7382782d1af..b46353fcef93 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts @@ -45,6 +45,7 @@ export async function syncBoundsData({ applyGlobalQuery: source.getApplyGlobalQuery(), applyGlobalTime: source.getApplyGlobalTime(), isFeatureEditorOpenForLayer, + executionContext: dataFilters.executionContext, }; let bounds = null; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx index 13a093ad5971..bff6a297fcd7 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx @@ -8,6 +8,7 @@ import React from 'react'; import { v4 as uuidv4 } from 'uuid'; import type { FilterSpecification, Map as MbMap, LayerSpecification } from '@kbn/mapbox-gl'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/data-plugin/common'; import { Feature, GeoJsonProperties, Geometry, Position } from 'geojson'; import _ from 'lodash'; @@ -94,7 +95,10 @@ export interface IVectorLayer extends ILayer { getSource(): IVectorSource; getFeatureId(feature: Feature): string | number | undefined; getFeatureById(id: string | number): Feature | null; - getPropertiesForTooltip(properties: GeoJsonProperties): Promise; + getPropertiesForTooltip( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise; hasJoins(): boolean; showJoinEditor(): boolean; canShowTooltip(): boolean; @@ -466,6 +470,7 @@ export class AbstractVectorLayer extends AbstractLayer implements IVectorLayer { timeFilters: nextMeta.timeFilters, searchSessionId: dataFilters.searchSessionId, inspectorAdapters, + executionContext: dataFilters.executionContext, }); stopLoading(dataRequestId, requestToken, styleMeta, nextMeta); @@ -931,13 +936,19 @@ export class AbstractVectorLayer extends AbstractLayer implements IVectorLayer { } } - async getPropertiesForTooltip(properties: GeoJsonProperties) { + async getPropertiesForTooltip( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ) { const vectorSource = this.getSource(); - let allProperties = await vectorSource.getTooltipProperties(properties); + let allProperties = await vectorSource.getTooltipProperties(properties, executionContext); this._addJoinsToSourceTooltips(allProperties); for (let i = 0; i < this.getJoins().length; i++) { - const propsFromJoin = await this.getJoins()[i].getTooltipProperties(properties); + const propsFromJoin = await this.getJoins()[i].getTooltipProperties( + properties, + executionContext + ); allProperties = [...allProperties, ...propsFromJoin]; } return allProperties; diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx index 70d5a9c54cc9..bc1289e10404 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx @@ -7,7 +7,7 @@ import React, { ReactElement } from 'react'; import { i18n } from '@kbn/i18n'; -import { Feature } from 'geojson'; +import { Feature, GeoJsonProperties } from 'geojson'; import { FileLayer } from '@elastic/ems-client'; import { ImmutableSourceProperty, SourceEditorArgs } from '../source'; import { AbstractVectorSource, GeoJsonWithMeta, IVectorSource } from '../vector_source'; @@ -199,7 +199,7 @@ export class EMSFileSource extends AbstractVectorSource implements IEmsFileSourc return this._tooltipFields.length > 0; } - async getTooltipProperties(properties: unknown): Promise { + async getTooltipProperties(properties: GeoJsonProperties): Promise { const promises = this._tooltipFields.map((field) => { // @ts-ignore const value = properties[field.getName()]; diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts index a7d0b4c9beb8..65325fed5d95 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts @@ -8,13 +8,14 @@ import { coreMock } from '@kbn/core/public/mocks'; import { MapExtent, VectorSourceRequestMeta } from '../../../../common/descriptor_types'; import { - getExecutionContext, + getExecutionContextService, getHttp, getIndexPatternService, getSearchService, } from '../../../kibana_services'; import { ESGeoGridSource } from './es_geo_grid_source'; import { + APP_ID, ES_GEO_FIELD_TYPE, GRID_RESOLUTION, RENDER_AS, @@ -139,7 +140,7 @@ describe('ESGeoGridSource', () => { name: 'some-app', }); // @ts-expect-error - getExecutionContext.mockReturnValue(coreStartMock.executionContext); + getExecutionContextService.mockReturnValue(coreStartMock.executionContext); }); afterEach(() => { @@ -177,6 +178,7 @@ describe('ESGeoGridSource', () => { zoom: 0, isForceRefresh: false, isFeatureEditorOpenForLayer: false, + executionContext: { name: APP_ID }, }; describe('getGeoJsonWithMeta', () => { @@ -310,9 +312,36 @@ describe('ESGeoGridSource', () => { it('getTileUrl', async () => { const tileUrl = await mvtGeogridSource.getTileUrl(vectorSourceRequestMeta, '1234', false, 5); - expect(tileUrl).toEqual( - "rootdir/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf?geometryFieldName=bar&index=foo-*&gridPrecision=8&hasLabels=false&buffer=5&requestBody=(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'1'%3A('0'%3Asize%2C'1'%3A0)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A''))%2C'6'%3A('0'%3Aaggs%2C'1'%3A())))&renderAs=heatmap&token=1234" + const urlParts = tileUrl.split('?'); + expect(urlParts[0]).toEqual('rootdir/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf'); + + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params)).toEqual({ + buffer: '5', + geometryFieldName: 'bar', + gridPrecision: '8', + hasLabels: 'false', + index: 'foo-*', + renderAs: 'heatmap', + requestBody: + "(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'1'%3A('0'%3Asize%2C'1'%3A0)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A''))%2C'6'%3A('0'%3Aaggs%2C'1'%3A())))", + token: '1234', + }); + }); + + it('getTileUrl should include executionContextId when provided', async () => { + const tileUrl = await mvtGeogridSource.getTileUrl( + { + ...vectorSourceRequestMeta, + executionContext: { name: APP_ID, id: 'map1234' }, + }, + '1234', + false, + 5 ); + const urlParts = tileUrl.split('?'); + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params).executionContextId).toEqual('map1234'); }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx index dea665f904b0..c19e715326bc 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx @@ -10,6 +10,7 @@ import React, { ReactElement } from 'react'; import { i18n } from '@kbn/i18n'; import { Feature } from 'geojson'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { ISearchSource } from '@kbn/data-plugin/common/search/search_source'; import { DataView } from '@kbn/data-plugin/common'; import { Adapters } from '@kbn/inspector-plugin/common/adapters'; @@ -50,7 +51,7 @@ import { } from '../../../../common/descriptor_types'; import { ImmutableSourceProperty, OnSourceChangeArgs, SourceEditorArgs } from '../source'; import { isValidStringConfig } from '../../util/valid_string_config'; -import { makePublicExecutionContext } from '../../../util'; +import { getExecutionContextId, mergeExecutionContext } from '../execution_context_utils'; import { isMvt } from './is_mvt'; import { VectorStyle } from '../../styles/vector/vector_style'; import { getIconSize } from './get_icon_size'; @@ -256,6 +257,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo isRequestStillActive, bufferedExtent, inspectorAdapters, + executionContext, }: { searchSource: ISearchSource; searchSessionId?: string; @@ -267,6 +269,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo isRequestStillActive: () => boolean; bufferedExtent: MapExtent; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }) { const gridsPerRequest: number = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); const aggs: any = { @@ -337,7 +340,10 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo } ), searchSessionId, - executionContext: makePublicExecutionContext('es_geo_grid_source:cluster_composite'), + executionContext: mergeExecutionContext( + { description: 'es_geo_grid_source:cluster_composite' }, + executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -365,6 +371,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo bufferedExtent, tooManyBuckets, inspectorAdapters, + executionContext, }: { searchSource: ISearchSource; searchSessionId?: string; @@ -375,6 +382,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo bufferedExtent: MapExtent; tooManyBuckets: boolean; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }): Promise { const valueAggsDsl = tooManyBuckets ? this.getValueAggsDsl(indexPattern, (metric) => { @@ -411,7 +419,10 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo defaultMessage: 'Elasticsearch geo grid aggregation request', }), searchSessionId, - executionContext: makePublicExecutionContext('es_geo_grid_source:cluster'), + executionContext: mergeExecutionContext( + { description: 'es_geo_grid_source:cluster' }, + executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -471,6 +482,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo isRequestStillActive, bufferedExtent: searchFilters.buffer, inspectorAdapters, + executionContext: searchFilters.executionContext, }) : await this._nonCompositeAggRequest({ searchSource, @@ -482,6 +494,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo bufferedExtent: searchFilters.buffer, tooManyBuckets, inspectorAdapters, + executionContext: searchFilters.executionContext, }); return { @@ -513,15 +526,21 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo `/${GIS_API_PATH}/${MVT_GETGRIDTILE_API_PATH}/{z}/{x}/{y}.pbf` ); - return `${mvtUrlServicePath}\ -?geometryFieldName=${this._descriptor.geoField}\ -&index=${dataView.getIndexPattern()}\ -&gridPrecision=${this._getGeoGridPrecisionResolutionDelta()}\ -&hasLabels=${hasLabels}\ -&buffer=${buffer}\ -&requestBody=${encodeMvtResponseBody(searchSource.getSearchRequestBody())}\ -&renderAs=${this._descriptor.requestType}\ -&token=${refreshToken}`; + const params = new URLSearchParams(); + params.set('geometryFieldName', this._descriptor.geoField); + params.set('index', dataView.getIndexPattern()); + params.set('gridPrecision', this._getGeoGridPrecisionResolutionDelta().toString()); + params.set('hasLabels', hasLabels.toString()); + params.set('buffer', buffer.toString()); + params.set('requestBody', encodeMvtResponseBody(searchSource.getSearchRequestBody())); + params.set('renderAs', this._descriptor.requestType); + params.set('token', refreshToken); + const executionContextId = getExecutionContextId(searchFilters.executionContext); + if (executionContextId) { + params.set('executionContextId', executionContextId); + } + + return `${mvtUrlServicePath}?${params.toString()}`; } isFilterByMapBounds(): boolean { diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx index 78aae064a655..33b2b2e38317 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx @@ -38,7 +38,7 @@ import { IField } from '../../fields/field'; import { ITooltipProperty, TooltipProperty } from '../../tooltips/tooltip_property'; import { getIsGoldPlus } from '../../../licensed_features'; import { LICENSED_FEATURES } from '../../../licensed_features'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; type ESGeoLineSourceSyncMeta = Pick; @@ -218,7 +218,10 @@ export class ESGeoLineSource extends AbstractESAggSource { defaultMessage: 'Elasticsearch terms request to fetch entities within map buffer.', }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_geo_line:entities'), + executionContext: mergeExecutionContext( + { description: 'es_geo_line:entities' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); const entityBuckets: Array<{ key: string; doc_count: number }> = _.get( @@ -291,7 +294,10 @@ export class ESGeoLineSource extends AbstractESAggSource { 'Elasticsearch geo_line request to fetch tracks for entities. Tracks are not filtered by map buffer.', }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_geo_line:tracks'), + executionContext: mergeExecutionContext( + { description: 'es_geo_line:tracks' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); const { featureCollection, numTrimmedTracks } = convertToGeoJson( diff --git a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx index dcec30b1f71a..ae00e00e58ae 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx @@ -26,7 +26,7 @@ import { AbstractESAggSource } from '../es_agg_source'; import { registerSource } from '../source_registry'; import { turfBboxToBounds } from '../../../../common/elasticsearch_util'; import { DataRequestAbortError } from '../../util/data_request'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; import { SourceEditorArgs } from '../source'; import { ESPewPewSourceDescriptor, @@ -184,7 +184,10 @@ export class ESPewPewSource extends AbstractESAggSource { defaultMessage: 'Source-destination connections request', }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_pew_pew_source:connections'), + executionContext: mergeExecutionContext( + { description: 'es_pew_pew_source:connections' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -229,7 +232,10 @@ export class ESPewPewSource extends AbstractESAggSource { searchSource.fetch$({ abortSignal: abortController.signal, legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_pew_pew_source:bounds'), + executionContext: mergeExecutionContext( + { description: 'es_pew_pew_source:bounds' }, + boundsFilters.executionContext + ), }) ); const destBounds = (esResp.aggregations?.destFitToBounds as AggregationsGeoBoundsAggregate) diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts index 2e9fa9e027f5..5157848cc576 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ES_GEO_FIELD_TYPE, SCALING_TYPES } from '../../../../common/constants'; +import { APP_ID, ES_GEO_FIELD_TYPE, SCALING_TYPES } from '../../../../common/constants'; jest.mock('../../../kibana_services'); jest.mock('./util/load_index_settings'); @@ -108,17 +108,48 @@ describe('ESSearchSource', () => { applyForceRefresh: true, isForceRefresh: false, isFeatureEditorOpenForLayer: false, + executionContext: { name: APP_ID }, }; - it('Should only include required props', async () => { + it('should include required props', async () => { const esSearchSource = new ESSearchSource({ geoField: geoFieldName, indexPatternId: 'ipId', }); const tileUrl = await esSearchSource.getTileUrl(searchFilters, '1234', false, 5); - expect(tileUrl).toBe( - `rootdir/api/maps/mvt/getTile/{z}/{x}/{y}.pbf?geometryFieldName=bar&index=foobar-title-*&hasLabels=false&buffer=5&requestBody=(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'1'%3A('0'%3Asize%2C'1'%3A1000)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A'tooltipField%3A%20foobar'))%2C'6'%3A('0'%3AfieldsFromSource%2C'1'%3A!(_id))%2C'7'%3A('0'%3Asource%2C'1'%3A!f)%2C'8'%3A('0'%3Afields%2C'1'%3A!(tooltipField%2CstyleField))))&token=1234` + + const urlParts = tileUrl.split('?'); + expect(urlParts[0]).toEqual('rootdir/api/maps/mvt/getTile/{z}/{x}/{y}.pbf'); + + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params)).toEqual({ + buffer: '5', + geometryFieldName: 'bar', + hasLabels: 'false', + index: 'foobar-title-*', + requestBody: + "(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'1'%3A('0'%3Asize%2C'1'%3A1000)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A'tooltipField%3A%20foobar'))%2C'6'%3A('0'%3AfieldsFromSource%2C'1'%3A!(_id))%2C'7'%3A('0'%3Asource%2C'1'%3A!f)%2C'8'%3A('0'%3Afields%2C'1'%3A!(tooltipField%2CstyleField))))", + token: '1234', + }); + }); + + it('should include executionContextId when provided', async () => { + const esSearchSource = new ESSearchSource({ + geoField: geoFieldName, + indexPatternId: 'ipId', + }); + const tileUrl = await esSearchSource.getTileUrl( + { + ...searchFilters, + executionContext: { name: APP_ID, id: 'map1234' }, + }, + '1234', + false, + 5 ); + const urlParts = tileUrl.split('?'); + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params).executionContextId).toEqual('map1234'); }); }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx index c5ed07a4c0e1..3b8ab273fbbc 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx @@ -9,6 +9,7 @@ import _ from 'lodash'; import React, { ReactElement } from 'react'; import { i18n } from '@kbn/i18n'; import { GeoJsonProperties, Geometry, Position } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { type Filter, buildPhraseFilter, type TimeRange } from '@kbn/es-query'; import type { DataViewField, DataView } from '@kbn/data-plugin/common'; import { lastValueFrom } from 'rxjs'; @@ -74,7 +75,7 @@ import { getIsDrawLayer, getMatchingIndexes, } from './util/feature_edit'; -import { makePublicExecutionContext } from '../../../util'; +import { getExecutionContextId, mergeExecutionContext } from '../execution_context_utils'; import { FeatureGeometryFilterForm } from '../../../connected_components/mb_map/tooltip_control/features_tooltip'; type ESSearchSourceSyncMeta = Pick< @@ -354,7 +355,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource registerCancelCallback, requestDescription: 'Elasticsearch document top hits request', searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_search_source:top_hits'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:top_hits' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -438,7 +442,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource registerCancelCallback, requestDescription: 'Elasticsearch document request', searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_search_source:doc_search'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:doc_search' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -574,7 +581,12 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource return this._tooltipFields.length > 0; } - async _loadTooltipProperties(docId: string | number, index: string, indexPattern: DataView) { + async _loadTooltipProperties( + docId: string | number, + index: string, + indexPattern: DataView, + executionContext: KibanaExecutionContext + ) { if (this._tooltipFields.length === 0) { return {}; } @@ -616,7 +628,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource const { rawResponse: resp } = await lastValueFrom( searchSource.fetch$({ legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_search_source:load_tooltip_properties'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:load_tooltip_properties' }, + executionContext + ), }) ); @@ -643,7 +658,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource return this._tooltipFields.map((field: IField) => field.getName()); } - async getTooltipProperties(properties: GeoJsonProperties): Promise { + async getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise { if (properties === null) { throw new Error('properties cannot be null'); } @@ -651,7 +669,8 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource const propertyValues = await this._loadTooltipProperties( properties._id, properties._index, - indexPattern + indexPattern, + executionContext ); const tooltipProperties = this._tooltipFields.map((field) => { const value = propertyValues[field.getName()]; @@ -869,13 +888,19 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource delete requestBody.script_fields; delete requestBody.stored_fields; - return `${mvtUrlServicePath}\ -?geometryFieldName=${this._descriptor.geoField}\ -&index=${dataView.getIndexPattern()}\ -&hasLabels=${hasLabels}\ -&buffer=${buffer}\ -&requestBody=${encodeMvtResponseBody(requestBody)}\ -&token=${refreshToken}`; + const params = new URLSearchParams(); + params.set('geometryFieldName', this._descriptor.geoField); + params.set('index', dataView.getIndexPattern()); + params.set('hasLabels', hasLabels.toString()); + params.set('buffer', buffer.toString()); + params.set('requestBody', encodeMvtResponseBody(requestBody)); + params.set('token', refreshToken); + const executionContextId = getExecutionContextId(searchFilters.executionContext); + if (executionContextId) { + params.set('executionContextId', executionContextId); + } + + return `${mvtUrlServicePath}?${params.toString()}`; } async getTimesliceMaskFieldName(): Promise { @@ -935,7 +960,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource abortSignal: abortController.signal, sessionId: searchFilters.searchSessionId, legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_search_source:all_doc_counts'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:all_doc_counts' }, + searchFilters.executionContext + ), }) ); return !isTotalHitsGreaterThan(resp.hits.total as unknown as TotalHits, maxResultWindow); diff --git a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts index 32d60cac8c7d..f941c610962d 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts @@ -41,7 +41,7 @@ import { IDynamicStyleProperty } from '../../styles/vector/properties/dynamic_st import { IField } from '../../fields/field'; import { FieldFormatter } from '../../../../common/constants'; import { isValidStringConfig } from '../../util/valid_string_config'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; export function isSearchSourceAbortError(error: Error) { return error.name === 'AbortError'; @@ -62,6 +62,7 @@ export interface IESSource extends IVectorSource { timeFilters, searchSessionId, inspectorAdapters, + executionContext, }: { layerName: string; style: IVectorStyle; @@ -71,6 +72,7 @@ export interface IESSource extends IVectorSource { timeFilters: TimeRange; searchSessionId?: string; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }): Promise; } @@ -310,7 +312,10 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource searchSource.fetch$({ abortSignal: abortController.signal, legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_source:bounds'), + executionContext: mergeExecutionContext( + { description: 'es_source:bounds' }, + boundsFilters.executionContext + ), }) ); @@ -451,6 +456,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource timeFilters, searchSessionId, inspectorAdapters, + executionContext, }: { layerName: string; style: IVectorStyle; @@ -460,6 +466,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource timeFilters: TimeRange; searchSessionId?: string; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }): Promise { const promises = dynamicStyleProps.map((dynamicStyleProp) => { return dynamicStyleProp.getFieldMetaRequest(); @@ -506,7 +513,10 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource } ), searchSessionId, - executionContext: makePublicExecutionContext('es_source:style_meta'), + executionContext: mergeExecutionContext( + { description: 'es_source:style_meta' }, + executionContext + ), requestsAdapter: inspectorAdapters.requests, }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts index a8a756eeb3ae..ce4f70c63561 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts @@ -32,7 +32,7 @@ import { PropertiesMap } from '../../../../common/elasticsearch_util'; import { isValidStringConfig } from '../../util/valid_string_config'; import { ITermJoinSource } from '../term_join_source'; import { IField } from '../../fields/field'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; const TERMS_AGG_NAME = 'join'; const TERMS_BUCKET_KEYS_TO_IGNORE = ['key', 'doc_count']; @@ -159,7 +159,10 @@ export class ESTermSource extends AbstractESAggSource implements ITermJoinSource }, }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_term_source:terms'), + executionContext: mergeExecutionContext( + { description: 'es_term_source:terms' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); diff --git a/x-pack/plugins/maps/public/classes/sources/execution_context_utils.test.ts b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.test.ts new file mode 100644 index 000000000000..5fc914337920 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APP_ID } from '../../../common/constants'; +import { getExecutionContextId, mergeExecutionContext } from './execution_context_utils'; + +describe('mergeExecutionContext', () => { + test('should merge with context', () => { + const mergeContext = { description: 'es_pew_pew_source:connections' }; + const context = { name: APP_ID }; + expect(mergeExecutionContext(mergeContext, context)).toEqual({ + description: 'es_pew_pew_source:connections', + name: APP_ID, + }); + }); + + test('should merge with hierarchical context', () => { + const mergeContext = { description: 'es_pew_pew_source:connections' }; + const context = { + name: 'dashboard', + child: { + name: APP_ID, + }, + }; + expect(mergeExecutionContext(mergeContext, context)).toEqual({ + name: 'dashboard', + child: { + description: 'es_pew_pew_source:connections', + name: APP_ID, + }, + }); + }); + + test('should not merge if "maps" context can not be found', () => { + const mergeContext = { description: 'es_pew_pew_source:connections' }; + const context = { + name: 'dashboard', + child: { + name: 'lens', + }, + }; + expect(mergeExecutionContext(mergeContext, context)).toEqual({ + name: 'dashboard', + child: { + name: 'lens', + }, + }); + }); +}); + +describe('getExecutionContextId', () => { + test('should return executionContextId', () => { + const context = { name: APP_ID, id: 'map1234' }; + expect(getExecutionContextId(context)).toBe('map1234'); + }); + + test('should return executionContextId with hierarchical context', () => { + const context = { + name: 'dashboard', + child: { + name: APP_ID, + id: 'map1234', + }, + }; + expect(getExecutionContextId(context)).toBe('map1234'); + }); + + test('should return undefined if "maps" context can not be found', () => { + const context = { + name: 'dashboard', + child: { + name: 'lens', + id: 'lens1234', + }, + }; + expect(getExecutionContextId(context)).toBeUndefined(); + }); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/execution_context_utils.ts b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.ts new file mode 100644 index 000000000000..4fda822e4199 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaExecutionContext } from '@kbn/core/public'; +import { APP_ID } from '../../../common/constants'; + +export function mergeExecutionContext( + mergeContext: Partial, + context: KibanaExecutionContext = {} +): KibanaExecutionContext { + if (isMapContext(context)) { + return { + ...context, + ...mergeContext, + }; + } + + if (context.child !== undefined) { + return { + ...context, + child: { + ...mergeExecutionContext(mergeContext, context.child), + }, + }; + } + + return context; +} + +export function getExecutionContextId(context: KibanaExecutionContext = {}): string | undefined { + if (isMapContext(context)) { + return context.id; + } + + if (context.child !== undefined) { + return getExecutionContextId(context.child); + } + + return undefined; +} + +function isMapContext(context: KibanaExecutionContext): boolean { + return context.name === APP_ID; +} diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx index 539503b1ebaf..a76d0268c836 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx @@ -208,10 +208,7 @@ export class MVTSingleLayerVectorSource extends AbstractSource implements IMvtVe return []; } - async getTooltipProperties( - properties: GeoJsonProperties, - featureId?: string | number - ): Promise { + async getTooltipProperties(properties: GeoJsonProperties): Promise { const tooltips = []; for (const key in properties) { if (properties.hasOwnProperty(key)) { diff --git a/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts b/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts index 9228fe1de449..6e3b12be53aa 100644 --- a/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts @@ -6,6 +6,7 @@ */ import { GeoJsonProperties } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Query } from '@kbn/data-plugin/common/query'; import { Adapters } from '@kbn/inspector-plugin/common/adapters'; import { IField } from '../../fields/field'; @@ -34,6 +35,9 @@ export interface ITermJoinSource extends ISource { getId(): string; getRightFields(): IField[]; - getTooltipProperties(properties: GeoJsonProperties): Promise; + getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise; getFieldByName(fieldName: string): IField | null; } diff --git a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx index 0620386fda4c..020b99fc429e 100644 --- a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx @@ -14,6 +14,7 @@ import { Polygon, Position, } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import type { Query } from '@kbn/data-plugin/common'; import type { MapGeoJSONFeature } from '@kbn/mapbox-gl'; @@ -62,6 +63,7 @@ export interface BoundsRequestMeta { timeslice?: Timeslice; isFeatureEditorOpenForLayer: boolean; joinKeyFilter?: Filter; + executionContext: KibanaExecutionContext; } export interface GetFeatureActionsArgs { @@ -84,7 +86,10 @@ export interface GetFeatureActionsArgs { export interface IVectorSource extends ISource { isMvt(): boolean; - getTooltipProperties(properties: GeoJsonProperties): Promise; + getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise; getBoundsForFilters( layerDataFilters: BoundsRequestMeta, registerCancelCallback: (callback: () => void) => void @@ -202,7 +207,10 @@ export class AbstractVectorSource extends AbstractSource implements IVectorSourc } // Allow source to filter and format feature properties before displaying to user - async getTooltipProperties(properties: GeoJsonProperties): Promise { + async getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise { const tooltipProperties: ITooltipProperty[] = []; for (const key in properties) { if (key.startsWith('__kbn')) { diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap index c5784cf9a309..de542ccfc8e8 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap @@ -6,6 +6,7 @@ exports[`TooltipControl render should render hover tooltip 1`] = ` void; + executionContext: KibanaExecutionContext; } export class TooltipControl extends Component { @@ -376,6 +378,7 @@ export class TooltipControl extends Component { isLocked={isLocked} index={index} loadFeatureGeometry={this._getFeatureGeometry} + executionContext={this.props.executionContext} /> ); }); diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx index cb96f364925b..849f052827eb 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx @@ -84,6 +84,7 @@ const defaultProps = { loadFeatureGeometry: () => { return null; }, + executionContext: {}, }; describe('TooltipPopover', () => { diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx index 5c1f8ef6a83a..0263c523f0fc 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx @@ -9,6 +9,7 @@ import React, { Component, RefObject } from 'react'; import { EuiPopover, EuiText } from '@elastic/eui'; import type { Map as MbMap } from '@kbn/mapbox-gl'; import { GeoJsonProperties, Geometry } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Filter } from '@kbn/es-query'; import { ActionExecutionContext, Action } from '@kbn/ui-actions-plugin/public'; import { FeaturesTooltip } from './features_tooltip'; @@ -39,6 +40,7 @@ interface Props { mbMap: MbMap; onSingleValueTrigger?: (actionId: string, key: string, value: RawValue) => void; renderTooltipContent?: RenderToolTipContent; + executionContext: KibanaExecutionContext; } interface State { @@ -98,7 +100,7 @@ export class TooltipPopover extends Component { return []; } - return await tooltipLayer.getPropertiesForTooltip(properties); + return await tooltipLayer.getPropertiesForTooltip(properties, this.props.executionContext); }; _getLayerName = async (layerId: string) => { diff --git a/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx b/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx index bcb5aea3cca8..1528290dcde2 100644 --- a/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx +++ b/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx @@ -15,6 +15,13 @@ import { MapSavedObjectAttributes } from '../../common/map_saved_object_type'; jest.mock('../kibana_services', () => { return { + getExecutionContextService() { + return { + get: () => { + return {}; + }, + }; + }, getHttp() { return { basePath: { diff --git a/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx b/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx index c109b4f940d8..3c6eaf77062c 100644 --- a/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx +++ b/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx @@ -21,6 +21,7 @@ import { startWith, } from 'rxjs/operators'; import { Unsubscribe } from 'redux'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { EuiEmptyPrompt } from '@elastic/eui'; import { type Filter } from '@kbn/es-query'; import { KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; @@ -46,6 +47,7 @@ import { updateLayerById, setGotoWithCenter, setEmbeddableSearchContext, + setExecutionContext, } from '../actions'; import { getIsLayerTOCOpen, getOpenTOCDetails } from '../selectors/ui_selectors'; import { @@ -80,13 +82,14 @@ import { } from '../../common/constants'; import { RenderToolTipContent } from '../classes/tooltips/tooltip_property'; import { - getUiActions, + getChartsPaletteServiceGetColor, getCoreI18n, + getExecutionContextService, getHttp, - getChartsPaletteServiceGetColor, - getSpacesApi, getSearchService, + getSpacesApi, getTheme, + getUiActions, } from '../kibana_services'; import { LayerDescriptor, MapExtent } from '../../common/descriptor_types'; import { MapContainer } from '../connected_components/map_container'; @@ -194,6 +197,8 @@ export class MapEmbeddable return; } + this._savedMap.getStore().dispatch(setExecutionContext(this.getExecutionContext())); + // deferred loading of this embeddable is complete this.setInitializationFinished(); @@ -203,6 +208,23 @@ export class MapEmbeddable } } + private getExecutionContext() { + const parentContext = getExecutionContextService().get(); + const mapContext: KibanaExecutionContext = { + type: APP_ID, + name: APP_ID, + id: this.id, + url: this.output.editPath, + }; + + return parentContext + ? { + ...parentContext, + child: mapContext, + } + : mapContext; + } + private _initializeStore() { this._dispatchSetChartsPaletteServiceGetColor(this.input.syncColors); diff --git a/x-pack/plugins/maps/public/kibana_services.ts b/x-pack/plugins/maps/public/kibana_services.ts index 0e4ee6d913e3..b9c64946f7fa 100644 --- a/x-pack/plugins/maps/public/kibana_services.ts +++ b/x-pack/plugins/maps/public/kibana_services.ts @@ -41,7 +41,7 @@ export const getIndexPatternSelectComponent = () => pluginsStart.unifiedSearch.ui.IndexPatternSelect; export const getSearchBar = () => pluginsStart.unifiedSearch.ui.SearchBar; export const getHttp = () => coreStart.http; -export const getExecutionContext = () => coreStart.executionContext; +export const getExecutionContextService = () => coreStart.executionContext; export const getTimeFilter = () => pluginsStart.data.query.timefilter.timefilter; export const getToasts = () => coreStart.notifications.toasts; export const getSavedObjectsClient = () => coreStart.savedObjects.client; diff --git a/x-pack/plugins/maps/public/reducers/map/map.ts b/x-pack/plugins/maps/public/reducers/map/map.ts index 41de8ab0b8b0..8215cf0fe7f9 100644 --- a/x-pack/plugins/maps/public/reducers/map/map.ts +++ b/x-pack/plugins/maps/public/reducers/map/map.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { APP_ID } from '../../../common/constants'; import { SET_SELECTED_LAYER, UPDATE_LAYER_ORDER, @@ -46,6 +47,7 @@ import { TRACK_MAP_SETTINGS, UPDATE_MAP_SETTING, UPDATE_EDIT_STATE, + SET_EXECUTION_CONTEXT, } from '../../actions/map_action_constants'; import { getDefaultMapSettings } from './default_map_settings'; @@ -63,6 +65,7 @@ import { startDataRequest, stopDataRequest, updateSourceDataRequest } from './da import { MapState } from './types'; export const DEFAULT_MAP_STATE: MapState = { + executionContext: { name: APP_ID }, ready: false, mapInitError: null, goto: null, @@ -322,6 +325,12 @@ export function map(state: MapState = DEFAULT_MAP_STATE, action: Record & { }; export type MapState = { + executionContext: KibanaExecutionContext; ready: boolean; mapInitError?: string | null; goto?: Goto | null; diff --git a/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx b/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx index 3dab6db00885..759189f509db 100644 --- a/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx +++ b/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx @@ -17,7 +17,7 @@ import { APP_ID, getEditPath, MAP_PATH, MAP_SAVED_OBJECT_TYPE } from '../../../c import { getMapsCapabilities, getCoreChrome, - getExecutionContext, + getExecutionContextService, getNavigateToApp, getSavedObjectsClient, getUiSettings, @@ -104,10 +104,10 @@ interface Props { } export function MapsListView(props: Props) { - getExecutionContext().set({ + getExecutionContextService().set({ type: 'application', + name: APP_ID, page: 'list', - id: '', }); const isReadOnly = !getMapsCapabilities().save; diff --git a/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts b/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts index af9054cd9957..649c5033fb62 100644 --- a/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts +++ b/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts @@ -8,6 +8,7 @@ import { connect } from 'react-redux'; import { ThunkDispatch } from 'redux-thunk'; import { AnyAction } from 'redux'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Filter } from '@kbn/es-query'; import type { Query, TimeRange } from '@kbn/es-query'; import { MapApp } from './map_app'; @@ -19,7 +20,7 @@ import { getTimeFilters, hasDirtyState, } from '../../../selectors/map_selectors'; -import { setQuery, enableFullScreen, openMapSettings } from '../../../actions'; +import { setQuery, setExecutionContext, enableFullScreen, openMapSettings } from '../../../actions'; import { FLYOUT_STATE } from '../../../reducers/ui'; import { getInspectorAdapters } from '../../../reducers/non_serializable_instances'; import { MapStoreState } from '../../../reducers/store'; @@ -65,6 +66,8 @@ function mapDispatchToProps(dispatch: ThunkDispatch dispatch(enableFullScreen()), openMapSettings: () => dispatch(openMapSettings()), + setExecutionContext: (executionContext: KibanaExecutionContext) => + dispatch(setExecutionContext(executionContext)), }; } diff --git a/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx b/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx index bf8963de0461..969334551adf 100644 --- a/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx +++ b/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx @@ -9,7 +9,12 @@ import React from 'react'; import _ from 'lodash'; import { finalize, switchMap, tap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; -import { AppLeaveAction, AppMountParameters, ScopedHistory } from '@kbn/core/public'; +import { + AppLeaveAction, + AppMountParameters, + KibanaExecutionContext, + ScopedHistory, +} from '@kbn/core/public'; import { Adapters } from '@kbn/embeddable-plugin/public'; import { Subscription } from 'rxjs'; import { type Filter, FilterStateStore, type Query, type TimeRange } from '@kbn/es-query'; @@ -29,7 +34,7 @@ import { } from '@kbn/kibana-utils-plugin/public'; import { getData, - getExecutionContext, + getExecutionContextService, getCoreChrome, getIndexPatternService, getMapsCapabilities, @@ -84,6 +89,7 @@ export interface Props { query: Query | undefined; setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; history: ScopedHistory; + setExecutionContext: (executionContext: KibanaExecutionContext) => void; } export interface State { @@ -123,11 +129,15 @@ export class MapApp extends React.Component { componentDidMount() { this._isMounted = true; - getExecutionContext().set({ + const executionContext = { type: 'application', - page: 'editor', + name: APP_ID, + url: window.location.pathname, id: this.props.savedMap.getSavedObjectId() || 'new', - }); + page: 'editor', + }; + getExecutionContextService().set(executionContext); // set execution context in core ExecutionContextStartService + this.props.setExecutionContext(executionContext); // set execution context in redux store this._autoRefreshSubscription = getTimeFilter() .getAutoRefreshFetch$() diff --git a/x-pack/plugins/maps/public/selectors/map_selectors.test.ts b/x-pack/plugins/maps/public/selectors/map_selectors.test.ts index dcd6a727238d..fc051d7a48a1 100644 --- a/x-pack/plugins/maps/public/selectors/map_selectors.test.ts +++ b/x-pack/plugins/maps/public/selectors/map_selectors.test.ts @@ -71,6 +71,7 @@ describe('getDataFilters', () => { minLon: -0.25, }; const isReadOnly = false; + const executionContext = {}; test('should set buffer as searchSessionMapBuffer when using searchSessionId', () => { const dataFilters = getDataFilters.resultFunc( @@ -84,7 +85,8 @@ describe('getDataFilters', () => { embeddableSearchContext, searchSessionId, searchSessionMapBuffer, - isReadOnly + isReadOnly, + executionContext ); expect(dataFilters.buffer).toEqual(searchSessionMapBuffer); }); @@ -101,7 +103,8 @@ describe('getDataFilters', () => { embeddableSearchContext, searchSessionId, undefined, - isReadOnly + isReadOnly, + executionContext ); expect(dataFilters.buffer).toEqual(mapBuffer); }); diff --git a/x-pack/plugins/maps/public/selectors/map_selectors.ts b/x-pack/plugins/maps/public/selectors/map_selectors.ts index e8aae1c61dda..798e24a6141e 100644 --- a/x-pack/plugins/maps/public/selectors/map_selectors.ts +++ b/x-pack/plugins/maps/public/selectors/map_selectors.ts @@ -8,6 +8,7 @@ import { createSelector } from 'reselect'; import { FeatureCollection } from 'geojson'; import _ from 'lodash'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/data-plugin/common'; import { Filter } from '@kbn/es-query'; import type { TimeRange } from '@kbn/es-query'; @@ -239,6 +240,10 @@ export function getDataRequestDescriptor(state: MapStoreState, layerId: string, }); } +export function getExecutionContext(state: MapStoreState): KibanaExecutionContext { + return state.map.executionContext; +} + export const getDataFilters = createSelector( getMapExtent, getMapBuffer, @@ -251,6 +256,7 @@ export const getDataFilters = createSelector( getSearchSessionId, getSearchSessionMapBuffer, getIsReadOnly, + getExecutionContext, ( mapExtent, mapBuffer, @@ -262,7 +268,8 @@ export const getDataFilters = createSelector( embeddableSearchContext, searchSessionId, searchSessionMapBuffer, - isReadOnly + isReadOnly, + executionContext ) => { return { extent: mapExtent, @@ -275,6 +282,7 @@ export const getDataFilters = createSelector( embeddableSearchContext, searchSessionId, isReadOnly, + executionContext, }; } ); diff --git a/x-pack/plugins/maps/public/util.test.js b/x-pack/plugins/maps/public/util.test.js deleted file mode 100644 index f07d60fc3d70..000000000000 --- a/x-pack/plugins/maps/public/util.test.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { makePublicExecutionContext } from './util'; - -describe('makePublicExecutionContext', () => { - let injectedContext = {}; - beforeAll(() => { - require('./kibana_services').getExecutionContext = () => ({ - get: () => injectedContext, - }); - }); - - test('creates basic context when no top level context is provided', () => { - const context = makePublicExecutionContext('test'); - expect(context).toStrictEqual({ - description: 'test', - name: 'maps', - type: 'application', - url: '/', - }); - }); - - test('merges with top level context if its from the same app', () => { - injectedContext = { - name: 'maps', - id: '1234', - }; - const context = makePublicExecutionContext('test'); - expect(context).toStrictEqual({ - description: 'test', - name: 'maps', - type: 'application', - url: '/', - id: '1234', - }); - }); - - test('nests inside top level context if its from a different app', () => { - injectedContext = { - name: 'other-app', - id: '1234', - }; - const context = makePublicExecutionContext('test'); - expect(context).toStrictEqual({ - name: 'other-app', - id: '1234', - child: { - description: 'test', - type: 'application', - name: 'maps', - url: '/', - }, - }); - }); -}); diff --git a/x-pack/plugins/maps/public/util.ts b/x-pack/plugins/maps/public/util.ts index 68a616f87000..364f72c0b564 100644 --- a/x-pack/plugins/maps/public/util.ts +++ b/x-pack/plugins/maps/public/util.ts @@ -6,15 +6,8 @@ */ import { EMSClient, FileLayer, TMSService } from '@elastic/ems-client'; -import type { KibanaExecutionContext } from '@kbn/core/public'; -import { - getTilemap, - getEMSSettings, - getMapsEmsStart, - getExecutionContext, -} from './kibana_services'; +import { getTilemap, getEMSSettings, getMapsEmsStart } from './kibana_services'; import { getLicenseId } from './licensed_features'; -import { makeExecutionContext } from '../common/execution_context'; export function getKibanaTileMap(): unknown { return getTilemap(); @@ -61,22 +54,3 @@ async function getEMSClient(): Promise { export function isRetina(): boolean { return window.devicePixelRatio === 2; } - -export function makePublicExecutionContext(description: string): KibanaExecutionContext { - const topLevelContext = getExecutionContext().get(); - const context = makeExecutionContext({ - url: window.location.pathname, - description, - }); - - // Distinguish between running in maps app vs. embedded - return topLevelContext.name !== undefined && topLevelContext.name !== context.name - ? { - ...topLevelContext, - child: context, - } - : { - ...topLevelContext, - ...context, - }; -} diff --git a/x-pack/plugins/maps/server/mvt/mvt_routes.ts b/x-pack/plugins/maps/server/mvt/mvt_routes.ts index 4ed26677b628..0b2e8c51d1eb 100644 --- a/x-pack/plugins/maps/server/mvt/mvt_routes.ts +++ b/x-pack/plugins/maps/server/mvt/mvt_routes.ts @@ -15,12 +15,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import { errors } from '@elastic/elasticsearch'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { + APP_ID, MVT_GETTILE_API_PATH, API_ROOT_PATH, MVT_GETGRIDTILE_API_PATH, RENDER_AS, } from '../../common/constants'; -import { makeExecutionContext } from '../../common/execution_context'; import { getAggsTileRequest, getHitsTileRequest } from '../../common/mvt_request_body'; const CACHE_TIMEOUT_SECONDS = 60 * 60; @@ -50,6 +50,7 @@ export function initMVTRoutes({ requestBody: schema.string(), index: schema.string(), token: schema.maybe(schema.string()), + executionContextId: schema.maybe(schema.string()), }), }, }, @@ -88,8 +89,11 @@ export function initMVTRoutes({ context, core, executionContext: makeExecutionContext({ + type: 'server', + name: APP_ID, description: 'mvt:get_hits_tile', url: `${API_ROOT_PATH}/${MVT_GETTILE_API_PATH}/${z}/${x}/${y}.pbf`, + id: query.executionContextId, }), logger, path: tileRequest.path, @@ -117,6 +121,7 @@ export function initMVTRoutes({ renderAs: schema.string(), token: schema.maybe(schema.string()), gridPrecision: schema.number(), + executionContextId: schema.maybe(schema.string()), }), }, }, @@ -157,8 +162,11 @@ export function initMVTRoutes({ context, core, executionContext: makeExecutionContext({ + type: 'server', + name: APP_ID, description: 'mvt:get_aggs_tile', url: `${API_ROOT_PATH}/${MVT_GETGRIDTILE_API_PATH}/${z}/${x}/${y}.pbf`, + id: query.executionContextId, }), logger, path: tileRequest.path, @@ -270,3 +278,32 @@ function makeAbortController( }); return abortController; } + +function makeExecutionContext({ + type, + name, + description, + url, + id, +}: { + type: string; + name: string; + description: string; + url: string; + id?: string; +}): KibanaExecutionContext { + return id !== undefined + ? { + type, + name, + description, + url, + id, + } + : { + type, + name, + description, + url, + }; +} diff --git a/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx b/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx index 2d95f0c97169..74f4bf66a0e0 100644 --- a/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx +++ b/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx @@ -76,6 +76,7 @@ export const NotificationsIndicator: FC = () => { = ({ node, type, height }) => { valueFormatter: (size: number) => bytesFormatter(size), }, shape: { - fillColor: (d: ShapeTreeNode) => getMemoryItemColor(d.dataName as JobType), + fillColor: (dataName) => getMemoryItemColor(dataName as JobType), }, }, { @@ -170,7 +169,7 @@ export const JobMemoryTreeMap: FC = ({ node, type, height }) => { }, }, shape: { - fillColor: (d: ShapeTreeNode) => { + fillColor: (dataName, index, d) => { // color the shape the same as its parent. const parentId = d.parent.path[d.parent.path.length - 1].value as JobType; return getMemoryItemColor(parentId); diff --git a/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/memory_preview_chart.tsx b/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/memory_preview_chart.tsx index f57fa18dc428..bb377cd60b54 100644 --- a/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/memory_preview_chart.tsx +++ b/x-pack/plugins/ml/public/application/memory_usage/nodes_overview/memory_preview_chart.tsx @@ -138,7 +138,15 @@ export const MemoryPreviewChart: FC = ({ memoryOverview }), }, ]} - marker={} + marker={ + + } markerPosition={Position.Top} /> diff --git a/x-pack/plugins/security_solution/cypress/screens/inspect.ts b/x-pack/plugins/security_solution/cypress/screens/inspect.ts index df509b06b081..76935b96b96f 100644 --- a/x-pack/plugins/security_solution/cypress/screens/inspect.ts +++ b/x-pack/plugins/security_solution/cypress/screens/inspect.ts @@ -99,13 +99,37 @@ export const INSPECT_BUTTONS_IN_SECURITY: InspectButtonMetadata[] = [ title: 'all hosts', panelSelector: HOSTS_VISUALIZATION, tab: ALL_HOSTS_TAB, - embeddableId: 'hostsKpiHostsQuery', + embeddableId: 'hostsKpiHostsQuery-hosts-metric-embeddable', + }, + { + title: 'all hosts', + panelSelector: HOSTS_VISUALIZATION, + tab: ALL_HOSTS_TAB, + embeddableId: 'hostsKpiHostsQuery-area-embeddable', + }, + { + title: 'Unique IPs', + panelSelector: UNIQUE_IPS_VISUALIZATIONS, + tab: ALL_HOSTS_TAB, + embeddableId: 'hostsKpiUniqueIpsQuery-uniqueSourceIps-metric-embeddable', }, { title: 'Unique IPs', panelSelector: UNIQUE_IPS_VISUALIZATIONS, tab: ALL_HOSTS_TAB, - embeddableId: 'hostsKpiUniqueIpsQuery', + embeddableId: 'hostsKpiUniqueIpsQuery-uniqueDestinationIps-metric-embeddable', + }, + { + title: 'Unique IPs', + panelSelector: UNIQUE_IPS_VISUALIZATIONS, + tab: ALL_HOSTS_TAB, + embeddableId: 'hostsKpiUniqueIpsQuery-bar-embeddable', + }, + { + title: 'Unique IPs', + panelSelector: UNIQUE_IPS_VISUALIZATIONS, + tab: ALL_HOSTS_TAB, + embeddableId: 'hostsKpiUniqueIpsQuery-area-embeddable', }, ], }, @@ -117,32 +141,39 @@ export const INSPECT_BUTTONS_IN_SECURITY: InspectButtonMetadata[] = [ title: 'Network events', panelSelector: NETWORK_EVENTS_VISUALIZATION, tab: NETWORK_FLOW_TAB, - embeddableId: 'networkKpiNetworkEventsQuery', + embeddableId: 'networkKpiNetworkEventsQuery-networkEvents-metric-embeddable', }, { title: 'DNS queries', panelSelector: NETWORK_DNS_VISUALIZATION, tab: NETWORK_FLOW_TAB, - embeddableId: 'networkKpiDnsQuery', + embeddableId: 'networkKpiDnsQuery-dnsQueries-metric-embeddable', }, { title: 'Unique flow IDs', panelSelector: NETWORK_UNIQUE_FLOW_VISUALIZATION, tab: NETWORK_FLOW_TAB, - embeddableId: 'networkKpiUniqueFlowsQuery', + embeddableId: 'networkKpiUniqueFlowsQuery-uniqueFlowId-metric-embeddable', }, { title: 'TLS handshakes', panelSelector: NETWORK_TLS_HANDSHAKE_VISUALIZATION, tab: NETWORK_FLOW_TAB, - embeddableId: 'networkKpiTlsHandshakesQuery', + embeddableId: 'networkKpiTlsHandshakesQuery-tlsHandshakes-metric-embeddable', }, { title: 'Unique private IPs', panelSelector: UNIQUE_IPS_VISUALIZATIONS, tab: NETWORK_FLOW_TAB, - embeddableId: 'networkKpiUniquePrivateIpsQuery', + embeddableId: 'networkKpiUniquePrivateIpsQuery-uniqueSourcePrivateIps-metric-embeddable', + }, + { + title: 'Unique private IPs', + panelSelector: UNIQUE_IPS_VISUALIZATIONS, + tab: NETWORK_FLOW_TAB, + embeddableId: + 'networkKpiUniquePrivateIpsQuery-uniqueDestinationPrivateIps-metric-embeddable', }, ], tables: [ @@ -196,13 +227,37 @@ export const INSPECT_BUTTONS_IN_SECURITY: InspectButtonMetadata[] = [ { title: 'Users', panelSelector: USERS_VISUALIZATION, - embeddableId: 'TotalUsersKpiQuery', + embeddableId: 'TotalUsersKpiQuery-users-metric-embeddable', + tab: ALL_USERS_TAB, + }, + { + title: 'Users', + panelSelector: USERS_VISUALIZATION, + embeddableId: 'TotalUsersKpiQuery-area-embeddable', + tab: ALL_USERS_TAB, + }, + { + title: 'User authentications', + panelSelector: AUTHENTICATION_VISUALIZATION, + embeddableId: 'usersKpiAuthenticationsQuery-authenticationsSuccess-metric-embeddable', + tab: ALL_USERS_TAB, + }, + { + title: 'User authentications', + panelSelector: AUTHENTICATION_VISUALIZATION, + embeddableId: 'usersKpiAuthenticationsQuery-authenticationsFailure-metric-embeddable', + tab: ALL_USERS_TAB, + }, + { + title: 'User authentications', + panelSelector: AUTHENTICATION_VISUALIZATION, + embeddableId: 'usersKpiAuthenticationsQuery-bar-embeddable', tab: ALL_USERS_TAB, }, { title: 'User authentications', panelSelector: AUTHENTICATION_VISUALIZATION, - embeddableId: 'usersKpiAuthenticationsQuery', + embeddableId: 'usersKpiAuthenticationsQuery-area-embeddable', tab: ALL_USERS_TAB, }, { diff --git a/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.test.ts b/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.test.ts index 216de0885139..f8404cc7d33e 100644 --- a/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.test.ts +++ b/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.test.ts @@ -7,8 +7,8 @@ import { getRiskScorePalette, RISK_SCORE_STEPS } from '../chart_palette'; import { maxRiskSubAggregations } from '../flatten/mocks/mock_buckets'; -import type { DataName, FillColorDatum, Path } from '.'; import { getGroupFromPath, getLayersOneDimension, getLayersMultiDimensional } from '.'; +import type { Key, ArrayNode } from '@elastic/charts'; describe('layers', () => { const colorPalette = getRiskScorePalette(RISK_SCORE_STEPS); @@ -16,36 +16,22 @@ describe('layers', () => { describe('getGroupFromPath', () => { it('returns the expected group from the path', () => { expect( - getGroupFromPath({ - path: [ - { index: 0, value: '__null_small_multiples_key__' }, - { index: 0, value: '__root_key__' }, - { index: 0, value: 'matches everything' }, - { index: 0, value: 'Host-k8iyfzraq9' }, - ], - }) + getGroupFromPath([ + { index: 0, value: '__null_small_multiples_key__' }, + { index: 0, value: '__root_key__' }, + { index: 0, value: 'matches everything' }, + { index: 0, value: 'Host-k8iyfzraq9' }, + ]) ).toEqual('matches everything'); }); - it('returns undefined when path is undefined', () => { - const datumWithUndefinedPath: FillColorDatum = {}; - - expect(getGroupFromPath(datumWithUndefinedPath)).toBeUndefined(); - }); - it('returns undefined when path is an empty array', () => { - expect( - getGroupFromPath({ - path: [], - }) - ).toBeUndefined(); + expect(getGroupFromPath([])).toBeUndefined(); }); it('returns undefined when path is an array with only one value', () => { expect( - getGroupFromPath({ - path: [{ index: 0, value: '__null_small_multiples_key__' }], - }) + getGroupFromPath([{ index: 0, value: '__null_small_multiples_key__' }]) ).toBeUndefined(); }); }); @@ -80,14 +66,14 @@ describe('layers', () => { }); it('returns the expected shape fillColor function', () => { - const dataName: DataName = { dataName: 'mimikatz process started' }; + const dataName = 'mimikatz process started'; expect( getLayersOneDimension({ colorPalette, maxRiskSubAggregations })[0].shape.fillColor(dataName) ).toEqual('#e7664c'); }); it('return the default fill color when dataName is not found in the maxRiskSubAggregations', () => { - const dataName: DataName = { dataName: 'this does not exist' }; + const dataName = 'this does not exist'; expect( getLayersOneDimension({ colorPalette, maxRiskSubAggregations })[0].shape.fillColor(dataName) ).toEqual('#54b399'); @@ -164,11 +150,14 @@ describe('layers', () => { colorPalette, layer0FillColor, maxRiskSubAggregations, - })[1].shape.fillColor as ({ dataName, path }: { dataName: string; path: Path[] }) => string; + })[1].shape.fillColor as ( + dataName: Key, + sortIndex: number, + node: Pick + ) => string; expect( - fillColorFn({ - dataName: 'Host-k8iyfzraq9', + fillColorFn('Host-k8iyfzraq9', 0, { path: [ { index: 0, value: '__null_small_multiples_key__' }, { index: 0, value: '__root_key__' }, @@ -184,11 +173,14 @@ describe('layers', () => { colorPalette, layer0FillColor, maxRiskSubAggregations, - })[1].shape.fillColor as ({ dataName, path }: { dataName: string; path: Path[] }) => string; + })[1].shape.fillColor as ( + dataName: Key, + sortIndex: number, + node: Pick + ) => string; expect( - fillColorFn({ - dataName: 'nope', + fillColorFn('nope', 0, { path: [ { index: 0, value: '__null_small_multiples_key__' }, { index: 0, value: '__root_key__' }, diff --git a/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.ts b/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.ts index 09a4d95bdcb0..788ac4c34027 100644 --- a/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.ts +++ b/x-pack/plugins/security_solution/public/common/components/alerts_treemap/lib/layers/index.ts @@ -5,24 +5,11 @@ * 2.0. */ -import type { Datum } from '@elastic/charts'; +import type { Datum, Key, ArrayNode } from '@elastic/charts'; import { getFillColor } from '../chart_palette'; import { getLabel } from '../labels'; -export interface DataName { - dataName: string; -} - -export interface Path { - index: number; - value: string; -} - -export interface FillColorDatum { - path?: Path[]; -} - // common functions used by getLayersOneDimension and getLayersMultiDimensional: const valueFormatter = (d: number) => `${d}`; const groupByRollup = (d: Datum) => d.key; @@ -30,13 +17,10 @@ const groupByRollup = (d: Datum) => d.key; /** * Extracts the first group name from the data representing the second group */ -export const getGroupFromPath = (datum: FillColorDatum): string | undefined => { +export const getGroupFromPath = (path: ArrayNode['path']): string | undefined => { const OFFSET_FROM_END = 2; // The offset from the end of the path array containing the group - - const pathLength = datum.path?.length ?? 0; - const groupIndex = pathLength - OFFSET_FROM_END; - - return Array.isArray(datum.path) && groupIndex > 0 ? datum.path[groupIndex].value : undefined; + const groupIndex = path.length - OFFSET_FROM_END; + return groupIndex > 0 ? path[groupIndex].value : undefined; }; export const getLayersOneDimension = ({ @@ -53,9 +37,9 @@ export const getLayersOneDimension = ({ groupByRollup, nodeLabel: (d: Datum) => getLabel({ baseLabel: d, riskScore: maxRiskSubAggregations[d] }), shape: { - fillColor: (d: DataName) => + fillColor: (dataName: Key) => getFillColor({ - riskScore: maxRiskSubAggregations[d.dataName] ?? 0, + riskScore: maxRiskSubAggregations[dataName] ?? 0, colorPalette, }), }, @@ -88,8 +72,8 @@ export const getLayersMultiDimensional = ({ groupByRollup: (d: Datum) => d.stackByField1Key, // different implementation than layer 0 nodeLabel: (d: Datum) => `${d}`, shape: { - fillColor: (d: FillColorDatum) => { - const groupFromPath = getGroupFromPath(d) ?? ''; + fillColor: (dataName: Key, sortIndex: number, node: Pick) => { + const groupFromPath = getGroupFromPath(node.path) ?? ''; return getFillColor({ riskScore: maxRiskSubAggregations[groupFromPath] ?? 0, diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_kpis/severity_level_panel/severity_level_chart.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_kpis/severity_level_panel/severity_level_chart.tsx index 5b2599a3a460..cdf770b2f3d7 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_kpis/severity_level_panel/severity_level_chart.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_kpis/severity_level_panel/severity_level_chart.tsx @@ -9,7 +9,6 @@ import { ALERT_SEVERITY } from '@kbn/rule-data-utils'; import styled from 'styled-components'; import { EuiFlexGroup, EuiFlexItem, EuiInMemoryTable, EuiLoadingSpinner } from '@elastic/eui'; import type { SortOrder } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import type { ShapeTreeNode } from '@elastic/charts'; import type { SeverityBuckets as SeverityData } from '../../../../overview/components/detection_response/alerts_by_status/types'; import type { FillColor } from '../../../../common/components/charts/donutchart'; import { DonutChart } from '../../../../common/components/charts/donutchart'; @@ -44,8 +43,8 @@ export const SeverityLevelChart: React.FC = ({ : 0; }, [data]); - const fillColor: FillColor = useCallback((d: ShapeTreeNode) => { - return getSeverityColor(d.dataName); + const fillColor: FillColor = useCallback((dataName) => { + return getSeverityColor(dataName); }, []); const sorting: { sort: { field: keyof SeverityData; direction: SortOrder } } = { diff --git a/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.test.tsx b/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.test.tsx index 87760cc856e7..61558b0be65f 100644 --- a/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.test.tsx +++ b/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.test.tsx @@ -15,11 +15,7 @@ import type { LensAttributes } from '../../../common/components/visualization_ac jest.mock('../../../common/components/visualization_actions/actions'); -jest.mock('../../../common/components/visualization_actions/lens_embeddable', () => { - return { - LensEmbeddable: () =>
, - }; -}); +jest.mock('../../../common/components/visualization_actions/visualization_embeddable'); describe('MetricEmbeddable', () => { const testProps = { @@ -58,7 +54,7 @@ describe('MetricEmbeddable', () => { }); it('render embeddables', () => { - expect(res.getAllByTestId('embeddable-metric')).toHaveLength(2); + expect(res.getAllByTestId('visualization-embeddable')).toHaveLength(2); }); it('render titles', () => { diff --git a/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.tsx b/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.tsx index d8e7ffbd6028..487d6775b496 100644 --- a/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.tsx +++ b/x-pack/plugins/security_solution/public/explore/components/stat_items/metric_embeddable.tsx @@ -8,7 +8,7 @@ import { EuiFlexGroup, EuiIcon } from '@elastic/eui'; import React from 'react'; import { FlexItem, MetricItem, StatValue } from './utils'; import type { MetricStatItem } from './types'; -import { LensEmbeddable } from '../../../common/components/visualization_actions/lens_embeddable'; +import { VisualizationEmbeddable } from '../../../common/components/visualization_actions/visualization_embeddable'; export interface MetricEmbeddableProps { fields: MetricStatItem[]; @@ -47,10 +47,10 @@ const MetricEmbeddableComponent = ({ {field.lensAttributes && (
- ({ useIsExperimentalFeatureEnabled: jest.fn(), })); -jest.mock('../../../common/components/visualization_actions/lens_embeddable'); +jest.mock('../../../common/components/visualization_actions/visualization_embeddable'); const mockSetToggle = jest.fn(); const mockUseIsExperimentalFeatureEnabled = useIsExperimentalFeatureEnabled as jest.Mock; @@ -252,7 +252,7 @@ describe('Stat Items Component', () => { }); test('renders Lens Embeddables', () => { - expect(wrapper.find('[data-test-subj="lens-embeddable"]').length).toEqual(4); + expect(wrapper.find('[data-test-subj="visualization-embeddable"]').length).toEqual(4); }); }); }); diff --git a/x-pack/plugins/security_solution/public/explore/components/stat_items/stat_items.tsx b/x-pack/plugins/security_solution/public/explore/components/stat_items/stat_items.tsx index bc51940c4fe2..1de63dff0490 100644 --- a/x-pack/plugins/security_solution/public/explore/components/stat_items/stat_items.tsx +++ b/x-pack/plugins/security_solution/public/explore/components/stat_items/stat_items.tsx @@ -27,7 +27,7 @@ import { areachartConfigs, barchartConfigs, FlexItem, ChartHeight } from './util import { Metric } from './metric'; import { MetricEmbeddable } from './metric_embeddable'; import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; -import { LensEmbeddable } from '../../../common/components/visualization_actions/lens_embeddable'; +import { VisualizationEmbeddable } from '../../../common/components/visualization_actions/visualization_embeddable'; export const StatItemsComponent = React.memo( ({ @@ -112,11 +112,11 @@ export const StatItemsComponent = React.memo( {enableBarChart && ( {isChartEmbeddablesEnabled && barChartLensAttributes ? ( - @@ -140,11 +140,11 @@ export const StatItemsComponent = React.memo( <> {isChartEmbeddablesEnabled && areaChartLensAttributes ? ( - diff --git a/x-pack/plugins/security_solution/public/overview/components/detection_response/alerts_by_status/alerts_by_status.tsx b/x-pack/plugins/security_solution/public/overview/components/detection_response/alerts_by_status/alerts_by_status.tsx index 0c67de7a2b07..131330f731a8 100644 --- a/x-pack/plugins/security_solution/public/overview/components/detection_response/alerts_by_status/alerts_by_status.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/detection_response/alerts_by_status/alerts_by_status.tsx @@ -16,7 +16,6 @@ import { useIsWithinMinBreakpoint, } from '@elastic/eui'; import React, { useCallback, useMemo } from 'react'; -import type { ShapeTreeNode } from '@elastic/charts'; import type { Severity } from '@kbn/securitysolution-io-ts-alerting-types'; import styled from 'styled-components'; @@ -213,8 +212,8 @@ export const AlertsByStatus = ({ const totalAlertsCount = isDonutChartEmbeddablesEnabled ? visualizationTotalAlerts : totalAlerts; - const fillColor: FillColor = useCallback((d: ShapeTreeNode) => { - return chartConfigs.find((cfg) => cfg.label === d.dataName)?.color ?? emptyDonutColor; + const fillColor: FillColor = useCallback((dataName) => { + return chartConfigs.find((cfg) => cfg.label === dataName)?.color ?? emptyDonutColor; }, []); return ( diff --git a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/common/risk_score_donut_chart.tsx b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/common/risk_score_donut_chart.tsx index ee92ff1dde3f..2ee81c42949b 100644 --- a/x-pack/plugins/security_solution/public/overview/components/entity_analytics/common/risk_score_donut_chart.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/entity_analytics/common/risk_score_donut_chart.tsx @@ -7,7 +7,6 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React from 'react'; -import type { ShapeTreeNode } from '@elastic/charts'; import styled from 'styled-components'; import type { SeverityCount } from '../../../../explore/components/risk_score/severity/types'; import { useRiskDonutChartData } from './use_risk_donut_chart_data'; @@ -22,8 +21,10 @@ import type { RiskSeverity } from '../../../../../common/search_strategy'; const DONUT_HEIGHT = 120; -const fillColor: FillColor = (d: ShapeTreeNode) => { - return RISK_SEVERITY_COLOUR[d.dataName as RiskSeverity] ?? emptyDonutColor; +const fillColor: FillColor = (dataName) => { + return Object.hasOwn(RISK_SEVERITY_COLOUR, dataName) + ? RISK_SEVERITY_COLOUR[dataName as RiskSeverity] + : emptyDonutColor; }; const DonutContainer = styled(EuiFlexItem)` diff --git a/x-pack/plugins/synthetics/public/apps/synthetics/components/step_details_page/step_timing_breakdown/network_timings_donut.tsx b/x-pack/plugins/synthetics/public/apps/synthetics/components/step_details_page/step_timing_breakdown/network_timings_donut.tsx index dd9a4312bf64..b51f5884f59e 100644 --- a/x-pack/plugins/synthetics/public/apps/synthetics/components/step_details_page/step_timing_breakdown/network_timings_donut.tsx +++ b/x-pack/plugins/synthetics/public/apps/synthetics/components/step_details_page/step_timing_breakdown/network_timings_donut.tsx @@ -80,7 +80,7 @@ export const NetworkTimingsDonut = () => { groupByRollup: (d: Datum) => d.label, nodeLabel: (d: Datum) => d, shape: { - fillColor: (d: Datum, index: number) => { + fillColor: (dataName, index) => { return (theme.eui as unknown as Record)[ `euiColorVis${index + 1}` ]; diff --git a/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/donut_chart.tsx b/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/donut_chart.tsx index 457819c5448c..ee31aad9a349 100644 --- a/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/donut_chart.tsx +++ b/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/donut_chart.tsx @@ -75,8 +75,8 @@ export const DonutChart = ({ height, down, up }: DonutChartProps) => { groupByRollup: (d: Datum) => d.label, nodeLabel: (d: Datum) => d, shape: { - fillColor: (d: Datum) => { - return d.dataName === 'Down' ? danger : gray; + fillColor: (dataName) => { + return dataName === 'Down' ? danger : gray; }, }, }, diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 7114cba4ea49..64d2a3a35e67 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -11248,7 +11248,6 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textClassificationModel": "De plus, la valeur_prévue (predicted_value) sera copiée sur \"{fieldName}\" si la probabilité de prédiction (prediction_probability) est supérieure à {probabilityThreshold}", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textEmbeddingModel": "De plus, la valeur_prévue (predicted_value) sera copiée sur \"{fieldName}\"", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.userProvided": "Cela attribue un nom au champ qui contient le résultat d'inférence. \"ml.inference\" sera ajouté comme préfixe, ml.inference.{targetField}", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.example.code": "Utiliser le format JSON : {code}", "xpack.enterpriseSearch.content.indices.pipelines.ingestPipelinesCard.customDescription": "Pipeline d'ingestion personnalisé pour {indexName}", "xpack.enterpriseSearch.content.indices.pipelines.ingestPipelinesCard.processorsDescription": "{processorsCount} processeurs", "xpack.enterpriseSearch.content.indices.pipelines.mlInferencePipelines.subtitleAPIindex": "Les pipelines d'inférence seront exécutés en tant que processeurs à partir du pipeline d'ingestion Enterprise Search. Afin d'utiliser ces pipelines dans des index basés sur des API, vous devrez référencer le pipeline {pipelineName} dans vos requêtes d'API.", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 11c83f035f1d..046de43e22aa 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -11247,7 +11247,6 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textClassificationModel": "さらに、predicted_probabilityが{probabilityThreshold}より大きい場合、predicted_valueは\"{fieldName}\"にコピーされます。", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textEmbeddingModel": "さらにpredicted_valueは\"{fieldName}\"にコピーされます。", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.userProvided": "これは、推論結果を保持するフィールドの名前を指定します。プレフィックスに\"ml.inference\"、ml.inference.{targetField}が付きます", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.example.code": "JSONフォーマットを使用: {code}", "xpack.enterpriseSearch.content.indices.pipelines.ingestPipelinesCard.customDescription": "{indexName}のカスタムインジェストパイプライン", "xpack.enterpriseSearch.content.indices.pipelines.ingestPipelinesCard.processorsDescription": "{processorsCount}プロセッサー", "xpack.enterpriseSearch.content.indices.pipelines.mlInferencePipelines.subtitleAPIindex": "推論パイプラインは、エンタープライズサーチインジェストパイプラインからのプロセッサーとして実行されます。APIベースのインデックスでこれらのパイプラインを使用するには、APIリクエストで{pipelineName}パイプラインを参照する必要があります。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index aa75fdff53bd..1efbbedc2d2a 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -11248,7 +11248,6 @@ "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textClassificationModel": "此外，如果 prediction_probability 大于 {probabilityThreshold}，则会将 predicted_value 复制到“{fieldName}”", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.textEmbeddingModel": "此外，还会将 predicted_value 复制到“{fieldName}”", "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.fields.targetField.helpText.userProvided": "这会命名存放推理结果的字段。它将加有“ml.inference”、ml.inference.{targetField} 前缀", - "xpack.enterpriseSearch.content.indices.pipelines.addInferencePipelineModal.steps.test.example.code": "使用 JSON 格式：{code}", "xpack.enterpriseSearch.content.indices.pipelines.ingestPipelinesCard.customDescription": "{indexName} 的定制采集管道", "xpack.enterpriseSearch.content.indices.pipelines.ingestPipelinesCard.processorsDescription": "{processorsCount} 个处理器", "xpack.enterpriseSearch.content.indices.pipelines.mlInferencePipelines.subtitleAPIindex": "推理管道将作为处理器从 Enterprise Search 采集管道中运行。要在基于 API 的索引上使用这些管道，您需要在 API 请求中引用 {pipelineName} 管道。", diff --git a/x-pack/test/accessibility/config.ts b/x-pack/test/accessibility/config.ts index 191a8b403c41..140d6fe6de10 100644 --- a/x-pack/test/accessibility/config.ts +++ b/x-pack/test/accessibility/config.ts @@ -23,7 +23,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./apps/grok_debugger'), require.resolve('./apps/search_profiler'), require.resolve('./apps/painless_lab'), - require.resolve('./apps/uptime'), + // https://github.com/elastic/kibana/issues/153601 + // require.resolve('./apps/uptime'), require.resolve('./apps/spaces'), require.resolve('./apps/advanced_settings'), require.resolve('./apps/dashboard_panel_options'), @@ -32,8 +33,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./apps/roles'), require.resolve('./apps/ingest_node_pipelines'), require.resolve('./apps/index_lifecycle_management'), - require.resolve('./apps/ml'), - require.resolve('./apps/transform'), + // https://github.com/elastic/kibana/issues/153596 + // https://github.com/elastic/kibana/issues/153592 + // require.resolve('./apps/ml'), + // require.resolve('./apps/transform'), require.resolve('./apps/lens'), require.resolve('./apps/upgrade_assistant'), require.resolve('./apps/canvas'), @@ -45,8 +48,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // Please make sure that the remote clusters, snapshot and restore and // CCR tests stay in that order. Their execution fails if rearranged. require.resolve('./apps/remote_clusters'), - require.resolve('./apps/snapshot_and_restore'), - require.resolve('./apps/cross_cluster_replication'), + // https://github.com/elastic/kibana/issues/153788 + // require.resolve('./apps/snapshot_and_restore'), + // https://github.com/elastic/kibana/issues/153599 + // require.resolve('./apps/cross_cluster_replication'), require.resolve('./apps/reporting'), require.resolve('./apps/enterprise_search'), require.resolve('./apps/license_management'), diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts index 89ba33449ec1..ee8f851489e7 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts @@ -74,9 +74,32 @@ export default function ({ getService }: FtrProviderContext) { expect(res.cspm.status).to.be('not-deployed'); expect(res.kspm.status).to.be('not-installed'); + expect(res.vuln_mgmt.status).to.be('not-installed'); expect(res.cspm.healthyAgents).to.be(0); expect(res.cspm.installedPackagePolicies).to.be(1); }); + + it(`Should return not-deployed when vuln_mgmt is not installed`, async () => { + await createPackagePolicy( + supertest, + agentPolicyId, + 'vuln_mgmt', + 'cloudbeat/vuln_mgmt_aws', + 'aws', + 'vuln_mgmt' + ); + + const { body: res }: { body: CspSetupStatus } = await supertest + .get(`/internal/cloud_security_posture/status`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + expect(res.cspm.status).to.be('not-installed'); + expect(res.kspm.status).to.be('not-installed'); + expect(res.vuln_mgmt.status).to.be('not-deployed'); + expect(res.vuln_mgmt.healthyAgents).to.be(0); + expect(res.vuln_mgmt.installedPackagePolicies).to.be(1); + }); }); } @@ -88,6 +111,26 @@ async function createPackagePolicy( deployment: string, posture: string ) { + const version = posture === 'kspm' || posture === 'cspm' ? '1.2.8' : '1.3.0-preview2'; + const title = 'Security Posture Management'; + const streams = [ + { + enabled: false, + data_stream: { + type: 'logs', + dataset: 'cloud_security_posture.vulnerabilities', + }, + }, + ]; + + const inputTemplate = { + enabled: true, + type: input, + policy_template: policyTemplate, + }; + + const inputs = posture === 'vuln_mgmt' ? { ...inputTemplate, streams } : { ...inputTemplate }; + const { body: postPackageResponse } = await supertest .post(`/api/fleet/package_policies`) .set('kbn-xsrf', 'xxxx') @@ -98,17 +141,11 @@ async function createPackagePolicy( namespace: 'default', policy_id: agentPolicyId, enabled: true, - inputs: [ - { - enabled: true, - type: input, - policy_template: policyTemplate, - }, - ], + inputs: [inputs], package: { name: 'cloud_security_posture', - title: 'Kubernetes Security Posture Management', - version: '1.2.8', + title, + version, }, vars: { deployment: { diff --git a/x-pack/test/cloud_security_posture_functional/pages/findings.ts b/x-pack/test/cloud_security_posture_functional/pages/findings.ts index 010bb788b8c7..99a399db9739 100644 --- a/x-pack/test/cloud_security_posture_functional/pages/findings.ts +++ b/x-pack/test/cloud_security_posture_functional/pages/findings.ts @@ -170,7 +170,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); }); - describe('Table Sort', () => { + // FLAKY: https://github.com/elastic/kibana/issues/152913 + describe.skip('Table Sort', () => { type SortingMethod = (a: string, b: string) => number; type SortDirection = 'asc' | 'desc'; // Sort by lexical order will sort by the first character of the string (case-sensitive) diff --git a/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js b/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js index c8f1835a1164..d4057948253f 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js @@ -44,13 +44,14 @@ export default function ({ getPageObjects, getService }) { delete searchParams.token; expect(searchParams).to.eql({ - buffer: 4, + buffer: '4', + executionContextId: '78116c8c-fd2a-11ea-adc1-0242ac120002', geometryFieldName: 'geo.coordinates', hasLabels: 'false', index: 'logstash-*', - gridPrecision: 8, + gridPrecision: '8', renderAs: 'grid', - requestBody: `(_source:(excludes:!()),aggs:(max_of_bytes:(max:(field:bytes))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:'doc[!'@timestamp!'].value.getHour()'))),size:0,stored_fields:!('*'))`, + requestBody: `(_source%3A(excludes%3A!())%2Caggs%3A(max_of_bytes%3A(max%3A(field%3Abytes)))%2Cfields%3A!((field%3A'%40timestamp'%2Cformat%3Adate_time)%2C(field%3A'relatedContent.article%3Amodified_time'%2Cformat%3Adate_time)%2C(field%3A'relatedContent.article%3Apublished_time'%2Cformat%3Adate_time)%2C(field%3Autc_time%2Cformat%3Adate_time))%2Cquery%3A(bool%3A(filter%3A!((range%3A('%40timestamp'%3A(format%3Astrict_date_optional_time%2Cgte%3A'2015-09-20T00%3A00%3A00.000Z'%2Clte%3A'2015-09-20T01%3A00%3A00.000Z'))))%2Cmust%3A!()%2Cmust_not%3A!()%2Cshould%3A!()))%2Cruntime_mappings%3A()%2Cscript_fields%3A(hour_of_day%3A(script%3A(lang%3Apainless%2Csource%3A'doc%5B!'%40timestamp!'%5D.value.getHour()')))%2Csize%3A0%2Cstored_fields%3A!('*'))`, }); //Should correctly load meta for style-rule (sigma is set to 1, opacity to 1) diff --git a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js index 80a7f8abc5d3..744547cfaf5e 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js @@ -49,12 +49,13 @@ export default function ({ getPageObjects, getService }) { delete searchParams.token; expect(searchParams).to.eql({ - buffer: 4, + buffer: '4', + executionContextId: 'bff99716-e3dc-11ea-87d0-0242ac130003', geometryFieldName: 'geometry', hasLabels: 'false', index: 'geo_shapes*', requestBody: - '(_source:!f,fields:!(prop1),query:(bool:(filter:!(),must:!(),must_not:!(),should:!())),runtime_mappings:(),size:10001)', + '(_source%3A!f%2Cfields%3A!(prop1)%2Cquery%3A(bool%3A(filter%3A!()%2Cmust%3A!()%2Cmust_not%3A!()%2Cshould%3A!()))%2Cruntime_mappings%3A()%2Csize%3A10001)', }); }); diff --git a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts index d7560a9533f3..ac3be7d1814a 100644 --- a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts +++ b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts @@ -11,37 +11,56 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const commonScreenshots = getService('commonScreenshots'); const screenshotDirectories = ['response_ops_docs', 'stack_alerting']; const pageObjects = getPageObjects(['common', 'header']); + const actions = getService('actions'); const rules = getService('rules'); const testSubjects = getService('testSubjects'); + const ruleName = 'kibana sites - low bytes'; describe('list view', function () { let ruleId: string; - const indexThresholdRule = { - consumer: 'alerts', - name: 'my rule', - notifyWhen: 'onActionGroupChange', - params: { - index: ['.test-index'], - timeField: '@timestamp', - aggType: 'count', - groupBy: 'all', - timeWindowSize: 5, - timeWindowUnit: 'd', - thresholdComparator: '>', - threshold: [1000], - }, - ruleTypeId: '.index-threshold', - schedule: { interval: '1m' }, - tags: [], - actions: [], - }; - + let connectorId: string; before(async () => { - ({ id: ruleId } = await rules.api.createRule(indexThresholdRule)); + ({ id: connectorId } = await actions.api.createConnector({ + name: 'my-server-log-connector', + config: {}, + secrets: {}, + connectorTypeId: '.server-log', + })); + ({ id: ruleId } = await rules.api.createRule({ + consumer: 'alerts', + name: ruleName, + notifyWhen: 'onActionGroupChange', + params: { + index: ['kibana_sample_data_logs'], + timeField: '@timestamp', + aggType: 'sum', + aggField: 'bytes', + groupBy: 'top', + termField: 'host.keyword', + termSize: 4, + timeWindowSize: 24, + timeWindowUnit: 'h', + thresholdComparator: '>', + threshold: [4200], + }, + ruleTypeId: '.index-threshold', + schedule: { interval: '1m' }, + actions: [ + { + group: 'threshold met', + id: connectorId, + params: { + level: 'info', + message: 'Test', + }, + }, + ], + })); }); after(async () => { await rules.api.deleteRule(ruleId); + await actions.api.deleteConnector(connectorId); }); it('rules list screenshot', async () => { @@ -71,5 +90,29 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await snoozeBadge.click(); await commonScreenshots.takeScreenshot('snooze-panel', screenshotDirectories, 1400, 1024); }); + + it('rule detail screenshots', async () => { + await pageObjects.common.navigateToApp('triggersActions'); + await pageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.setValue('ruleSearchField', ruleName); + const rulesList = await testSubjects.find('rulesList'); + const alertRule = await rulesList.findByCssSelector(`[title="${ruleName}"]`); + await alertRule.click(); + await pageObjects.header.waitUntilLoadingHasFinished(); + await commonScreenshots.takeScreenshot( + 'rule-details-alerts-active', + screenshotDirectories, + 1400, + 1024 + ); + const actionsButton = await testSubjects.find('ruleActionsButton'); + await actionsButton.click(); + await commonScreenshots.takeScreenshot( + 'rule-details-disabling', + screenshotDirectories, + 1400, + 1024 + ); + }); }); } diff --git a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts index aa5ac66aa9e2..247b25d2be2c 100644 --- a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts +++ b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts @@ -14,7 +14,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const actions = getService('actions'); const testSubjects = getService('testSubjects'); const comboBox = getService('comboBox'); - const es = getService('es'); const testIndex = `test-index`; const indexDocument = `{\n` + @@ -43,19 +42,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('index connector screenshots', async () => { - await es.indices.create({ - index: testIndex, - body: { - mappings: { - properties: { - date_updated: { - type: 'date', - format: 'epoch_millis', - }, - }, - }, - }, - }); await pageObjects.common.navigateToApp('connectors'); await pageObjects.header.waitUntilLoadingHasFinished(); await actions.common.openNewConnectorForm('index'); @@ -71,8 +57,5 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const flyOutCancelButton = await testSubjects.find('euiFlyoutCloseButton'); await flyOutCancelButton.click(); }); - after(async () => { - await es.indices.delete({ index: testIndex }); - }); }); } diff --git a/yarn.lock b/yarn.lock index a136e9b5d784..c72382dced30 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1460,10 +1460,10 @@ dependencies: object-hash "^1.3.0" -"@elastic/charts@53.1.0": - version "53.1.0" - resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-53.1.0.tgz#4ad12f01044b0102e78e6895b7b5f1997d11f3da" - integrity sha512-VcUoU4hl8wZfU5m32qgecgNP9Z3Lq0aIYkFxvwC16nCbkX4AtmVEEcj4Ld62MhjcBVQPNXb/qEPBpggxsdBEOg== +"@elastic/charts@54.0.0": + version "54.0.0" + resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-54.0.0.tgz#97e19c87c94d4282c12440e9d5703048232bc2b8" + integrity sha512-gyAgBrRKRg+QxaOluAy1tJXm3gv95IuZRL+/QMUR7tuAwqfD+Bi3eFUoqMhVJCRmhYJa2iDmLMjb7Mkb7HZJgw== dependencies: "@popperjs/core" "^2.4.0" bezier-easing "^2.1.0" @@ -10661,6 +10661,11 @@ axe-core@^4.2.0: resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.3.5.tgz#78d6911ba317a8262bfee292aeafcc1e04b49cc5" integrity sha512-WKTW1+xAzhMS5dJsxWkliixlO/PqC4VhmO9T4juNYcaTg9jzWiJsou6m5pxWYGfigWbwzJWeFY6z47a+4neRXA== +axe-core@^4.6.1: + version "4.6.1" + resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.6.1.tgz#79cccdee3e3ab61a8f42c458d4123a6768e6fbce" + integrity sha512-lCZN5XRuOnpG4bpMq8v0khrWtUOn+i8lZSb6wHZH56ZfbIEv6XwJV84AAueh9/zi7qPVJ/E4yz6fmsiyOmXR4w== + axios@^0.21.1: version "0.21.4" resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"